@yolk_vat-y/dsh-project-memory 0.1.3

package/src/lazy.js

@@ -0,0 +1,174 @@
+import path from 'node:path'
+import { tmpdir } from 'node:os'
+import { existsSync, readdirSync, readFileSync, statSync } from 'node:fs'
+import { isSupportedCode, isSupportedDoc, memoryRootFor, relativePath, sha256OfFile, storeKey } from './util/fs.js'
+import { buildDocEntries } from './doc-pipeline.js'
+import { scanSymbols } from './symbols.js'
+import { linkEntries } from './link.js'
+import { ProjectMemoryStore, withStoreLock } from './store.js'
+
+const STRONG_MARKERS = ['.git', '.hg', '.svn']
+
+const WEAK_MARKERS = [
+  '.dsh-project-memory',
+  'package.json',
+  'go.mod',
+  'Cargo.toml',
+  'pyproject.toml',
+  'requirements.txt',
+  'pom.xml',
+]
+
+const SOURCE_DIR_NAMES = new Set([
+  'src', 'app', 'lib', 'libs', 'tools', 'include', 'core', 'modules',
+  'scripts', 'components', 'assets', 'utils', 'shared', 'common', 'server', 'client',
+])
+
+function looksLikeProjectRoot(dir) {
+  let hasReadmeFile = false
+  let sourceDirs = 0
+  try {
+    for (const entry of readdirSync(dir, { withFileTypes: true })) {
+      const name = entry.name.toLowerCase()
+      if (entry.isDirectory()) {
+        if (SOURCE_DIR_NAMES.has(name)) sourceDirs++
+      } else if (name.startsWith('readme')) {
+        hasReadmeFile = true
+      }
+    }
+  } catch {
+    return false
+  }
+  return sourceDirs >= 2 || (hasReadmeFile && sourceDirs >= 1)
+}
+
+function sameDir(a, b) {
+  return storeKey(path.resolve(a)) === storeKey(path.resolve(b))
+}
+
+export function findProjectRoot(filePath, ceiling = path.resolve(tmpdir())) {
+  let dir = path.dirname(filePath)
+  for (;;) {
+    if (!sameDir(dir, ceiling)) {
+      for (const marker of STRONG_MARKERS) {
+        if (existsSync(path.join(dir, marker))) return dir
+      }
+    }
+    const parent = path.dirname(dir)
+    if (parent === dir || sameDir(parent, ceiling)) break
+    dir = parent
+  }
+
+  dir = path.dirname(filePath)
+  let best = null
+  for (;;) {
+    if (!sameDir(dir, ceiling)) {
+      for (const marker of WEAK_MARKERS) {
+        if (existsSync(path.join(dir, marker))) return dir
+      }
+      if (!best && looksLikeProjectRoot(dir)) best = dir
+    }
+    const parent = path.dirname(dir)
+    if (parent === dir || sameDir(parent, ceiling)) break
+    dir = parent
+  }
+  return best || path.dirname(filePath)
+}
+
+export async function indexFile(ctx, config, filePath, watchManager = null) {
+  const ext = path.extname(filePath).toLowerCase()
+  if (!isSupportedDoc(ext) && !isSupportedCode(ext)) return false
+  const root = findProjectRoot(filePath)
+  if (!root) return false
+
+  const memoryDir = memoryRootFor(root, config.memoryDir)
+  return withStoreLock(memoryDir, async () => {
+    const store = new ProjectMemoryStore(memoryDir).load()
+    const rel = storeKey(relativePath(root, filePath))
+    const existing = store.fileRecord(rel)
+    let hash
+    let size
+    try {
+      if (isSupportedCode(ext) && config.maxFileSizeMb && statSync(filePath).size > config.maxFileSizeMb * 1024 * 1024) {
+        return false
+      }
+      ;({ hash, size } = await sha256OfFile(filePath))
+    } catch {
+      return false
+    }
+    if (existing && existing.sha256 === hash) return false
+
+    if (watchManager) {
+      watchManager.addRoot(root)
+      store.addWatch(root)
+    }
+
+    let entries
+    if (isSupportedCode(ext)) {
+      entries = scanSymbols(filePath, readFileSync(filePath, 'utf8'))
+      store.markFile(rel, { sha256: hash, size, type: 'code', indexedAt: new Date().toISOString() })
+    } else {
+      entries = await buildDocEntries(ctx.llm, filePath, {
+        chunkChars: config.chunkChars,
+        maxChunks: config.maxChunksPerFile,
+        maxFileSizeMb: config.maxFileSizeMb,
+        maxPdfPages: config.maxPdfPages,
+      })
+      if (entries === null) {
+        store.removeFile(rel)
+        store.save()
+        return false
+      }
+      store.markFile(rel, { sha256: hash, size, type: 'doc', indexedAt: new Date().toISOString() })
+    }
+    store.setEntries(rel, entries)
+    store.save()
+    const links = linkEntries(store)
+    if (links) store.save()
+    return true
+  })
+}
+
+export function codeFirst(paths) {
+  return [...paths].sort((a, b) => {
+    const aCode = isSupportedCode(path.extname(a).toLowerCase()) ? 0 : 1
+    const bCode = isSupportedCode(path.extname(b).toLowerCase()) ? 0 : 1
+    return aCode - bCode
+  })
+}
+
+export function setupLazyIndexing(ctx, config, watchManager = null) {
+  const pending = new Map()
+  let timer = null
+
+  const flush = async () => {
+    timer = null
+    const batch = codeFirst([...pending.keys()])
+    pending.clear()
+    for (const filePath of batch) {
+      try {
+        await indexFile(ctx, config, filePath, watchManager)
+      } catch (err) {
+        console.error(`[dsh-project-memory] lazy index failed for ${filePath}: ${err.message}`)
+      }
+    }
+  }
+
+  const queue = (filePath) => {
+    pending.set(filePath, true)
+    if (!timer) {
+      timer = setTimeout(flush, 300)
+      if (timer.unref) timer.unref()
+    }
+  }
+
+  ctx.on('fs/observed', (target, observation) => {
+    if (!target || !observation || observation.kind !== 'present') return
+    if (typeof target.displayPath !== 'string' || !target.displayPath) return
+    queue(target.displayPath)
+  })
+
+  ctx.effect(() => () => {
+    if (timer) clearTimeout(timer)
+  })
+}

package/src/link.js

@@ -0,0 +1,31 @@
+export function linkEntries(store) {
+  const all = store.allEntries()
+  const symbols = all.filter((e) => e.type === 'symbol')
+  const docs = all.filter((e) => e.type === 'doc')
+  if (!symbols.length || !docs.length) return 0
+
+  const symbolByName = new Map()
+  for (const s of symbols) {
+    const name = s.keywords[0]
+    if (!name || name.length < 3) continue
+    if (!symbolByName.has(name)) symbolByName.set(name, [])
+    symbolByName.get(name).push(s)
+  }
+
+  let links = 0
+  for (const doc of docs) {
+    const linked = new Set()
+    const haystack = `${doc.title || ''} ${doc.summary || ''} ${doc.keywords ? doc.keywords.join(' ') : ''}`.toLowerCase()
+    for (const [name, syms] of symbolByName) {
+      if (haystack.includes(name.toLowerCase())) {
+        for (const s of syms) {
+          const before = linked.size
+          linked.add(s.id)
+          if (linked.size > before) links++
+        }
+      }
+    }
+    doc.linkedSymbols = linked.size ? [...linked] : undefined
+  }
+  return links
+}

package/src/llm.js

@@ -0,0 +1,130 @@
+import { BlockAssembler, createUserMessage } from '@deepseek-ai/dsh-llm'
+import { tokenize } from './util/search.js'
+
+function systemMessage(text) {
+  return { role: 'system', content: [{ type: 'text', text }] }
+}
+
+function textOf(message) {
+  const blocks = message.content || []
+  return blocks
+    .filter((b) => b.type === 'text' && typeof b.text === 'string')
+    .map((b) => b.text)
+    .join('\n')
+}
+
+const MAX_SUMMARY = 300
+
+export function summarizeText(text, max = MAX_SUMMARY) {
+  const flat = String(text || '').replace(/\s+/g, ' ').trim()
+  if (!flat) return ''
+  if (flat.length <= max) return flat
+  const clip = max - 1
+  const clipped = flat.slice(0, clip)
+  const lastBreak = Math.max(clipped.lastIndexOf('。'), clipped.lastIndexOf('.'), clipped.lastIndexOf(';'))
+  return lastBreak > clip * 0.4 ? clipped.slice(0, lastBreak + 1) : clipped + '…'
+}
+
+export async function chatText(llm, system, user, { timeoutMs = 120000 } = {}) {
+  const assembler = new BlockAssembler()
+  const controller = new AbortController()
+  const timer = setTimeout(() => controller.abort(), timeoutMs)
+  try {
+    for await (const chunk of llm.stream({
+      messages: [systemMessage(system), createUserMessage({ content: [{ type: 'text', text: user }] })],
+      signal: controller.signal,
+    })) {
+      assembler.push(chunk)
+    }
+  } finally {
+    clearTimeout(timer)
+  }
+  return textOf(assembler.message())
+}
+
+export function parseStructuredJson(text) {
+  return parseJson(text, (parsed) => parsed && typeof parsed === 'object' && !Array.isArray(parsed))
+}
+
+export function parseJsonArray(text) {
+  return parseJson(text, (parsed) => Array.isArray(parsed))
+}
+
+function parseJson(text, validate) {
+  if (!text) return null
+  let candidate = text.trim()
+  const fence = candidate.match(/```(?:json)?\s*([\s\S]*?)```/i)
+  if (fence) candidate = fence[1].trim()
+  const first = candidate.indexOf('[')
+  const firstObj = candidate.indexOf('{')
+  let start = firstObj
+  if (first >= 0 && (firstObj < 0 || first < firstObj)) start = first
+  const end = candidate.lastIndexOf(start === first ? ']' : '}')
+  if (start >= 0 && end > start) {
+    candidate = candidate.slice(start, end + 1)
+  }
+  try {
+    const parsed = JSON.parse(candidate)
+    if (validate(parsed)) return parsed
+  } catch {
+    // fall through
+  }
+  return null
+}
+
+export async function expandQuery(llm, query, count = 6) {
+  if (!llm) return [query]
+  const system =
+    'You are a search-query expander for a codebase/document memory search engine. ' +
+    'Given a user query, return a STRICT JSON array of alternative search queries that ' +
+    'capture the same intent with different words: synonyms, English/Chinese equivalents, ' +
+    'code identifier guesses, and narrower/longer phrasings. Include the original query first. ' +
+    'Output only the JSON array of strings, no fences, no commentary.'
+  try {
+    const raw = await chatText(llm, system, `Query: "${query}"\n\nReturn the JSON array.`)
+    const parsed = parseJsonArray(raw)
+    if (Array.isArray(parsed) && parsed.length) {
+      const variants = parsed.map(String).filter((s) => s.trim()).slice(0, count)
+      if (variants.length) return variants
+    }
+  } catch {
+    // fall through to the raw query
+  }
+  return [query]
+}
+
+export async function extractDocEntry(llm, chunk, sourcePath) {
+  const system =
+    'You are a project-documentation indexer. Given a chunk of a project document, ' +
+    'return a STRICT JSON object with exactly three fields: ' +
+    '"title" (short section title, string), "summary" (2-4 sentence dense summary of what this section covers, ' +
+    'mentioning concrete names, decisions and constraints), "keywords" (array of 5-10 searchable strings: ' +
+    'cover the document\'s own language AND English equivalents, so a query in either language can match). ' +
+    'Do not include markdown fences, do not add commentary, output only the JSON object.'
+
+  const user =
+    `Document: ${sourcePath}\nSection: ${chunk.title || '(untitled)'}\n\n` +
+    `Content:\n${chunk.text.slice(0, 6000)}\n\nReturn the JSON object.`
+
+  const fallback = () => ({
+    title: chunk.title || sourcePath,
+    summary: summarizeText(chunk.text),
+    keywords: tokenize(chunk.title).slice(0, 5),
+  })
+
+  if (!llm) return fallback()
+
+  try {
+    const raw = await chatText(llm, system, user)
+    const parsed = parseStructuredJson(raw)
+    if (!parsed || typeof parsed.summary !== 'string' || !parsed.summary.trim()) return fallback()
+    const kw = Array.isArray(parsed.keywords) ? parsed.keywords.map(String).filter((k) => k).slice(0, 8) : []
+    return {
+      title: typeof parsed.title === 'string' && parsed.title.trim() ? parsed.title.trim() : chunk.title || sourcePath,
+      summary: summarizeText(parsed.summary),
+      keywords: kw.length ? kw : tokenize(chunk.title).slice(0, 5),
+    }
+  } catch {
+    return fallback()
+  }
+}

package/src/parsers/pdfjs-parser.js

@@ -0,0 +1,119 @@
+import { readFile } from 'node:fs/promises'
+import { getDocument, GlobalWorkerOptions, OPS } from 'pdfjs-dist/legacy/build/pdf.mjs'
+
+const PDFJS_OPTIONS = {
+  useSystemFonts: true,
+  isEvalSupported: false,
+  useWorkerFetch: false,
+  useWorker: false,
+}
+
+function buildMarkdown(pages) {
+  return pages
+    .map((p) => (pages.length > 1 ? `## Page ${p.page}\n\n${p.text}` : p.text))
+    .join('\n\n')
+}
+
+function extractPageText(items) {
+  const lines = []
+  let currentY = null
+  let currentLine = ''
+  for (const item of items) {
+    if (!item.str || !item.str.trim()) continue
+    const y = item.transform ? item.transform[5] : 0
+    if (currentY === null || Math.abs(y - currentY) < 2) {
+      currentY = y
+      currentLine += (currentLine && !currentLine.endsWith(' ') ? ' ' : '') + item.str
+    } else {
+      lines.push(currentLine)
+      currentY = y
+      currentLine = item.str
+    }
+  }
+  if (currentLine) lines.push(currentLine)
+  return lines.join('\n')
+}
+
+export async function parsePdf(filePath, { pages = null, maxPages = 1000, backend = 'pdfjs', collectLayoutStats = false } = {}) {
+  const data = new Uint8Array(await readFile(filePath))
+  const loadingTask = getDocument({ data, ...PDFJS_OPTIONS })
+  const doc = await loadingTask.promise
+
+  try {
+    const total = doc.numPages
+    if (total > maxPages) {
+      throw new Error(`PDF has ${total} pages, over the maxPages limit of ${maxPages}`)
+    }
+
+    const pageList = []
+    let imageCount = 0
+    let sampledPages = 0
+    for (let n = 1; n <= total; n++) {
+      if (pages && !pages.has(n)) continue
+      const page = await doc.getPage(n)
+      const content = await page.getTextContent()
+      pageList.push({ page: n, text: extractPageText(content.items) })
+
+      if (collectLayoutStats && sampledPages < 5) {
+        const opList = await page.getOperatorList()
+        for (const fn of opList.fnArray) {
+          if (fn === OPS.paintImageXObject || fn === OPS.paintInlineImageXObject) imageCount++
+        }
+        sampledPages++
+      }
+    }
+    if (pageList.length === 0) {
+      throw new Error('No pages matched the selection, or the PDF contains no extractable text')
+    }
+
+    const totalTextChars = pageList.reduce((sum, p) => sum + p.text.length, 0)
+    return {
+      pageCount: pageList.length,
+      pages: pageList,
+      markdown: buildMarkdown(pageList),
+      backend,
+      stats: collectLayoutStats
+        ? {
+            totalTextChars,
+            avgImagesPerPage: sampledPages ? imageCount / sampledPages : 0,
+          }
+        : undefined,
+    }
+  } finally {
+    await loadingTask.destroy()
+  }
+}
+
+export async function parsePdfInfo(filePath, maxPages = 1000) {
+  const data = new Uint8Array(await readFile(filePath))
+  const loadingTask = getDocument({ data, ...PDFJS_OPTIONS })
+  const doc = await loadingTask.promise
+
+  try {
+    if (doc.numPages > maxPages) {
+      throw new Error(`PDF has ${doc.numPages} pages, over the maxPages limit of ${maxPages}`)
+    }
+    let meta = {}
+    try {
+      meta = await doc.getMetadata()
+    } catch {
+      // metadata is optional
+    }
+    const info = meta.info || {}
+    return {
+      pageCount: doc.numPages,
+      title: info.Title ?? null,
+      author: info.Author ?? null,
+      subject: info.Subject ?? null,
+      created: info.CreationDate ?? null,
+      modified: info.ModDate ?? null,
+      encrypted: doc.isEncrypted,
+    }
+  } finally {
+    await loadingTask.destroy()
+  }
+}
+
+export function configurePdfjsWorker(workerSrc) {
+  GlobalWorkerOptions.workerSrc = workerSrc
+}

package/src/store.js

@@ -0,0 +1,207 @@
+import { mkdirSync, readFileSync, readdirSync, renameSync, statSync, unlinkSync, writeFileSync } from 'node:fs'
+import { randomUUID } from 'node:crypto'
+import path from 'node:path'
+import { rankEntries, rankExperience, tokenize } from './util/search.js'
+
+const INDEX_FILE = 'index.json'
+const ENTRIES_FILE = 'entries.json'
+const EXPERIENCE_FILE = 'experience.json'
+const WATCH_FILE = 'watch.json'
+
+const dirLocks = new Map()
+
+export async function withStoreLock(memoryDir, fn) {
+  const key = path.resolve(memoryDir)
+  const prev = dirLocks.get(key) || Promise.resolve()
+  let release
+  const cur = new Promise((resolve) => {
+    release = resolve
+  })
+  const chain = prev.then(() => cur)
+  dirLocks.set(key, chain)
+  await prev
+  try {
+    return await fn()
+  } finally {
+    release()
+    if (dirLocks.get(key) === chain) dirLocks.delete(key)
+  }
+}
+
+function loadJson(filePath, fallback) {
+  try {
+    return JSON.parse(readFileSync(filePath, 'utf8'))
+  } catch {
+    return fallback
+  }
+}
+
+function writeJsonAtomic(filePath, data) {
+  const tmp = `${filePath}.${process.pid}.tmp`
+  try {
+    writeFileSync(tmp, JSON.stringify(data))
+    renameSync(tmp, filePath)
+  } catch (err) {
+    try {
+      unlinkSync(tmp)
+    } catch {
+      // tmp already gone (rename succeeded) or undeletable; nothing to do
+    }
+    throw err
+  }
+}
+
+export class ProjectMemoryStore {
+  constructor(memoryDir) {
+    this.dir = memoryDir
+    this.files = {}
+    this.entries = {}
+    this.experience = []
+    this.watchlist = []
+  }
+
+  load() {
+    const index = loadJson(path.join(this.dir, INDEX_FILE), {})
+    this.files = index.files || {}
+    this.entries = loadJson(path.join(this.dir, ENTRIES_FILE), {})
+    this.experience = loadJson(path.join(this.dir, EXPERIENCE_FILE), [])
+    this.watchlist = loadJson(path.join(this.dir, WATCH_FILE), [])
+    return this
+  }
+
+  cleanStaleTmp() {
+    let entries
+    try {
+      entries = readdirSync(this.dir)
+    } catch {
+      return
+    }
+    const now = Date.now()
+    for (const name of entries) {
+      if (!name.endsWith('.tmp')) continue
+      try {
+        if (now - statSync(path.join(this.dir, name)).mtimeMs > 60000) unlinkSync(path.join(this.dir, name))
+      } catch {
+        // already gone or locked; skip
+      }
+    }
+  }
+
+  save() {
+    mkdirSync(this.dir, { recursive: true })
+    this.cleanStaleTmp()
+    writeJsonAtomic(path.join(this.dir, INDEX_FILE), { version: 1, files: this.files })
+    writeJsonAtomic(path.join(this.dir, ENTRIES_FILE), this.entries)
+    writeJsonAtomic(path.join(this.dir, EXPERIENCE_FILE), this.experience)
+    writeJsonAtomic(path.join(this.dir, WATCH_FILE), this.watchlist)
+  }
+
+  addWatch(root) {
+    if (!this.watchlist.includes(root)) {
+      this.watchlist.push(root)
+      return true
+    }
+    return false
+  }
+
+  fileRecord(relPath) {
+    return this.files[relPath]
+  }
+
+  markFile(relPath, record) {
+    this.files[relPath] = record
+  }
+
+  setEntries(relPath, entries) {
+    if (entries.length) {
+      this.entries[relPath] = entries
+    } else {
+      delete this.entries[relPath]
+    }
+  }
+
+  removeFile(relPath) {
+    delete this.files[relPath]
+    delete this.entries[relPath]
+  }
+
+  allEntries() {
+    const out = []
+    for (const list of Object.values(this.entries)) {
+      for (const entry of list) out.push(entry)
+    }
+    return out
+  }
+
+  searchEntries(query, limit = 8) {
+    return rankEntries(this.allEntries(), query, limit)
+  }
+
+  addExperience({ problem, solution, sourceFile }) {
+    const existing = this.findSupersede(problem)
+    const now = new Date().toISOString()
+    if (existing) {
+      existing.problem = problem
+      existing.solution = solution
+      if (sourceFile) existing.sourceFile = sourceFile
+      existing.updatedAt = now
+      return { id: existing.id, superseded: true }
+    }
+    const id = randomUUID()
+    this.experience.push({ id, problem, solution, sourceFile, createdAt: now, updatedAt: now })
+    this.pruneExperience()
+    return { id, superseded: false }
+  }
+
+  pruneExperience() {
+    const indexedFileCount = Object.keys(this.files).length
+    const max = Math.max(100, Math.min(2000, indexedFileCount * 2))
+    if (this.experience.length <= max) return 0
+    const sorted = [...this.experience].sort((a, b) => (a.updatedAt < b.updatedAt ? -1 : 1))
+    const victims = new Set(sorted.slice(0, this.experience.length - max).map((e) => e.id))
+    this.experience = this.experience.filter((e) => !victims.has(e.id))
+    return victims.size
+  }
+
+  findSupersede(problem) {
+    const tokens = tokenize(problem)
+    if (!tokens.length) return null
+    let best = null
+    let bestOverlap = 0
+    for (const item of this.experience) {
+      const itemTokens = tokenize(item.problem)
+      const overlap = itemTokens.filter((t) => tokens.includes(t)).length
+      const base = Math.min(tokens.length, itemTokens.length)
+      if (base && overlap / base >= 0.6 && overlap > bestOverlap) {
+        best = item
+        bestOverlap = overlap
+      }
+    }
+    return best
+  }
+
+  removeExperience(idOrQuery) {
+    const before = this.experience.length
+    if (idOrQuery && this.experience.some((item) => item.id === idOrQuery)) {
+      this.experience = this.experience.filter((item) => item.id !== idOrQuery)
+    } else {
+      const tokens = tokenize(idOrQuery)
+      if (!tokens.length) return 0
+      this.experience = this.experience.filter((item) => {
+        const itemTokens = tokenize(`${item.problem} ${item.solution}`)
+        if (!itemTokens.length) return true
+        const overlap = itemTokens.filter((t) => tokens.includes(t)).length
+        return overlap / Math.min(tokens.length, itemTokens.length) < 0.5
+      })
+    }
+    return before - this.experience.length
+  }
+
+  stats() {
+    return {
+      files: Object.keys(this.files).length,
+      entries: this.allEntries().length,
+      experience: this.experience.length,
+    }
+  }
+}