@yolk_vat-y/dsh-project-memory 0.1.3

package/src/util/search.js

@@ -0,0 +1,138 @@
+export const CJK_RANGE =
+  /[\u3400-\u9fff\uf900-\ufaff\u3040-\u309f\u30a0-\u30ff\uac00-\ud7af]/
+
+export function tokenizeRaw(text) {
+  if (!text) return []
+  const lower = text.toLowerCase()
+  const tokens = []
+  let cjkRun = ''
+  const flushCjk = () => {
+    if (cjkRun.length === 1) {
+      tokens.push(cjkRun)
+    } else {
+      for (let i = 0; i < cjkRun.length - 1; i++) tokens.push(cjkRun.slice(i, i + 2))
+    }
+    cjkRun = ''
+  }
+  const latin = lower.match(/[a-z0-9_]+/g) || []
+  for (const tok of latin) tokens.push(tok)
+  for (const ch of lower) {
+    if (CJK_RANGE.test(ch)) cjkRun += ch
+    else flushCjk()
+  }
+  flushCjk()
+  return tokens
+}
+
+export function tokenize(text) {
+  return [...new Set(tokenizeRaw(text))]
+}
+
+const K1 = 1.5
+const B = 0.75
+const DELTA = 1
+
+function avgDocLen(docs) {
+  if (!docs.length) return 1
+  return docs.reduce((sum, d) => sum + d.length, 0) / docs.length
+}
+
+export function buildBm25(docs, getFieldText) {
+  const documents = docs.map((doc) => {
+    const text = getFieldText(doc)
+    const terms = tokenizeRaw(text)
+    const tf = {}
+    for (const t of terms) tf[t] = (tf[t] || 0) + 1
+    return { doc, length: terms.length, tf }
+  })
+  const df = {}
+  for (const d of documents) {
+    for (const t of Object.keys(d.tf)) df[t] = (df[t] || 0) + 1
+  }
+  const N = documents.length
+  const avgdl = avgDocLen(documents)
+  const idf = (t) => Math.log(1 + (N - (df[t] || 0) + 0.5) / ((df[t] || 0) + 0.5))
+  return {
+    idf,
+    score(query) {
+      const q = tokenize(query)
+      if (!q.length) return []
+      const avgLen = avgdl
+      return documents
+        .map((d) => {
+          const len = d.length || 1
+          let score = 0
+          for (const t of q) {
+            const tf = d.tf[t] || 0
+            if (!tf) continue
+            score += idf(t) * ((tf * (K1 + 1)) / (tf + K1 * (1 - B + (B * len) / avgLen)))
+          }
+          return { doc: d.doc, score }
+        })
+        .filter((r) => r.score > 0)
+        .sort((a, b) => b.score - a.score)
+    },
+  }
+}
+
+export function weightedFieldText(entry) {
+  const parts = []
+  for (let i = 0; i < 5; i++) parts.push(entry.title || '')
+  parts.push((entry.keywords || []).join(' '))
+  parts.push(entry.summary || '')
+  parts.push(entry.sourcePath || '')
+  return parts.join(' ')
+}
+
+export function rankEntries(entries, query, limit = 8) {
+  const bm25 = buildBm25(entries, weightedFieldText)
+  const scored = bm25.score(query)
+  return scored.slice(0, limit).map((r) => r.doc)
+}
+
+export function rankEntriesMerged(entries, queries, limit = 8) {
+  return rankEntriesMergedScored(entries, queries, limit).map((r) => r.entry)
+}
+
+export function rankEntriesMergedScored(entries, queries, limit = 8) {
+  if (!queries.length) return entries.slice(0, limit).map((entry) => ({ entry, score: 0 }))
+  const bm25 = buildBm25(entries, weightedFieldText)
+  const merged = new Map()
+  for (const query of queries) {
+    for (const r of bm25.score(query)) {
+      const id = r.doc.id || r.doc.sourcePath
+      if (merged.has(id)) {
+        if (r.score > merged.get(id).score) merged.get(id).score = r.score
+      } else {
+        merged.set(id, { entry: r.doc, score: r.score })
+      }
+    }
+  }
+  return [...merged.values()]
+    .sort((a, b) => b.score - a.score)
+    .slice(0, limit)
+}
+
+export function rankExperience(items, queryOrQueries, limit = 5) {
+  return rankExperienceScored(items, queryOrQueries, limit).map((r) => r.item)
+}
+
+export function rankExperienceScored(items, queryOrQueries, limit = 5) {
+  const bm25 = buildBm25(items, (item) =>
+    `${item.problem} ${item.problem} ${item.problem} ${item.solution} ${item.sourceFile || ''}`,
+  )
+  const queries = Array.isArray(queryOrQueries) ? queryOrQueries : [queryOrQueries]
+  const merged = new Map()
+  for (const query of queries) {
+    for (const r of bm25.score(query)) {
+      if (merged.has(r.doc.id)) {
+        if (r.score > merged.get(r.doc.id).score) merged.get(r.doc.id).score = r.score
+      } else {
+        merged.set(r.doc.id, { item: r.doc, score: r.score })
+      }
+    }
+  }
+  return [...merged.values()]
+    .sort((a, b) => b.score - a.score)
+    .slice(0, limit)
+}

package/src/util/text.js

@@ -0,0 +1,5 @@
+export function truncate(text, maxChars) {
+  if (typeof text !== 'string') return String(text)
+  if (text.length <= maxChars) return text
+  return text.slice(0, maxChars) + `\n\n...[truncated at ${maxChars} chars]`
+}

package/src/watch.js

@@ -0,0 +1,135 @@
+import { existsSync, readFileSync, statSync } from 'node:fs'
+import path from 'node:path'
+import { isSupportedCode, isSupportedDoc, memoryRootFor, relativePath, sha256OfFile, storeKey, walkDir } from './util/fs.js'
+import { buildDocEntries } from './doc-pipeline.js'
+import { scanSymbols } from './symbols.js'
+import { linkEntries } from './link.js'
+import { ProjectMemoryStore, withStoreLock } from './store.js'
+
+export class WatchManager {
+  constructor(ctx, config) {
+    this.ctx = ctx
+    this.config = config
+    this.roots = new Map()
+    this.timer = null
+  }
+
+  restorePersisted() {
+    const cwd = process.cwd()
+    const store = new ProjectMemoryStore(memoryRootFor(cwd, this.config.memoryDir))
+    if (existsSync(store.dir)) {
+      store.load()
+      for (const root of store.watchlist) {
+        if (typeof root === 'string' && root) this.addRoot(root)
+      }
+    }
+  }
+
+  addRoot(root) {
+    if (!this.roots.has(root)) {
+      this.roots.set(root, {
+        store: new ProjectMemoryStore(memoryRootFor(root, this.config.memoryDir)).load(),
+        snapshot: {},
+      })
+      return true
+    }
+    return false
+  }
+
+  removeRoot(root) {
+    return this.roots.delete(root)
+  }
+
+  start(intervalMs = 15000) {
+    if (this.timer) return
+    this.timer = setInterval(() => this.poll(), Math.max(intervalMs, 1000))
+    if (this.timer.unref) this.timer.unref()
+  }
+
+  stop() {
+    if (this.timer) clearInterval(this.timer)
+    this.timer = null
+  }
+
+  async poll() {
+    for (const [root, state] of this.roots) {
+      try {
+        await this.pollRoot(root, state)
+      } catch (err) {
+        console.error(`[dsh-project-memory] watch poll failed for ${root}: ${err.message}`)
+      }
+    }
+  }
+
+  async pollRoot(root, state) {
+    const memoryDir = memoryRootFor(root, this.config.memoryDir)
+    await withStoreLock(memoryDir, async () => {
+      const files = walkDir(root)
+      const seen = new Set()
+      let changed = 0
+
+      for (const filePath of files) {
+        const rel = storeKey(relativePath(root, filePath))
+        seen.add(rel)
+        const ext = path.extname(filePath).toLowerCase()
+        if (!isSupportedDoc(ext) && !isSupportedCode(ext)) continue
+
+        let stats
+        try {
+          stats = statSync(filePath)
+        } catch {
+          continue
+        }
+        const sig = `${stats.mtimeMs}:${stats.size}`
+        if (state.snapshot[rel] === sig) continue
+        state.snapshot[rel] = sig
+
+        if (isSupportedCode(ext) && this.config.maxFileSizeMb && stats.size > this.config.maxFileSizeMb * 1024 * 1024) {
+          continue
+        }
+
+        const { hash } = await sha256OfFile(filePath)
+        const existing = state.store.fileRecord(rel)
+        if (existing && existing.sha256 === hash) continue
+
+        try {
+          let entries
+          if (isSupportedCode(ext)) {
+            entries = scanSymbols(filePath, readFileSync(filePath, 'utf8'))
+            state.store.markFile(rel, { sha256: hash, size: stats.size, type: 'code', indexedAt: new Date().toISOString() })
+          } else {
+            entries = await buildDocEntries(this.ctx.llm, filePath, {
+              chunkChars: this.config.chunkChars,
+              maxChunks: this.config.maxChunksPerFile,
+              maxFileSizeMb: this.config.maxFileSizeMb,
+              maxPdfPages: this.config.maxPdfPages,
+            })
+            if (entries === null) {
+              state.store.removeFile(rel)
+              changed++
+              continue
+            }
+            state.store.markFile(rel, { sha256: hash, size: stats.size, type: 'doc', indexedAt: new Date().toISOString() })
+          }
+          state.store.setEntries(rel, entries)
+          changed++
+        } catch (err) {
+          delete state.snapshot[rel]
+          console.error(`[dsh-project-memory] re-index failed for ${rel}: ${err.message}`)
+        }
+      }
+
+      for (const rel of Object.keys(state.store.files)) {
+        if (!seen.has(rel)) {
+          state.store.removeFile(rel)
+          changed++
+        }
+      }
+
+      if (changed) {
+        linkEntries(state.store)
+        state.store.save()
+      }
+    })
+  }
+}