codesynapt 0.0.0

package/packages/core/lib/embedding.cjs

@@ -0,0 +1,89 @@
+// Semantic embeddings via @xenova/transformers (MiniLM-L6-v2, ~25 MB
+// quantized ONNX model). Runs entirely locally — no network call at
+// query time, no telemetry. The model itself is fetched once on
+// first load and cached under the OS HF cache directory.
+//
+// Used by /symbol/explore to give "auth" ↔ "login" / "signIn" /
+// "verifyJWT" the synonym match that keyword-only scoring misses.
+//
+// All exports are lazy: until the first call, this module doesn't
+// even import @xenova/transformers — so the 45 MB dep cost only
+// hits processes that actually opt into embedding mode.
+
+'use strict'
+
+let _pipelinePromise = null
+let _ready = false
+let _failed = false
+
+async function loadPipeline() {
+  if (_pipelinePromise) return _pipelinePromise
+  if (_failed) return null
+  _pipelinePromise = (async () => {
+    try {
+      const mod = await import('@xenova/transformers')
+      const p = await mod.pipeline(
+        'feature-extraction',
+        'Xenova/all-MiniLM-L6-v2',
+        { quantized: true }
+      )
+      _ready = true
+      return p
+    } catch (e) {
+      console.warn('[embedding] pipeline init failed — falling back to keyword-only:', e.message)
+      _failed = true
+      _pipelinePromise = null
+      return null
+    }
+  })()
+  return _pipelinePromise
+}
+
+// Single text → 384-d normalized vector. Caller should await loadPipeline
+// at least once before calling embed in a tight loop.
+async function embed(text) {
+  const p = await loadPipeline()
+  if (!p) return null
+  try {
+    const out = await p(text, { pooling: 'mean', normalize: true })
+    return Array.from(out.data)
+  } catch { return null }
+}
+
+// Batch of N texts → N vectors. Faster than N single calls thanks
+// to ONNX op fusion. CHUNK keeps peak memory bounded.
+async function embedBatch(texts) {
+  const p = await loadPipeline()
+  if (!p) return null
+  try {
+    const out = await p(texts, { pooling: 'mean', normalize: true })
+    const D = out.dims[1]
+    const N = out.dims[0]
+    const result = []
+    for (let i = 0; i < N; i++) {
+      result.push(Array.from(out.data.slice(i * D, (i + 1) * D)))
+    }
+    return result
+  } catch (e) {
+    console.warn('[embedding] batch failed:', e.message)
+    return null
+  }
+}
+
+// Both vectors come back L2-normalized from the pipeline, so dot
+// product is already cosine similarity in [-1, 1].
+function cosineSim(a, b) {
+  if (!a || !b || a.length !== b.length) return 0
+  let dot = 0
+  for (let i = 0; i < a.length; i++) dot += a[i] * b[i]
+  return dot
+}
+
+module.exports = {
+  loadPipeline,
+  embed,
+  embedBatch,
+  cosineSim,
+  isReady: () => _ready,
+  isFailed: () => _failed,
+}

package/packages/core/lib/logger.cjs

@@ -0,0 +1,63 @@
+// logger.cjs — minimal structured logger (NDJSON file + stderr).
+//
+// Why not pino: pino adds a runtime dep (we currently have 3 deps total —
+// keeping it lean matters). This is a 50-line equivalent for our actual
+// needs: ts/level/module/msg as one JSON line per entry, stderr fallback
+// for errors, no rotation (callers pick the file path).
+//
+// Usage:
+//   const { createLogger } = require('./logger.cjs')
+//   const log = createLogger({ file: '/var/log/cs.jsonl', module: 'search' })
+//   log.info('search started', { q: 'foo' })
+//   log.warn('cache eviction', { freed: 12345 })
+//   log.error('disk full', { errno: 28 })
+
+const fs = require('fs')
+const path = require('path')
+
+const LEVELS = { trace: 10, debug: 20, info: 30, warn: 40, error: 50, fatal: 60 }
+
+function createLogger(opts = {}) {
+  const file = opts.file || null
+  const module_ = opts.module || 'cs'
+  const minLevel = LEVELS[opts.level || 'info'] || 30
+  const echoStderr = opts.echoStderr ?? 'warn'   // 'never' | 'warn' | 'always'
+
+  // Ensure log directory exists once
+  if (file) {
+    try { fs.mkdirSync(path.dirname(file), { recursive: true }) } catch {}
+  }
+
+  function emit(level, msg, meta) {
+    if (LEVELS[level] < minLevel) return
+    const entry = {
+      ts: new Date().toISOString(),
+      level,
+      module: module_,
+      msg,
+      ...(meta || {}),
+    }
+    const line = JSON.stringify(entry) + '\n'
+    if (file) {
+      try { fs.appendFileSync(file, line) }
+      catch (e) { process.stderr.write(`[logger] append failed: ${e.message}\n`) }
+    }
+    const shouldEcho = echoStderr === 'always'
+                     || (echoStderr === 'warn' && LEVELS[level] >= LEVELS.warn)
+    if (shouldEcho) {
+      process.stderr.write(`[${module_}] ${level.toUpperCase()} ${msg}\n`)
+    }
+  }
+
+  return {
+    trace: (m, x) => emit('trace', m, x),
+    debug: (m, x) => emit('debug', m, x),
+    info:  (m, x) => emit('info',  m, x),
+    warn:  (m, x) => emit('warn',  m, x),
+    error: (m, x) => emit('error', m, x),
+    fatal: (m, x) => emit('fatal', m, x),
+    child: (extra) => createLogger({ ...opts, module: extra.module || module_ }),
+  }
+}
+
+module.exports = { createLogger, LEVELS }

package/packages/core/lib/search-cache.cjs

@@ -0,0 +1,140 @@
+// search-cache.cjs — LRU file content cache for full-text search.
+//
+// Design goals:
+//   1) Zero misses: scanner emits add/change/remove → cache reacts.
+//      mtime is also checked on every read as a belt-and-suspenders guard.
+//   2) Fast: second search re-reads ~0 files; everything's in memory.
+//   3) Bounded: LRU evicts least-recent files when over MAX_BYTES.
+//
+// Usage:
+//   const cache = createSearchCache(scanner, { maxBytes: 100 * 1024 * 1024 })
+//   const buf = await cache.read(id, absPath)   // returns Buffer
+//   cache.invalidate(id)                         // manual nuke
+//   cache.clear()                                 // wipe all
+//   cache.stats()                                 // { entries, bytes, hits, misses, evictions }
+
+const fs = require('fs')
+
+const DEFAULT_MAX_BYTES = 100 * 1024 * 1024  // 100 MB
+const DEFAULT_MAX_FILE_BYTES = 2 * 1024 * 1024  // skip caching files > 2 MB
+
+function createSearchCache(scanner, opts = {}) {
+  const maxBytes = opts.maxBytes || DEFAULT_MAX_BYTES
+  const maxFileBytes = opts.maxFileBytes || DEFAULT_MAX_FILE_BYTES
+
+  // id → { mtime, buf, lastAccess }
+  const cache = new Map()
+  let bytesUsed = 0
+  const stats = { hits: 0, misses: 0, evictions: 0, stales: 0 }
+
+  // Hook scanner events for proactive invalidation
+  if (scanner && typeof scanner.on === 'function') {
+    scanner.on('file-changed', ({ id }) => invalidate(id))
+    scanner.on('file-removed', ({ id }) => invalidate(id))
+    // file-added: no action — next read will populate
+  }
+
+  function invalidate(id) {
+    const entry = cache.get(id)
+    if (entry) {
+      bytesUsed -= entry.buf.length
+      cache.delete(id)
+    }
+  }
+
+  function clear() {
+    cache.clear()
+    bytesUsed = 0
+  }
+
+  function touch(id, entry) {
+    // Move to end (most recently used) by re-inserting
+    cache.delete(id)
+    entry.lastAccess = Date.now()
+    cache.set(id, entry)
+  }
+
+  function evictUntilFits(needed) {
+    // Map iteration order = insertion order; oldest is at the front
+    for (const [id, entry] of cache) {
+      if (bytesUsed + needed <= maxBytes) break
+      bytesUsed -= entry.buf.length
+      cache.delete(id)
+      stats.evictions++
+    }
+  }
+
+  async function read(id, absPath) {
+    // 1) stat for mtime — ~0.1ms
+    let stat
+    try { stat = await fs.promises.stat(absPath) }
+    catch (e) { invalidate(id); throw e }
+
+    // 2) cache hit if mtime unchanged
+    const cached = cache.get(id)
+    if (cached && cached.mtime === stat.mtimeMs) {
+      stats.hits++
+      touch(id, cached)
+      return cached.buf
+    }
+    if (cached) stats.stales++   // mtime differed → stale
+
+    // 3) read fresh
+    let buf
+    try { buf = await fs.promises.readFile(absPath) }
+    catch (e) { invalidate(id); throw e }
+    stats.misses++
+
+    // 4) cache the new content if it fits the per-file limit
+    if (buf.length <= maxFileBytes) {
+      if (cached) {
+        bytesUsed -= cached.buf.length
+        cache.delete(id)
+      }
+      if (buf.length <= maxBytes) {
+        evictUntilFits(buf.length)
+        cache.set(id, { mtime: stat.mtimeMs, buf, lastAccess: Date.now() })
+        bytesUsed += buf.length
+      }
+    } else if (cached) {
+      // too big to cache anymore — drop old entry
+      bytesUsed -= cached.buf.length
+      cache.delete(id)
+    }
+
+    return buf
+  }
+
+  function getStats() {
+    return {
+      entries:   cache.size,
+      bytes:     bytesUsed,
+      bytesMb:   +(bytesUsed / 1024 / 1024).toFixed(1),
+      maxBytes,
+      hits:      stats.hits,
+      misses:    stats.misses,
+      stales:    stats.stales,
+      evictions: stats.evictions,
+      hitRate:   stats.hits + stats.misses === 0 ? null
+                  : +(stats.hits / (stats.hits + stats.misses)).toFixed(3),
+    }
+  }
+
+  // getText — like read, but returns the utf-8 decoded string. Caches the
+  // decoded text on the entry so repeat searches skip the (CPU-bound) decode.
+  // mtime invalidation also drops the text since the buf is replaced.
+  async function getText(id, absPath) {
+    const buf = await read(id, absPath)
+    const entry = cache.get(id)
+    if (entry) {
+      if (!entry.text) entry.text = buf.toString('utf8')
+      return entry.text
+    }
+    // Too large to cache — decode without storing
+    return buf.toString('utf8')
+  }
+
+  return { read, getText, invalidate, clear, stats: getStats }
+}
+
+module.exports = { createSearchCache }

package/packages/core/lib/search-worker.cjs

@@ -0,0 +1,255 @@
+// search-worker.cjs — runs in a Node worker_thread, isolated from the main
+// thread (which is busy with scanner / chokidar / babel parse).
+//
+// Protocol (messages from main):
+//   { type: 'search', id, files: [{id, absPath}], q, regex, caseSensitive, max, maxPerFile, concurrency, fileTimeoutMs }
+//   { type: 'invalidate', id }     — drop one cache entry
+//   { type: 'clear' }              — drop all cache
+//
+// Reply (messages to main):
+//   { type: 'result', id, payload: { matches, skipped, filesScanned, filesMatched, ms, cacheStats, truncated } }
+//   { type: 'error',  id, error: string }
+
+const { parentPort, workerData } = require('worker_threads')
+const fs = require('fs')
+
+if (!parentPort) {
+  throw new Error('search-worker.cjs must be loaded as a worker_thread')
+}
+
+const _reqId = workerData?.reqId ?? '?'
+const _tracePath = workerData?.tracePath
+function _trace(msg) {
+  if (!_tracePath) return
+  try { fs.appendFileSync(_tracePath, `${new Date().toISOString()} [#${_reqId}] [worker] ${msg}\n`) } catch {}
+}
+_trace('boot')
+
+const DEFAULT_MAX = 100
+// 8 is the sweet spot with the size-gate in place (large files are skipped
+// without holding threads). Going higher gives diminishing returns.
+const DEFAULT_CONCURRENCY = 8
+const DEFAULT_FILE_TIMEOUT_MS = 5000
+const SNIPPET_CONTEXT = 50
+const MAX_CACHE_BYTES = 100 * 1024 * 1024
+const MAX_FILE_BYTES = 2 * 1024 * 1024
+// Files larger than this are SKIPPED entirely (not even attempted).
+// Reason: AI model tokenizer JSONs (tens of MB) read slowly enough that
+// they hold a libuv thread for seconds, blocking every other concurrent
+// task. Better to skip them honestly than hang the whole search.
+const MAX_SEARCH_BYTES = 5 * 1024 * 1024  // 5 MB
+
+// Cache: id → { mtime, buf, text? }
+const cache = new Map()
+let bytesUsed = 0
+const cacheStats = { hits: 0, misses: 0, stales: 0, evictions: 0 }
+
+function invalidate(id) {
+  const entry = cache.get(id)
+  if (entry) {
+    bytesUsed -= entry.buf.length
+    cache.delete(id)
+  }
+}
+function clearCache() { cache.clear(); bytesUsed = 0 }
+
+function evictUntilFits(needed) {
+  for (const [id, entry] of cache) {
+    if (bytesUsed + needed <= MAX_CACHE_BYTES) break
+    bytesUsed -= entry.buf.length
+    cache.delete(id)
+    cacheStats.evictions++
+  }
+}
+
+async function getText(id, absPath) {
+  // Fast path: cache hit, no stat call. Chokidar invalidation messages
+  // from main keep us honest. Skipping stat saves ~0.5ms × N files.
+  const cached = cache.get(id)
+  if (cached) {
+    cacheStats.hits++
+    cache.delete(id); cache.set(id, cached)   // LRU touch
+    if (!cached.text) cached.text = cached.buf.toString('utf8')
+    return cached.text
+  }
+
+  let stat
+  try { stat = await fs.promises.stat(absPath) }
+  catch (e) { invalidate(id); throw e }
+
+  // Hard size gate — prevents tokenizer.json (50+ MB) from stalling pool.
+  if (stat.size > MAX_SEARCH_BYTES) {
+    throw new Error(`too-large: ${stat.size} bytes (cap ${MAX_SEARCH_BYTES})`)
+  }
+
+  let buf
+  try { buf = await fs.promises.readFile(absPath) }
+  catch (e) { invalidate(id); throw e }
+  cacheStats.misses++
+
+  if (buf.length <= MAX_FILE_BYTES) {
+    if (cached) { bytesUsed -= cached.buf.length; cache.delete(id) }
+    evictUntilFits(buf.length)
+    cache.set(id, { mtime: stat.mtimeMs, buf, text: null })
+    bytesUsed += buf.length
+  }
+  return buf.toString('utf8')
+}
+
+function withTimeout(promise, ms, label) {
+  let to
+  const timeout = new Promise((_, reject) => {
+    to = setTimeout(() => reject(new Error(`timeout ${ms}ms: ${label}`)), ms)
+  })
+  return Promise.race([promise, timeout]).finally(() => clearTimeout(to))
+}
+
+function scanContent(text, q, opts, maxPerFile) {
+  const matches = []
+  if (opts.regex) {
+    const flags = opts.caseSensitive ? 'g' : 'gi'
+    let re
+    try { re = new RegExp(q, flags) } catch { return [] }
+    let m
+    while ((m = re.exec(text)) !== null) {
+      const idx = m.index
+      const before = text.lastIndexOf('\n', idx - 1)
+      const line = (text.slice(0, idx).match(/\n/g) || []).length + 1
+      const col = idx - (before + 1) + 1
+      const sStart = Math.max(0, idx - SNIPPET_CONTEXT)
+      const sEnd   = Math.min(text.length, idx + m[0].length + SNIPPET_CONTEXT)
+      matches.push({ line, col, snippet: text.slice(sStart, sEnd).replace(/\r?\n/g, ' ') })
+      if (matches.length >= maxPerFile) break
+      if (m.index === re.lastIndex) re.lastIndex++
+    }
+    return matches
+  }
+  const needle = opts.caseSensitive ? q : q.toLowerCase()
+  const hay    = opts.caseSensitive ? text : text.toLowerCase()
+  // Fast reject: most files have no match. One indexOf call vs O(lines).
+  if (hay.indexOf(needle) === -1) return []
+  let lineStart = 0, lineNo = 1
+  while (lineStart < hay.length) {
+    let lineEnd = hay.indexOf('\n', lineStart)
+    if (lineEnd === -1) lineEnd = hay.length
+    let from = lineStart
+    while (true) {
+      const idx = hay.indexOf(needle, from)
+      if (idx === -1 || idx >= lineEnd) break
+      const sStart = Math.max(0, idx - SNIPPET_CONTEXT)
+      const sEnd   = Math.min(text.length, idx + needle.length + SNIPPET_CONTEXT)
+      matches.push({ line: lineNo, col: idx - lineStart + 1, snippet: text.slice(sStart, sEnd).replace(/\r?\n/g, ' ') })
+      if (matches.length >= maxPerFile) return matches
+      from = idx + needle.length
+    }
+    lineStart = lineEnd + 1
+    lineNo++
+  }
+  return matches
+}
+
+async function runConcurrent(tasks, concurrency, shouldStop) {
+  let i = 0
+  async function worker() {
+    while (i < tasks.length) {
+      if (shouldStop && shouldStop()) return
+      const idx = i++
+      await tasks[idx]()
+    }
+  }
+  const workers = Array.from({ length: Math.min(concurrency, tasks.length) }, worker)
+  await Promise.all(workers)
+}
+
+async function doSearch(req) {
+  const t0 = Date.now()
+  const {
+    files, q,
+    regex = false, caseSensitive = false,
+    max = DEFAULT_MAX, maxPerFile = 10,
+    concurrency = DEFAULT_CONCURRENCY, fileTimeoutMs = DEFAULT_FILE_TIMEOUT_MS,
+  } = req
+
+  if (!q) throw new Error('q (query) is required')
+  if (regex) { try { new RegExp(q) } catch (e) { throw new Error(`invalid regex: ${e.message}`) } }
+
+  const totalFiles = files.length
+  const matches = []
+  const skipped = []
+  let filesScanned = 0, filesMatched = 0, stopFlag = false
+  let _completed = 0
+  _trace(`doSearch begin: q=${q} totalFiles=${totalFiles} concurrency=${concurrency}`)
+
+  const tasks = files.map((f, taskIdx) => async () => {
+    if (stopFlag) return
+    let text
+    const tRead = Date.now()
+    try {
+      text = await withTimeout(getText(f.id, f.absPath), fileTimeoutMs, f.id)
+    } catch (e) {
+      _trace(`  SKIP task[${taskIdx}] ${f.id} after ${Date.now()-tRead}ms — ${e.message}`)
+      let reason = 'read-error'
+      if (e.message.startsWith('timeout')) reason = 'timeout'
+      else if (e.message.startsWith('too-large')) reason = 'too-large'
+      skipped.push({ id: f.id, reason })
+      return
+    }
+    const rDur = Date.now() - tRead
+    if (rDur > 500) _trace(`  SLOW task[${taskIdx}] ${f.id} read took ${rDur}ms`)
+    if (stopFlag) return
+    filesScanned++
+    let fileMatches
+    try {
+      fileMatches = scanContent(text, q, { regex, caseSensitive }, maxPerFile)
+    } catch {
+      skipped.push({ id: f.id, reason: 'scan-error' })
+      return
+    }
+    _completed++
+    if (_completed % 200 === 0) _trace(`  progress: ${_completed}/${totalFiles} matches=${matches.length}`)
+    if (fileMatches.length === 0) return
+    filesMatched++
+    for (const m of fileMatches) {
+      matches.push({ id: f.id, line: m.line, col: m.col, snippet: m.snippet, totalInFile: fileMatches.length })
+      if (matches.length >= max) { stopFlag = true; break }
+    }
+  })
+
+  _trace(`runConcurrent starting`)
+  await runConcurrent(tasks, concurrency, () => stopFlag)
+  _trace(`runConcurrent finished: completed=${_completed} matches=${matches.length} skipped=${skipped.length}`)
+
+  return {
+    query: q, regex, caseSensitive,
+    totalFiles, filesScanned, filesMatched,
+    matches, skipped, truncated: stopFlag,
+    ms: Date.now() - t0,
+    cacheStats: { ...cacheStats, entries: cache.size, bytesMb: +(bytesUsed/1024/1024).toFixed(1) },
+  }
+}
+
+parentPort.on('message', async (msg) => {
+  _trace(`recv ${msg.type}${msg.q ? ` q=${msg.q}`:''}${msg.files ? ` files=${msg.files.length}` : ''}`)
+  try {
+    if (msg.type === 'search') {
+      const t = Date.now()
+      const payload = await doSearch(msg)
+      _trace(`doSearch returned in ${Date.now() - t}ms — matches=${payload.matches.length} scanned=${payload.filesScanned}`)
+      parentPort.postMessage({ type: 'result', id: msg.id, payload })
+      _trace('postMessage result done')
+    } else if (msg.type === 'invalidate') {
+      invalidate(msg.id)
+    } else if (msg.type === 'clear') {
+      clearCache()
+    } else {
+      parentPort.postMessage({ type: 'error', id: msg.id, error: `unknown msg type: ${msg.type}` })
+    }
+  } catch (e) {
+    _trace(`error: ${e.message}`)
+    parentPort.postMessage({ type: 'error', id: msg.id, error: e.message || String(e) })
+  }
+})
+
+_trace('parentPort handler set, sending ready')
+parentPort.postMessage({ type: 'ready' })
+_trace('ready sent')