@rekal/mem 0.0.0

package/src/runtime.bun.ts

@@ -0,0 +1,33 @@
+import type { Database } from "bun:sqlite"
+
+let didInit = false
+
+export async function openDatabase(path: string) {
+  const [{ load: sqliteVec }, { Database }] = await Promise.all([
+    import("sqlite-vec"),
+    import("bun:sqlite"),
+  ])
+  if (!didInit) {
+    didInit = true
+    // See: https://bun.com/docs/runtime/sqlite#setcustomsqlite
+    if (process.platform === "darwin")
+      for (const p of [
+        "/opt/homebrew/opt/sqlite/lib/libsqlite3.dylib", // Apple Silicon
+        "/usr/local/opt/sqlite/lib/libsqlite3.dylib", // Intel
+      ]) {
+        try {
+          Database.setCustomSQLite(p)
+          break
+        } catch {}
+      }
+  }
+  const db = new Database(path, { strict: true })
+  sqliteVec(db)
+  return db
+}
+
+export type { Database }
+
+export function parseYaml(content: string): unknown {
+  return Bun.YAML.parse(content)
+}

package/src/runtime.node.ts

@@ -0,0 +1,47 @@
+import type { Database as BetterDB } from "better-sqlite3"
+import type { Database } from "bun:sqlite"
+
+let DB: undefined | typeof Database
+
+async function dbInit() {
+  const { default: BetterDatabase } = await import("better-sqlite3")
+
+  // Extend better-sqlite3 to mimic Bun's Database API
+  return class extends BetterDatabase {
+    private prepareCache = new Map<string, ReturnType<BetterDB["prepare"]>>()
+
+    // oxlint-disable-next-line no-useless-constructor
+    constructor(filename?: string) {
+      super(filename)
+    }
+
+    run(...args: Parameters<BetterDB["exec"]>) {
+      return this.exec(...args)
+    }
+
+    query(source: string) {
+      let ret = this.prepareCache.get(source)
+      if (!ret) {
+        ret = this.prepare(source)
+        this.prepareCache.set(source, ret)
+      }
+      return ret
+    }
+  } as unknown as typeof Database
+}
+
+export async function openDatabase(path: string) {
+  DB ??= await dbInit()
+  const { load: sqliteVec } = await import("sqlite-vec")
+  const db = new DB(path, { strict: true })
+  sqliteVec(db)
+  return db
+}
+
+export type { Database }
+
+const { load: loadYaml } = await import("js-yaml")
+
+export function parseYaml(content: string): unknown {
+  return loadYaml(content)
+}

package/src/search.ts

@@ -0,0 +1,230 @@
+import type { Context } from "./context.ts"
+import type { Db, DocRow, FTSResult, VecResult } from "./db.ts"
+import type { VfsEntry } from "./vfs.ts"
+
+import { toFts } from "./query.ts"
+import { parentUri } from "./uri.ts"
+import { hash } from "./util.ts"
+
+export type SearchMode = "hybrid" | "vec" | "fts"
+
+export type SearchScore = {
+  score: number
+  display_score?: number
+  rank: number
+}
+
+export type SearchResult = {
+  uri: string
+  path: string
+  doc: DocRow
+  scores: Partial<Record<SearchMode, SearchScore>>
+  match: { fts?: FTSResult; vec?: VecResult }
+} & VfsEntry
+
+type SearchResultMap = {
+  hybrid: HybridSR
+  vec: VecSR
+  fts: FtsSR
+}
+
+export type HybridSR = SearchResult & { scores: { hybrid: SearchScore } }
+export type VecSR = SearchResult & { scores: { vec: SearchScore }; match: { vec: VecResult } }
+export type FtsSR = SearchResult & { scores: { fts: SearchScore }; match: { fts: FTSResult } }
+
+export type SearchOptions = {
+  limit?: number
+  uri?: string
+  mode?: SearchMode
+}
+
+export type FtsSearchOptions = Omit<SearchOptions, "mode"> & {
+  op?: "AND" | "OR"
+}
+
+// Description chunks (seq=0) get boosted in vector scoring
+const DESC_BOOST = 0.2
+const PARENT_BOOST = 0.3 // how much to boost a chunk based on its parent's score, vs its own score, in [0, 1]
+const RRF_K = 60
+const RRF_LIMIT = 50
+const VEC_OVERSAMPLE = 4
+
+export class Search {
+  private constructor(
+    public db: Db,
+    public ctx: Context
+  ) {}
+
+  static async load(ctx: Context) {
+    return new Search(await ctx.db(), ctx)
+  }
+
+  async search(query: string, opts: SearchOptions = {}): Promise<SearchResult[]> {
+    const mode = opts.mode ?? "hybrid"
+    if (mode === "fts") return this.searchFts(query, opts)
+    if (mode === "vec") return this.searchVec(query, opts)
+
+    const limit = opts.limit ?? 20
+
+    // Hybrid: run both, fuse with RRF — need enough candidates for good fusion
+    const subLimit = Math.max(RRF_LIMIT, limit * 2)
+    const [fts, vec] = await Promise.all([
+      this.searchFts(query, { ...opts, limit: subLimit }),
+      this.searchVec(query, { ...opts, limit: subLimit, slice: false }),
+    ])
+
+    return this.fuse(fts, vec, limit)
+  }
+
+  async searchVec(
+    query: string,
+    opts: Omit<SearchOptions, "mode"> & { slice?: boolean } = {}
+  ): Promise<VecSR[]> {
+    const cacheKey = hash(`embed:${query}`)
+    const embedder = await this.ctx.embedder()
+    const vfs = await this.ctx.vfs()
+
+    const embedding =
+      this.db.cacheGet<number[]>(cacheKey) ??
+      this.db.cacheSet(cacheKey, await embedder.embed(query))
+
+    const scope = vfs.getScope(opts.uri)
+    const limit = opts.limit ?? 20
+
+    // Oversample for post-filtering when scoped
+    const results = this.db.searchVec(embedding, {
+      limit: Math.max(limit, RRF_LIMIT) * VEC_OVERSAMPLE,
+    })
+
+    // Group by doc, take best chunk per doc
+    const best = new Map<number, VecResult & { uri: string; hiscore: number }>()
+    for (const vec of results) {
+      const uri = scope.map(vec.path)
+      if (!uri) continue
+      vec.score = vec.seq === 0 ? vec.score + DESC_BOOST * (1 - vec.score) : vec.score
+      const existing = best.get(vec.doc_id)?.score ?? -Infinity
+      if (vec.score > existing) best.set(vec.doc_id, Object.assign(vec, { hiscore: 0, uri }))
+    }
+    const scores = new Map<string, number>(best.values().map((vec) => [vec.uri, vec.score]))
+    const parentScores = new Map<string, number>()
+    const getParentScore = (uri: string): number => {
+      const parent = parentUri(uri)
+      if (!parent) return 0
+      let score = parentScores.get(parent)
+      if (score !== undefined) return score
+      score = (scores.get(parent) ?? 0) * 0.5 + getParentScore(parent) * 0.5
+      parentScores.set(parent, score)
+      return score
+    }
+
+    for (const vec of best.values()) {
+      const parentScore = getParentScore(vec.uri)
+      vec.score += PARENT_BOOST * parentScore * (1 - vec.score)
+    }
+
+    let bestResults = [...best.values()].toSorted((a, b) => b.score - a.score)
+    bestResults = opts.slice === false ? bestResults : bestResults.slice(0, limit)
+
+    const docs = this.db.getDocs(bestResults.map((r) => r.doc_id))
+    const ret: VecSR[] = []
+    for (const vec of bestResults) {
+      const doc = docs.get(vec.doc_id)
+      if (doc)
+        ret.push({
+          doc,
+          match: { vec },
+          path: vec.path,
+          scores: { vec: { rank: 0, score: vec.score } },
+          uri: vec.uri,
+        })
+    }
+
+    return this.rank("vec", ret)
+  }
+
+  async searchFts(query: string, opts: FtsSearchOptions = {}): Promise<FtsSR[]> {
+    const vfs = await this.ctx.vfs()
+    const scope = vfs.getScope(opts.uri)
+    const results = this.db.searchFts(toFts(query, opts.op ?? "OR"), {
+      limit: opts.limit ?? 20,
+      scope: scope.paths.map((p) => p.path),
+    })
+    const docs = this.db.getDocs(results.map((r) => r.rowid))
+    const ret: FtsSR[] = []
+    for (const fts of results) {
+      fts.score = Math.abs(fts.score) / (1 + Math.abs(fts.score))
+      const doc = docs.get(fts.rowid)
+      const uri = scope.map(doc?.path ?? "")
+      if (doc && uri)
+        ret.push({
+          doc,
+          match: { fts },
+          path: doc.path,
+          scores: { fts: { rank: 0, score: fts.score } },
+          uri,
+        })
+    }
+    return this.rank("fts", ret)
+  }
+
+  rank<M extends SearchMode>(mode: M, results: SearchResultMap[M][]): SearchResultMap[M][] {
+    const score = (r: SearchResult) => (r.scores as Record<string, SearchScore>)[mode]
+    return results
+      .toSorted(
+        (a, b) =>
+          score(b).score - score(a).score ||
+          (score(b).display_score ?? 0) - (score(a).display_score ?? 0)
+      )
+      .map((r, i) => {
+        score(r).rank = i + 1
+        return r
+      })
+  }
+
+  /** Reciprocal Rank Fusion: merge FTS and vector results */
+  private fuse(ftsResults: FtsSR[], vecResults: VecSR[], limit: number): HybridSR[] {
+    const merged = new Map<number, { uri: string; vec?: VecSR; fts?: FtsSR }>()
+
+    const minVecScore = vecResults.length
+      ? vecResults[vecResults.length - 1]?.scores.vec.score
+      : undefined
+    const minFtsScore = ftsResults.length
+      ? ftsResults[ftsResults.length - 1]?.scores.fts.score
+      : undefined
+    const minScore = Math.min(minVecScore ?? 1, minFtsScore ?? 1)
+
+    for (const fts of ftsResults) merged.set(fts.doc.id, { fts, uri: fts.uri })
+    for (const vec of vecResults)
+      merged.set(vec.doc.id, { ...merged.get(vec.doc.id), uri: vec.uri, vec })
+
+    let ret: HybridSR[] = [...merged.values()].map(({ uri, fts, vec }) => {
+      const ftsScore = fts?.scores.fts
+      const vecScore = vec?.scores.vec
+      const score =
+        (ftsScore?.rank !== undefined ? 1 / (RRF_K + ftsScore.rank) : 0) +
+        (vecScore?.rank !== undefined ? 1 / (RRF_K + vecScore.rank) : 0)
+      const display_score =
+        0.6 * (vecScore?.score ?? minScore) + 0.4 * (ftsScore?.score ?? minScore)
+      const doc = (fts?.doc ?? vec?.doc)!
+      return {
+        doc,
+        match: { ...fts?.match, ...vec?.match },
+        path: doc.path,
+        scores: {
+          ...fts?.scores,
+          ...vec?.scores,
+          hybrid: { display_score, rank: 0, score },
+        },
+        uri,
+      }
+    })
+
+    ret = this.rank("hybrid", ret).slice(0, limit)
+
+    // Normalize scores to [0, 1]
+    const bestScore = ret[0]?.scores.hybrid.score ?? 1
+    for (const r of ret) r.scores.hybrid.score /= bestScore
+
+    return ret
+  }
+}

package/src/snippet.ts

@@ -0,0 +1,248 @@
+// oxfmt-ignore
+export type Token = {
+  text: string
+  lower: string
+}
+
+export type TokenWithScore = Token & { score: number }
+
+export type SnippetOptions = {
+  query: string
+  lines?: number
+  stopWords?: Map<string, number>
+}
+
+export type SnippetWindow = {
+  start: number
+  heat: number
+  coverage: number
+  score: number
+}
+
+export type SnippetResult = {
+  lines: string[]
+  tokens: Token[][]
+  scores: number[]
+  windows: SnippetWindow[]
+  best: SnippetWindow
+  heat: number[]
+  snippet: string[]
+}
+
+export const WORD_REGEX = /[\p{L}\p{N}]+/gu // regex to match words (unicode letters and numbers)
+
+const SCORE_EXACT = 3
+const SCORE_LOWER = 2
+const SCORE_QUERY_PREFIX = 1.5 // query "child" matches doc "children" — query is prefix of doc
+const SCORE_DOC_PREFIX = 1 // query "children" matches doc "child" — doc is prefix of query
+const SCORE_OVERLAP = 1
+const MIN_PREFIX_LENGTH = 3 // minimum prefix length to consider for scoring
+const STOPWORD_MIN = 0 // min stopword score for a term
+const STOPWORD_MAX = 1 // max stopword score for a term
+const HEAT_SPREAD = 5 // how many lines to spread the heat on each side of a match (should be < lines/2)
+const WEIGHT_EMPTY = 0.1 // score multiplier for empty lines
+const WEIGHT_NONWORD = 0.3 // score multiplier for lines without any word characters
+const WEIGHT_REPETITION = 0.7 // score multiplier for repeated terms in the same line (to de-emphasize boilerplate)
+const decayLinear = (d: number, r: number) => Math.max(0, 1 - d / r) // linear decay function
+// const decayHyperbolic = (d: number, _r: number) => 1 / (1 + d) // hyperbolic decay function
+// const decayExponential = (d: number, r: number) => Math.exp(-d / r) // exponential decay function
+
+// Common English stop words — used for post-processing snippets.
+// Kept minimal: only the highest-frequency words that add no search value.
+// oxfmt-ignore
+const STOP_WORDS = new Set([
+  "a", "an", "and", "are", "as", "at", "be", "but", "by", "do", "for", "from",
+  "had", "has", "have", "he", "her", "his", "how", "i", "if", "in", "is", "it",
+  "its", "my", "no", "not", "of", "on", "or", "our", "she", "so", "than",
+  "that", "the", "their", "them", "then", "there", "these", "they", "this",
+  "to", "up", "us", "was", "we", "what", "when", "which", "who", "will",
+  "with", "you", "your",
+])
+
+export function isStopWord(word: string): boolean {
+  return STOP_WORDS.has(word.toLowerCase())
+}
+
+export class Snippet {
+  query: (Token & { score: number })[] = []
+  prefixes = new Set<string>()
+  prefixRegex: RegExp
+  opts: Required<SnippetOptions>
+
+  constructor(opts: SnippetOptions) {
+    this.opts = { ...opts, lines: 5, stopWords: new Map() }
+    const stopwords = new Map<string, number>([...STOP_WORDS].map((w) => [w, 0]))
+    this.opts.stopWords.forEach((s, w) =>
+      stopwords.set(w, Math.max(STOPWORD_MIN, Math.min(STOPWORD_MAX, s)))
+    )
+
+    const tokens = this.tokenize(opts.query, false)
+    for (const tok of tokens) {
+      const score = stopwords.get(tok.lower) ?? 2
+      if (score === 0) continue
+      this.prefixes.add(tok.lower.slice(0, 2))
+      this.query.push({ ...tok, score })
+    }
+    this.prefixRegex =
+      this.prefixes.size > 0 ? new RegExp(`(${[...this.prefixes].join("|")})`, "i") : /(?!)/ // never matches
+  }
+
+  normalize(text: string): string {
+    // Handle diacritics
+    text = text.normalize("NFD").replace(/\p{M}/gu, "")
+    // Drop possessive 's (and smart quote ’s) completely to avoid orphaned "s" tokens
+    text = text.replace(/['’]s\b/gi, "")
+    // Globally replace any remaining single quotes/apostrophes with a space
+    text = text.replace(/['’]/g, " ")
+    return text
+  }
+
+  tokenize(text: string, queryOnly = true): Token[] {
+    if (queryOnly && !this.prefixRegex.test(text)) return []
+    // only keep unicode letters and numbers
+    const tokens = this.normalize(text).match(WORD_REGEX) ?? []
+    const ret: Token[] = []
+    for (const token of tokens) {
+      const lower = token.toLowerCase()
+      ret.push({ lower, text: token })
+    }
+    return ret
+  }
+
+  score(token: Token, queryToken: Token): number {
+    const tl = token.lower.length
+    if (token.text === queryToken.text) return SCORE_EXACT
+    if (token.lower === queryToken.lower) return SCORE_LOWER
+    if (token.lower.startsWith(queryToken.lower)) return SCORE_QUERY_PREFIX
+    if (queryToken.lower.startsWith(token.lower) && tl >= MIN_PREFIX_LENGTH) return SCORE_DOC_PREFIX
+    let prefix = 0
+    for (let i = 0; i < token.lower.length; i++) {
+      if (token.lower[i] !== queryToken.lower[i]) break
+      prefix++
+    }
+    return prefix >= MIN_PREFIX_LENGTH
+      ? SCORE_OVERLAP * (prefix / Math.max(token.lower.length, queryToken.lower.length))
+      : 0
+  }
+
+  match(input: Token | string): TokenWithScore | undefined {
+    let token: Token
+    if (typeof input === "string") {
+      const tok = this.normalize(input)
+      token = { lower: tok.toLowerCase(), text: input }
+    } else token = input
+
+    let [bestScore, bestTok] = [0, this.query[0]]
+    // oxlint-disable-next-line typescript/prefer-for-of
+    for (let t = 0; t < this.query.length; t++) {
+      const queryTok = this.query[t]
+      const s = this.score(token, queryTok)
+      if (s > bestScore) {
+        ;[bestScore, bestTok] = [s, queryTok]
+      }
+    }
+    if (bestScore > 0) return { ...bestTok, score: bestTok.score * bestScore }
+  }
+
+  // get initial scores/coverage for each line
+  scores(tokens: Token[][]) {
+    const coverage: Set<string>[] = tokens.map(() => new Set())
+    const scores = tokens.map((line, l) => {
+      let lineScore = 0
+      for (const token of line) {
+        const queryTok = this.match(token)
+        if (!queryTok) continue
+        let score = queryTok.score
+        if (coverage[l].has(queryTok.lower)) score *= WEIGHT_REPETITION
+        coverage[l].add(queryTok.lower)
+        lineScore += score
+      }
+      return lineScore
+    })
+    return { coverage, scores }
+  }
+
+  // Build heatmap using a bounded spread (O(N * radius))
+  heat(lines: string[], scores: number[]): number[] {
+    const spread = Math.max(HEAT_SPREAD, Math.ceil(this.opts.lines / 2))
+    const heat = new Float64Array(scores.length)
+    for (let i = 0; i < scores.length; i++) {
+      if (scores[i] === 0) continue
+      const spreadStart = Math.max(0, i - spread)
+      const spreadEnd = Math.min(scores.length - 1, i + spread)
+      for (let j = spreadStart; j <= spreadEnd; j++) {
+        let weight = 1
+
+        // NOTE: de-emphasize lines without any word characters (e.g. code blocks, separators)
+        if (!lines[j].trim()) weight *= WEIGHT_EMPTY
+        else if (!lines[j].match(/\p{L}/u)) weight *= WEIGHT_NONWORD
+
+        weight *= decayLinear(Math.abs(i - j), spread)
+
+        heat[j] += scores[i] * weight
+      }
+    }
+    return [...heat]
+  }
+
+  extract(text: string): SnippetResult {
+    const lines = text.split("\n")
+    const radius = Math.min(this.opts.lines, lines.length)
+    const tokens = lines.map((line) => this.tokenize(line))
+
+    const { scores, coverage } = this.scores(tokens)
+    const heat = this.heat(lines, scores)
+
+    // Find the window with highest heat × term coverage
+    const windows: SnippetWindow[] = []
+
+    for (let i = 0; i <= scores.length - radius; i++) {
+      if (heat[i] === 0 && windows.length > 0) continue // skip windows that don't start with any heat to save computation
+      let heatSum = 0
+      // Count how many distinct query terms appear in this window
+      const covered = new Set<string>()
+      for (let j = i; j < i + radius; j++) {
+        heatSum += heat[j]
+        coverage[j].forEach((t) => covered.add(t))
+      }
+      const cov = this.query.length === 0 ? 1 : covered.size / this.query.length
+      windows.push({ coverage: cov, heat: heatSum, score: heatSum * cov, start: i })
+    }
+
+    let best = windows[0] ?? { coverage: 0, heat: 0, score: 0, start: 0 }
+    for (let i = 1; i < windows.length; i++) {
+      if (windows[i].score > best.score) best = windows[i]
+    }
+
+    const snippet = lines.slice(best.start, best.start + radius)
+    return { best, heat: [...heat], lines, scores, snippet, tokens, windows }
+  }
+
+  debug(result: SnippetResult) {
+    // oxlint-disable-next-line no-console
+    console.info("Options:", this.opts)
+    // oxlint-disable-next-line no-console
+    console.info("Query :", this.query)
+    // oxlint-disable-next-line unicorn/consistent-function-scoping
+    const score = (n?: number, f = 1) => (n !== undefined ? n.toFixed(f).padEnd(4) : " ".repeat(4))
+    result.lines.forEach((line, i) => {
+      const isBest = (l: number) =>
+        l >= result.best.start && l < result.best.start + this.opts.lines
+
+      const lineScore = score(result.scores[i])
+      const lineHeat = score(result.heat[i])
+      const windowHeat = score(result.windows[i]?.heat ?? 0)
+      const windowScore = score(result.windows[i]?.score ?? 0)
+      const coverage = ((result.windows[i]?.coverage ?? 0) * 100).toFixed(0).padStart(3)
+
+      // oxlint-disable-next-line no-console
+      console.log(
+        `s:${lineScore} h:${lineHeat} wh:${windowHeat} ws:${windowScore} ${coverage}% ${isBest(i) ? ">" : " "} ${line}`
+      )
+    })
+  }
+
+  highlight(text: string, hl: (word: string, offset: number) => string): string {
+    return text.replace(WORD_REGEX, (word, offset) => (this.match(word) ? hl(word, offset) : word))
+  }
+}

package/src/sqlite.ts

@@ -0,0 +1 @@
+export { openDatabase, type Database } from "#runtime"