@rekal/mem 0.0.0

package/src/db.ts

@@ -0,0 +1,464 @@
+import type { EmbedderChunk } from "./embed/index.ts"
+import type { Database } from "./sqlite.ts"
+import type { StoreChunk } from "./store.ts"
+
+import { openDatabase } from "./sqlite.ts"
+
+export type { Database }
+
+export type DocRow = {
+  id: number
+  path: string
+  hash: string
+  body: string
+  vec_hash?: string
+  description: string
+  title: string
+  tags: string
+  entities: string
+  updated_at: string
+  synced_at?: string
+  deadline?: number
+}
+
+export type VecResult = {
+  doc_id: number
+  path: string
+  seq: number
+  distance: number
+  score: number
+  rank?: number
+}
+
+export type FTSResult = {
+  rowid: number
+  score: number
+  rank?: number
+}
+
+export type DbSearchOptions = {
+  limit?: number
+  scope?: string[] // path prefixes to limit search to (e.g. ["folder1/", "folder2/sub"])
+}
+
+const SEARCH_LIMIT = 20
+const STOPWORD_THRESHOLD = 0.3 // terms in >50% of docs are candidates
+const STOPWORD_MIN_DOCS = 10 // terms must be in at least 5 docs to be considered stop words
+const STOPWORD_LIMIT = 1000 // max number of stop words to return
+
+export function hasEmbedding<T extends EmbedderChunk>(c: T): c is T & { embedding: number[] } {
+  return Array.isArray(c.embedding)
+}
+
+export function assertEmbeddings<T extends EmbedderChunk>(
+  chunks: T[]
+): asserts chunks is (T & { embedding: number[] })[] {
+  for (const c of chunks) {
+    if (!hasEmbedding(c)) throw new Error(`Chunk is missing embedding: ${JSON.stringify(c)}`)
+  }
+}
+
+export class Db {
+  #db: Database
+  #vec?: { exists: boolean; dims?: number; init?: boolean }
+
+  private constructor(db: Database) {
+    this.#db = db
+    this.init()
+  }
+
+  static async load(dbPath: string) {
+    return new Db(await openDatabase(dbPath))
+  }
+
+  private init() {
+    this.#db.run("PRAGMA journal_mode = WAL")
+    this.#db.run("PRAGMA foreign_keys = ON")
+    this.#db.run("PRAGMA busy_timeout = 5000")
+
+    this.#db.run(`
+      CREATE TABLE IF NOT EXISTS docs (
+        id INTEGER PRIMARY KEY AUTOINCREMENT,
+        path TEXT NOT NULL UNIQUE,
+        hash TEXT NOT NULL,
+        vec_hash TEXT,
+        body TEXT NOT NULL DEFAULT '',
+        description TEXT NOT NULL DEFAULT '',
+        title TEXT NOT NULL DEFAULT '',
+        tags TEXT NOT NULL DEFAULT '',
+        entities TEXT NOT NULL DEFAULT '',
+        updated_at TEXT NOT NULL,
+        synced_at TEXT,
+        deadline REAL
+      )
+    `)
+
+    this.#db.run(`CREATE INDEX IF NOT EXISTS idx_docs_path ON docs(path)`)
+    this.#db.run(`CREATE INDEX IF NOT EXISTS idx_docs_hash ON docs(hash)`)
+
+    // Content-synced FTS5: reads content from docs table, no duplication.
+    // Fields ordered by BM25 weight: entities(10), tags(8), description(5), title(3), body(1)
+    this.#db.run(`
+      CREATE VIRTUAL TABLE IF NOT EXISTS docs_fts USING fts5(
+        entities, tags, description, title, body,
+        content='docs',
+        content_rowid='id',
+        tokenize='porter unicode61'
+      )
+    `)
+
+    // Triggers to keep FTS in sync with docs table
+    this.#db.run(`
+      CREATE TRIGGER IF NOT EXISTS docs_fts_insert AFTER INSERT ON docs BEGIN
+        INSERT INTO docs_fts(rowid, entities, tags, description, title, body)
+        VALUES (new.id, new.entities, new.tags, new.description, new.title, new.body);
+      END
+    `)
+
+    this.#db.run(`
+      CREATE TRIGGER IF NOT EXISTS docs_fts_delete AFTER DELETE ON docs BEGIN
+        INSERT INTO docs_fts(docs_fts, rowid, entities, tags, description, title, body)
+        VALUES ('delete', old.id, old.entities, old.tags, old.description, old.title, old.body);
+      END
+    `)
+
+    this.#db.run(`
+      CREATE TRIGGER IF NOT EXISTS docs_fts_update AFTER UPDATE ON docs
+      WHEN old.body != new.body
+        OR old.title != new.title
+        OR old.description != new.description
+        OR old.tags != new.tags
+        OR old.entities != new.entities
+      BEGIN
+        INSERT INTO docs_fts(docs_fts, rowid, entities, tags, description, title, body)
+        VALUES ('delete', old.id, old.entities, old.tags, old.description, old.title, old.body);
+        INSERT INTO docs_fts(rowid, entities, tags, description, title, body)
+        VALUES (new.id, new.entities, new.tags, new.description, new.title, new.body);
+      END
+    `)
+
+    // FTS5 vocabulary table for IDF-based term weighting
+    this.#db.run(`CREATE VIRTUAL TABLE IF NOT EXISTS docs_vocab USING fts5vocab('docs_fts', 'row')`)
+
+    this.#db.run(`
+      CREATE TABLE IF NOT EXISTS meta (
+        key TEXT PRIMARY KEY,
+        value TEXT
+      )
+    `)
+
+    this.#db.run(`
+      CREATE TABLE IF NOT EXISTS cache (
+        key TEXT PRIMARY KEY,
+        value TEXT NOT NULL,
+        accessed_at TEXT NOT NULL
+      )
+    `)
+  }
+
+  reset() {
+    // Drop triggers first, then FTS (which references docs), then docs
+    this.#db.run(`DROP TRIGGER IF EXISTS docs_fts_insert`)
+    this.#db.run(`DROP TRIGGER IF EXISTS docs_fts_delete`)
+    this.#db.run(`DROP TRIGGER IF EXISTS docs_fts_update`)
+    this.#db.run(`DROP TABLE IF EXISTS docs_fts`)
+    this.#db.run(`DROP TABLE IF EXISTS vec`)
+    this.#db.run(`DROP TABLE IF EXISTS cache`)
+    this.#db.run(`DROP TABLE IF EXISTS docs`)
+    this.#db.run(`DROP TABLE IF EXISTS meta`)
+    this.#db.run(`VACUUM`)
+    this.#vec = { exists: false }
+    this.init()
+  }
+
+  private initVec(dims: number) {
+    if (this.vec.init) return
+    const existingDims = this.vec.dims
+    if (existingDims && existingDims !== dims)
+      throw new Error(
+        `Vector dimension mismatch: existing **vec** has \`${existingDims}\` dims, but got \`${dims}\`.\n` +
+          `Run \`rekal reset\` and \`rekal sync\` to recreate with the correct dimensions.`
+      )
+    this.#db.run(
+      `CREATE VIRTUAL TABLE IF NOT EXISTS vec USING vec0(
+        doc_id INTEGER NOT NULL,
+        seq INTEGER NOT NULL,
+        +path TEXT NOT NULL,
+        embedding float[${dims}] distance_metric=cosine
+      )`
+    )
+    this.#vec = { dims, exists: true, init: true }
+  }
+
+  // --- Docs ---
+
+  getDoc(from: string | number) {
+    const field = typeof from === "number" ? "id" : "path"
+    return this.#db.query(`SELECT * FROM docs WHERE ${field} = ?`).get(from) as DocRow | undefined
+  }
+
+  getDocs(from?: (string | number)[]) {
+    let ret: DocRow[]
+
+    if (!from) ret = this.#db.query(`SELECT * FROM docs`).all() as DocRow[]
+    else {
+      const field = typeof from[0] === "number" ? "id" : "path"
+      const placeholders = from.map(() => "?").join(",")
+      ret = this.#db
+        .query(`SELECT * FROM docs WHERE ${field} IN (${placeholders})`)
+        .all(...from) as DocRow[]
+    }
+    return new Map(ret.map((row) => [row.id, row]))
+  }
+
+  addDoc(row: Omit<DocRow, "id">) {
+    const result = this.#db
+      .query(
+        `INSERT INTO docs (path, hash, body, description, title, tags, entities, updated_at, synced_at)
+        VALUES($path, $hash, $body, $description, $title, $tags, $entities, $updated_at, $synced_at)
+        ON CONFLICT(path) DO UPDATE SET
+          hash = excluded.hash,
+          body = excluded.body,
+          description = excluded.description,
+          title = excluded.title,
+          tags = excluded.tags,
+          entities = excluded.entities,
+          updated_at = excluded.updated_at,
+          synced_at = excluded.synced_at
+        RETURNING id`
+      )
+      .get(row) as { id: number }
+    return result.id
+  }
+
+  deleteDoc(id: number, tables: { docs?: boolean; vec?: boolean } = {}) {
+    // FTS is auto-synced via triggers when docs are deleted/updated
+    if (tables.vec) this.deleteEmbeddings(id)
+    if (tables.docs) this.#db.query(`DELETE FROM docs WHERE id = ?`).run(id)
+  }
+
+  get vec() {
+    if (this.#vec) return this.#vec
+    const row = this.#db
+      .query(`SELECT sql FROM sqlite_master WHERE type = 'table' AND name = 'vec'`)
+      .get() as { sql: string } | undefined
+    const match = row?.sql.match(/embedding float\[(\d+)\]/)
+    this.#vec = { dims: match ? parseInt(match[1]) : undefined, exists: !!row?.sql }
+    return this.#vec
+  }
+
+  getStatus() {
+    const count = (sql: string) => (this.#db.query(sql).get() as { n: number }).n
+    return {
+      cache: count(`SELECT count(*) as n FROM cache`),
+      dbSize: (
+        this.#db
+          .query(`SELECT page_count * page_size as n FROM pragma_page_count, pragma_page_size`)
+          .get() as { n: number }
+      ).n,
+      docs: count(`SELECT count(*) as n FROM docs`),
+      docsWithDescription: count(`SELECT count(*) as n FROM docs WHERE description != ''`),
+      lastSync: (
+        this.#db.query(`SELECT max(synced_at) as t FROM docs`).get() as { t: string | null }
+      ).t,
+      unembedded: count(
+        `SELECT count(*) as n FROM docs WHERE vec_hash IS NULL OR vec_hash != hash`
+      ),
+      vecDims: this.vec.dims,
+      vecs: this.vec.exists ? count(`SELECT count(*) as n FROM vec`) : 0,
+      vocabTerms: count(`SELECT count(DISTINCT term) as n FROM docs_vocab`),
+    }
+  }
+
+  transaction<A extends any[], T>(fn: (...args: A) => T) {
+    return this.#db.transaction(fn)
+  }
+
+  getUnembeddedDocs(): DocRow[] {
+    return this.#db
+      .query(`SELECT * FROM docs
+        WHERE vec_hash IS NULL OR vec_hash != hash
+        ORDER BY path`)
+      .all() as DocRow[]
+  }
+
+  touchDoc(id: number) {
+    this.#db.query(`UPDATE docs SET synced_at = ? WHERE id = ?`).run(new Date().toISOString(), id)
+  }
+
+  markEmbedded(id: number, docHash: string) {
+    this.#db.query(`UPDATE docs SET vec_hash = ? WHERE id = ?`).run(docHash, id)
+  }
+
+  /** Delete docs not seen since the given sync timestamp, optionally scoped to a path prefix. */
+  deleteStaleDocs(syncedBefore: string, prefix?: string): number {
+    let query = `SELECT id FROM docs WHERE synced_at IS NULL OR synced_at < ?`
+    const params = [syncedBefore]
+    if (prefix) {
+      query += ` AND path LIKE ? || '%'`
+      params.push(prefix)
+    }
+    const stale = this.#db.query(query).all(...params) as { id: number }[]
+    for (const { id } of stale) {
+      this.deleteDoc(id, { docs: true, vec: true })
+    }
+    return stale.length
+  }
+
+  // --- FTS ---
+  // FTS is auto-synced via triggers on the docs table.
+
+  /** Scoped FTS search: only match docs whose path starts with one of the given prefixes */
+  searchFts(query: string, opts?: DbSearchOptions): FTSResult[] {
+    if (opts?.scope?.length === 0) return [] // empty scope means no results
+    const scope = opts?.scope ?? []
+    const scopeQuery =
+      scope.length === 0 ? "" : `AND (${scope.map(() => `d.path LIKE ? || '%'`).join(" OR ")})`
+    return this.#db
+      .query(
+        `SELECT f.rowid, bm25(docs_fts, 10, 8, 5, 3, 1) as score
+        FROM docs_fts f
+        ${scope.length > 0 ? "JOIN docs d ON d.id = f.rowid" : ""}
+        WHERE docs_fts MATCH ?
+          ${scopeQuery}
+        ORDER BY score
+        LIMIT ?`
+      )
+      .all(query, ...scope, opts?.limit ?? SEARCH_LIMIT) as FTSResult[]
+  }
+
+  /** * Gets weights for high-frequency terms.
+   * Note: Truly common words will result in an IDF of 0 or less.
+   */
+  getStopWords(): Map<string, number> {
+    // 1. Get total doc count (N) first
+    const totalDocs =
+      (this.#db.query("SELECT count(*) as n FROM docs").get() as { n: number } | undefined)?.n ?? 0
+
+    if (totalDocs === 0) return new Map()
+
+    // 2. Fetch the high-frequency terms
+    const rows = this.#db
+      .query(
+        `SELECT v.term, v.doc 
+       FROM docs_vocab v
+       WHERE v.doc > ? AND v.doc > ?
+       ORDER BY v.doc DESC
+       LIMIT ?`
+      )
+      .all(totalDocs * STOPWORD_THRESHOLD, STOPWORD_MIN_DOCS, STOPWORD_LIMIT) as {
+      term: string
+      doc: number
+    }[]
+
+    return new Map(
+      rows.map((r) => {
+        // Calculate IDF
+        const idf = Math.log((totalDocs - r.doc + 0.5) / (r.doc + 0.5))
+
+        // For stop words, we usually want to clamp at 0.
+        // If a word is in >50% of docs, the formula goes negative.
+        return [r.term, Math.max(0, idf)]
+      })
+    )
+  }
+
+  getWeights(terms: string[]): number[] {
+    if (terms.length === 0) return []
+    const total = (this.#db.query(`SELECT count(*) as n FROM docs`).get() as { n: number }).n
+    const placeholders = terms.map(() => "?").join(",")
+    const rows = this.#db
+      .query(`SELECT term, doc FROM docs_vocab WHERE term IN (${placeholders})`)
+      .all(...terms) as { term: string; doc: number }[]
+    const df = new Map(rows.map((r) => [r.term, r.doc]))
+    return terms.map((t) => Math.log((total - (df.get(t) ?? 0) + 0.5) / ((df.get(t) ?? 0) + 0.5)))
+  }
+
+  // --- Vector ---
+
+  /** Insert embeddings into the vec table */
+  insertEmbeddings(chunks: StoreChunk[]) {
+    assertEmbeddings(chunks)
+    if (chunks.length === 0) return
+    this.initVec(chunks[0].embedding.length)
+    const stmt = this.#db.query(`INSERT INTO vec(doc_id, seq, path, embedding) VALUES (?, ?, ?, ?)`)
+    for (const chunk of chunks) {
+      stmt.run(chunk.doc_id, chunk.seq, chunk.doc.path, JSON.stringify(chunk.embedding))
+    }
+  }
+
+  /** Delete all vec entries for a doc */
+  deleteEmbeddings(docId: number) {
+    if (this.vec.exists) this.#db.query(`DELETE FROM vec WHERE doc_id = ?`).run(docId)
+  }
+
+  /** Global KNN search, returns top results across all docs */
+  searchVec(embedding: number[], opts?: DbSearchOptions): VecResult[] {
+    if (!this.vec.exists) return []
+    const limit = opts?.limit ?? SEARCH_LIMIT
+    return this.#db
+      .query(
+        `SELECT doc_id, seq, path, distance, (1 - distance/2) as score
+        FROM vec
+        WHERE embedding MATCH ?
+          AND k = ?
+        ORDER BY distance`
+      )
+      .all(JSON.stringify(embedding), limit) as VecResult[]
+  }
+
+  // --- Frecency ---
+
+  setDeadline(docId: number, deadline: number) {
+    this.#db.query(`UPDATE docs SET deadline = ? WHERE id = ?`).run(deadline, docId)
+  }
+
+  // --- Meta ---
+
+  getMeta(key: string) {
+    return (
+      this.#db.query(`SELECT value FROM meta WHERE key = ?`).get(key) as
+        | { value: string }
+        | undefined
+    )?.value
+  }
+
+  setMeta(key: string, value: string) {
+    this.#db
+      .query(`INSERT INTO meta (key, value) VALUES (?, ?) ON CONFLICT(key) DO UPDATE SET value = ?`)
+      .run(key, value, value)
+  }
+
+  // --- Cache ---
+
+  cacheGet<T>(key: string): T | undefined {
+    const row = this.#db.query(`SELECT value FROM cache WHERE key = ?`).get(key) as
+      | { value: string }
+      | undefined
+    if (!row) return
+    this.#db
+      .query(`UPDATE cache SET accessed_at = ? WHERE key = ?`)
+      .run(new Date().toISOString(), key)
+    return JSON.parse(row.value) as T
+  }
+
+  cacheSet<T>(key: string, value: T): T {
+    this.#db
+      .query(
+        `INSERT INTO cache (key, value, accessed_at) VALUES (?, ?, ?)
+        ON CONFLICT(key) DO UPDATE SET value = excluded.value, accessed_at = excluded.accessed_at`
+      )
+      .run(key, JSON.stringify(value), new Date().toISOString())
+    return value
+  }
+
+  cachePrune(maxEntries = 10_000) {
+    this.#db
+      .query(
+        `DELETE FROM cache WHERE key NOT IN (
+          SELECT key FROM cache ORDER BY accessed_at DESC LIMIT ?
+        )`
+      )
+      .run(maxEntries)
+  }
+}

package/src/doc.ts

@@ -0,0 +1,163 @@
+import type { Frontmatter, MarkdownDoc } from "./md.ts"
+import type { VfsEntry } from "./vfs.ts"
+
+import { readFile } from "node:fs/promises"
+import { basename, join, resolve } from "pathe"
+import { astat } from "./fs.ts"
+import { parseMarkdown } from "./md.ts"
+import { normUri } from "./uri.ts"
+import { hash } from "./util.ts"
+
+const INDEX = "index.md"
+const MAX_DESC_LENGTH = 30 * 4 // roughly 30 tokens
+
+export type DocFrontmatter = {
+  description?: string
+  tags?: string[]
+  entities?: string[]
+} & Frontmatter
+
+type DocHeading = {
+  level: number
+  text: string
+}
+
+export class Doc {
+  #isDir = false
+  #name = ""
+  hash = ""
+  updated = new Date(0)
+  headings: DocHeading[] = []
+  parsed: MarkdownDoc = { body: "", bodyOffset: 0, frontmatter: {}, sections: [], text: "" }
+
+  protected constructor(
+    public uri: string,
+    public path: string
+  ) {
+    this.path = resolve(path)
+    this.#name = basename(this.path, ".md")
+  }
+
+  /** Full original markdown text, including frontmatter. */
+  get text() {
+    return this.parsed.text
+  }
+
+  /** Markdown body without frontmatter. */
+  get body() {
+    return this.parsed.body
+  }
+
+  /** Parsed frontmatter as an object. */
+  get fm() {
+    return this.parsed.frontmatter as DocFrontmatter
+  }
+
+  /** Name of the doc, derived from the file or folder name, without extension */
+  get name(): string {
+    return this.#name
+  }
+
+  // Actual (non-empty) description from frontmatter
+  get $description() {
+    const ret = this.fm.description?.trim()
+    return ret?.length ? ret : undefined
+  }
+
+  /** Desc from frontmatter or packed from headings. */
+  get description() {
+    const desc = this.$description
+    if (desc || this.headings.length === 0) return desc
+
+    const headings: (DocHeading & { used?: boolean })[] = this.headings.map((h) => ({ ...h }))
+    const minLevel = Math.min(...headings.map((h) => h.level))
+    const maxLevel = Math.max(...headings.map((h) => h.level))
+
+    // Pack by level until we reach maximum description length
+    let chars = 0
+    for (let level = minLevel; level <= maxLevel; level++) {
+      for (const h of headings) {
+        if (h.level !== level) continue
+        if (chars !== 0 && chars + h.text.length > MAX_DESC_LENGTH) continue
+        h.used = true
+        chars += h.text.length
+      }
+    }
+
+    return headings
+      .filter((h) => h.used)
+      .map((h) => h.text)
+      .join(", ")
+      .trim()
+  }
+
+  /** Title from fontmatter or first heading */
+  get $title(): string | undefined {
+    const title = this.fm.title
+    if (typeof title === "string" && title.trim().length > 0) return title.trim()
+    return this.headings[0]?.text
+  }
+
+  /** `$title` if it doesn't contain the name, otherwise `name - $title` */
+  get title() {
+    const title = this.$title
+    if (!(title ?? "").length) return this.name
+    return title?.toLowerCase().includes(this.name.toLowerCase())
+      ? title
+      : `${this.name} - ${title}`
+  }
+
+  get tags(): string[] {
+    return this.fm.tags ?? []
+  }
+
+  get entities(): string[] {
+    return this.fm.entities ?? []
+  }
+
+  get isDir(): boolean {
+    return this.#isDir
+  }
+
+  protected async load(): Promise<Doc | undefined> {
+    const name = basename(this.path)
+
+    // Quick validation. It's up to the caller to ensure the path
+    // is either a markdown file or a directory.
+    if (name === INDEX) throw new Error(`Doc path cannot end with \`${INDEX}\`:\n\`${this.path}\``)
+
+    let s = await astat(this.path)
+    let mdPath = this.path
+
+    if (s?.isDirectory()) {
+      this.#isDir = true
+      mdPath = join(this.path, INDEX)
+      s = await astat(mdPath)
+    }
+
+    if (!s && !this.#isDir) return
+
+    this.uri = normUri(this.uri, this.isDir)
+
+    // read file and normalize line endings to LF
+    const text = (s ? await readFile(mdPath, "utf8") : "").replace(/\r\n/g, "\n")
+
+    this.updated = s?.mtime ?? new Date(0)
+    this.hash = hash(text)
+    this.parsed = parseMarkdown(text)
+    this.headings = this.parsed.sections
+      .filter((section) => section.level > 0 && section.heading.trim().length > 0)
+      .map((section) => ({
+        level: section.level,
+        text: section.heading,
+      }))
+    return this
+  }
+
+  static async load(entry: string | VfsEntry): Promise<Doc | undefined>
+  static async load(uri: string, path?: string): Promise<Doc | undefined>
+  static async load(uri: string | VfsEntry, path?: string): Promise<Doc | undefined> {
+    const e = typeof uri === "string" ? { path: path, uri } : uri
+    return e.path ? await new Doc(e.uri, e.path).load() : undefined
+  }
+}