@rekal/mem 0.0.0

package/src/store.ts

@@ -0,0 +1,180 @@
+import type { Context } from "./context.ts"
+import type { Db } from "./db.ts"
+import type { EmbedderChunk } from "./embed/index.ts"
+
+import { performance } from "node:perf_hooks"
+import { Doc } from "./doc.ts"
+import { Progress } from "./progress.ts"
+export type StoreChunk = EmbedderChunk & {
+  doc_id: number
+  doc: Doc
+}
+
+export class Store {
+  private constructor(
+    public db: Db,
+    public ctx: Context
+  ) {}
+
+  static async load(ctx: Context) {
+    return new Store(await ctx.db(), ctx)
+  }
+
+  // Add or update a node in the store (docs + FTS via triggers, no embeddings)
+  add(doc: Doc) {
+    const row = this.db.getDoc(doc.path)
+
+    if (row?.hash === doc.hash) {
+      this.db.touchDoc(row.id)
+      return row.id
+    }
+
+    // Document changed, so delete old vec
+    if (row) this.db.deleteDoc(row.id, { vec: true })
+
+    const now = new Date().toISOString()
+    const id = this.db.addDoc({
+      body: doc.body,
+      description: doc.$description ?? "",
+      entities: doc.entities.join(","),
+      hash: doc.hash,
+      path: doc.path,
+      synced_at: now,
+      tags: doc.tags.join(","),
+      title: doc.title,
+      updated_at: now,
+    })
+
+    return id
+  }
+
+  // Chunk a doc for embedding
+  async chunk(id: number, doc: Doc): Promise<StoreChunk[]> {
+    const title = doc.title.trim()
+    const description = (doc.description ?? "").trim()
+    const body = doc.body.trim()
+
+    const chunks: EmbedderChunk[] = []
+
+    const embedder = await this.ctx.embedder()
+    if (description.length > 0) chunks.push(...(await embedder.chunk({ text: description, title })))
+
+    if (body.length > 0) {
+      // seq=0 is reserved for description, so offset body seq by at least 1
+      const offset = Math.max(1, chunks.length)
+      const bodyChunks = await embedder.chunk({ text: body, title })
+      chunks.push(
+        ...bodyChunks.map((c) =>
+          Object.assign(c, {
+            seq: c.seq + offset,
+          })
+        )
+      )
+    }
+    return chunks.map((chunk) => Object.assign(chunk, { doc, doc_id: id }))
+  }
+
+  async index() {
+    const docs: Promise<Doc | undefined>[] = []
+    const nodes = new Map<number, Doc>()
+
+    const vfs = await this.ctx.vfs()
+    for await (const entry of vfs.find()) {
+      docs.push(Doc.load(entry))
+    }
+
+    const loaded = await Promise.all(docs)
+    this.db.transaction(() => {
+      for (const doc of loaded) {
+        if (doc) nodes.set(this.add(doc), doc)
+      }
+    })()
+
+    this.ctx.success(`Indexed ${nodes.size} docs from disk`)
+    return nodes
+  }
+
+  async embed(docs: Map<number, Doc>) {
+    const todo = this.db.getUnembeddedDocs()
+    if (todo.length === 0) {
+      this.ctx.success("All docs are already embedded")
+      return
+    }
+    this.ctx.info(`Sync found ${todo.length} unembedded docs`)
+
+    let doneBytes = 0
+    let doneDocs = 0
+    const queue: StoreChunk[] = []
+    const embedder = await this.ctx.embedder()
+    await embedder.backend() // load the embedder before starting the progress bar
+    const start = performance.now()
+    const progress = new Progress("Embedding", { max: todo.length })
+    this.ctx.events.emit("progress", progress)
+
+    const updateProgress = () => {
+      const secs = (performance.now() - start) / 1000
+      const kbPerSec = (doneBytes / secs / 1024).toFixed(0)
+      progress.set({
+        status: `${progress.max - todo.length}/${progress.max} docs embedded ${kbPerSec}kb/s...`,
+        value: doneDocs,
+      })
+    }
+
+    const markEmbedded = (id: number) => {
+      doneDocs++
+      const hash = docs.get(id)?.hash
+      if (hash) this.db.markEmbedded(id, hash)
+    }
+
+    const embed = async (flush?: boolean) => {
+      while (queue.length >= (flush ? 1 : embedder.opts.batchSize)) {
+        const batch = queue.splice(0, embedder.opts.batchSize)
+        // oxlint-disable-next-line no-await-in-loop
+        const embeddings = await embedder.embed(batch)
+        doneBytes += batch.reduce((sum, c) => sum + c.prompt.length, 0)
+        batch.forEach((chunk, i) => (chunk.embedding = embeddings[i]))
+        this.db.insertEmbeddings(batch)
+
+        updateProgress()
+
+        const completed = new Set(batch.map((c) => c.doc_id))
+        for (const c of queue) completed.delete(c.doc_id)
+        completed.forEach((id) => markEmbedded(id))
+      }
+    }
+
+    while (todo.length > 0) {
+      const { id } = todo.pop()!
+      const doc = docs.get(id)
+      if (!doc) continue
+      this.db.deleteEmbeddings(id)
+      // oxlint-disable-next-line no-await-in-loop
+      const chunks = await this.chunk(id, doc)
+      queue.push(...chunks)
+      if (chunks.length === 0) markEmbedded(id) // mark as embedded even if there are no chunks to embed
+      // oxlint-disable-next-line no-await-in-loop
+      await embed()
+    }
+    await embed(true) // embed any remaining chunks in the queue
+
+    progress.stop()
+    this.ctx.success("Sync complete")
+  }
+
+  async sync(opts?: { embed?: boolean }) {
+    const syncStart = new Date().toISOString()
+    const docs = await this.index()
+    await this.prune(syncStart)
+    if (opts?.embed) await this.embed(docs)
+  }
+
+  // Remove docs that no longer exist on disk
+  async prune(syncStart: string) {
+    let dels = 0
+    const vfs = await this.ctx.vfs()
+    for (const { path } of vfs.folders) {
+      dels += this.db.deleteStaleDocs(syncStart, path)
+    }
+    if (dels > 0) this.ctx.warn(`Removed ${dels} stale docs`)
+  }
+}

package/src/uri.ts

@@ -0,0 +1,28 @@
+export const URI_PREFIX = "rekal://"
+
+export function assertUri(uri: string) {
+  if (!uri.startsWith(URI_PREFIX)) throw new Error(`URI must start with ${URI_PREFIX}, got: ${uri}`)
+}
+
+export function normUri(uri?: string, dir?: boolean): string {
+  if (uri === undefined) return URI_PREFIX
+  if (typeof uri !== "string") throw new Error(`URI must be a string, got: ${JSON.stringify(uri)}`)
+  uri = uri.trim()
+  uri = uri.replace(/^rekall?:/, "") // protocol
+  uri = uri.replace(/[\\/]+/g, "/") // normalize slashes
+  uri = uri.replace(/^\/+/, "") // leading slashes
+  if (uri === "") return URI_PREFIX
+  uri = URI_PREFIX + uri
+  if (uri.endsWith("/index.md")) return uri.replace(/\/index\.md$/, "/") // index.md implies directory
+  uri = dir ? uri.replace(/\/?$/, "/") : uri // trailing slash for directories
+  uri = dir === false ? uri.replace(/\/?$/, "") : uri // remove trailing slash for files
+  return uri
+}
+
+export function parentUri(uri: string): string | undefined {
+  uri = normUri(uri)
+  if (uri === URI_PREFIX) return
+  uri = uri.replace(/\/?$/, "") // remove trailing slash
+  uri = uri.replace(/\/[^/]+$/, "") // remove last segment
+  return uri === URI_PREFIX ? URI_PREFIX : `${uri}/`
+}

package/src/util.ts

@@ -0,0 +1,21 @@
+import { parseYaml } from "#runtime"
+import { createHash } from "node:crypto"
+
+export { parseYaml }
+
+export function hash(content: string): string {
+  return createHash("sha256").update(content).digest("hex")
+}
+
+export function toError(err: unknown): Error {
+  return err instanceof Error ? err : new Error(String(err))
+}
+
+export type Events = Record<string, unknown[]>
+
+export type TypedEmitter<T extends Events> = {
+  on<K extends keyof T>(event: K, fn: (...args: T[K]) => void): TypedEmitter<T>
+  off<K extends keyof T>(event: K, fn: (...args: T[K]) => void): TypedEmitter<T>
+  once<K extends keyof T>(event: K, fn: (...args: T[K]) => void): TypedEmitter<T>
+  emit<K extends keyof T>(event: K, ...args: T[K]): boolean
+}

package/src/vfs.ts

@@ -0,0 +1,257 @@
+import type { Context } from "./context.ts"
+import type { Doc } from "./doc.ts"
+
+import { basename, join, relative } from "pathe"
+import { normPath, sstat } from "./fs.ts"
+import { URI_PREFIX, normUri } from "./uri.ts"
+
+export class Node {
+  constructor(
+    public uri: string,
+    public doc: Doc
+  ) {}
+}
+
+const DEFAULT_EXCLUDE = [".git", "node_modules/"]
+
+export type VfsFolder = {
+  uri: string
+  path: string
+  merge?: boolean // TODO: whether this path should be merged with others in the same URI, defaults to false
+}
+
+export type VfsEntry = {
+  uri: string
+  path?: string
+}
+
+export type VfsPath = {
+  node: VfsNode
+  path: string
+}
+
+export type VfsNode = {
+  name: string
+  parent?: VfsNode
+  uri: string
+  paths: string[]
+  depth: number
+  children: Map<string, VfsNode>
+}
+
+export type VfsFindOptions = {
+  /** URI to start from, defaults to root */
+  uri?: string
+  depth?: number // max depth to search, defaults to Infinity
+  pattern?: string // extra regex pattern to match URIs against
+  ignoreCase?: boolean // whether pattern matching should ignore case. When not set smart case is used.
+  limit?: number // max results to return, defaults to Infinity
+  type?: "file" | "directory" // filter by type
+}
+
+export type VfsView = {
+  uri: string // URI of the scope, defaults to rekal://
+  node: VfsNode // the node representing the resolved URI
+  paths: VfsPath[] // paths leading up to this node and to descendant nodes
+}
+
+export type VfsScope = VfsView & {
+  // map a path to the shortest URI in this scope, if it exists
+  map: (path: string) => string | undefined
+}
+
+export class Vfs {
+  #folders = new Map<string, VfsFolder[]>() // map of paths to folders
+  #root: VfsNode = { children: new Map(), depth: 0, name: "#root", paths: [], uri: URI_PREFIX }
+
+  public constructor(public ctx: Context) {
+    for (const folder of ctx.opts.folders ?? []) this.addFolder(folder)
+  }
+
+  get folders(): VfsFolder[] {
+    return [...this.#folders.values()].flat()
+  }
+
+  isFolder(path: string): boolean {
+    path = normPath(path).replace(/\/?$/, "/")
+    return this.#folders.has(path)
+  }
+
+  getScope(uri?: string, opts?: { children?: boolean }): VfsScope {
+    uri = normUri(uri, true)
+    const view = this.resolve(uri, opts)
+    return {
+      ...view,
+      map: (path: string) => {
+        path = normPath(path)
+        let best: string | undefined
+        for (const p of view.paths) {
+          const rel = relative(p.path, path)
+          if (rel.startsWith("..")) continue
+          const candidate = p.node.uri + rel
+          if (!best || candidate.length < best.length) best = candidate
+        }
+        return best
+      },
+    }
+  }
+
+  getNode(uri: string, create = false) {
+    uri = normUri(uri)
+    let node = this.#root
+    const parts = uri.slice(URI_PREFIX.length).split("/").filter(Boolean)
+    for (const part of parts) {
+      let child = node.children.get(part)
+      if (!child) {
+        child = {
+          children: new Map(),
+          depth: node.depth + 1,
+          name: part,
+          parent: node,
+          paths: [],
+          uri: `${node.uri}${part}/`,
+        }
+        if (create) node.children.set(part, child)
+      }
+      node = child
+    }
+    return node
+  }
+
+  resolve(node: VfsNode | string, opts?: { children?: boolean }): VfsView {
+    node = typeof node === "string" ? this.getNode(node) : node
+    const nodes = [node]
+    let { parent } = node
+    while (parent) {
+      nodes.unshift(parent)
+      parent = parent.parent
+    }
+
+    const paths: VfsPath[] = [] // paths to this node or to descendant nodes
+    const folders: VfsPath[] = [] // folders used by ancestor and descendant nodes
+
+    // resolve paths and folders to this node
+    for (const n of nodes) {
+      for (const p of paths) p.path = join(p.path, n.name)
+      for (const path of n.paths) {
+        paths.push({ node, path })
+        if (this.isFolder(path)) folders.push({ node: n, path })
+      }
+    }
+
+    // resolve paths and folders to descendant nodes
+    const stack = opts?.children === false ? [] : [...node.children.values()]
+    while (stack.length > 0) {
+      const n = stack.pop()!
+      for (const path of n.paths) {
+        paths.push({ node: n, path })
+        if (this.isFolder(path)) folders.push({ node: n, path })
+      }
+      stack.push(...n.children.values())
+    }
+
+    paths.sort((a, b) => a.node.uri.localeCompare(b.node.uri))
+    return { node, paths, uri: node.uri }
+  }
+
+  addFolder(folder: VfsFolder) {
+    folder.uri = normUri(folder.uri, true)
+    folder.path = normPath(folder.path).replace(/\/?$/, "/")
+    const folders = this.#folders.get(folder.path) ?? []
+    this.#folders.set(folder.path, [...folders, folder])
+    const node = this.getNode(folder.uri, true)
+    node.paths.push(folder.path)
+  }
+
+  matcher(opts?: VfsFindOptions): (uri: string) => boolean {
+    const pattern = opts?.pattern ?? ""
+    if (!pattern.length) return () => true
+    const ignoreCase = opts?.ignoreCase ?? !/[A-Z]/.test(pattern)
+    const re = new RegExp(pattern, ignoreCase ? "i" : "")
+    return (uri: string) => re.test(uri)
+  }
+
+  async *find(opts: VfsFindOptions = {}): AsyncGenerator<VfsEntry> {
+    const { glob } = await import("./glob.ts")
+    const uri = normUri(opts.uri ?? URI_PREFIX, true)
+    const root = this.resolve(uri)
+    const maxDepth = root.node.depth + (opts.depth ?? Infinity)
+    const visited = new Set<string>()
+    const filter = this.matcher(opts)
+
+    const stop = () => opts.limit !== undefined && visited.size >= opts.limit
+
+    const use = (p: VfsEntry) => {
+      if (p.uri.endsWith("/") && opts.type === "file") return false
+      if (!filter(p.uri)) return false
+      if (stop()) return false
+      const key = `${p.uri}:${p.path ?? ""}`
+      if (visited.has(key)) return false
+      visited.add(key)
+      return true
+    }
+
+    // add virtual internal uris
+    function* yieldVirtual(p: VfsPath) {
+      if (p.node === root.node || opts.type === "file") return
+      let parent = p.node.parent
+      while (parent && parent !== root.node && !stop()) {
+        const virtual =
+          parent.depth <= maxDepth &&
+          !root.paths.some((rp) => parent?.uri.startsWith(rp.node.uri)) &&
+          use({ uri: parent.uri })
+        if (virtual) yield { uri: parent.uri }
+        parent = parent.parent
+      }
+    }
+
+    scan: for (const p of root.paths) {
+      yield* yieldVirtual(p)
+      if (p.node.depth > maxDepth) continue
+      const e = { path: p.path, uri: p.node.uri }
+      if (p.node !== root.node && use(e)) yield e
+
+      const cwd = p.path
+      const todo = glob({
+        cwd,
+        depth: maxDepth - p.node.depth,
+        empty: false,
+        exclude: DEFAULT_EXCLUDE,
+        glob: ["**/*.md"],
+        type: opts.type,
+      })
+
+      // oxlint-disable-next-line no-await-in-loop
+      for await (const childPath of todo) {
+        if (basename(childPath) === "index.md") continue
+        const path = { path: join(cwd, childPath), uri: p.node.uri + childPath }
+        if (use(path)) yield path
+        if (stop()) break scan
+      }
+    }
+  }
+
+  async *ls(opts?: Omit<VfsFindOptions, "depth">) {
+    yield* this.find({ ...opts, depth: 1 })
+  }
+
+  /** Normalizes the URI and path to a real path if it exists **/
+  normPath(p: VfsPath): VfsEntry {
+    const transforms = [
+      { from: /\/index\.md$/, to: "/" },
+      { from: /\/index$/, to: "/" },
+      { from: /\.md$/, to: "" },
+      { from: /(?!\.md)$/, to: ".md" },
+    ]
+    if (!sstat(p.path) || basename(p.path) === "index.md") {
+      const root = this.getScope()
+      for (const t of transforms) {
+        const path = p.path.replace(t.from, t.to)
+        const uri = root.map(path)
+        if (sstat(path) && uri) return { path, uri }
+      }
+      return { uri: p.node.uri }
+    }
+    return { path: p.path, uri: p.node.uri }
+  }
+}

package/test/doc.test.ts

@@ -0,0 +1,61 @@
+import { fileURLToPath } from "node:url"
+import { join } from "pathe"
+import { describe, expect, test } from "vitest"
+import { Doc } from "../src/doc.ts"
+
+const FIXTURES = join(fileURLToPath(import.meta.url), "..", "fixtures/store")
+
+describe("Doc.load", () => {
+  test("loads a markdown file", async () => {
+    const doc = (await Doc.load("", join(FIXTURES, "user/family.md")))!
+    expect(doc).toBeDefined()
+    expect(doc.description).toContain("Douglas's family")
+    expect(doc.tags).toContain("family")
+    expect(doc.entities).toContain("Melina")
+    expect(doc.entities).toContain("Lori Quaid")
+    expect(doc.title).toBeTruthy()
+    expect(doc.body).toContain("Lori")
+  })
+
+  test("handles directory with index.md", async () => {
+    const doc = await Doc.load("", join(FIXTURES, "concepts"))
+    expect(doc?.isDir).toBe(true)
+  })
+
+  test("throws for nonexistent file", async () => {
+    expect(await Doc.load("", join(FIXTURES, "nonexistent.md"))).toBeUndefined()
+  })
+
+  test("extracts title from H1", async () => {
+    const doc = await Doc.load("", join(FIXTURES, "concepts/frecency.md"))
+    expect(doc?.title).toBe("Frecency Algorithm")
+  })
+
+  test("handles file without frontmatter", async () => {
+    // Create a doc from raw content to simulate
+    const doc = await Doc.load("", join(FIXTURES, "user/family.md"))
+    expect(doc?.tags.length).toBeGreaterThan(0)
+  })
+})
+
+describe("Doc properties", () => {
+  test("name for regular file", async () => {
+    const doc = await Doc.load("", join(FIXTURES, "user/family.md"))
+    expect(doc?.name).toBe("family")
+  })
+
+  test("name for directory (index.md)", async () => {
+    const doc = await Doc.load("", join(FIXTURES, "user"))
+    expect(doc?.name).toBe("user")
+  })
+
+  test("isDir for index.md", async () => {
+    const doc = await Doc.load("", join(FIXTURES, "user"))
+    expect(doc?.isDir).toBe(true)
+  })
+
+  test("isDir for regular file", async () => {
+    const doc = await Doc.load("", join(FIXTURES, "user/family.md"))
+    expect(doc?.isDir).toBe(false)
+  })
+})

package/test/fixtures/store/agent/index.md

@@ -0,0 +1,9 @@
+---
+description: "Agent configuration — personality, operational notes, and accumulated lessons"
+tags: [agent, config]
+entities: [Douglas Quaid]
+---
+
+# Agent Configuration
+
+Configuration and learned behaviors for the AI agent. Defines personality, tool-specific notes, and lessons from past interactions.

package/test/fixtures/store/agent/lessons.md

@@ -0,0 +1,21 @@
+---
+description: "Lessons learned from past agent mistakes — verification, attribution, skill priority"
+tags: [agent, lessons, corrections]
+entities: []
+---
+
+# Lessons Learned from Agent Mistakes
+
+Accumulated lessons from past failures. Key themes: always verify before speaking, use specialized tools before general search, and watch for cognitive biases in content generation.
+
+## Verification First
+
+Never state an action is "done" before actually executing the tool. Always call the tool first and verify facts before speaking. Multiple failures from premature confirmation.
+
+## Attribution Accuracy
+
+Messed up a news report by confusing two country names — caused by contextual bias from recent conversations. Must carefully cross-reference entity names in headlines before reporting.
+
+## Skill Priority
+
+Defaulted to generic web search for "AI news" instead of using the specialized news skill. Always check defined skills and sources first to ensure high-signal, persona-aligned updates before falling back to general search.

package/test/fixtures/store/agent/soul.md

@@ -0,0 +1,28 @@
+---
+description: "Agent personality — be genuine, have opinions, earn trust through competence"
+tags: [agent, personality, identity]
+entities: []
+---
+
+# Agent Personality
+
+Core principles: genuinely helpful over performatively helpful, resourceful before asking, careful with external actions, bold with internal ones.
+
+## Core Truths
+
+- Be genuinely helpful, not performatively helpful. Skip filler words, just help.
+- Have opinions. Disagree, prefer things, find stuff amusing or boring.
+- Be resourceful before asking. Read the file, check the context, search for it.
+- Earn trust through competence. Be careful with external actions, bold with internal ones.
+- Remember you're a guest. Access to someone's life is intimacy — treat it with respect.
+
+## Boundaries
+
+- Private things stay private
+- Ask before acting externally when in doubt
+- Never send half-baked replies to messaging surfaces
+- Not the user's voice — careful in group chats
+
+## Vibe
+
+Concise when needed, thorough when it matters. Not a corporate drone. Not a sycophant.

package/test/fixtures/store/agent/tools.md

@@ -0,0 +1,25 @@
+---
+description: "Agent tool-specific notes — calendar, email, home automation quirks"
+tags: [agent, tools, operations]
+entities: [Home Assistant]
+---
+
+# Agent Tool Notes
+
+Operational notes for tools the agent uses. Includes calendar patterns, email filtering, and home automation quirks.
+
+## Calendar
+
+- Always check all calendars by default
+- Focus on urgent and personal events
+
+## Email
+
+- Focus on urgent/personal email, ignore newsletters and promotions
+- PDF extraction: convert to Google Docs as workaround for parsing
+
+## Home Automation
+
+- Use status scripts for entity overview
+- Some integrations go offline periodically — check connection status before reporting errors
+- Ignore known false alarms: offline VMs, sensors on units without hardware

package/test/fixtures/store/concepts/frecency.md

@@ -0,0 +1,30 @@
+---
+description: "Frecency algorithm — frequency × recency scoring with exponential decay"
+tags: [algorithm, scoring, frecency]
+entities: [Firefox]
+---
+
+# Frecency Algorithm
+
+Frecency combines frequency and recency into a single score. Items accessed often and recently rank highest. Used in Firefox's URL bar and rekal's memory injection.
+
+## Core Idea
+
+Each access bumps a score. Scores decay exponentially over time. The result is a ranking that naturally balances "how often" with "how recently."
+
+## Exponential Decay
+
+```
+score(t) = Σ e^(-λ * (now - access_i))
+```
+
+Where λ controls how fast old accesses fade. Higher λ = more recency-biased.
+
+## Deadline Trick
+
+Instead of recomputing all scores on every access, store a "deadline" — the time at which the score would drop below a threshold. On read, only decay if the deadline has passed. This makes frecency O(1) amortized per access.
+
+## Applications
+
+- **Firefox:** URL bar suggestions ranked by frecency
+- **rekal:** Memory nodes scored by frecency for auto-injection into agent context

package/test/fixtures/store/concepts/index.md

@@ -0,0 +1,9 @@
+---
+description: "Technical concepts — frecency, RAG, memory coherence, retrieval methods"
+tags: [concepts, reference]
+entities: []
+---
+
+# Technical Concepts
+
+General-purpose technical concepts that appear across multiple projects. Includes retrieval methods, scoring algorithms, memory systems, and AI patterns.