@rekal/mem 0.0.0 → 0.0.2

package/dist/{openai-j2_2GM4J.mjs → openai-ALl6_YhI.mjs}

@@ -1,4 +1,4 @@
-import { n as parseModelUri } from "./models-DFQSgBNr.mjs";
+import { n as parseModelUri } from "./models-Bo6czhQe.mjs";
 //#region src/embed/openai.ts
 const OPENAI_EMBEDDING_URL = "https://api.openai.com/v1/embeddings";
 const MODEL_INFO = {
@@ -74,3 +74,5 @@ var OpenAIBackend = class OpenAIBackend {
 };
 //#endregion
 export { OpenAIBackend };
+
+//# sourceMappingURL=openai-ALl6_YhI.mjs.map

package/dist/openai-ALl6_YhI.mjs.map

@@ -0,0 +1 @@
+{"version":3,"file":"openai-ALl6_YhI.mjs","names":["#model","#apiKey","#tokenizer","#ctx"],"sources":["../src/embed/openai.ts"],"sourcesContent":["import type { encode as Tokenize } from \"gpt-tokenizer\"\nimport type { EmbedderBackend, EmbedderContext } from \"./index.ts\"\n\nimport { parseModelUri } from \"./models.ts\"\n\nconst OPENAI_EMBEDDING_URL = \"https://api.openai.com/v1/embeddings\"\n\n// Context sizes for known OpenAI embedding models\nconst MODEL_INFO: Record<string, { contextSize: number; vectorSize: number } | undefined> = {\n  \"text-embedding-3-large\": { contextSize: 8191, vectorSize: 3072 },\n  \"text-embedding-3-small\": { contextSize: 8191, vectorSize: 1536 },\n  \"text-embedding-ada-002\": { contextSize: 8191, vectorSize: 1536 },\n}\n\nexport class OpenAIBackend implements EmbedderBackend {\n  device = \"api\" as const\n  maxTokens: number\n  dims: number\n  #model: string\n  #apiKey: string\n  #tokenizer: typeof Tokenize\n  #ctx: EmbedderContext\n\n  // oxlint-disable-next-line max-params\n  private constructor(\n    model: string,\n    apiKey: string,\n    info: { contextSize: number; vectorSize: number },\n    tokenizer: typeof Tokenize,\n    ctx: EmbedderContext\n  ) {\n    this.#model = model\n    this.#apiKey = apiKey\n    this.maxTokens = info.contextSize\n    this.dims = info.vectorSize\n    this.#tokenizer = tokenizer\n    this.#ctx = ctx\n  }\n\n  static async load(this: void, ctx: EmbedderContext): Promise<OpenAIBackend> {\n    const { model } = parseModelUri(ctx.opts.model.uri)\n    const apiKey = process.env.OPENAI_API_KEY\n    if (!apiKey)\n      throw new Error(\"Missing `OPENAI_API_KEY` environment variable for OpenAI embeddings.\")\n\n    const { encode } = await import(`gpt-tokenizer/model/${model}`)\n    // Probe the model for dimensions if not in our known list\n    let info = MODEL_INFO[model]\n    if (!info) {\n      ctx.status.status = \"probing model dimensions...\"\n      const backend = new OpenAIBackend(\n        model,\n        apiKey,\n        { contextSize: 8191, vectorSize: 0 },\n        encode,\n        ctx\n      )\n      const result = await backend.embed([\"test\"])\n      info = { contextSize: 8191, vectorSize: result[0].length }\n    }\n\n    return new OpenAIBackend(model, apiKey, info, encode, ctx)\n  }\n\n  async embed(texts: string[]): Promise<number[][]> {\n    const response = await fetch(OPENAI_EMBEDDING_URL, {\n      body: JSON.stringify({\n        dimensions: this.#ctx.opts.maxDims,\n        input: texts,\n        model: this.#model,\n      }),\n      headers: {\n        Authorization: `Bearer ${this.#apiKey}`,\n        \"Content-Type\": \"application/json\",\n      },\n      method: \"POST\",\n    })\n\n    if (!response.ok) {\n      const error = await response.text()\n      throw new Error(`OpenAI embedding API error (${response.status}): ${error}`)\n    }\n\n    const data = (await response.json()) as {\n      data: { embedding: number[]; index: number }[]\n    }\n\n    // Sort by index to maintain input order\n    return data.data.toSorted((a, b) => a.index - b.index).map((d) => d.embedding)\n  }\n\n  toks(input: string): number {\n    return this.#tokenizer(input).length\n  }\n}\n"],"mappings":";;AAKA,MAAM,uBAAuB;AAG7B,MAAM,aAAsF;CAC1F,0BAA0B;EAAE,aAAa;EAAM,YAAY;EAAM;CACjE,0BAA0B;EAAE,aAAa;EAAM,YAAY;EAAM;CACjE,0BAA0B;EAAE,aAAa;EAAM,YAAY;EAAM;CAClE;AAED,IAAa,gBAAb,MAAa,cAAyC;CACpD,SAAS;CACT;CACA;CACA;CACA;CACA;CACA;CAGA,YACE,OACA,QACA,MACA,WACA,KACA;AACA,QAAA,QAAc;AACd,QAAA,SAAe;AACf,OAAK,YAAY,KAAK;AACtB,OAAK,OAAO,KAAK;AACjB,QAAA,YAAkB;AAClB,QAAA,MAAY;;CAGd,aAAa,KAAiB,KAA8C;EAC1E,MAAM,EAAE,UAAU,cAAc,IAAI,KAAK,MAAM,IAAI;EACnD,MAAM,SAAS,QAAQ,IAAI;AAC3B,MAAI,CAAC,OACH,OAAM,IAAI,MAAM,uEAAuE;EAEzF,MAAM,EAAE,WAAW,MAAM,OAAO,uBAAuB;EAEvD,IAAI,OAAO,WAAW;AACtB,MAAI,CAAC,MAAM;AACT,OAAI,OAAO,SAAS;AASpB,UAAO;IAAE,aAAa;IAAM,aADb,MAPC,IAAI,cAClB,OACA,QACA;KAAE,aAAa;KAAM,YAAY;KAAG,EACpC,QACA,IACD,CAC4B,MAAM,CAAC,OAAO,CAAC,EACG,GAAG;IAAQ;;AAG5D,SAAO,IAAI,cAAc,OAAO,QAAQ,MAAM,QAAQ,IAAI;;CAG5D,MAAM,MAAM,OAAsC;EAChD,MAAM,WAAW,MAAM,MAAM,sBAAsB;GACjD,MAAM,KAAK,UAAU;IACnB,YAAY,MAAA,IAAU,KAAK;IAC3B,OAAO;IACP,OAAO,MAAA;IACR,CAAC;GACF,SAAS;IACP,eAAe,UAAU,MAAA;IACzB,gBAAgB;IACjB;GACD,QAAQ;GACT,CAAC;AAEF,MAAI,CAAC,SAAS,IAAI;GAChB,MAAM,QAAQ,MAAM,SAAS,MAAM;AACnC,SAAM,IAAI,MAAM,+BAA+B,SAAS,OAAO,KAAK,QAAQ;;AAQ9E,UALc,MAAM,SAAS,MAAM,EAKvB,KAAK,UAAU,GAAG,MAAM,EAAE,QAAQ,EAAE,MAAM,CAAC,KAAK,MAAM,EAAE,UAAU;;CAGhF,KAAK,OAAuB;AAC1B,SAAO,MAAA,UAAgB,MAAM,CAAC"}

package/dist/progress-B1JdNapX.mjs

@@ -261,3 +261,5 @@ var Progress = class Progress extends EventEmitter {
 };
 //#endregion
 export { parseMarkdown as a, parseFrontmatter as i, chunkMarkdown as n, parseSections as o, chunkText as r, Progress as t };
+
+//# sourceMappingURL=progress-B1JdNapX.mjs.map

package/dist/progress-B1JdNapX.mjs.map

@@ -0,0 +1 @@
+{"version":3,"file":"progress-B1JdNapX.mjs","names":["#chars","#toks","#children","#done","#value","#max","#status"],"sources":["../src/md.ts","../src/progress.ts"],"sourcesContent":["import type { TokenCounter } from \"./embed/index.ts\"\n\nimport { parseYaml } from \"#runtime\"\n\n// NOTE: all markdown parsing expects normalized line endings (\\n)\n\n// average chars per token across common models,\n// used for estimating token counts without actual tokenization\nconst CHARS_PER_TOKEN = 3\n\nexport type MarkdownSection = {\n  content: string[]\n  context: string[] // parent headings for context, e.g. [\"# Chapter 1\", \"## Section 1.2\"]\n  /** full heading text with markdown syntax, e.g. \"## Section 1.2\" */\n  headingText: string\n  /** heading without markdown syntax, e.g. \"Section 1.2\" */\n  heading: string\n  level: number\n  offset: number // 0-indexed line offset of the section in the original markdown body, used for mapping back to source\n}\n\nexport type MarkdownDoc = {\n  body: string\n  bodyOffset: number // line offset of the body start (after frontmatter) in the original markdown\n  frontmatter: Frontmatter\n  frontmatterText?: string\n  sections: MarkdownSection[]\n  text: string\n}\n\nexport type Frontmatter = Record<string, unknown>\n\nexport function parseFrontmatter(text: string): Omit<MarkdownDoc, \"sections\"> {\n  const match = text.match(/^---\\n([\\s\\S]*?)\\n---\\n?/)\n  const body = match ? text.slice(match[0].length) : text\n  return {\n    body,\n    bodyOffset: match?.[0].trim().split(\"\\n\").length ?? 0,\n    frontmatter: match ? (parseYaml(match[1]) as Record<string, unknown>) : {},\n    frontmatterText: match?.[0],\n    text,\n  }\n}\n\nexport function parseMarkdown(text: string): MarkdownDoc {\n  const ret = parseFrontmatter(text)\n  return { ...ret, sections: parseSections(ret.body) }\n}\n\nexport function parseSections(md: string): MarkdownSection[] {\n  const lines = md.split(/\\n/)\n  let current: MarkdownSection = {\n    content: [],\n    context: [],\n    heading: \"\",\n    headingText: \"\",\n    level: 0,\n    offset: 0,\n  }\n  const sections: MarkdownSection[] = [current]\n  let codeBlock: string | undefined = undefined\n  for (const [i, line] of lines.entries()) {\n    const match = line.match(/^(#+)\\s+(.*)/)\n    const fenceMatch = line.match(/^\\s*(`{3,}|~{3,})/)\n\n    if (codeBlock && line.startsWith(codeBlock)) {\n      codeBlock = undefined // end of code block\n    } else if (!codeBlock && fenceMatch) {\n      codeBlock = fenceMatch[1] // start of code block\n    }\n\n    if (!codeBlock && match) {\n      if (current.content.length === 0) sections.pop() // discard empty sections.\n      const level = match[1].length\n      current = {\n        content: [line],\n        context: [],\n        heading: match[2].trim(),\n        headingText: match[0].trim(),\n        level,\n        offset: i,\n      }\n      sections.push(current)\n    } else current.content.push(line)\n  }\n\n  const stack: MarkdownSection[] = []\n  for (const section of sections) {\n    // Track parent sections\n    while ((stack.at(-1)?.level ?? -1) >= section.level) stack.pop()\n    section.context = stack.map((s) => s.headingText)\n    if (section.level > 0) stack.push(section)\n  }\n  return sections\n}\n\nfunction findSplit(slice: string) {\n  for (const sub of [\"\\n\\n\", \"\\n\", \"  \", \"\\t\", \" \"]) {\n    const i = slice.lastIndexOf(sub)\n    if (i > slice.length * 0.8) {\n      return i\n    }\n  }\n  return slice.length\n}\n\nclass SafeCounter {\n  static #chars = 0\n  static #toks = 0\n\n  constructor(\n    public tok: TokenCounter,\n    public maxTokens = 500\n  ) {}\n\n  get charsPerToken() {\n    return SafeCounter.#toks > this.maxTokens * 2\n      ? SafeCounter.#chars / SafeCounter.#toks\n      : CHARS_PER_TOKEN\n  }\n\n  estimate(text: string) {\n    return Math.ceil(text.length / this.charsPerToken)\n  }\n\n  // Returns the actual token count, unless the estimated token count based\n  // on character length is much higher than the maxTokens\n  toks(text: string) {\n    if (text.length === 0) return { count: 0, estimated: false }\n    let count = this.estimate(text) * 0.9 // add 10% buffer to account for variance in chars per token\n    if (count > this.maxTokens) return { count, estimated: true }\n    count = this.tok.toks(text)\n    SafeCounter.#chars += text.length\n    SafeCounter.#toks += count\n    return { count, estimated: false }\n  }\n}\n\nexport function chunkText(text: string, tok: TokenCounter, size = 500): string[] {\n  const counter = new SafeCounter(tok, size)\n  const chunks: string[] = []\n  while (text.length) {\n    let next = text\n    let toks = counter.toks(next)\n\n    if (toks.count <= size) {\n      chunks.push(next)\n      break\n    }\n    let maxChars = size * counter.charsPerToken * 0.8\n    // oxlint-disable-next-line typescript/no-unnecessary-condition\n    while (true) {\n      maxChars = Math.min(maxChars, next.length)\n      const split = findSplit(next.slice(0, maxChars))\n      next = next.slice(0, split)\n      toks = counter.toks(next)\n      if (toks.count <= size) break\n      maxChars *= (size / toks.count) * 0.8\n    }\n\n    chunks.push(next)\n    text = text.slice(next.length)\n  }\n  return chunks\n}\n\nexport function chunkMarkdown(md: string, tok: TokenCounter, size = 500): string[] {\n  const sections = parseSections(md)\n  type Chunk = { content: string[]; tokens: number; context: string[] }\n  const chunks: Chunk[] = [{ content: [], context: [], tokens: 0 }]\n  const counter = new SafeCounter(tok, size)\n\n  for (const section of sections) {\n    const chunk = chunks.at(-1) as Chunk\n    // Include parent headings in the content to preserve context\n    const content = [...section.context, ...section.content]\n    const text = content.join(\"\\n\")\n    const toks = counter.toks(text).count\n\n    if (chunk.tokens + toks <= size) {\n      // only add parent headings that aren't already in the chunk for context\n      const context = section.context.filter((h, c) => chunk.context[c] !== h)\n      chunk.content.push(...context)\n      chunk.content.push(...section.content)\n      chunk.context = [...section.context, section.headingText]\n      chunk.tokens += toks\n    } else if (toks <= size) {\n      chunks.push({ content, context: [...section.context, section.headingText], tokens: toks })\n    } else {\n      const context = section.context.join(\"\\n\")\n      const toksCtx = counter.toks(context)\n      chunks.push(\n        ...chunkText(section.content.join(\"\\n\"), tok, size - toksCtx.count).map((c) => ({\n          content: (context.length ? `${context}\\n${c}` : c).split(\"\\n\"),\n          context: [],\n          tokens: 0, // we don't track tokens for these sub-chunks since they're already guaranteed to fit\n        }))\n      )\n      chunks.push({ content: [], context: [], tokens: 0 }) // start a new chunk for the next section\n    }\n  }\n\n  return chunks.map((c) => c.content.join(\"\\n\").trim()).filter(Boolean)\n}\n","import type { TypedEmitter } from \"./util.ts\"\n\nimport { EventEmitter } from \"node:events\"\nimport { inspect } from \"node:util\"\n\nexport type ProgressOpts = { max?: number; status?: string; value?: number }\n\ntype ProgressEvents = {\n  update: [progress: Progress]\n  done: [progress: Progress]\n}\n\nexport class Progress extends (EventEmitter as new () => TypedEmitter<ProgressEvents>) {\n  #max = 100\n  #value = 0\n  #children = new Map<string, Progress>()\n  #status?: string\n  #done = false\n\n  constructor(\n    public name: string,\n    opts: ProgressOpts = {}\n  ) {\n    super()\n    this.set(opts)\n  }\n\n  get group() {\n    return this.#children.size > 0\n  }\n\n  set(opts: ProgressOpts | number): this {\n    if (this.#done) return this\n    if (typeof opts === \"number\") this.#value = opts\n    else {\n      this.#max = opts.max ?? this.#max\n      this.#status = opts.status ?? this.#status\n      this.#value = opts.value ?? this.#value\n    }\n    this.emit(\"update\", this)\n    if (this.#value >= this.#max) this.stop()\n    return this\n  }\n\n  get status() {\n    return this.#status ?? this.name\n  }\n\n  set status(status: string) {\n    this.set({ status })\n  }\n\n  set value(value: number) {\n    this.set(value)\n  }\n\n  get value(): number {\n    return !this.group ? this.#value : this.#children.values().reduce((sum, c) => sum + c.value, 0)\n  }\n\n  set max(max: number) {\n    this.set({ max })\n  }\n\n  get max(): number {\n    return !this.group ? this.#max : this.#children.values().reduce((sum, c) => sum + c.max, 0)\n  }\n\n  get done() {\n    return this.#done\n  }\n\n  get ratio(): number {\n    return this.max === 0 ? 0 : Math.min(1, this.value / this.max)\n  }\n\n  get pct(): number {\n    return this.ratio * 100\n  }\n\n  stop() {\n    if (this.#done) return\n    this.#done = true\n    if (!this.group) this.#value = this.#max\n    this.#children.forEach((c) => c.stop())\n    this.emit(\"done\", this)\n  }\n\n  children() {\n    return [...this.#children.values()]\n  }\n\n  child(name: string, opts: ProgressOpts = {}): Progress {\n    if (this.#value > 0)\n      throw new Error(\"Cannot add child to Progress that has already made progress\")\n    let child = this.#children.get(name)\n    if (!child) {\n      child = new Progress(name, opts)\n      child.on(\"update\", () => this.emit(\"update\", this))\n      child.on(\"done\", () => {\n        if (!this.done && this.children().every((c) => c.done)) this.stop()\n      })\n      this.#children.set(name, child)\n    }\n    return child\n  }\n\n  [inspect.custom](_depth: number, _options: object): string {\n    return this.toString()\n  }\n\n  override toString(indent = 0): string {\n    const pad = \"  \".repeat(indent)\n    const pct = `${this.pct.toFixed(0)}%`.padStart(4)\n    const status = this.#status ? ` ${this.#status}` : \"\"\n    const line = `${pad}${pct} ${this.name}${status}`\n    if (!this.group) return line\n    const children = [...this.#children.values()].map((c) => c.toString(indent + 1))\n    return [line, ...children].join(\"\\n\")\n  }\n}\n"],"mappings":";;;;AAQA,MAAM,kBAAkB;AAwBxB,SAAgB,iBAAiB,MAA6C;CAC5E,MAAM,QAAQ,KAAK,MAAM,2BAA2B;AAEpD,QAAO;EACL,MAFW,QAAQ,KAAK,MAAM,MAAM,GAAG,OAAO,GAAG;EAGjD,YAAY,QAAQ,GAAG,MAAM,CAAC,MAAM,KAAK,CAAC,UAAU;EACpD,aAAa,QAAS,UAAU,MAAM,GAAG,GAA+B,EAAE;EAC1E,iBAAiB,QAAQ;EACzB;EACD;;AAGH,SAAgB,cAAc,MAA2B;CACvD,MAAM,MAAM,iBAAiB,KAAK;AAClC,QAAO;EAAE,GAAG;EAAK,UAAU,cAAc,IAAI,KAAK;EAAE;;AAGtD,SAAgB,cAAc,IAA+B;CAC3D,MAAM,QAAQ,GAAG,MAAM,KAAK;CAC5B,IAAI,UAA2B;EAC7B,SAAS,EAAE;EACX,SAAS,EAAE;EACX,SAAS;EACT,aAAa;EACb,OAAO;EACP,QAAQ;EACT;CACD,MAAM,WAA8B,CAAC,QAAQ;CAC7C,IAAI,YAAgC,KAAA;AACpC,MAAK,MAAM,CAAC,GAAG,SAAS,MAAM,SAAS,EAAE;EACvC,MAAM,QAAQ,KAAK,MAAM,eAAe;EACxC,MAAM,aAAa,KAAK,MAAM,oBAAoB;AAElD,MAAI,aAAa,KAAK,WAAW,UAAU,CACzC,aAAY,KAAA;WACH,CAAC,aAAa,WACvB,aAAY,WAAW;AAGzB,MAAI,CAAC,aAAa,OAAO;AACvB,OAAI,QAAQ,QAAQ,WAAW,EAAG,UAAS,KAAK;GAChD,MAAM,QAAQ,MAAM,GAAG;AACvB,aAAU;IACR,SAAS,CAAC,KAAK;IACf,SAAS,EAAE;IACX,SAAS,MAAM,GAAG,MAAM;IACxB,aAAa,MAAM,GAAG,MAAM;IAC5B;IACA,QAAQ;IACT;AACD,YAAS,KAAK,QAAQ;QACjB,SAAQ,QAAQ,KAAK,KAAK;;CAGnC,MAAM,QAA2B,EAAE;AACnC,MAAK,MAAM,WAAW,UAAU;AAE9B,UAAQ,MAAM,GAAG,GAAG,EAAE,SAAS,OAAO,QAAQ,MAAO,OAAM,KAAK;AAChE,UAAQ,UAAU,MAAM,KAAK,MAAM,EAAE,YAAY;AACjD,MAAI,QAAQ,QAAQ,EAAG,OAAM,KAAK,QAAQ;;AAE5C,QAAO;;AAGT,SAAS,UAAU,OAAe;AAChC,MAAK,MAAM,OAAO;EAAC;EAAQ;EAAM;EAAM;EAAM;EAAI,EAAE;EACjD,MAAM,IAAI,MAAM,YAAY,IAAI;AAChC,MAAI,IAAI,MAAM,SAAS,GACrB,QAAO;;AAGX,QAAO,MAAM;;AAGf,IAAM,cAAN,MAAM,YAAY;CAChB,QAAA,QAAgB;CAChB,QAAA,OAAe;CAEf,YACE,KACA,YAAmB,KACnB;AAFO,OAAA,MAAA;AACA,OAAA,YAAA;;CAGT,IAAI,gBAAgB;AAClB,SAAO,aAAA,OAAoB,KAAK,YAAY,IACxC,aAAA,QAAqB,aAAA,OACrB;;CAGN,SAAS,MAAc;AACrB,SAAO,KAAK,KAAK,KAAK,SAAS,KAAK,cAAc;;CAKpD,KAAK,MAAc;AACjB,MAAI,KAAK,WAAW,EAAG,QAAO;GAAE,OAAO;GAAG,WAAW;GAAO;EAC5D,IAAI,QAAQ,KAAK,SAAS,KAAK,GAAG;AAClC,MAAI,QAAQ,KAAK,UAAW,QAAO;GAAE;GAAO,WAAW;GAAM;AAC7D,UAAQ,KAAK,IAAI,KAAK,KAAK;AAC3B,eAAA,SAAsB,KAAK;AAC3B,eAAA,QAAqB;AACrB,SAAO;GAAE;GAAO,WAAW;GAAO;;;AAItC,SAAgB,UAAU,MAAc,KAAmB,OAAO,KAAe;CAC/E,MAAM,UAAU,IAAI,YAAY,KAAK,KAAK;CAC1C,MAAM,SAAmB,EAAE;AAC3B,QAAO,KAAK,QAAQ;EAClB,IAAI,OAAO;EACX,IAAI,OAAO,QAAQ,KAAK,KAAK;AAE7B,MAAI,KAAK,SAAS,MAAM;AACtB,UAAO,KAAK,KAAK;AACjB;;EAEF,IAAI,WAAW,OAAO,QAAQ,gBAAgB;AAE9C,SAAO,MAAM;AACX,cAAW,KAAK,IAAI,UAAU,KAAK,OAAO;GAC1C,MAAM,QAAQ,UAAU,KAAK,MAAM,GAAG,SAAS,CAAC;AAChD,UAAO,KAAK,MAAM,GAAG,MAAM;AAC3B,UAAO,QAAQ,KAAK,KAAK;AACzB,OAAI,KAAK,SAAS,KAAM;AACxB,eAAa,OAAO,KAAK,QAAS;;AAGpC,SAAO,KAAK,KAAK;AACjB,SAAO,KAAK,MAAM,KAAK,OAAO;;AAEhC,QAAO;;AAGT,SAAgB,cAAc,IAAY,KAAmB,OAAO,KAAe;CACjF,MAAM,WAAW,cAAc,GAAG;CAElC,MAAM,SAAkB,CAAC;EAAE,SAAS,EAAE;EAAE,SAAS,EAAE;EAAE,QAAQ;EAAG,CAAC;CACjE,MAAM,UAAU,IAAI,YAAY,KAAK,KAAK;AAE1C,MAAK,MAAM,WAAW,UAAU;EAC9B,MAAM,QAAQ,OAAO,GAAG,GAAG;EAE3B,MAAM,UAAU,CAAC,GAAG,QAAQ,SAAS,GAAG,QAAQ,QAAQ;EACxD,MAAM,OAAO,QAAQ,KAAK,KAAK;EAC/B,MAAM,OAAO,QAAQ,KAAK,KAAK,CAAC;AAEhC,MAAI,MAAM,SAAS,QAAQ,MAAM;GAE/B,MAAM,UAAU,QAAQ,QAAQ,QAAQ,GAAG,MAAM,MAAM,QAAQ,OAAO,EAAE;AACxE,SAAM,QAAQ,KAAK,GAAG,QAAQ;AAC9B,SAAM,QAAQ,KAAK,GAAG,QAAQ,QAAQ;AACtC,SAAM,UAAU,CAAC,GAAG,QAAQ,SAAS,QAAQ,YAAY;AACzD,SAAM,UAAU;aACP,QAAQ,KACjB,QAAO,KAAK;GAAE;GAAS,SAAS,CAAC,GAAG,QAAQ,SAAS,QAAQ,YAAY;GAAE,QAAQ;GAAM,CAAC;OACrF;GACL,MAAM,UAAU,QAAQ,QAAQ,KAAK,KAAK;GAC1C,MAAM,UAAU,QAAQ,KAAK,QAAQ;AACrC,UAAO,KACL,GAAG,UAAU,QAAQ,QAAQ,KAAK,KAAK,EAAE,KAAK,OAAO,QAAQ,MAAM,CAAC,KAAK,OAAO;IAC9E,UAAU,QAAQ,SAAS,GAAG,QAAQ,IAAI,MAAM,GAAG,MAAM,KAAK;IAC9D,SAAS,EAAE;IACX,QAAQ;IACT,EAAE,CACJ;AACD,UAAO,KAAK;IAAE,SAAS,EAAE;IAAE,SAAS,EAAE;IAAE,QAAQ;IAAG,CAAC;;;AAIxD,QAAO,OAAO,KAAK,MAAM,EAAE,QAAQ,KAAK,KAAK,CAAC,MAAM,CAAC,CAAC,OAAO,QAAQ;;;;AC9LvE,IAAa,WAAb,MAAa,iBAAkB,aAAwD;CACrF,OAAO;CACP,SAAS;CACT,4BAAY,IAAI,KAAuB;CACvC;CACA,QAAQ;CAER,YACE,MACA,OAAqB,EAAE,EACvB;AACA,SAAO;AAHA,OAAA,OAAA;AAIP,OAAK,IAAI,KAAK;;CAGhB,IAAI,QAAQ;AACV,SAAO,MAAA,SAAe,OAAO;;CAG/B,IAAI,MAAmC;AACrC,MAAI,MAAA,KAAY,QAAO;AACvB,MAAI,OAAO,SAAS,SAAU,OAAA,QAAc;OACvC;AACH,SAAA,MAAY,KAAK,OAAO,MAAA;AACxB,SAAA,SAAe,KAAK,UAAU,MAAA;AAC9B,SAAA,QAAc,KAAK,SAAS,MAAA;;AAE9B,OAAK,KAAK,UAAU,KAAK;AACzB,MAAI,MAAA,SAAe,MAAA,IAAW,MAAK,MAAM;AACzC,SAAO;;CAGT,IAAI,SAAS;AACX,SAAO,MAAA,UAAgB,KAAK;;CAG9B,IAAI,OAAO,QAAgB;AACzB,OAAK,IAAI,EAAE,QAAQ,CAAC;;CAGtB,IAAI,MAAM,OAAe;AACvB,OAAK,IAAI,MAAM;;CAGjB,IAAI,QAAgB;AAClB,SAAO,CAAC,KAAK,QAAQ,MAAA,QAAc,MAAA,SAAe,QAAQ,CAAC,QAAQ,KAAK,MAAM,MAAM,EAAE,OAAO,EAAE;;CAGjG,IAAI,IAAI,KAAa;AACnB,OAAK,IAAI,EAAE,KAAK,CAAC;;CAGnB,IAAI,MAAc;AAChB,SAAO,CAAC,KAAK,QAAQ,MAAA,MAAY,MAAA,SAAe,QAAQ,CAAC,QAAQ,KAAK,MAAM,MAAM,EAAE,KAAK,EAAE;;CAG7F,IAAI,OAAO;AACT,SAAO,MAAA;;CAGT,IAAI,QAAgB;AAClB,SAAO,KAAK,QAAQ,IAAI,IAAI,KAAK,IAAI,GAAG,KAAK,QAAQ,KAAK,IAAI;;CAGhE,IAAI,MAAc;AAChB,SAAO,KAAK,QAAQ;;CAGtB,OAAO;AACL,MAAI,MAAA,KAAY;AAChB,QAAA,OAAa;AACb,MAAI,CAAC,KAAK,MAAO,OAAA,QAAc,MAAA;AAC/B,QAAA,SAAe,SAAS,MAAM,EAAE,MAAM,CAAC;AACvC,OAAK,KAAK,QAAQ,KAAK;;CAGzB,WAAW;AACT,SAAO,CAAC,GAAG,MAAA,SAAe,QAAQ,CAAC;;CAGrC,MAAM,MAAc,OAAqB,EAAE,EAAY;AACrD,MAAI,MAAA,QAAc,EAChB,OAAM,IAAI,MAAM,8DAA8D;EAChF,IAAI,QAAQ,MAAA,SAAe,IAAI,KAAK;AACpC,MAAI,CAAC,OAAO;AACV,WAAQ,IAAI,SAAS,MAAM,KAAK;AAChC,SAAM,GAAG,gBAAgB,KAAK,KAAK,UAAU,KAAK,CAAC;AACnD,SAAM,GAAG,cAAc;AACrB,QAAI,CAAC,KAAK,QAAQ,KAAK,UAAU,CAAC,OAAO,MAAM,EAAE,KAAK,CAAE,MAAK,MAAM;KACnE;AACF,SAAA,SAAe,IAAI,MAAM,MAAM;;AAEjC,SAAO;;CAGT,CAAC,QAAQ,QAAQ,QAAgB,UAA0B;AACzD,SAAO,KAAK,UAAU;;CAGxB,SAAkB,SAAS,GAAW;EACpC,MAAM,MAAM,KAAK,OAAO,OAAO;EAC/B,MAAM,MAAM,GAAG,KAAK,IAAI,QAAQ,EAAE,CAAC,GAAG,SAAS,EAAE;EACjD,MAAM,SAAS,MAAA,SAAe,IAAI,MAAA,WAAiB;EACnD,MAAM,OAAO,GAAG,MAAM,IAAI,GAAG,KAAK,OAAO;AACzC,MAAI,CAAC,KAAK,MAAO,QAAO;AAExB,SAAO,CAAC,MAAM,GADG,CAAC,GAAG,MAAA,SAAe,QAAQ,CAAC,CAAC,KAAK,MAAM,EAAE,SAAS,SAAS,EAAE,CAAC,CACtD,CAAC,KAAK,KAAK"}

package/dist/query-VFSpErTB.mjs

@@ -123,3 +123,5 @@ function toFts(input, defaultOp = "OR") {
 }
 //#endregion
 export { tokenize as n, toFts as t };
+
+//# sourceMappingURL=query-VFSpErTB.mjs.map

package/dist/query-VFSpErTB.mjs.map

@@ -0,0 +1 @@
+{"version":3,"file":"query-VFSpErTB.mjs","names":[],"sources":["../src/query.ts"],"sourcesContent":["type Token =\n  | { type: \"term\"; value: string; neg?: boolean; req?: boolean; field?: string }\n  | { type: \"op\"; value: \"AND\" | \"OR\" }\n  | { type: \"paren\"; value: \"(\" | \")\" }\n\nconst FTS_FIELDS = new Set([\"entities\", \"tags\", \"description\", \"title\", \"body\"])\n\nexport function tokenize(input: string): Token[] {\n  const tokens: Token[] = []\n  let i = 0\n\n  while (i < input.length) {\n    while (i < input.length && input[i] === \" \") i++\n    if (i >= input.length) break\n\n    const ch = input[i]\n\n    if (ch === \"(\" || ch === \")\") {\n      tokens.push({ type: \"paren\", value: ch })\n      i++\n    } else if (ch === \"|\") {\n      tokens.push({ type: \"op\", value: \"OR\" })\n      i++\n    } else if ((ch === '\"' || ch === \"'\") && (i === 0 || input[i - 1] === \" \")) {\n      const quote = ch\n      i++\n      const start = i\n      while (i < input.length && input[i] !== quote) i++\n      if (start < i) tokens.push({ type: \"term\", value: input.slice(start, i) })\n      if (i < input.length) i++\n    } else {\n      const neg = ch === \"-\"\n      const req = ch === \"+\"\n      if (neg || req) i++\n      const start = i\n      while (i < input.length && !' \"()|'.includes(input[i])) i++\n      if (start < i) {\n        const raw = input.slice(start, i)\n        const colon = raw.indexOf(\":\")\n        if (colon > 0 && FTS_FIELDS.has(raw.slice(0, colon))) {\n          tokens.push({\n            field: raw.slice(0, colon),\n            neg: neg || undefined,\n            req: req || undefined,\n            type: \"term\",\n            value: raw.slice(colon + 1),\n          })\n        } else {\n          tokens.push({ neg: neg || undefined, req: req || undefined, type: \"term\", value: raw })\n        }\n      }\n    }\n  }\n  return tokens\n}\n\n/** Sanitize a term for FTS5 — strip non-word/non-apostrophe chars, preserve colons */\nfunction sanitize(term: string): string {\n  return term.replace(/[^\\p{L}\\p{N}\\s':]/gu, \"\").trim()\n}\n\nfunction buildTerm(token: Extract<Token, { type: \"term\" }>): string | undefined {\n  const clean = sanitize(token.value)\n  if (!clean) return\n  const isPrefix = token.value.endsWith(\"*\")\n  const phrase = `\"${clean}\"${isPrefix ? \"*\" : \"\"}`\n  const scoped = token.field ? `${token.field} : ${phrase}` : phrase\n  return token.neg ? `NOT ${scoped}` : scoped\n}\n\nfunction joinParts(parts: string[], op: string): string {\n  return parts.join(` ${op} `)\n}\n\n/** Build an FTS5 query string from user input */\nexport function toFts(input: string, defaultOp: \"AND\" | \"OR\" = \"OR\"): string {\n  const tokens = tokenize(input)\n  const hasRequired = tokens.some((t) => t.type === \"term\" && t.req)\n\n  // If no required terms, build normally\n  if (!hasRequired) {\n    const parts: string[] = []\n    let needsOp = false\n\n    for (const token of tokens) {\n      if (token.type === \"paren\") {\n        if (token.value === \"(\") {\n          if (needsOp) parts.push(defaultOp)\n          parts.push(token.value)\n          needsOp = false\n        } else {\n          parts.push(token.value)\n          needsOp = true\n        }\n        continue\n      }\n      if (token.type === \"op\") {\n        parts.push(token.value)\n        needsOp = false\n        continue\n      }\n      const term = buildTerm(token)\n      if (!term) continue\n      if (needsOp) parts.push(defaultOp)\n      parts.push(term)\n      needsOp = true\n    }\n\n    return parts.join(\" \").replace(/\\( /g, \"(\").replace(/ \\)/g, \")\")\n  }\n\n  // With required terms: required1 AND required2 AND (all terms joined with OR)\n  const required: string[] = []\n  const all: string[] = []\n\n  for (const token of tokens) {\n    if (token.type !== \"term\") continue\n    const term = buildTerm(token)\n    if (!term) continue\n    all.push(term)\n    if (token.req) required.push(term)\n  }\n\n  const requiredPart = joinParts(required, \"AND\")\n  const allPart = joinParts(all, \"OR\")\n\n  // If everything is required, no need for the OR group\n  if (required.length === all.length) return requiredPart\n\n  return `${requiredPart} AND (${allPart})`\n}\n"],"mappings":";AAKA,MAAM,aAAa,IAAI,IAAI;CAAC;CAAY;CAAQ;CAAe;CAAS;CAAO,CAAC;AAEhF,SAAgB,SAAS,OAAwB;CAC/C,MAAM,SAAkB,EAAE;CAC1B,IAAI,IAAI;AAER,QAAO,IAAI,MAAM,QAAQ;AACvB,SAAO,IAAI,MAAM,UAAU,MAAM,OAAO,IAAK;AAC7C,MAAI,KAAK,MAAM,OAAQ;EAEvB,MAAM,KAAK,MAAM;AAEjB,MAAI,OAAO,OAAO,OAAO,KAAK;AAC5B,UAAO,KAAK;IAAE,MAAM;IAAS,OAAO;IAAI,CAAC;AACzC;aACS,OAAO,KAAK;AACrB,UAAO,KAAK;IAAE,MAAM;IAAM,OAAO;IAAM,CAAC;AACxC;cACU,OAAO,QAAO,OAAO,SAAS,MAAM,KAAK,MAAM,IAAI,OAAO,MAAM;GAC1E,MAAM,QAAQ;AACd;GACA,MAAM,QAAQ;AACd,UAAO,IAAI,MAAM,UAAU,MAAM,OAAO,MAAO;AAC/C,OAAI,QAAQ,EAAG,QAAO,KAAK;IAAE,MAAM;IAAQ,OAAO,MAAM,MAAM,OAAO,EAAE;IAAE,CAAC;AAC1E,OAAI,IAAI,MAAM,OAAQ;SACjB;GACL,MAAM,MAAM,OAAO;GACnB,MAAM,MAAM,OAAO;AACnB,OAAI,OAAO,IAAK;GAChB,MAAM,QAAQ;AACd,UAAO,IAAI,MAAM,UAAU,CAAC,SAAQ,SAAS,MAAM,GAAG,CAAE;AACxD,OAAI,QAAQ,GAAG;IACb,MAAM,MAAM,MAAM,MAAM,OAAO,EAAE;IACjC,MAAM,QAAQ,IAAI,QAAQ,IAAI;AAC9B,QAAI,QAAQ,KAAK,WAAW,IAAI,IAAI,MAAM,GAAG,MAAM,CAAC,CAClD,QAAO,KAAK;KACV,OAAO,IAAI,MAAM,GAAG,MAAM;KAC1B,KAAK,OAAO,KAAA;KACZ,KAAK,OAAO,KAAA;KACZ,MAAM;KACN,OAAO,IAAI,MAAM,QAAQ,EAAE;KAC5B,CAAC;QAEF,QAAO,KAAK;KAAE,KAAK,OAAO,KAAA;KAAW,KAAK,OAAO,KAAA;KAAW,MAAM;KAAQ,OAAO;KAAK,CAAC;;;;AAK/F,QAAO;;;AAIT,SAAS,SAAS,MAAsB;AACtC,QAAO,KAAK,QAAQ,uBAAuB,GAAG,CAAC,MAAM;;AAGvD,SAAS,UAAU,OAA6D;CAC9E,MAAM,QAAQ,SAAS,MAAM,MAAM;AACnC,KAAI,CAAC,MAAO;CAEZ,MAAM,SAAS,IAAI,MAAM,GADR,MAAM,MAAM,SAAS,IAAI,GACH,MAAM;CAC7C,MAAM,SAAS,MAAM,QAAQ,GAAG,MAAM,MAAM,KAAK,WAAW;AAC5D,QAAO,MAAM,MAAM,OAAO,WAAW;;AAGvC,SAAS,UAAU,OAAiB,IAAoB;AACtD,QAAO,MAAM,KAAK,IAAI,GAAG,GAAG;;;AAI9B,SAAgB,MAAM,OAAe,YAA0B,MAAc;CAC3E,MAAM,SAAS,SAAS,MAAM;AAI9B,KAAI,CAHgB,OAAO,MAAM,MAAM,EAAE,SAAS,UAAU,EAAE,IAAI,EAGhD;EAChB,MAAM,QAAkB,EAAE;EAC1B,IAAI,UAAU;AAEd,OAAK,MAAM,SAAS,QAAQ;AAC1B,OAAI,MAAM,SAAS,SAAS;AAC1B,QAAI,MAAM,UAAU,KAAK;AACvB,SAAI,QAAS,OAAM,KAAK,UAAU;AAClC,WAAM,KAAK,MAAM,MAAM;AACvB,eAAU;WACL;AACL,WAAM,KAAK,MAAM,MAAM;AACvB,eAAU;;AAEZ;;AAEF,OAAI,MAAM,SAAS,MAAM;AACvB,UAAM,KAAK,MAAM,MAAM;AACvB,cAAU;AACV;;GAEF,MAAM,OAAO,UAAU,MAAM;AAC7B,OAAI,CAAC,KAAM;AACX,OAAI,QAAS,OAAM,KAAK,UAAU;AAClC,SAAM,KAAK,KAAK;AAChB,aAAU;;AAGZ,SAAO,MAAM,KAAK,IAAI,CAAC,QAAQ,QAAQ,IAAI,CAAC,QAAQ,QAAQ,IAAI;;CAIlE,MAAM,WAAqB,EAAE;CAC7B,MAAM,MAAgB,EAAE;AAExB,MAAK,MAAM,SAAS,QAAQ;AAC1B,MAAI,MAAM,SAAS,OAAQ;EAC3B,MAAM,OAAO,UAAU,MAAM;AAC7B,MAAI,CAAC,KAAM;AACX,MAAI,KAAK,KAAK;AACd,MAAI,MAAM,IAAK,UAAS,KAAK,KAAK;;CAGpC,MAAM,eAAe,UAAU,UAAU,MAAM;CAC/C,MAAM,UAAU,UAAU,KAAK,KAAK;AAGpC,KAAI,SAAS,WAAW,IAAI,OAAQ,QAAO;AAE3C,QAAO,GAAG,aAAa,QAAQ,QAAQ"}

package/dist/runtime.node-DlQPaGrV.mjs

@@ -33,3 +33,5 @@ function parseYaml(content) {
 }
 //#endregion
 export { parseYaml as n, openDatabase as t };
+
+//# sourceMappingURL=runtime.node-DlQPaGrV.mjs.map

package/dist/runtime.node-DlQPaGrV.mjs.map

@@ -0,0 +1 @@
+{"version":3,"file":"runtime.node-DlQPaGrV.mjs","names":[],"sources":["../src/runtime.node.ts"],"sourcesContent":["import type { Database as BetterDB } from \"better-sqlite3\"\nimport type { Database } from \"bun:sqlite\"\n\nlet DB: undefined | typeof Database\n\nasync function dbInit() {\n  const { default: BetterDatabase } = await import(\"better-sqlite3\")\n\n  // Extend better-sqlite3 to mimic Bun's Database API\n  return class extends BetterDatabase {\n    private prepareCache = new Map<string, ReturnType<BetterDB[\"prepare\"]>>()\n\n    // oxlint-disable-next-line no-useless-constructor\n    constructor(filename?: string) {\n      super(filename)\n    }\n\n    run(...args: Parameters<BetterDB[\"exec\"]>) {\n      return this.exec(...args)\n    }\n\n    query(source: string) {\n      let ret = this.prepareCache.get(source)\n      if (!ret) {\n        ret = this.prepare(source)\n        this.prepareCache.set(source, ret)\n      }\n      return ret\n    }\n  } as unknown as typeof Database\n}\n\nexport async function openDatabase(path: string) {\n  DB ??= await dbInit()\n  const { load: sqliteVec } = await import(\"sqlite-vec\")\n  const db = new DB(path, { strict: true })\n  sqliteVec(db)\n  return db\n}\n\nexport type { Database }\n\nconst { load: loadYaml } = await import(\"js-yaml\")\n\nexport function parseYaml(content: string): unknown {\n  return loadYaml(content)\n}\n"],"mappings":";AAGA,IAAI;AAEJ,eAAe,SAAS;CACtB,MAAM,EAAE,SAAS,mBAAmB,MAAM,OAAO;AAGjD,QAAO,cAAc,eAAe;EAClC,+BAAuB,IAAI,KAA8C;EAGzE,YAAY,UAAmB;AAC7B,SAAM,SAAS;;EAGjB,IAAI,GAAG,MAAoC;AACzC,UAAO,KAAK,KAAK,GAAG,KAAK;;EAG3B,MAAM,QAAgB;GACpB,IAAI,MAAM,KAAK,aAAa,IAAI,OAAO;AACvC,OAAI,CAAC,KAAK;AACR,UAAM,KAAK,QAAQ,OAAO;AAC1B,SAAK,aAAa,IAAI,QAAQ,IAAI;;AAEpC,UAAO;;;;AAKb,eAAsB,aAAa,MAAc;AAC/C,QAAO,MAAM,QAAQ;CACrB,MAAM,EAAE,MAAM,cAAc,MAAM,OAAO;CACzC,MAAM,KAAK,IAAI,GAAG,MAAM,EAAE,QAAQ,MAAM,CAAC;AACzC,WAAU,GAAG;AACb,QAAO;;AAKT,MAAM,EAAE,MAAM,aAAa,MAAM,OAAO;AAExC,SAAgB,UAAU,SAA0B;AAClD,QAAO,SAAS,QAAQ"}

package/dist/{search-BllHWtZF.mjs → search-DsVjB-9f.mjs}

@@ -164,3 +164,5 @@ var Search = class Search {
 };
 //#endregion
 export { Search };
+
+//# sourceMappingURL=search-DsVjB-9f.mjs.map

package/dist/search-DsVjB-9f.mjs.map

@@ -0,0 +1 @@
+{"version":3,"file":"search-DsVjB-9f.mjs","names":[],"sources":["../src/search.ts"],"sourcesContent":["import type { Context } from \"./context.ts\"\nimport type { Db, DocRow, FTSResult, VecResult } from \"./db.ts\"\nimport type { VfsEntry } from \"./vfs.ts\"\n\nimport { toFts } from \"./query.ts\"\nimport { parentUri } from \"./uri.ts\"\nimport { hash } from \"./util.ts\"\n\nexport type SearchMode = \"hybrid\" | \"vec\" | \"fts\"\n\nexport type SearchScore = {\n  score: number\n  display_score?: number\n  rank: number\n}\n\nexport type SearchResult = {\n  uri: string\n  path: string\n  doc: DocRow\n  scores: Partial<Record<SearchMode, SearchScore>>\n  match: { fts?: FTSResult; vec?: VecResult }\n} & VfsEntry\n\ntype SearchResultMap = {\n  hybrid: HybridSR\n  vec: VecSR\n  fts: FtsSR\n}\n\nexport type HybridSR = SearchResult & { scores: { hybrid: SearchScore } }\nexport type VecSR = SearchResult & { scores: { vec: SearchScore }; match: { vec: VecResult } }\nexport type FtsSR = SearchResult & { scores: { fts: SearchScore }; match: { fts: FTSResult } }\n\nexport type SearchOptions = {\n  limit?: number\n  uri?: string\n  mode?: SearchMode\n}\n\nexport type FtsSearchOptions = Omit<SearchOptions, \"mode\"> & {\n  op?: \"AND\" | \"OR\"\n}\n\n// Description chunks (seq=0) get boosted in vector scoring\nconst DESC_BOOST = 0.2\nconst PARENT_BOOST = 0.3 // how much to boost a chunk based on its parent's score, vs its own score, in [0, 1]\nconst RRF_K = 60\nconst RRF_LIMIT = 50\nconst VEC_OVERSAMPLE = 4\n\nexport class Search {\n  private constructor(\n    public db: Db,\n    public ctx: Context\n  ) {}\n\n  static async load(ctx: Context) {\n    return new Search(await ctx.db(), ctx)\n  }\n\n  async search(query: string, opts: SearchOptions = {}): Promise<SearchResult[]> {\n    const mode = opts.mode ?? \"hybrid\"\n    if (mode === \"fts\") return this.searchFts(query, opts)\n    if (mode === \"vec\") return this.searchVec(query, opts)\n\n    const limit = opts.limit ?? 20\n\n    // Hybrid: run both, fuse with RRF — need enough candidates for good fusion\n    const subLimit = Math.max(RRF_LIMIT, limit * 2)\n    const [fts, vec] = await Promise.all([\n      this.searchFts(query, { ...opts, limit: subLimit }),\n      this.searchVec(query, { ...opts, limit: subLimit, slice: false }),\n    ])\n\n    return this.fuse(fts, vec, limit)\n  }\n\n  async searchVec(\n    query: string,\n    opts: Omit<SearchOptions, \"mode\"> & { slice?: boolean } = {}\n  ): Promise<VecSR[]> {\n    const cacheKey = hash(`embed:${query}`)\n    const embedder = await this.ctx.embedder()\n    const vfs = await this.ctx.vfs()\n\n    const embedding =\n      this.db.cacheGet<number[]>(cacheKey) ??\n      this.db.cacheSet(cacheKey, await embedder.embed(query))\n\n    const scope = vfs.getScope(opts.uri)\n    const limit = opts.limit ?? 20\n\n    // Oversample for post-filtering when scoped\n    const results = this.db.searchVec(embedding, {\n      limit: Math.max(limit, RRF_LIMIT) * VEC_OVERSAMPLE,\n    })\n\n    // Group by doc, take best chunk per doc\n    const best = new Map<number, VecResult & { uri: string; hiscore: number }>()\n    for (const vec of results) {\n      const uri = scope.map(vec.path)\n      if (!uri) continue\n      vec.score = vec.seq === 0 ? vec.score + DESC_BOOST * (1 - vec.score) : vec.score\n      const existing = best.get(vec.doc_id)?.score ?? -Infinity\n      if (vec.score > existing) best.set(vec.doc_id, Object.assign(vec, { hiscore: 0, uri }))\n    }\n    const scores = new Map<string, number>(best.values().map((vec) => [vec.uri, vec.score]))\n    const parentScores = new Map<string, number>()\n    const getParentScore = (uri: string): number => {\n      const parent = parentUri(uri)\n      if (!parent) return 0\n      let score = parentScores.get(parent)\n      if (score !== undefined) return score\n      score = (scores.get(parent) ?? 0) * 0.5 + getParentScore(parent) * 0.5\n      parentScores.set(parent, score)\n      return score\n    }\n\n    for (const vec of best.values()) {\n      const parentScore = getParentScore(vec.uri)\n      vec.score += PARENT_BOOST * parentScore * (1 - vec.score)\n    }\n\n    let bestResults = [...best.values()].toSorted((a, b) => b.score - a.score)\n    bestResults = opts.slice === false ? bestResults : bestResults.slice(0, limit)\n\n    const docs = this.db.getDocs(bestResults.map((r) => r.doc_id))\n    const ret: VecSR[] = []\n    for (const vec of bestResults) {\n      const doc = docs.get(vec.doc_id)\n      if (doc)\n        ret.push({\n          doc,\n          match: { vec },\n          path: vec.path,\n          scores: { vec: { rank: 0, score: vec.score } },\n          uri: vec.uri,\n        })\n    }\n\n    return this.rank(\"vec\", ret)\n  }\n\n  async searchFts(query: string, opts: FtsSearchOptions = {}): Promise<FtsSR[]> {\n    const vfs = await this.ctx.vfs()\n    const scope = vfs.getScope(opts.uri)\n    const results = this.db.searchFts(toFts(query, opts.op ?? \"OR\"), {\n      limit: opts.limit ?? 20,\n      scope: scope.paths.map((p) => p.path),\n    })\n    const docs = this.db.getDocs(results.map((r) => r.rowid))\n    const ret: FtsSR[] = []\n    for (const fts of results) {\n      fts.score = Math.abs(fts.score) / (1 + Math.abs(fts.score))\n      const doc = docs.get(fts.rowid)\n      const uri = scope.map(doc?.path ?? \"\")\n      if (doc && uri)\n        ret.push({\n          doc,\n          match: { fts },\n          path: doc.path,\n          scores: { fts: { rank: 0, score: fts.score } },\n          uri,\n        })\n    }\n    return this.rank(\"fts\", ret)\n  }\n\n  rank<M extends SearchMode>(mode: M, results: SearchResultMap[M][]): SearchResultMap[M][] {\n    const score = (r: SearchResult) => (r.scores as Record<string, SearchScore>)[mode]\n    return results\n      .toSorted(\n        (a, b) =>\n          score(b).score - score(a).score ||\n          (score(b).display_score ?? 0) - (score(a).display_score ?? 0)\n      )\n      .map((r, i) => {\n        score(r).rank = i + 1\n        return r\n      })\n  }\n\n  /** Reciprocal Rank Fusion: merge FTS and vector results */\n  private fuse(ftsResults: FtsSR[], vecResults: VecSR[], limit: number): HybridSR[] {\n    const merged = new Map<number, { uri: string; vec?: VecSR; fts?: FtsSR }>()\n\n    const minVecScore = vecResults.length\n      ? vecResults[vecResults.length - 1]?.scores.vec.score\n      : undefined\n    const minFtsScore = ftsResults.length\n      ? ftsResults[ftsResults.length - 1]?.scores.fts.score\n      : undefined\n    const minScore = Math.min(minVecScore ?? 1, minFtsScore ?? 1)\n\n    for (const fts of ftsResults) merged.set(fts.doc.id, { fts, uri: fts.uri })\n    for (const vec of vecResults)\n      merged.set(vec.doc.id, { ...merged.get(vec.doc.id), uri: vec.uri, vec })\n\n    let ret: HybridSR[] = [...merged.values()].map(({ uri, fts, vec }) => {\n      const ftsScore = fts?.scores.fts\n      const vecScore = vec?.scores.vec\n      const score =\n        (ftsScore?.rank !== undefined ? 1 / (RRF_K + ftsScore.rank) : 0) +\n        (vecScore?.rank !== undefined ? 1 / (RRF_K + vecScore.rank) : 0)\n      const display_score =\n        0.6 * (vecScore?.score ?? minScore) + 0.4 * (ftsScore?.score ?? minScore)\n      const doc = (fts?.doc ?? vec?.doc)!\n      return {\n        doc,\n        match: { ...fts?.match, ...vec?.match },\n        path: doc.path,\n        scores: {\n          ...fts?.scores,\n          ...vec?.scores,\n          hybrid: { display_score, rank: 0, score },\n        },\n        uri,\n      }\n    })\n\n    ret = this.rank(\"hybrid\", ret).slice(0, limit)\n\n    // Normalize scores to [0, 1]\n    const bestScore = ret[0]?.scores.hybrid.score ?? 1\n    for (const r of ret) r.scores.hybrid.score /= bestScore\n\n    return ret\n  }\n}\n"],"mappings":";;;;AA6CA,MAAM,aAAa;AACnB,MAAM,eAAe;AACrB,MAAM,QAAQ;AACd,MAAM,YAAY;AAClB,MAAM,iBAAiB;AAEvB,IAAa,SAAb,MAAa,OAAO;CAClB,YACE,IACA,KACA;AAFO,OAAA,KAAA;AACA,OAAA,MAAA;;CAGT,aAAa,KAAK,KAAc;AAC9B,SAAO,IAAI,OAAO,MAAM,IAAI,IAAI,EAAE,IAAI;;CAGxC,MAAM,OAAO,OAAe,OAAsB,EAAE,EAA2B;EAC7E,MAAM,OAAO,KAAK,QAAQ;AAC1B,MAAI,SAAS,MAAO,QAAO,KAAK,UAAU,OAAO,KAAK;AACtD,MAAI,SAAS,MAAO,QAAO,KAAK,UAAU,OAAO,KAAK;EAEtD,MAAM,QAAQ,KAAK,SAAS;EAG5B,MAAM,WAAW,KAAK,IAAI,WAAW,QAAQ,EAAE;EAC/C,MAAM,CAAC,KAAK,OAAO,MAAM,QAAQ,IAAI,CACnC,KAAK,UAAU,OAAO;GAAE,GAAG;GAAM,OAAO;GAAU,CAAC,EACnD,KAAK,UAAU,OAAO;GAAE,GAAG;GAAM,OAAO;GAAU,OAAO;GAAO,CAAC,CAClE,CAAC;AAEF,SAAO,KAAK,KAAK,KAAK,KAAK,MAAM;;CAGnC,MAAM,UACJ,OACA,OAA0D,EAAE,EAC1C;EAClB,MAAM,WAAW,KAAK,SAAS,QAAQ;EACvC,MAAM,WAAW,MAAM,KAAK,IAAI,UAAU;EAC1C,MAAM,MAAM,MAAM,KAAK,IAAI,KAAK;EAEhC,MAAM,YACJ,KAAK,GAAG,SAAmB,SAAS,IACpC,KAAK,GAAG,SAAS,UAAU,MAAM,SAAS,MAAM,MAAM,CAAC;EAEzD,MAAM,QAAQ,IAAI,SAAS,KAAK,IAAI;EACpC,MAAM,QAAQ,KAAK,SAAS;EAG5B,MAAM,UAAU,KAAK,GAAG,UAAU,WAAW,EAC3C,OAAO,KAAK,IAAI,OAAO,UAAU,GAAG,gBACrC,CAAC;EAGF,MAAM,uBAAO,IAAI,KAA2D;AAC5E,OAAK,MAAM,OAAO,SAAS;GACzB,MAAM,MAAM,MAAM,IAAI,IAAI,KAAK;AAC/B,OAAI,CAAC,IAAK;AACV,OAAI,QAAQ,IAAI,QAAQ,IAAI,IAAI,QAAQ,cAAc,IAAI,IAAI,SAAS,IAAI;GAC3E,MAAM,WAAW,KAAK,IAAI,IAAI,OAAO,EAAE,SAAS;AAChD,OAAI,IAAI,QAAQ,SAAU,MAAK,IAAI,IAAI,QAAQ,OAAO,OAAO,KAAK;IAAE,SAAS;IAAG;IAAK,CAAC,CAAC;;EAEzF,MAAM,SAAS,IAAI,IAAoB,KAAK,QAAQ,CAAC,KAAK,QAAQ,CAAC,IAAI,KAAK,IAAI,MAAM,CAAC,CAAC;EACxF,MAAM,+BAAe,IAAI,KAAqB;EAC9C,MAAM,kBAAkB,QAAwB;GAC9C,MAAM,SAAS,UAAU,IAAI;AAC7B,OAAI,CAAC,OAAQ,QAAO;GACpB,IAAI,QAAQ,aAAa,IAAI,OAAO;AACpC,OAAI,UAAU,KAAA,EAAW,QAAO;AAChC,YAAS,OAAO,IAAI,OAAO,IAAI,KAAK,KAAM,eAAe,OAAO,GAAG;AACnE,gBAAa,IAAI,QAAQ,MAAM;AAC/B,UAAO;;AAGT,OAAK,MAAM,OAAO,KAAK,QAAQ,EAAE;GAC/B,MAAM,cAAc,eAAe,IAAI,IAAI;AAC3C,OAAI,SAAS,eAAe,eAAe,IAAI,IAAI;;EAGrD,IAAI,cAAc,CAAC,GAAG,KAAK,QAAQ,CAAC,CAAC,UAAU,GAAG,MAAM,EAAE,QAAQ,EAAE,MAAM;AAC1E,gBAAc,KAAK,UAAU,QAAQ,cAAc,YAAY,MAAM,GAAG,MAAM;EAE9E,MAAM,OAAO,KAAK,GAAG,QAAQ,YAAY,KAAK,MAAM,EAAE,OAAO,CAAC;EAC9D,MAAM,MAAe,EAAE;AACvB,OAAK,MAAM,OAAO,aAAa;GAC7B,MAAM,MAAM,KAAK,IAAI,IAAI,OAAO;AAChC,OAAI,IACF,KAAI,KAAK;IACP;IACA,OAAO,EAAE,KAAK;IACd,MAAM,IAAI;IACV,QAAQ,EAAE,KAAK;KAAE,MAAM;KAAG,OAAO,IAAI;KAAO,EAAE;IAC9C,KAAK,IAAI;IACV,CAAC;;AAGN,SAAO,KAAK,KAAK,OAAO,IAAI;;CAG9B,MAAM,UAAU,OAAe,OAAyB,EAAE,EAAoB;EAE5E,MAAM,SADM,MAAM,KAAK,IAAI,KAAK,EACd,SAAS,KAAK,IAAI;EACpC,MAAM,UAAU,KAAK,GAAG,UAAU,MAAM,OAAO,KAAK,MAAM,KAAK,EAAE;GAC/D,OAAO,KAAK,SAAS;GACrB,OAAO,MAAM,MAAM,KAAK,MAAM,EAAE,KAAK;GACtC,CAAC;EACF,MAAM,OAAO,KAAK,GAAG,QAAQ,QAAQ,KAAK,MAAM,EAAE,MAAM,CAAC;EACzD,MAAM,MAAe,EAAE;AACvB,OAAK,MAAM,OAAO,SAAS;AACzB,OAAI,QAAQ,KAAK,IAAI,IAAI,MAAM,IAAI,IAAI,KAAK,IAAI,IAAI,MAAM;GAC1D,MAAM,MAAM,KAAK,IAAI,IAAI,MAAM;GAC/B,MAAM,MAAM,MAAM,IAAI,KAAK,QAAQ,GAAG;AACtC,OAAI,OAAO,IACT,KAAI,KAAK;IACP;IACA,OAAO,EAAE,KAAK;IACd,MAAM,IAAI;IACV,QAAQ,EAAE,KAAK;KAAE,MAAM;KAAG,OAAO,IAAI;KAAO,EAAE;IAC9C;IACD,CAAC;;AAEN,SAAO,KAAK,KAAK,OAAO,IAAI;;CAG9B,KAA2B,MAAS,SAAqD;EACvF,MAAM,SAAS,MAAqB,EAAE,OAAuC;AAC7E,SAAO,QACJ,UACE,GAAG,MACF,MAAM,EAAE,CAAC,QAAQ,MAAM,EAAE,CAAC,UACzB,MAAM,EAAE,CAAC,iBAAiB,MAAM,MAAM,EAAE,CAAC,iBAAiB,GAC9D,CACA,KAAK,GAAG,MAAM;AACb,SAAM,EAAE,CAAC,OAAO,IAAI;AACpB,UAAO;IACP;;;CAIN,KAAa,YAAqB,YAAqB,OAA2B;EAChF,MAAM,yBAAS,IAAI,KAAwD;EAE3E,MAAM,cAAc,WAAW,SAC3B,WAAW,WAAW,SAAS,IAAI,OAAO,IAAI,QAC9C,KAAA;EACJ,MAAM,cAAc,WAAW,SAC3B,WAAW,WAAW,SAAS,IAAI,OAAO,IAAI,QAC9C,KAAA;EACJ,MAAM,WAAW,KAAK,IAAI,eAAe,GAAG,eAAe,EAAE;AAE7D,OAAK,MAAM,OAAO,WAAY,QAAO,IAAI,IAAI,IAAI,IAAI;GAAE;GAAK,KAAK,IAAI;GAAK,CAAC;AAC3E,OAAK,MAAM,OAAO,WAChB,QAAO,IAAI,IAAI,IAAI,IAAI;GAAE,GAAG,OAAO,IAAI,IAAI,IAAI,GAAG;GAAE,KAAK,IAAI;GAAK;GAAK,CAAC;EAE1E,IAAI,MAAkB,CAAC,GAAG,OAAO,QAAQ,CAAC,CAAC,KAAK,EAAE,KAAK,KAAK,UAAU;GACpE,MAAM,WAAW,KAAK,OAAO;GAC7B,MAAM,WAAW,KAAK,OAAO;GAC7B,MAAM,SACH,UAAU,SAAS,KAAA,IAAY,KAAK,QAAQ,SAAS,QAAQ,MAC7D,UAAU,SAAS,KAAA,IAAY,KAAK,QAAQ,SAAS,QAAQ;GAChE,MAAM,gBACJ,MAAO,UAAU,SAAS,YAAY,MAAO,UAAU,SAAS;GAClE,MAAM,MAAO,KAAK,OAAO,KAAK;AAC9B,UAAO;IACL;IACA,OAAO;KAAE,GAAG,KAAK;KAAO,GAAG,KAAK;KAAO;IACvC,MAAM,IAAI;IACV,QAAQ;KACN,GAAG,KAAK;KACR,GAAG,KAAK;KACR,QAAQ;MAAE;MAAe,MAAM;MAAG;MAAO;KAC1C;IACD;IACD;IACD;AAEF,QAAM,KAAK,KAAK,UAAU,IAAI,CAAC,MAAM,GAAG,MAAM;EAG9C,MAAM,YAAY,IAAI,IAAI,OAAO,OAAO,SAAS;AACjD,OAAK,MAAM,KAAK,IAAK,GAAE,OAAO,OAAO,SAAS;AAE9C,SAAO"}

package/dist/{store-DE7S35SS.mjs → store-I5nVEYxK.mjs}

@@ -1,5 +1,6 @@
 import { t as Progress } from "./progress-B1JdNapX.mjs";
 import { t as Doc } from "./doc-DnYN4jAU.mjs";
+import { n as toDeadline, t as addVisit } from "./frecency-CiaqPIOy.mjs";
 import { performance } from "node:perf_hooks";
 //#region src/store.ts
 var Store = class Store {
@@ -17,17 +18,18 @@ var Store = class Store {
 			return row.id;
 		}
 		if (row) this.db.deleteDoc(row.id, { vec: true });
-		const now = (/* @__PURE__ */ new Date()).toISOString();
+		const frecency = addVisit(row?.frecency ?? 0, row ? "updated" : "new", doc.updated.getTime() / 1e3);
 		return this.db.addDoc({
 			body: doc.body,
+			deadline: toDeadline(frecency),
 			description: doc.$description ?? "",
-			entities: doc.entities.join(","),
+			entities: doc.entities,
 			hash: doc.hash,
 			path: doc.path,
-			synced_at: now,
-			tags: doc.tags.join(","),
+			synced_at: /* @__PURE__ */ new Date(),
+			tags: doc.tags,
 			title: doc.title,
-			updated_at: now
+			updated_at: doc.updated
 		});
 	}
 	async chunk(id, doc) {
@@ -121,7 +123,7 @@ var Store = class Store {
 		this.ctx.success("Sync complete");
 	}
 	async sync(opts) {
-		const syncStart = (/* @__PURE__ */ new Date()).toISOString();
+		const syncStart = /* @__PURE__ */ new Date();
 		const docs = await this.index();
 		await this.prune(syncStart);
 		if (opts?.embed) await this.embed(docs);
@@ -135,3 +137,5 @@ var Store = class Store {
 };
 //#endregion
 export { Store };
+
+//# sourceMappingURL=store-I5nVEYxK.mjs.map

package/dist/store-I5nVEYxK.mjs.map

@@ -0,0 +1 @@
+{"version":3,"file":"store-I5nVEYxK.mjs","names":[],"sources":["../src/store.ts"],"sourcesContent":["import type { Context } from \"./context.ts\"\nimport type { Db } from \"./db.ts\"\nimport type { EmbedderChunk } from \"./embed/index.ts\"\n\nimport { performance } from \"node:perf_hooks\"\nimport { Doc } from \"./doc.ts\"\nimport { addVisit, toDeadline } from \"./frecency.ts\"\nimport { Progress } from \"./progress.ts\"\nexport type StoreChunk = EmbedderChunk & {\n  doc_id: number\n  doc: Doc\n}\n\nexport class Store {\n  private constructor(\n    public db: Db,\n    public ctx: Context\n  ) {}\n\n  static async load(ctx: Context) {\n    return new Store(await ctx.db(), ctx)\n  }\n\n  // Add or update a node in the store (docs + FTS via triggers, no embeddings)\n  add(doc: Doc) {\n    const row = this.db.getDoc(doc.path)\n\n    if (row?.hash === doc.hash) {\n      this.db.touchDoc(row.id)\n      return row.id\n    }\n\n    // Document changed, so delete old vec\n    if (row) this.db.deleteDoc(row.id, { vec: true })\n\n    const frecency = addVisit(\n      row?.frecency ?? 0,\n      row ? \"updated\" : \"new\",\n      doc.updated.getTime() / 1000\n    )\n    const id = this.db.addDoc({\n      body: doc.body,\n      deadline: toDeadline(frecency),\n      description: doc.$description ?? \"\",\n      entities: doc.entities,\n      hash: doc.hash,\n      path: doc.path,\n      synced_at: new Date(),\n      tags: doc.tags,\n      title: doc.title,\n      updated_at: doc.updated,\n    })\n\n    return id\n  }\n\n  // Chunk a doc for embedding\n  async chunk(id: number, doc: Doc): Promise<StoreChunk[]> {\n    const title = doc.title.trim()\n    const description = (doc.description ?? \"\").trim()\n    const body = doc.body.trim()\n\n    const chunks: EmbedderChunk[] = []\n\n    const embedder = await this.ctx.embedder()\n    if (description.length > 0) chunks.push(...(await embedder.chunk({ text: description, title })))\n\n    if (body.length > 0) {\n      // seq=0 is reserved for description, so offset body seq by at least 1\n      const offset = Math.max(1, chunks.length)\n      const bodyChunks = await embedder.chunk({ text: body, title })\n      chunks.push(\n        ...bodyChunks.map((c) =>\n          Object.assign(c, {\n            seq: c.seq + offset,\n          })\n        )\n      )\n    }\n    return chunks.map((chunk) => Object.assign(chunk, { doc, doc_id: id }))\n  }\n\n  async index() {\n    const docs: Promise<Doc | undefined>[] = []\n    const nodes = new Map<number, Doc>()\n\n    const vfs = await this.ctx.vfs()\n    for await (const entry of vfs.find()) {\n      docs.push(Doc.load(entry))\n    }\n\n    const loaded = await Promise.all(docs)\n    this.db.transaction(() => {\n      for (const doc of loaded) {\n        if (doc) nodes.set(this.add(doc), doc)\n      }\n    })()\n\n    this.ctx.success(`Indexed ${nodes.size} docs from disk`)\n    return nodes\n  }\n\n  async embed(docs: Map<number, Doc>) {\n    const todo = this.db.getUnembeddedDocs()\n    if (todo.length === 0) {\n      this.ctx.success(\"All docs are already embedded\")\n      return\n    }\n    this.ctx.info(`Sync found ${todo.length} unembedded docs`)\n\n    let doneBytes = 0\n    let doneDocs = 0\n    const queue: StoreChunk[] = []\n    const embedder = await this.ctx.embedder()\n    await embedder.backend() // load the embedder before starting the progress bar\n    const start = performance.now()\n    const progress = new Progress(\"Embedding\", { max: todo.length })\n    this.ctx.events.emit(\"progress\", progress)\n\n    const updateProgress = () => {\n      const secs = (performance.now() - start) / 1000\n      const kbPerSec = (doneBytes / secs / 1024).toFixed(0)\n      progress.set({\n        status: `${progress.max - todo.length}/${progress.max} docs embedded ${kbPerSec}kb/s...`,\n        value: doneDocs,\n      })\n    }\n\n    const markEmbedded = (id: number) => {\n      doneDocs++\n      const hash = docs.get(id)?.hash\n      if (hash) this.db.markEmbedded(id, hash)\n    }\n\n    const embed = async (flush?: boolean) => {\n      while (queue.length >= (flush ? 1 : embedder.opts.batchSize)) {\n        const batch = queue.splice(0, embedder.opts.batchSize)\n        // oxlint-disable-next-line no-await-in-loop\n        const embeddings = await embedder.embed(batch)\n        doneBytes += batch.reduce((sum, c) => sum + c.prompt.length, 0)\n        batch.forEach((chunk, i) => (chunk.embedding = embeddings[i]))\n        this.db.insertEmbeddings(batch)\n\n        updateProgress()\n\n        const completed = new Set(batch.map((c) => c.doc_id))\n        for (const c of queue) completed.delete(c.doc_id)\n        completed.forEach((id) => markEmbedded(id))\n      }\n    }\n\n    while (todo.length > 0) {\n      const { id } = todo.pop()!\n      const doc = docs.get(id)\n      if (!doc) continue\n      this.db.deleteEmbeddings(id)\n      // oxlint-disable-next-line no-await-in-loop\n      const chunks = await this.chunk(id, doc)\n      queue.push(...chunks)\n      if (chunks.length === 0) markEmbedded(id) // mark as embedded even if there are no chunks to embed\n      // oxlint-disable-next-line no-await-in-loop\n      await embed()\n    }\n    await embed(true) // embed any remaining chunks in the queue\n\n    progress.stop()\n    this.ctx.success(\"Sync complete\")\n  }\n\n  async sync(opts?: { embed?: boolean }) {\n    const syncStart = new Date()\n    const docs = await this.index()\n    await this.prune(syncStart)\n    if (opts?.embed) await this.embed(docs)\n  }\n\n  // Remove docs that no longer exist on disk\n  async prune(syncStart: Date) {\n    let dels = 0\n    const vfs = await this.ctx.vfs()\n    for (const { path } of vfs.folders) {\n      dels += this.db.deleteStaleDocs(syncStart, path)\n    }\n    if (dels > 0) this.ctx.warn(`Removed ${dels} stale docs`)\n  }\n}\n"],"mappings":";;;;;AAaA,IAAa,QAAb,MAAa,MAAM;CACjB,YACE,IACA,KACA;AAFO,OAAA,KAAA;AACA,OAAA,MAAA;;CAGT,aAAa,KAAK,KAAc;AAC9B,SAAO,IAAI,MAAM,MAAM,IAAI,IAAI,EAAE,IAAI;;CAIvC,IAAI,KAAU;EACZ,MAAM,MAAM,KAAK,GAAG,OAAO,IAAI,KAAK;AAEpC,MAAI,KAAK,SAAS,IAAI,MAAM;AAC1B,QAAK,GAAG,SAAS,IAAI,GAAG;AACxB,UAAO,IAAI;;AAIb,MAAI,IAAK,MAAK,GAAG,UAAU,IAAI,IAAI,EAAE,KAAK,MAAM,CAAC;EAEjD,MAAM,WAAW,SACf,KAAK,YAAY,GACjB,MAAM,YAAY,OAClB,IAAI,QAAQ,SAAS,GAAG,IACzB;AAcD,SAbW,KAAK,GAAG,OAAO;GACxB,MAAM,IAAI;GACV,UAAU,WAAW,SAAS;GAC9B,aAAa,IAAI,gBAAgB;GACjC,UAAU,IAAI;GACd,MAAM,IAAI;GACV,MAAM,IAAI;GACV,2BAAW,IAAI,MAAM;GACrB,MAAM,IAAI;GACV,OAAO,IAAI;GACX,YAAY,IAAI;GACjB,CAAC;;CAMJ,MAAM,MAAM,IAAY,KAAiC;EACvD,MAAM,QAAQ,IAAI,MAAM,MAAM;EAC9B,MAAM,eAAe,IAAI,eAAe,IAAI,MAAM;EAClD,MAAM,OAAO,IAAI,KAAK,MAAM;EAE5B,MAAM,SAA0B,EAAE;EAElC,MAAM,WAAW,MAAM,KAAK,IAAI,UAAU;AAC1C,MAAI,YAAY,SAAS,EAAG,QAAO,KAAK,GAAI,MAAM,SAAS,MAAM;GAAE,MAAM;GAAa;GAAO,CAAC,CAAE;AAEhG,MAAI,KAAK,SAAS,GAAG;GAEnB,MAAM,SAAS,KAAK,IAAI,GAAG,OAAO,OAAO;GACzC,MAAM,aAAa,MAAM,SAAS,MAAM;IAAE,MAAM;IAAM;IAAO,CAAC;AAC9D,UAAO,KACL,GAAG,WAAW,KAAK,MACjB,OAAO,OAAO,GAAG,EACf,KAAK,EAAE,MAAM,QACd,CAAC,CACH,CACF;;AAEH,SAAO,OAAO,KAAK,UAAU,OAAO,OAAO,OAAO;GAAE;GAAK,QAAQ;GAAI,CAAC,CAAC;;CAGzE,MAAM,QAAQ;EACZ,MAAM,OAAmC,EAAE;EAC3C,MAAM,wBAAQ,IAAI,KAAkB;EAEpC,MAAM,MAAM,MAAM,KAAK,IAAI,KAAK;AAChC,aAAW,MAAM,SAAS,IAAI,MAAM,CAClC,MAAK,KAAK,IAAI,KAAK,MAAM,CAAC;EAG5B,MAAM,SAAS,MAAM,QAAQ,IAAI,KAAK;AACtC,OAAK,GAAG,kBAAkB;AACxB,QAAK,MAAM,OAAO,OAChB,KAAI,IAAK,OAAM,IAAI,KAAK,IAAI,IAAI,EAAE,IAAI;IAExC,EAAE;AAEJ,OAAK,IAAI,QAAQ,WAAW,MAAM,KAAK,iBAAiB;AACxD,SAAO;;CAGT,MAAM,MAAM,MAAwB;EAClC,MAAM,OAAO,KAAK,GAAG,mBAAmB;AACxC,MAAI,KAAK,WAAW,GAAG;AACrB,QAAK,IAAI,QAAQ,gCAAgC;AACjD;;AAEF,OAAK,IAAI,KAAK,cAAc,KAAK,OAAO,kBAAkB;EAE1D,IAAI,YAAY;EAChB,IAAI,WAAW;EACf,MAAM,QAAsB,EAAE;EAC9B,MAAM,WAAW,MAAM,KAAK,IAAI,UAAU;AAC1C,QAAM,SAAS,SAAS;EACxB,MAAM,QAAQ,YAAY,KAAK;EAC/B,MAAM,WAAW,IAAI,SAAS,aAAa,EAAE,KAAK,KAAK,QAAQ,CAAC;AAChE,OAAK,IAAI,OAAO,KAAK,YAAY,SAAS;EAE1C,MAAM,uBAAuB;GAC3B,MAAM,QAAQ,YAAY,KAAK,GAAG,SAAS;GAC3C,MAAM,YAAY,YAAY,OAAO,MAAM,QAAQ,EAAE;AACrD,YAAS,IAAI;IACX,QAAQ,GAAG,SAAS,MAAM,KAAK,OAAO,GAAG,SAAS,IAAI,iBAAiB,SAAS;IAChF,OAAO;IACR,CAAC;;EAGJ,MAAM,gBAAgB,OAAe;AACnC;GACA,MAAM,OAAO,KAAK,IAAI,GAAG,EAAE;AAC3B,OAAI,KAAM,MAAK,GAAG,aAAa,IAAI,KAAK;;EAG1C,MAAM,QAAQ,OAAO,UAAoB;AACvC,UAAO,MAAM,WAAW,QAAQ,IAAI,SAAS,KAAK,YAAY;IAC5D,MAAM,QAAQ,MAAM,OAAO,GAAG,SAAS,KAAK,UAAU;IAEtD,MAAM,aAAa,MAAM,SAAS,MAAM,MAAM;AAC9C,iBAAa,MAAM,QAAQ,KAAK,MAAM,MAAM,EAAE,OAAO,QAAQ,EAAE;AAC/D,UAAM,SAAS,OAAO,MAAO,MAAM,YAAY,WAAW,GAAI;AAC9D,SAAK,GAAG,iBAAiB,MAAM;AAE/B,oBAAgB;IAEhB,MAAM,YAAY,IAAI,IAAI,MAAM,KAAK,MAAM,EAAE,OAAO,CAAC;AACrD,SAAK,MAAM,KAAK,MAAO,WAAU,OAAO,EAAE,OAAO;AACjD,cAAU,SAAS,OAAO,aAAa,GAAG,CAAC;;;AAI/C,SAAO,KAAK,SAAS,GAAG;GACtB,MAAM,EAAE,OAAO,KAAK,KAAK;GACzB,MAAM,MAAM,KAAK,IAAI,GAAG;AACxB,OAAI,CAAC,IAAK;AACV,QAAK,GAAG,iBAAiB,GAAG;GAE5B,MAAM,SAAS,MAAM,KAAK,MAAM,IAAI,IAAI;AACxC,SAAM,KAAK,GAAG,OAAO;AACrB,OAAI,OAAO,WAAW,EAAG,cAAa,GAAG;AAEzC,SAAM,OAAO;;AAEf,QAAM,MAAM,KAAK;AAEjB,WAAS,MAAM;AACf,OAAK,IAAI,QAAQ,gBAAgB;;CAGnC,MAAM,KAAK,MAA4B;EACrC,MAAM,4BAAY,IAAI,MAAM;EAC5B,MAAM,OAAO,MAAM,KAAK,OAAO;AAC/B,QAAM,KAAK,MAAM,UAAU;AAC3B,MAAI,MAAM,MAAO,OAAM,KAAK,MAAM,KAAK;;CAIzC,MAAM,MAAM,WAAiB;EAC3B,IAAI,OAAO;EACX,MAAM,MAAM,MAAM,KAAK,IAAI,KAAK;AAChC,OAAK,MAAM,EAAE,UAAU,IAAI,QACzB,SAAQ,KAAK,GAAG,gBAAgB,WAAW,KAAK;AAElD,MAAI,OAAO,EAAG,MAAK,IAAI,KAAK,WAAW,KAAK,aAAa"}

package/dist/{transformers-CJ3QA2PK.mjs → transformers-Df56Nq9G.mjs}

@@ -1,4 +1,4 @@
-import { n as parseModelUri } from "./models-DFQSgBNr.mjs";
+import { n as parseModelUri } from "./models-Bo6czhQe.mjs";
 //#region src/embed/transformers.ts
 var TransformersBackend = class TransformersBackend {
 	device = "cpu";
@@ -53,3 +53,5 @@ var TransformersBackend = class TransformersBackend {
 };
 //#endregion
 export { TransformersBackend };
+
+//# sourceMappingURL=transformers-Df56Nq9G.mjs.map

package/dist/transformers-Df56Nq9G.mjs.map

@@ -0,0 +1 @@
+{"version":3,"file":"transformers-Df56Nq9G.mjs","names":["#pipeline","#ctx"],"sources":["../src/embed/transformers.ts"],"sourcesContent":["import type {\n  DataType,\n  FeatureExtractionPipeline,\n  ProgressInfo,\n  Tensor,\n} from \"@huggingface/transformers\"\nimport type { EmbedderBackend, EmbedderContext, EmbedderDevice } from \"./index.ts\"\n\nimport { parseModelUri } from \"./models.ts\"\n\nexport class TransformersBackend implements EmbedderBackend {\n  device: EmbedderDevice = \"cpu\"\n  maxTokens: number\n  dims: number\n  #pipeline: FeatureExtractionPipeline\n  #ctx: EmbedderContext\n  private normalize?: (tensor: Tensor) => Tensor // optional normalization function, e.g. for L2 normalization after truncation\n\n  private constructor(pipeline: FeatureExtractionPipeline, ctx: EmbedderContext) {\n    this.#pipeline = pipeline\n    this.#ctx = ctx\n    this.maxTokens = pipeline.model.config.max_position_embeddings\n    this.dims = (pipeline.model.config as { hidden_size?: number }).hidden_size ?? 0\n    this.device = pipeline.model.sessions.model?.config?.device ?? \"cpu\"\n  }\n\n  static async load(this: void, ctx: EmbedderContext) {\n    const parsed = parseModelUri(ctx.opts.model.uri)\n    const { pipeline, layer_norm } = await import(\"@huggingface/transformers\")\n\n    const extractor = await pipeline(\"feature-extraction\", parsed.model, {\n      // device: \"webgpu\",\n      dtype: (parsed.variant ?? \"auto\") as DataType,\n      progress_callback: (event) => TransformersBackend.onProgress(ctx, event),\n      session_options: { intraOpNumThreads: ctx.opts.threads },\n    })\n    const backend = new TransformersBackend(extractor, ctx)\n\n    // Matryoshka: layer_norm → truncate\n    const dims = ctx.opts.maxDims\n    if (dims < backend.dims)\n      backend.normalize = (output) =>\n        layer_norm(output, [output.dims[1] ?? 0])\n          // oxlint-disable-next-line unicorn/no-null\n          .slice(null, [0, dims])\n          .normalize(2, -1)\n\n    return backend\n  }\n\n  async embed(texts: string[]): Promise<number[][]> {\n    const output = await this.#pipeline(texts, {\n      normalize: !this.normalize,\n      pooling: this.#ctx.opts.model.pooling,\n    })\n    return (this.normalize?.(output) ?? output).tolist() as number[][]\n  }\n\n  toks(input: string) {\n    return this.#pipeline.tokenizer.tokenize(input).length\n  }\n\n  static onProgress(ctx: EmbedderContext, event: ProgressInfo) {\n    if (event.status === \"initiate\") {\n      ctx.status.child(event.name).child(event.file).status = event.status\n    } else if (event.status === \"download\") {\n      ctx.status.child(event.name).child(event.file).status = event.status\n    } else if (event.status === \"progress\") {\n      ctx.status.child(event.name).child(event.file).set({\n        max: event.total,\n        status: event.status,\n        value: event.loaded,\n      })\n    } else if (event.status === \"done\") {\n      ctx.status.child(event.name).child(event.file).set({ status: event.status }).stop()\n    } else if (event.status === \"ready\") {\n      ctx.status.name = `model \\`${ctx.opts.model.uri}\\` loaded`\n      ctx.status.child(event.task).set({ status: event.status }).stop()\n    }\n  }\n}\n"],"mappings":";;AAUA,IAAa,sBAAb,MAAa,oBAA+C;CAC1D,SAAyB;CACzB;CACA;CACA;CACA;CACA;CAEA,YAAoB,UAAqC,KAAsB;AAC7E,QAAA,WAAiB;AACjB,QAAA,MAAY;AACZ,OAAK,YAAY,SAAS,MAAM,OAAO;AACvC,OAAK,OAAQ,SAAS,MAAM,OAAoC,eAAe;AAC/E,OAAK,SAAS,SAAS,MAAM,SAAS,OAAO,QAAQ,UAAU;;CAGjE,aAAa,KAAiB,KAAsB;EAClD,MAAM,SAAS,cAAc,IAAI,KAAK,MAAM,IAAI;EAChD,MAAM,EAAE,UAAU,eAAe,MAAM,OAAO;EAQ9C,MAAM,UAAU,IAAI,oBANF,MAAM,SAAS,sBAAsB,OAAO,OAAO;GAEnE,OAAQ,OAAO,WAAW;GAC1B,oBAAoB,UAAU,oBAAoB,WAAW,KAAK,MAAM;GACxE,iBAAiB,EAAE,mBAAmB,IAAI,KAAK,SAAS;GACzD,CAAC,EACiD,IAAI;EAGvD,MAAM,OAAO,IAAI,KAAK;AACtB,MAAI,OAAO,QAAQ,KACjB,SAAQ,aAAa,WACnB,WAAW,QAAQ,CAAC,OAAO,KAAK,MAAM,EAAE,CAAC,CAEtC,MAAM,MAAM,CAAC,GAAG,KAAK,CAAC,CACtB,UAAU,GAAG,GAAG;AAEvB,SAAO;;CAGT,MAAM,MAAM,OAAsC;EAChD,MAAM,SAAS,MAAM,MAAA,SAAe,OAAO;GACzC,WAAW,CAAC,KAAK;GACjB,SAAS,MAAA,IAAU,KAAK,MAAM;GAC/B,CAAC;AACF,UAAQ,KAAK,YAAY,OAAO,IAAI,QAAQ,QAAQ;;CAGtD,KAAK,OAAe;AAClB,SAAO,MAAA,SAAe,UAAU,SAAS,MAAM,CAAC;;CAGlD,OAAO,WAAW,KAAsB,OAAqB;AAC3D,MAAI,MAAM,WAAW,WACnB,KAAI,OAAO,MAAM,MAAM,KAAK,CAAC,MAAM,MAAM,KAAK,CAAC,SAAS,MAAM;WACrD,MAAM,WAAW,WAC1B,KAAI,OAAO,MAAM,MAAM,KAAK,CAAC,MAAM,MAAM,KAAK,CAAC,SAAS,MAAM;WACrD,MAAM,WAAW,WAC1B,KAAI,OAAO,MAAM,MAAM,KAAK,CAAC,MAAM,MAAM,KAAK,CAAC,IAAI;GACjD,KAAK,MAAM;GACX,QAAQ,MAAM;GACd,OAAO,MAAM;GACd,CAAC;WACO,MAAM,WAAW,OAC1B,KAAI,OAAO,MAAM,MAAM,KAAK,CAAC,MAAM,MAAM,KAAK,CAAC,IAAI,EAAE,QAAQ,MAAM,QAAQ,CAAC,CAAC,MAAM;WAC1E,MAAM,WAAW,SAAS;AACnC,OAAI,OAAO,OAAO,WAAW,IAAI,KAAK,MAAM,IAAI;AAChD,OAAI,OAAO,MAAM,MAAM,KAAK,CAAC,IAAI,EAAE,QAAQ,MAAM,QAAQ,CAAC,CAAC,MAAM"}

package/dist/uri-CehXVDGB.mjs

@@ -26,3 +26,5 @@ function parentUri(uri) {
 }
 //#endregion
 export { parentUri as i, assertUri as n, normUri as r, URI_PREFIX as t };
+
+//# sourceMappingURL=uri-CehXVDGB.mjs.map

package/dist/uri-CehXVDGB.mjs.map

@@ -0,0 +1 @@
+{"version":3,"file":"uri-CehXVDGB.mjs","names":[],"sources":["../src/uri.ts"],"sourcesContent":["export const URI_PREFIX = \"rekal://\"\n\nexport function assertUri(uri: string) {\n  if (!uri.startsWith(URI_PREFIX)) throw new Error(`URI must start with ${URI_PREFIX}, got: ${uri}`)\n}\n\nexport function normUri(uri?: string, dir?: boolean): string {\n  if (uri === undefined) return URI_PREFIX\n  if (typeof uri !== \"string\") throw new Error(`URI must be a string, got: ${JSON.stringify(uri)}`)\n  uri = uri.trim()\n  uri = uri.replace(/^rekall?:/, \"\") // protocol\n  uri = uri.replace(/[\\\\/]+/g, \"/\") // normalize slashes\n  uri = uri.replace(/^\\/+/, \"\") // leading slashes\n  if (uri === \"\") return URI_PREFIX\n  uri = URI_PREFIX + uri\n  if (uri.endsWith(\"/index.md\")) return uri.replace(/\\/index\\.md$/, \"/\") // index.md implies directory\n  uri = dir ? uri.replace(/\\/?$/, \"/\") : uri // trailing slash for directories\n  uri = dir === false ? uri.replace(/\\/?$/, \"\") : uri // remove trailing slash for files\n  return uri\n}\n\nexport function parentUri(uri: string): string | undefined {\n  uri = normUri(uri)\n  if (uri === URI_PREFIX) return\n  uri = uri.replace(/\\/?$/, \"\") // remove trailing slash\n  uri = uri.replace(/\\/[^/]+$/, \"\") // remove last segment\n  return uri === URI_PREFIX ? URI_PREFIX : `${uri}/`\n}\n"],"mappings":";AAAA,MAAa,aAAa;AAE1B,SAAgB,UAAU,KAAa;AACrC,KAAI,CAAC,IAAI,WAAA,WAAsB,CAAE,OAAM,IAAI,MAAM,uBAAuB,WAAW,SAAS,MAAM;;AAGpG,SAAgB,QAAQ,KAAc,KAAuB;AAC3D,KAAI,QAAQ,KAAA,EAAW,QAAO;AAC9B,KAAI,OAAO,QAAQ,SAAU,OAAM,IAAI,MAAM,8BAA8B,KAAK,UAAU,IAAI,GAAG;AACjG,OAAM,IAAI,MAAM;AAChB,OAAM,IAAI,QAAQ,aAAa,GAAG;AAClC,OAAM,IAAI,QAAQ,WAAW,IAAI;AACjC,OAAM,IAAI,QAAQ,QAAQ,GAAG;AAC7B,KAAI,QAAQ,GAAI,QAAO;AACvB,OAAM,aAAa;AACnB,KAAI,IAAI,SAAS,YAAY,CAAE,QAAO,IAAI,QAAQ,gBAAgB,IAAI;AACtE,OAAM,MAAM,IAAI,QAAQ,QAAQ,IAAI,GAAG;AACvC,OAAM,QAAQ,QAAQ,IAAI,QAAQ,QAAQ,GAAG,GAAG;AAChD,QAAO;;AAGT,SAAgB,UAAU,KAAiC;AACzD,OAAM,QAAQ,IAAI;AAClB,KAAI,QAAA,WAAoB;AACxB,OAAM,IAAI,QAAQ,QAAQ,GAAG;AAC7B,OAAM,IAAI,QAAQ,YAAY,GAAG;AACjC,QAAO,QAAA,aAAqB,aAAa,GAAG,IAAI"}

package/dist/util-DNyrmcA3.mjs

@@ -9,3 +9,5 @@ function toError(err) {
 }
 //#endregion
 export { toError as n, hash as t };
+
+//# sourceMappingURL=util-DNyrmcA3.mjs.map

package/dist/util-DNyrmcA3.mjs.map

@@ -0,0 +1 @@
+{"version":3,"file":"util-DNyrmcA3.mjs","names":[],"sources":["../src/util.ts"],"sourcesContent":["import { parseYaml } from \"#runtime\"\nimport { createHash } from \"node:crypto\"\n\nexport { parseYaml }\n\nexport function hash(content: string): string {\n  return createHash(\"sha256\").update(content).digest(\"hex\")\n}\n\nexport function toError(err: unknown): Error {\n  return err instanceof Error ? err : new Error(String(err))\n}\n\nexport type Events = Record<string, unknown[]>\n\nexport type TypedEmitter<T extends Events> = {\n  on<K extends keyof T>(event: K, fn: (...args: T[K]) => void): TypedEmitter<T>\n  off<K extends keyof T>(event: K, fn: (...args: T[K]) => void): TypedEmitter<T>\n  once<K extends keyof T>(event: K, fn: (...args: T[K]) => void): TypedEmitter<T>\n  emit<K extends keyof T>(event: K, ...args: T[K]): boolean\n}\n"],"mappings":";;;AAKA,SAAgB,KAAK,SAAyB;AAC5C,QAAO,WAAW,SAAS,CAAC,OAAO,QAAQ,CAAC,OAAO,MAAM;;AAG3D,SAAgB,QAAQ,KAAqB;AAC3C,QAAO,eAAe,QAAQ,MAAM,IAAI,MAAM,OAAO,IAAI,CAAC"}

package/dist/{vfs-CNQbkhsf.mjs → vfs-QUP1rnSI.mjs}

@@ -220,3 +220,5 @@ var Vfs = class {
 };
 //#endregion
 export { Vfs };
+
+//# sourceMappingURL=vfs-QUP1rnSI.mjs.map

package/dist/vfs-QUP1rnSI.mjs.map

@@ -0,0 +1 @@
+{"version":3,"file":"vfs-QUP1rnSI.mjs","names":["#folders","#root"],"sources":["../src/vfs.ts"],"sourcesContent":["import type { Context } from \"./context.ts\"\nimport type { Doc } from \"./doc.ts\"\n\nimport { basename, join, relative } from \"pathe\"\nimport { normPath, sstat } from \"./fs.ts\"\nimport { URI_PREFIX, normUri } from \"./uri.ts\"\n\nexport class Node {\n  constructor(\n    public uri: string,\n    public doc: Doc\n  ) {}\n}\n\nconst DEFAULT_EXCLUDE = [\".git\", \"node_modules/\"]\n\nexport type VfsFolder = {\n  uri: string\n  path: string\n  merge?: boolean // TODO: whether this path should be merged with others in the same URI, defaults to false\n}\n\nexport type VfsEntry = {\n  uri: string\n  path?: string\n}\n\nexport type VfsPath = {\n  node: VfsNode\n  path: string\n}\n\nexport type VfsNode = {\n  name: string\n  parent?: VfsNode\n  uri: string\n  paths: string[]\n  depth: number\n  children: Map<string, VfsNode>\n}\n\nexport type VfsFindOptions = {\n  /** URI to start from, defaults to root */\n  uri?: string\n  depth?: number // max depth to search, defaults to Infinity\n  pattern?: string // extra regex pattern to match URIs against\n  ignoreCase?: boolean // whether pattern matching should ignore case. When not set smart case is used.\n  limit?: number // max results to return, defaults to Infinity\n  type?: \"file\" | \"directory\" // filter by type\n}\n\nexport type VfsView = {\n  uri: string // URI of the scope, defaults to rekal://\n  node: VfsNode // the node representing the resolved URI\n  paths: VfsPath[] // paths leading up to this node and to descendant nodes\n}\n\nexport type VfsScope = VfsView & {\n  // map a path to the shortest URI in this scope, if it exists\n  map: (path: string) => string | undefined\n}\n\nexport class Vfs {\n  #folders = new Map<string, VfsFolder[]>() // map of paths to folders\n  #root: VfsNode = { children: new Map(), depth: 0, name: \"#root\", paths: [], uri: URI_PREFIX }\n\n  public constructor(public ctx: Context) {\n    for (const folder of ctx.opts.folders ?? []) this.addFolder(folder)\n  }\n\n  get folders(): VfsFolder[] {\n    return [...this.#folders.values()].flat()\n  }\n\n  isFolder(path: string): boolean {\n    path = normPath(path).replace(/\\/?$/, \"/\")\n    return this.#folders.has(path)\n  }\n\n  getScope(uri?: string, opts?: { children?: boolean }): VfsScope {\n    uri = normUri(uri, true)\n    const view = this.resolve(uri, opts)\n    return {\n      ...view,\n      map: (path: string) => {\n        path = normPath(path)\n        let best: string | undefined\n        for (const p of view.paths) {\n          const rel = relative(p.path, path)\n          if (rel.startsWith(\"..\")) continue\n          const candidate = p.node.uri + rel\n          if (!best || candidate.length < best.length) best = candidate\n        }\n        return best\n      },\n    }\n  }\n\n  getNode(uri: string, create = false) {\n    uri = normUri(uri)\n    let node = this.#root\n    const parts = uri.slice(URI_PREFIX.length).split(\"/\").filter(Boolean)\n    for (const part of parts) {\n      let child = node.children.get(part)\n      if (!child) {\n        child = {\n          children: new Map(),\n          depth: node.depth + 1,\n          name: part,\n          parent: node,\n          paths: [],\n          uri: `${node.uri}${part}/`,\n        }\n        if (create) node.children.set(part, child)\n      }\n      node = child\n    }\n    return node\n  }\n\n  resolve(node: VfsNode | string, opts?: { children?: boolean }): VfsView {\n    node = typeof node === \"string\" ? this.getNode(node) : node\n    const nodes = [node]\n    let { parent } = node\n    while (parent) {\n      nodes.unshift(parent)\n      parent = parent.parent\n    }\n\n    const paths: VfsPath[] = [] // paths to this node or to descendant nodes\n    const folders: VfsPath[] = [] // folders used by ancestor and descendant nodes\n\n    // resolve paths and folders to this node\n    for (const n of nodes) {\n      for (const p of paths) p.path = join(p.path, n.name)\n      for (const path of n.paths) {\n        paths.push({ node, path })\n        if (this.isFolder(path)) folders.push({ node: n, path })\n      }\n    }\n\n    // resolve paths and folders to descendant nodes\n    const stack = opts?.children === false ? [] : [...node.children.values()]\n    while (stack.length > 0) {\n      const n = stack.pop()!\n      for (const path of n.paths) {\n        paths.push({ node: n, path })\n        if (this.isFolder(path)) folders.push({ node: n, path })\n      }\n      stack.push(...n.children.values())\n    }\n\n    paths.sort((a, b) => a.node.uri.localeCompare(b.node.uri))\n    return { node, paths, uri: node.uri }\n  }\n\n  addFolder(folder: VfsFolder) {\n    folder.uri = normUri(folder.uri, true)\n    folder.path = normPath(folder.path).replace(/\\/?$/, \"/\")\n    const folders = this.#folders.get(folder.path) ?? []\n    this.#folders.set(folder.path, [...folders, folder])\n    const node = this.getNode(folder.uri, true)\n    node.paths.push(folder.path)\n  }\n\n  matcher(opts?: VfsFindOptions): (uri: string) => boolean {\n    const pattern = opts?.pattern ?? \"\"\n    if (!pattern.length) return () => true\n    const ignoreCase = opts?.ignoreCase ?? !/[A-Z]/.test(pattern)\n    const re = new RegExp(pattern, ignoreCase ? \"i\" : \"\")\n    return (uri: string) => re.test(uri)\n  }\n\n  async *find(opts: VfsFindOptions = {}): AsyncGenerator<VfsEntry> {\n    const { glob } = await import(\"./glob.ts\")\n    const uri = normUri(opts.uri ?? URI_PREFIX, true)\n    const root = this.resolve(uri)\n    const maxDepth = root.node.depth + (opts.depth ?? Infinity)\n    const visited = new Set<string>()\n    const filter = this.matcher(opts)\n\n    const stop = () => opts.limit !== undefined && visited.size >= opts.limit\n\n    const use = (p: VfsEntry) => {\n      if (p.uri.endsWith(\"/\") && opts.type === \"file\") return false\n      if (!filter(p.uri)) return false\n      if (stop()) return false\n      const key = `${p.uri}:${p.path ?? \"\"}`\n      if (visited.has(key)) return false\n      visited.add(key)\n      return true\n    }\n\n    // add virtual internal uris\n    function* yieldVirtual(p: VfsPath) {\n      if (p.node === root.node || opts.type === \"file\") return\n      let parent = p.node.parent\n      while (parent && parent !== root.node && !stop()) {\n        const virtual =\n          parent.depth <= maxDepth &&\n          !root.paths.some((rp) => parent?.uri.startsWith(rp.node.uri)) &&\n          use({ uri: parent.uri })\n        if (virtual) yield { uri: parent.uri }\n        parent = parent.parent\n      }\n    }\n\n    scan: for (const p of root.paths) {\n      yield* yieldVirtual(p)\n      if (p.node.depth > maxDepth) continue\n      const e = { path: p.path, uri: p.node.uri }\n      if (p.node !== root.node && use(e)) yield e\n\n      const cwd = p.path\n      const todo = glob({\n        cwd,\n        depth: maxDepth - p.node.depth,\n        empty: false,\n        exclude: DEFAULT_EXCLUDE,\n        glob: [\"**/*.md\"],\n        type: opts.type,\n      })\n\n      // oxlint-disable-next-line no-await-in-loop\n      for await (const childPath of todo) {\n        if (basename(childPath) === \"index.md\") continue\n        const path = { path: join(cwd, childPath), uri: p.node.uri + childPath }\n        if (use(path)) yield path\n        if (stop()) break scan\n      }\n    }\n  }\n\n  async *ls(opts?: Omit<VfsFindOptions, \"depth\">) {\n    yield* this.find({ ...opts, depth: 1 })\n  }\n\n  /** Normalizes the URI and path to a real path if it exists **/\n  normPath(p: VfsPath): VfsEntry {\n    const transforms = [\n      { from: /\\/index\\.md$/, to: \"/\" },\n      { from: /\\/index$/, to: \"/\" },\n      { from: /\\.md$/, to: \"\" },\n      { from: /(?!\\.md)$/, to: \".md\" },\n    ]\n    if (!sstat(p.path) || basename(p.path) === \"index.md\") {\n      const root = this.getScope()\n      for (const t of transforms) {\n        const path = p.path.replace(t.from, t.to)\n        const uri = root.map(path)\n        if (sstat(path) && uri) return { path, uri }\n      }\n      return { uri: p.node.uri }\n    }\n    return { path: p.path, uri: p.node.uri }\n  }\n}\n"],"mappings":";;;;AAcA,MAAM,kBAAkB,CAAC,QAAQ,gBAAgB;AAgDjD,IAAa,MAAb,MAAiB;CACf,2BAAW,IAAI,KAA0B;CACzC,QAAiB;EAAE,0BAAU,IAAI,KAAK;EAAE,OAAO;EAAG,MAAM;EAAS,OAAO,EAAE;EAAE,KAAK;EAAY;CAE7F,YAAmB,KAAqB;AAAd,OAAA,MAAA;AACxB,OAAK,MAAM,UAAU,IAAI,KAAK,WAAW,EAAE,CAAE,MAAK,UAAU,OAAO;;CAGrE,IAAI,UAAuB;AACzB,SAAO,CAAC,GAAG,MAAA,QAAc,QAAQ,CAAC,CAAC,MAAM;;CAG3C,SAAS,MAAuB;AAC9B,SAAO,SAAS,KAAK,CAAC,QAAQ,QAAQ,IAAI;AAC1C,SAAO,MAAA,QAAc,IAAI,KAAK;;CAGhC,SAAS,KAAc,MAAyC;AAC9D,QAAM,QAAQ,KAAK,KAAK;EACxB,MAAM,OAAO,KAAK,QAAQ,KAAK,KAAK;AACpC,SAAO;GACL,GAAG;GACH,MAAM,SAAiB;AACrB,WAAO,SAAS,KAAK;IACrB,IAAI;AACJ,SAAK,MAAM,KAAK,KAAK,OAAO;KAC1B,MAAM,MAAM,SAAS,EAAE,MAAM,KAAK;AAClC,SAAI,IAAI,WAAW,KAAK,CAAE;KAC1B,MAAM,YAAY,EAAE,KAAK,MAAM;AAC/B,SAAI,CAAC,QAAQ,UAAU,SAAS,KAAK,OAAQ,QAAO;;AAEtD,WAAO;;GAEV;;CAGH,QAAQ,KAAa,SAAS,OAAO;AACnC,QAAM,QAAQ,IAAI;EAClB,IAAI,OAAO,MAAA;EACX,MAAM,QAAQ,IAAI,MAAM,WAAW,OAAO,CAAC,MAAM,IAAI,CAAC,OAAO,QAAQ;AACrE,OAAK,MAAM,QAAQ,OAAO;GACxB,IAAI,QAAQ,KAAK,SAAS,IAAI,KAAK;AACnC,OAAI,CAAC,OAAO;AACV,YAAQ;KACN,0BAAU,IAAI,KAAK;KACnB,OAAO,KAAK,QAAQ;KACpB,MAAM;KACN,QAAQ;KACR,OAAO,EAAE;KACT,KAAK,GAAG,KAAK,MAAM,KAAK;KACzB;AACD,QAAI,OAAQ,MAAK,SAAS,IAAI,MAAM,MAAM;;AAE5C,UAAO;;AAET,SAAO;;CAGT,QAAQ,MAAwB,MAAwC;AACtE,SAAO,OAAO,SAAS,WAAW,KAAK,QAAQ,KAAK,GAAG;EACvD,MAAM,QAAQ,CAAC,KAAK;EACpB,IAAI,EAAE,WAAW;AACjB,SAAO,QAAQ;AACb,SAAM,QAAQ,OAAO;AACrB,YAAS,OAAO;;EAGlB,MAAM,QAAmB,EAAE;EAC3B,MAAM,UAAqB,EAAE;AAG7B,OAAK,MAAM,KAAK,OAAO;AACrB,QAAK,MAAM,KAAK,MAAO,GAAE,OAAO,KAAK,EAAE,MAAM,EAAE,KAAK;AACpD,QAAK,MAAM,QAAQ,EAAE,OAAO;AAC1B,UAAM,KAAK;KAAE;KAAM;KAAM,CAAC;AAC1B,QAAI,KAAK,SAAS,KAAK,CAAE,SAAQ,KAAK;KAAE,MAAM;KAAG;KAAM,CAAC;;;EAK5D,MAAM,QAAQ,MAAM,aAAa,QAAQ,EAAE,GAAG,CAAC,GAAG,KAAK,SAAS,QAAQ,CAAC;AACzE,SAAO,MAAM,SAAS,GAAG;GACvB,MAAM,IAAI,MAAM,KAAK;AACrB,QAAK,MAAM,QAAQ,EAAE,OAAO;AAC1B,UAAM,KAAK;KAAE,MAAM;KAAG;KAAM,CAAC;AAC7B,QAAI,KAAK,SAAS,KAAK,CAAE,SAAQ,KAAK;KAAE,MAAM;KAAG;KAAM,CAAC;;AAE1D,SAAM,KAAK,GAAG,EAAE,SAAS,QAAQ,CAAC;;AAGpC,QAAM,MAAM,GAAG,MAAM,EAAE,KAAK,IAAI,cAAc,EAAE,KAAK,IAAI,CAAC;AAC1D,SAAO;GAAE;GAAM;GAAO,KAAK,KAAK;GAAK;;CAGvC,UAAU,QAAmB;AAC3B,SAAO,MAAM,QAAQ,OAAO,KAAK,KAAK;AACtC,SAAO,OAAO,SAAS,OAAO,KAAK,CAAC,QAAQ,QAAQ,IAAI;EACxD,MAAM,UAAU,MAAA,QAAc,IAAI,OAAO,KAAK,IAAI,EAAE;AACpD,QAAA,QAAc,IAAI,OAAO,MAAM,CAAC,GAAG,SAAS,OAAO,CAAC;AACvC,OAAK,QAAQ,OAAO,KAAK,KAAK,CACtC,MAAM,KAAK,OAAO,KAAK;;CAG9B,QAAQ,MAAiD;EACvD,MAAM,UAAU,MAAM,WAAW;AACjC,MAAI,CAAC,QAAQ,OAAQ,cAAa;EAClC,MAAM,aAAa,MAAM,cAAc,CAAC,QAAQ,KAAK,QAAQ;EAC7D,MAAM,KAAK,IAAI,OAAO,SAAS,aAAa,MAAM,GAAG;AACrD,UAAQ,QAAgB,GAAG,KAAK,IAAI;;CAGtC,OAAO,KAAK,OAAuB,EAAE,EAA4B;EAC/D,MAAM,EAAE,SAAS,MAAM,OAAO;EAC9B,MAAM,MAAM,QAAQ,KAAK,OAAA,YAAmB,KAAK;EACjD,MAAM,OAAO,KAAK,QAAQ,IAAI;EAC9B,MAAM,WAAW,KAAK,KAAK,SAAS,KAAK,SAAS;EAClD,MAAM,0BAAU,IAAI,KAAa;EACjC,MAAM,SAAS,KAAK,QAAQ,KAAK;EAEjC,MAAM,aAAa,KAAK,UAAU,KAAA,KAAa,QAAQ,QAAQ,KAAK;EAEpE,MAAM,OAAO,MAAgB;AAC3B,OAAI,EAAE,IAAI,SAAS,IAAI,IAAI,KAAK,SAAS,OAAQ,QAAO;AACxD,OAAI,CAAC,OAAO,EAAE,IAAI,CAAE,QAAO;AAC3B,OAAI,MAAM,CAAE,QAAO;GACnB,MAAM,MAAM,GAAG,EAAE,IAAI,GAAG,EAAE,QAAQ;AAClC,OAAI,QAAQ,IAAI,IAAI,CAAE,QAAO;AAC7B,WAAQ,IAAI,IAAI;AAChB,UAAO;;EAIT,UAAU,aAAa,GAAY;AACjC,OAAI,EAAE,SAAS,KAAK,QAAQ,KAAK,SAAS,OAAQ;GAClD,IAAI,SAAS,EAAE,KAAK;AACpB,UAAO,UAAU,WAAW,KAAK,QAAQ,CAAC,MAAM,EAAE;AAKhD,QAHE,OAAO,SAAS,YAChB,CAAC,KAAK,MAAM,MAAM,OAAO,QAAQ,IAAI,WAAW,GAAG,KAAK,IAAI,CAAC,IAC7D,IAAI,EAAE,KAAK,OAAO,KAAK,CAAC,CACb,OAAM,EAAE,KAAK,OAAO,KAAK;AACtC,aAAS,OAAO;;;AAIpB,OAAM,MAAK,MAAM,KAAK,KAAK,OAAO;AAChC,UAAO,aAAa,EAAE;AACtB,OAAI,EAAE,KAAK,QAAQ,SAAU;GAC7B,MAAM,IAAI;IAAE,MAAM,EAAE;IAAM,KAAK,EAAE,KAAK;IAAK;AAC3C,OAAI,EAAE,SAAS,KAAK,QAAQ,IAAI,EAAE,CAAE,OAAM;GAE1C,MAAM,MAAM,EAAE;GACd,MAAM,OAAO,KAAK;IAChB;IACA,OAAO,WAAW,EAAE,KAAK;IACzB,OAAO;IACP,SAAS;IACT,MAAM,CAAC,UAAU;IACjB,MAAM,KAAK;IACZ,CAAC;AAGF,cAAW,MAAM,aAAa,MAAM;AAClC,QAAI,SAAS,UAAU,KAAK,WAAY;IACxC,MAAM,OAAO;KAAE,MAAM,KAAK,KAAK,UAAU;KAAE,KAAK,EAAE,KAAK,MAAM;KAAW;AACxE,QAAI,IAAI,KAAK,CAAE,OAAM;AACrB,QAAI,MAAM,CAAE,OAAM;;;;CAKxB,OAAO,GAAG,MAAsC;AAC9C,SAAO,KAAK,KAAK;GAAE,GAAG;GAAM,OAAO;GAAG,CAAC;;;CAIzC,SAAS,GAAsB;EAC7B,MAAM,aAAa;GACjB;IAAE,MAAM;IAAgB,IAAI;IAAK;GACjC;IAAE,MAAM;IAAY,IAAI;IAAK;GAC7B;IAAE,MAAM;IAAS,IAAI;IAAI;GACzB;IAAE,MAAM;IAAa,IAAI;IAAO;GACjC;AACD,MAAI,CAAC,MAAM,EAAE,KAAK,IAAI,SAAS,EAAE,KAAK,KAAK,YAAY;GACrD,MAAM,OAAO,KAAK,UAAU;AAC5B,QAAK,MAAM,KAAK,YAAY;IAC1B,MAAM,OAAO,EAAE,KAAK,QAAQ,EAAE,MAAM,EAAE,GAAG;IACzC,MAAM,MAAM,KAAK,IAAI,KAAK;AAC1B,QAAI,MAAM,KAAK,IAAI,IAAK,QAAO;KAAE;KAAM;KAAK;;AAE9C,UAAO,EAAE,KAAK,EAAE,KAAK,KAAK;;AAE5B,SAAO;GAAE,MAAM,EAAE;GAAM,KAAK,EAAE,KAAK;GAAK"}

package/package.json

@@ -1,6 +1,14 @@
 {
   "name": "@rekal/mem",
-  "version": "0.0.0",
+  "version": "0.0.2",
+  "repository": {
+    "type": "git",
+    "url": "git+https://github.com/folke/rekal.git"
+  },
+  "files": [
+    "src/**/*",
+    "dist/**/*"
+  ],
   "type": "module",
   "imports": {
     "#runtime": {
@@ -9,31 +17,12 @@
     }
   },
   "exports": {
-    ".": {
-      "bun": "./src/index.ts",
-      "default": "./dist/index.mjs"
-    },
-    "./glob": {
-      "bun": "./src/glob.ts",
-      "default": "./dist/glob.mjs"
-    },
+    ".": "./dist/index.mjs",
+    "./glob": "./dist/glob.mjs",
     "./package.json": "./package.json"
   },
-  "publishConfig": {
-    "exports": {
-      ".": "./dist/index.mjs",
-      "./glob": "./dist/glob.mjs",
-      "./package.json": "./package.json"
-    }
-  },
-  "scripts": {
-    "build": "tsdown --cwd ../../ --filter @rekal/mem",
-    "test": "bun test:node && bun test:bun",
-    "test:node": "cd ../../; vitest --project @rekal/mem run",
-    "test:bun": "bun test --only-failures"
-  },
   "dependencies": {
-    "@huggingface/transformers": "^4.0.0",
+    "@huggingface/transformers": "^4.0.1",
     "better-sqlite3": "^12.8.0",
     "defu": "^6.1.6",
     "gpt-tokenizer": "^3.4.0",
@@ -46,16 +35,27 @@
   "devDependencies": {
     "@arethetypeswrong/cli": "^0.18.2",
     "@types/better-sqlite3": "^7.6.13",
+    "@types/js-yaml": "^4.0.9",
     "publint": "^0.3.18",
     "tsdown": "^0.21.7",
     "type-fest": "^5.5.0",
     "typescript": "^6.0.2"
   },
+  "engines": {
+    "bun": ">=1.2",
+    "node": ">=24"
+  },
   "inlinedDependencies": {
     "type-fest": "5.5.0"
   },
   "trustedDependencies": [
     "sqlite-vec",
     "better-sqlite3"
-  ]
-}
+  ],
+  "scripts": {
+    "build": "tsdown --cwd ../../ --filter @rekal/mem",
+    "test": "bun test:node && bun test:bun",
+    "test:node": "cd ../../; vitest --project @rekal/mem run",
+    "test:bun": "bun test --only-failures"
+  }
+}

package/src/db.ts

@@ -1,7 +1,9 @@
 import type { EmbedderChunk } from "./embed/index.ts"
+import type { FrecencyScore } from "./frecency.ts"
 import type { Database } from "./sqlite.ts"
 import type { StoreChunk } from "./store.ts"
 
+import { addVisit, toDeadline, toScore } from "./frecency.ts"
 import { openDatabase } from "./sqlite.ts"
 
 export type { Database }
@@ -14,11 +16,36 @@ export type DocRow = {
   vec_hash?: string
   description: string
   title: string
-  tags: string
-  entities: string
+  tags: string[]
+  entities: string[]
+  updated_at: Date
+  synced_at?: Date
+  deadline: number
+  frecency: number
+}
+
+type RawDocRow = Omit<DocRow, "updated_at" | "synced_at" | "tags" | "entities" | "frecency"> & {
   updated_at: string
   synced_at?: string
-  deadline?: number
+  tags: string
+  entities: string
+}
+
+function splitCsv(s: string): string[] {
+  return s ? s.split(",") : []
+}
+
+function toDocRow(rows: RawDocRow[]): DocRow[] {
+  return rows.map(
+    (row) =>
+      Object.assign(row, {
+        entities: splitCsv(row.entities),
+        frecency: toScore(row.deadline),
+        synced_at: row.synced_at ? new Date(row.synced_at) : undefined,
+        tags: splitCsv(row.tags),
+        updated_at: new Date(row.updated_at),
+      }) as unknown as DocRow
+  )
 }
 
 export type VecResult = {
@@ -89,7 +116,7 @@ export class Db {
         entities TEXT NOT NULL DEFAULT '',
         updated_at TEXT NOT NULL,
         synced_at TEXT,
-        deadline REAL
+        deadline REAL NOT NULL DEFAULT 0
       )
     `)
 
@@ -194,28 +221,42 @@ export class Db {
 
   getDoc(from: string | number) {
     const field = typeof from === "number" ? "id" : "path"
-    return this.#db.query(`SELECT * FROM docs WHERE ${field} = ?`).get(from) as DocRow | undefined
+    const row = this.#db.query(`SELECT * FROM docs WHERE ${field} = ?`).get(from) as
+      | RawDocRow
+      | undefined
+    return row ? toDocRow([row])[0] : undefined
   }
 
   getDocs(from?: (string | number)[]) {
     let ret: DocRow[]
 
-    if (!from) ret = this.#db.query(`SELECT * FROM docs`).all() as DocRow[]
+    if (!from) ret = toDocRow(this.#db.query(`SELECT * FROM docs`).all() as RawDocRow[])
     else {
       const field = typeof from[0] === "number" ? "id" : "path"
       const placeholders = from.map(() => "?").join(",")
-      ret = this.#db
-        .query(`SELECT * FROM docs WHERE ${field} IN (${placeholders})`)
-        .all(...from) as DocRow[]
+      ret = toDocRow(
+        this.#db
+          .query(`SELECT * FROM docs WHERE ${field} IN (${placeholders})`)
+          .all(...from) as RawDocRow[]
+      )
     }
     return new Map(ret.map((row) => [row.id, row]))
   }
 
-  addDoc(row: Omit<DocRow, "id">) {
+  addDoc(row: Omit<DocRow, "id" | "frecency">) {
+    const raw = {
+      ...row,
+      deadline: row.deadline,
+      entities: row.entities.join(","),
+      // oxlint-disable-next-line unicorn/no-null
+      synced_at: row.synced_at?.toISOString() ?? null,
+      tags: row.tags.join(","),
+      updated_at: row.updated_at.toISOString(),
+    }
     const result = this.#db
       .query(
-        `INSERT INTO docs (path, hash, body, description, title, tags, entities, updated_at, synced_at)
-        VALUES($path, $hash, $body, $description, $title, $tags, $entities, $updated_at, $synced_at)
+        `INSERT INTO docs (path, hash, body, description, title, tags, entities, updated_at, synced_at, deadline)
+        VALUES($path, $hash, $body, $description, $title, $tags, $entities, $updated_at, $synced_at, $deadline)
         ON CONFLICT(path) DO UPDATE SET
           hash = excluded.hash,
           body = excluded.body,
@@ -224,10 +265,11 @@ export class Db {
           tags = excluded.tags,
           entities = excluded.entities,
           updated_at = excluded.updated_at,
-          synced_at = excluded.synced_at
+          synced_at = excluded.synced_at,
+          deadline = excluded.deadline
         RETURNING id`
       )
-      .get(row) as { id: number }
+      .get(raw) as { id: number }
     return result.id
   }
 
@@ -275,15 +317,17 @@ export class Db {
   }
 
   getUnembeddedDocs(): DocRow[] {
-    return this.#db
-      .query(`SELECT * FROM docs
+    return toDocRow(
+      this.#db
+        .query(`SELECT * FROM docs
         WHERE vec_hash IS NULL OR vec_hash != hash
         ORDER BY path`)
-      .all() as DocRow[]
+        .all() as RawDocRow[]
+    )
   }
 
-  touchDoc(id: number) {
-    this.#db.query(`UPDATE docs SET synced_at = ? WHERE id = ?`).run(new Date().toISOString(), id)
+  touchDoc(id: number, syncedAt = new Date()) {
+    this.#db.query(`UPDATE docs SET synced_at = ? WHERE id = ?`).run(syncedAt.toISOString(), id)
   }
 
   markEmbedded(id: number, docHash: string) {
@@ -291,9 +335,9 @@ export class Db {
   }
 
   /** Delete docs not seen since the given sync timestamp, optionally scoped to a path prefix. */
-  deleteStaleDocs(syncedBefore: string, prefix?: string): number {
+  deleteStaleDocs(syncedBefore: Date, prefix?: string): number {
     let query = `SELECT id FROM docs WHERE synced_at IS NULL OR synced_at < ?`
-    const params = [syncedBefore]
+    const params = [syncedBefore.toISOString()]
     if (prefix) {
       query += ` AND path LIKE ? || '%'`
       params.push(prefix)
@@ -409,8 +453,14 @@ export class Db {
 
   // --- Frecency ---
 
-  setDeadline(docId: number, deadline: number) {
-    this.#db.query(`UPDATE docs SET deadline = ? WHERE id = ?`).run(deadline, docId)
+  visit(doc: DocRow | number, value?: FrecencyScore | number) {
+    if (typeof doc === "number") {
+      const row = this.getDoc(doc)
+      if (!row) return
+      doc = row
+    }
+    const frecency = addVisit(doc.frecency, value)
+    this.#db.query(`UPDATE docs SET deadline = ? WHERE id = ?`).run(toDeadline(frecency), doc.id)
   }
 
   // --- Meta ---