@rekal/mem 0.0.0

package/src/glob.ts

@@ -0,0 +1,163 @@
+import type { Ignore } from "ignore"
+import type { Dirent } from "node:fs"
+
+import { readFileSync } from "node:fs"
+import { readdir } from "node:fs/promises"
+import { join } from "pathe"
+import { findUp, normPath, sstat } from "./fs.ts"
+import { toError } from "./util.ts"
+
+export type GlobSort = (a: Dirent, b: Dirent) => number
+
+const sorters = {
+  name: (a, b) => a.name.localeCompare(b.name),
+  none: () => 0,
+  type: (a, b) => {
+    if (a.isDirectory() && !b.isDirectory()) return -1
+    if (!a.isDirectory() && b.isDirectory()) return 1
+    return a.name.localeCompare(b.name)
+  },
+} satisfies Record<string, GlobSort>
+
+export type GlobOptions = {
+  cwd: string | string[]
+  glob?: string | string[] // optional glob patterns to filter files (e.g. "*.js")
+  follow: boolean // follow symlinks
+  hidden: boolean // include hidden files (those starting with a dot)
+  ignore: boolean // respect ignore files
+  type?: "file" | "directory" // filter by type
+  empty: boolean // include empty directories
+  depth: number // maximum depth to traverse
+  ignoreFiles: string[] // names of ignore files to look for in each directory
+  exclude: string[] // additional ignore rules to apply globally
+  onVisit?: (rel: string) => void
+  onError?: (path: string, error: Error) => void
+  sort?: GlobSort | keyof typeof sorters
+}
+
+const defaults: GlobOptions = {
+  cwd: ".",
+  depth: Infinity,
+  empty: false,
+  exclude: [".git", "node_modules/"],
+  follow: false,
+  hidden: false,
+  ignore: true,
+  ignoreFiles: [".gitignore", ".ignore"],
+  sort: "name",
+}
+
+type GlobEntry = {
+  path: string
+  rel: string
+  ignore?: IgnoreTree
+  depth: number
+  dir: boolean
+}
+
+class IgnoreTree {
+  parent?: IgnoreTree
+
+  constructor(
+    public ig: Ignore,
+    public rel = ""
+  ) {}
+
+  extend(ig: Ignore, rel: string) {
+    const child = new IgnoreTree(ig, rel)
+    child.parent = this
+    return child
+  }
+
+  ignores(rel: string): boolean {
+    const test = this.ig.test(rel.slice(this.rel.length))
+    if (test.ignored) return true
+    if (test.unignored) return false
+    return this.parent?.ignores(rel) ?? false
+  }
+}
+
+export async function* glob(opts: Partial<GlobOptions> = {}): AsyncGenerator<string> {
+  if (opts.depth && opts.depth < 1) return // fast path for zero results
+
+  const { default: ignore } = await import("ignore")
+  const o: GlobOptions = { ...defaults, ...opts }
+  if (Array.isArray(o.cwd)) {
+    for (const cwd of o.cwd) yield* glob({ ...o, cwd })
+    return
+  }
+  const root = normPath(o.cwd)
+  const ignoreFiles = new Set(o.ignoreFiles)
+  const rootIgnore = ignore().add([...o.exclude, ...ignoreFiles])
+  const globIgnore = ignore().add(o.glob ?? [])
+  const sorter = (typeof o.sort === "string" ? sorters[o.sort] : o.sort) ?? sorters.name
+  const visited = new Set<string>()
+
+  if (o.ignore)
+    for (const igf of ignoreFiles) {
+      const igPath = findUp(root, igf, ".git")
+      if (igPath) rootIgnore.add(readFileSync(igPath, "utf8"))
+    }
+
+  async function ls(dir: GlobEntry) {
+    if (visited.has(dir.path)) return
+    visited.add(dir.path)
+    let entries
+    try {
+      const dirents = await readdir(dir.path, { withFileTypes: true })
+      entries = dirents.toSorted(sorter).toReversed()
+    } catch (error) {
+      return o.onError?.(dir.path, toError(error))
+    }
+
+    let ig = dir.ignore
+    const children: GlobEntry[] = []
+
+    for (const entry of entries) {
+      const path = join(entry.parentPath, entry.name)
+      if (o.ignore && entry.isFile() && ignoreFiles.has(entry.name)) {
+        const fig = ignore().add(readFileSync(path, "utf8"))
+        ig = ig ? ig.extend(fig, dir.rel) : new IgnoreTree(fig, dir.rel)
+      } else if (!o.hidden && entry.name.startsWith(".")) {
+        continue
+      } else {
+        let isDirectory = entry.isDirectory()
+        isDirectory ||= o.follow && entry.isSymbolicLink() && (sstat(path)?.isDirectory() ?? false)
+        const rel = dir.rel + entry.name + (isDirectory ? "/" : "")
+        const depth = dir.depth + 1
+        children.push({ depth, dir: isDirectory, path, rel })
+      }
+    }
+
+    for (const child of children) {
+      o.onVisit?.(child.rel)
+      if (o.ignore && ig?.ignores(child.rel)) continue
+      if (o.glob && !child.dir && !globIgnore.ignores(child.rel)) continue
+      stack.push({ ...child, ignore: ig })
+    }
+  }
+
+  const stack: GlobEntry[] = [
+    { depth: 0, dir: true, ignore: new IgnoreTree(rootIgnore), path: root, rel: "" },
+  ]
+  const parents: GlobEntry[] = []
+
+  while (stack.length > 0) {
+    const entry = stack.pop()!
+
+    if (o.type !== "file" && entry.depth !== 0) {
+      while (!o.empty && parents.length > 0 && parents[parents.length - 1].depth >= entry.depth)
+        parents.pop()
+      if (entry.dir && entry.depth < o.depth) {
+        parents.push(entry)
+      } else {
+        for (const p of parents) yield p.rel
+        parents.length = 0
+        if (o.type !== "directory") yield entry.rel
+      }
+    } else if (!entry.dir) yield entry.rel
+
+    // oxlint-disable-next-line no-await-in-loop
+    if (entry.dir && entry.depth < o.depth) await ls(entry)
+  }
+}

package/src/index.ts

@@ -0,0 +1,15 @@
+export type * from "./db.ts"
+export type * from "./embed/index.ts"
+export type * from "./search.ts"
+export type * from "./store.ts"
+export type * from "./vfs.ts"
+export * from "./context.ts"
+export * from "./doc.ts"
+export * from "./fs.ts"
+export * from "./log.ts"
+export * from "./progress.ts"
+export * from "./query.ts"
+export * from "./uri.ts"
+export * from "./util.ts"
+export * from "./md.ts"
+export * from "./snippet.ts"

package/src/log.ts

@@ -0,0 +1,60 @@
+export type LogLevel = (typeof LOG_LEVELS)[number]
+export type LogFn<T = void> = (...msg: unknown[]) => T
+export type Logger<T = void> = Record<LogLevel, LogFn<T>>
+
+export const LOG_LEVELS = [
+  "cancel",
+  "info",
+  "success",
+  "warn",
+  "error",
+  "debug",
+  "fatal",
+  "prompt",
+  "log",
+  "trace",
+] as const
+
+// oxlint-disable-next-line sort-keys
+const LOG_PRIORITY: Record<LogLevel, number> = {
+  trace: 0,
+  debug: 1,
+  log: 2,
+  info: 2,
+  prompt: 2,
+  success: 2,
+  cancel: 2,
+  warn: 3,
+  error: 4,
+  fatal: 5,
+}
+
+export function isLogLevel(level: string): level is LogLevel {
+  return LOG_LEVELS.includes(level as LogLevel)
+}
+
+export function shouldLog(level: string, minLevel?: LogLevel): boolean {
+  if (!isLogLevel(level)) return true
+  return LOG_PRIORITY[level] >= LOG_PRIORITY[minLevel ?? "log"]
+}
+
+export abstract class LoggerBase<T = void> implements Logger<T> {
+  cancel!: LogFn<T>
+  info!: LogFn<T>
+  success!: LogFn<T>
+  warn!: LogFn<T>
+  error!: LogFn<T>
+  debug!: LogFn<T>
+  fatal!: LogFn<T>
+  prompt!: LogFn<T>
+  log!: LogFn<T>
+  trace!: LogFn<T>
+
+  constructor() {
+    for (const level of LOG_LEVELS) {
+      this[level] = (...msg: unknown[]) => this._log(level, ...msg)
+    }
+  }
+
+  protected abstract _log(level: LogLevel, ...msg: unknown[]): T
+}

package/src/md.ts

@@ -0,0 +1,204 @@
+import type { TokenCounter } from "./embed/index.ts"
+
+import { parseYaml } from "#runtime"
+
+// NOTE: all markdown parsing expects normalized line endings (\n)
+
+// average chars per token across common models,
+// used for estimating token counts without actual tokenization
+const CHARS_PER_TOKEN = 3
+
+export type MarkdownSection = {
+  content: string[]
+  context: string[] // parent headings for context, e.g. ["# Chapter 1", "## Section 1.2"]
+  /** full heading text with markdown syntax, e.g. "## Section 1.2" */
+  headingText: string
+  /** heading without markdown syntax, e.g. "Section 1.2" */
+  heading: string
+  level: number
+  offset: number // 0-indexed line offset of the section in the original markdown body, used for mapping back to source
+}
+
+export type MarkdownDoc = {
+  body: string
+  bodyOffset: number // line offset of the body start (after frontmatter) in the original markdown
+  frontmatter: Frontmatter
+  frontmatterText?: string
+  sections: MarkdownSection[]
+  text: string
+}
+
+export type Frontmatter = Record<string, unknown>
+
+export function parseFrontmatter(text: string): Omit<MarkdownDoc, "sections"> {
+  const match = text.match(/^---\n([\s\S]*?)\n---\n?/)
+  const body = match ? text.slice(match[0].length) : text
+  return {
+    body,
+    bodyOffset: match?.[0].trim().split("\n").length ?? 0,
+    frontmatter: match ? (parseYaml(match[1]) as Record<string, unknown>) : {},
+    frontmatterText: match?.[0],
+    text,
+  }
+}
+
+export function parseMarkdown(text: string): MarkdownDoc {
+  const ret = parseFrontmatter(text)
+  return { ...ret, sections: parseSections(ret.body) }
+}
+
+export function parseSections(md: string): MarkdownSection[] {
+  const lines = md.split(/\n/)
+  let current: MarkdownSection = {
+    content: [],
+    context: [],
+    heading: "",
+    headingText: "",
+    level: 0,
+    offset: 0,
+  }
+  const sections: MarkdownSection[] = [current]
+  let codeBlock: string | undefined = undefined
+  for (const [i, line] of lines.entries()) {
+    const match = line.match(/^(#+)\s+(.*)/)
+    const fenceMatch = line.match(/^\s*(`{3,}|~{3,})/)
+
+    if (codeBlock && line.startsWith(codeBlock)) {
+      codeBlock = undefined // end of code block
+    } else if (!codeBlock && fenceMatch) {
+      codeBlock = fenceMatch[1] // start of code block
+    }
+
+    if (!codeBlock && match) {
+      if (current.content.length === 0) sections.pop() // discard empty sections.
+      const level = match[1].length
+      current = {
+        content: [line],
+        context: [],
+        heading: match[2].trim(),
+        headingText: match[0].trim(),
+        level,
+        offset: i,
+      }
+      sections.push(current)
+    } else current.content.push(line)
+  }
+
+  const stack: MarkdownSection[] = []
+  for (const section of sections) {
+    // Track parent sections
+    while ((stack.at(-1)?.level ?? -1) >= section.level) stack.pop()
+    section.context = stack.map((s) => s.headingText)
+    if (section.level > 0) stack.push(section)
+  }
+  return sections
+}
+
+function findSplit(slice: string) {
+  for (const sub of ["\n\n", "\n", "  ", "\t", " "]) {
+    const i = slice.lastIndexOf(sub)
+    if (i > slice.length * 0.8) {
+      return i
+    }
+  }
+  return slice.length
+}
+
+class SafeCounter {
+  static #chars = 0
+  static #toks = 0
+
+  constructor(
+    public tok: TokenCounter,
+    public maxTokens = 500
+  ) {}
+
+  get charsPerToken() {
+    return SafeCounter.#toks > this.maxTokens * 2
+      ? SafeCounter.#chars / SafeCounter.#toks
+      : CHARS_PER_TOKEN
+  }
+
+  estimate(text: string) {
+    return Math.ceil(text.length / this.charsPerToken)
+  }
+
+  // Returns the actual token count, unless the estimated token count based
+  // on character length is much higher than the maxTokens
+  toks(text: string) {
+    if (text.length === 0) return { count: 0, estimated: false }
+    let count = this.estimate(text) * 0.9 // add 10% buffer to account for variance in chars per token
+    if (count > this.maxTokens) return { count, estimated: true }
+    count = this.tok.toks(text)
+    SafeCounter.#chars += text.length
+    SafeCounter.#toks += count
+    return { count, estimated: false }
+  }
+}
+
+export function chunkText(text: string, tok: TokenCounter, size = 500): string[] {
+  const counter = new SafeCounter(tok, size)
+  const chunks: string[] = []
+  while (text.length) {
+    let next = text
+    let toks = counter.toks(next)
+
+    if (toks.count <= size) {
+      chunks.push(next)
+      break
+    }
+    let maxChars = size * counter.charsPerToken * 0.8
+    // oxlint-disable-next-line typescript/no-unnecessary-condition
+    while (true) {
+      maxChars = Math.min(maxChars, next.length)
+      const split = findSplit(next.slice(0, maxChars))
+      next = next.slice(0, split)
+      toks = counter.toks(next)
+      if (toks.count <= size) break
+      maxChars *= (size / toks.count) * 0.8
+    }
+
+    chunks.push(next)
+    text = text.slice(next.length)
+  }
+  return chunks
+}
+
+export function chunkMarkdown(md: string, tok: TokenCounter, size = 500): string[] {
+  const sections = parseSections(md)
+  type Chunk = { content: string[]; tokens: number; context: string[] }
+  const chunks: Chunk[] = [{ content: [], context: [], tokens: 0 }]
+  const counter = new SafeCounter(tok, size)
+
+  for (const section of sections) {
+    const chunk = chunks.at(-1) as Chunk
+    // Include parent headings in the content to preserve context
+    const content = [...section.context, ...section.content]
+    const text = content.join("\n")
+    const toks = counter.toks(text).count
+
+    if (chunk.tokens + toks <= size) {
+      // only add parent headings that aren't already in the chunk for context
+      const context = section.context.filter((h, c) => chunk.context[c] !== h)
+      chunk.content.push(...context)
+      chunk.content.push(...section.content)
+      chunk.context = [...section.context, section.headingText]
+      chunk.tokens += toks
+    } else if (toks <= size) {
+      chunks.push({ content, context: [...section.context, section.headingText], tokens: toks })
+    } else {
+      const context = section.context.join("\n")
+      const toksCtx = counter.toks(context)
+      chunks.push(
+        ...chunkText(section.content.join("\n"), tok, size - toksCtx.count).map((c) => ({
+          content: (context.length ? `${context}\n${c}` : c).split("\n"),
+          context: [],
+          tokens: 0, // we don't track tokens for these sub-chunks since they're already guaranteed to fit
+        }))
+      )
+      chunks.push({ content: [], context: [], tokens: 0 }) // start a new chunk for the next section
+    }
+  }
+
+  return chunks.map((c) => c.content.join("\n").trim()).filter(Boolean)
+}

package/src/progress.ts

@@ -0,0 +1,121 @@
+import type { TypedEmitter } from "./util.ts"
+
+import { EventEmitter } from "node:events"
+import { inspect } from "node:util"
+
+export type ProgressOpts = { max?: number; status?: string; value?: number }
+
+type ProgressEvents = {
+  update: [progress: Progress]
+  done: [progress: Progress]
+}
+
+export class Progress extends (EventEmitter as new () => TypedEmitter<ProgressEvents>) {
+  #max = 100
+  #value = 0
+  #children = new Map<string, Progress>()
+  #status?: string
+  #done = false
+
+  constructor(
+    public name: string,
+    opts: ProgressOpts = {}
+  ) {
+    super()
+    this.set(opts)
+  }
+
+  get group() {
+    return this.#children.size > 0
+  }
+
+  set(opts: ProgressOpts | number): this {
+    if (this.#done) return this
+    if (typeof opts === "number") this.#value = opts
+    else {
+      this.#max = opts.max ?? this.#max
+      this.#status = opts.status ?? this.#status
+      this.#value = opts.value ?? this.#value
+    }
+    this.emit("update", this)
+    if (this.#value >= this.#max) this.stop()
+    return this
+  }
+
+  get status() {
+    return this.#status ?? this.name
+  }
+
+  set status(status: string) {
+    this.set({ status })
+  }
+
+  set value(value: number) {
+    this.set(value)
+  }
+
+  get value(): number {
+    return !this.group ? this.#value : this.#children.values().reduce((sum, c) => sum + c.value, 0)
+  }
+
+  set max(max: number) {
+    this.set({ max })
+  }
+
+  get max(): number {
+    return !this.group ? this.#max : this.#children.values().reduce((sum, c) => sum + c.max, 0)
+  }
+
+  get done() {
+    return this.#done
+  }
+
+  get ratio(): number {
+    return this.max === 0 ? 0 : Math.min(1, this.value / this.max)
+  }
+
+  get pct(): number {
+    return this.ratio * 100
+  }
+
+  stop() {
+    if (this.#done) return
+    this.#done = true
+    if (!this.group) this.#value = this.#max
+    this.#children.forEach((c) => c.stop())
+    this.emit("done", this)
+  }
+
+  children() {
+    return [...this.#children.values()]
+  }
+
+  child(name: string, opts: ProgressOpts = {}): Progress {
+    if (this.#value > 0)
+      throw new Error("Cannot add child to Progress that has already made progress")
+    let child = this.#children.get(name)
+    if (!child) {
+      child = new Progress(name, opts)
+      child.on("update", () => this.emit("update", this))
+      child.on("done", () => {
+        if (!this.done && this.children().every((c) => c.done)) this.stop()
+      })
+      this.#children.set(name, child)
+    }
+    return child
+  }
+
+  [inspect.custom](_depth: number, _options: object): string {
+    return this.toString()
+  }
+
+  override toString(indent = 0): string {
+    const pad = "  ".repeat(indent)
+    const pct = `${this.pct.toFixed(0)}%`.padStart(4)
+    const status = this.#status ? ` ${this.#status}` : ""
+    const line = `${pad}${pct} ${this.name}${status}`
+    if (!this.group) return line
+    const children = [...this.#children.values()].map((c) => c.toString(indent + 1))
+    return [line, ...children].join("\n")
+  }
+}

package/src/query.ts

@@ -0,0 +1,131 @@
+type Token =
+  | { type: "term"; value: string; neg?: boolean; req?: boolean; field?: string }
+  | { type: "op"; value: "AND" | "OR" }
+  | { type: "paren"; value: "(" | ")" }
+
+const FTS_FIELDS = new Set(["entities", "tags", "description", "title", "body"])
+
+export function tokenize(input: string): Token[] {
+  const tokens: Token[] = []
+  let i = 0
+
+  while (i < input.length) {
+    while (i < input.length && input[i] === " ") i++
+    if (i >= input.length) break
+
+    const ch = input[i]
+
+    if (ch === "(" || ch === ")") {
+      tokens.push({ type: "paren", value: ch })
+      i++
+    } else if (ch === "|") {
+      tokens.push({ type: "op", value: "OR" })
+      i++
+    } else if ((ch === '"' || ch === "'") && (i === 0 || input[i - 1] === " ")) {
+      const quote = ch
+      i++
+      const start = i
+      while (i < input.length && input[i] !== quote) i++
+      if (start < i) tokens.push({ type: "term", value: input.slice(start, i) })
+      if (i < input.length) i++
+    } else {
+      const neg = ch === "-"
+      const req = ch === "+"
+      if (neg || req) i++
+      const start = i
+      while (i < input.length && !' "()|'.includes(input[i])) i++
+      if (start < i) {
+        const raw = input.slice(start, i)
+        const colon = raw.indexOf(":")
+        if (colon > 0 && FTS_FIELDS.has(raw.slice(0, colon))) {
+          tokens.push({
+            field: raw.slice(0, colon),
+            neg: neg || undefined,
+            req: req || undefined,
+            type: "term",
+            value: raw.slice(colon + 1),
+          })
+        } else {
+          tokens.push({ neg: neg || undefined, req: req || undefined, type: "term", value: raw })
+        }
+      }
+    }
+  }
+  return tokens
+}
+
+/** Sanitize a term for FTS5 — strip non-word/non-apostrophe chars, preserve colons */
+function sanitize(term: string): string {
+  return term.replace(/[^\p{L}\p{N}\s':]/gu, "").trim()
+}
+
+function buildTerm(token: Extract<Token, { type: "term" }>): string | undefined {
+  const clean = sanitize(token.value)
+  if (!clean) return
+  const isPrefix = token.value.endsWith("*")
+  const phrase = `"${clean}"${isPrefix ? "*" : ""}`
+  const scoped = token.field ? `${token.field} : ${phrase}` : phrase
+  return token.neg ? `NOT ${scoped}` : scoped
+}
+
+function joinParts(parts: string[], op: string): string {
+  return parts.join(` ${op} `)
+}
+
+/** Build an FTS5 query string from user input */
+export function toFts(input: string, defaultOp: "AND" | "OR" = "OR"): string {
+  const tokens = tokenize(input)
+  const hasRequired = tokens.some((t) => t.type === "term" && t.req)
+
+  // If no required terms, build normally
+  if (!hasRequired) {
+    const parts: string[] = []
+    let needsOp = false
+
+    for (const token of tokens) {
+      if (token.type === "paren") {
+        if (token.value === "(") {
+          if (needsOp) parts.push(defaultOp)
+          parts.push(token.value)
+          needsOp = false
+        } else {
+          parts.push(token.value)
+          needsOp = true
+        }
+        continue
+      }
+      if (token.type === "op") {
+        parts.push(token.value)
+        needsOp = false
+        continue
+      }
+      const term = buildTerm(token)
+      if (!term) continue
+      if (needsOp) parts.push(defaultOp)
+      parts.push(term)
+      needsOp = true
+    }
+
+    return parts.join(" ").replace(/\( /g, "(").replace(/ \)/g, ")")
+  }
+
+  // With required terms: required1 AND required2 AND (all terms joined with OR)
+  const required: string[] = []
+  const all: string[] = []
+
+  for (const token of tokens) {
+    if (token.type !== "term") continue
+    const term = buildTerm(token)
+    if (!term) continue
+    all.push(term)
+    if (token.req) required.push(term)
+  }
+
+  const requiredPart = joinParts(required, "AND")
+  const allPart = joinParts(all, "OR")
+
+  // If everything is required, no need for the OR group
+  if (required.length === all.length) return requiredPart
+
+  return `${requiredPart} AND (${allPart})`
+}