@rekal/mem 0.0.0

package/src/embed/base.ts

@@ -0,0 +1,122 @@
+import type { Context } from "../context.ts"
+import type {
+  EmbedderBackend,
+  EmbedderChunk,
+  EmbedderDoc,
+  EmbedderOptions,
+  ModelBackend,
+  ResolvedEmbedderModel,
+  ResolvedEmbedderOptions,
+} from "./index.ts"
+
+import { availableParallelism } from "node:os"
+import { chunkMarkdown } from "../md.ts"
+import { Progress } from "../progress.ts"
+import { loadModel, parseModelUri, resolveModel } from "./models.ts"
+
+const defaults = {
+  batchSize: 0, // 0 = auto
+  maxDims: 512,
+  maxTokens: 512,
+  useGpu: true,
+} satisfies EmbedderOptions
+
+const backend_defaults: Record<ModelBackend, EmbedderOptions> = {
+  llama: {},
+  openai: { batchSize: 50 },
+  transformers: {},
+}
+
+function isChunk(input: any): input is EmbedderChunk {
+  return typeof (input as EmbedderChunk | undefined)?.prompt === "string"
+}
+
+export class Embedder {
+  #loading?: Promise<EmbedderBackend>
+  opts: ResolvedEmbedderOptions
+  model: ResolvedEmbedderModel
+  status = new Progress("embedder")
+  #backend?: EmbedderBackend
+
+  constructor(public ctx: Context) {
+    const opts = ctx.opts.embedder ?? {}
+    this.model = resolveModel(opts.model)
+    const { backend } = parseModelUri(this.model.uri)
+    const base = { ...defaults, ...backend_defaults[backend] }
+    const threads = Math.max(1, opts.threads ?? Math.min(8, availableParallelism() - 2))
+    this.opts = {
+      threads,
+      ...base,
+      ...opts,
+      model: this.model,
+    }
+  }
+
+  info() {
+    return parseModelUri(this.model.uri)
+  }
+
+  async backend() {
+    this.#loading ??= (async () => {
+      this.status.name = `Loading model \`${this.model.uri}\``
+
+      const t = setTimeout(() => {
+        this.opts.onProgress?.(this.status)
+      }, 500)
+
+      this.#backend = await loadModel({
+        logger: this.ctx,
+        opts: this.opts,
+        root: this.ctx.root,
+        status: this.status,
+      })
+
+      clearTimeout(t)
+
+      this.opts.maxTokens = Math.min(this.opts.maxTokens, this.#backend.maxTokens)
+      this.opts.maxDims = Math.min(this.opts.maxDims, this.#backend.dims)
+      if (this.opts.batchSize === 0) this.opts.batchSize = this.#backend.device === "gpu" ? 50 : 1
+      this.ctx.debug({
+        batchSize: this.opts.batchSize,
+        device: this.#backend.device,
+        threads: this.opts.threads,
+        useGpu: this.opts.useGpu,
+      })
+      this.status.stop()
+      return this.#backend
+    })()
+    return (this.#backend ??= await this.#loading)
+  }
+
+  transform(input: string | EmbedderDoc | EmbedderChunk): string {
+    if (isChunk(input)) return input.prompt
+    const { prompt } = this.model
+    return typeof input === "string" ? prompt.query(input) : prompt.document(input)
+  }
+
+  async embed(input: string | EmbedderDoc | EmbedderChunk): Promise<number[]>
+  async embed(input: (string | EmbedderDoc | EmbedderChunk)[]): Promise<number[][]>
+  async embed(
+    input: string | EmbedderDoc | EmbedderChunk | (string | EmbedderDoc | EmbedderChunk)[]
+  ): Promise<number[][] | number[]> {
+    const single = !Array.isArray(input)
+    const todo = single ? [input] : input
+    const backend = await this.backend()
+    const ret = await backend.embed(todo.map((item) => this.transform(item)))
+    return single ? ret[0] : ret
+  }
+
+  async chunk(input: string | EmbedderDoc): Promise<EmbedderChunk[]>
+  async chunk(input: string | EmbedderDoc): Promise<EmbedderChunk[]> {
+    const backend = await this.backend()
+    const isQuery = typeof input === "string"
+    const fixed = this.transform(isQuery ? "" : { text: "", title: input.title })
+    const chunkText = isQuery ? input : input.text
+    const tokens = this.opts.maxTokens - backend.toks(fixed)
+    return chunkMarkdown(chunkText, backend, tokens).map((text, seq) => ({
+      prompt: isQuery ? this.transform(text) : this.transform({ text, title: input.title }),
+      seq,
+      text,
+    }))
+  }
+}

package/src/embed/index.ts

@@ -0,0 +1,67 @@
+import type { SetOptional, Merge } from "type-fest"
+import type { Logger } from "../log.ts"
+import type { Progress } from "../progress.ts"
+
+export * from "./base.ts"
+
+export type TokenCounter = {
+  toks(input: string): number
+}
+
+export type EmbedderDevice = "cpu" | "gpu" | "api"
+export interface EmbedderBackend extends TokenCounter {
+  device: EmbedderDevice // what hardware/service this backend uses
+  maxTokens: number // maximum number of tokens that can be embedded
+  dims: number // dimensionality of the output vectors
+  embed(input: string[]): Promise<number[][]>
+}
+
+export type EmbedderDoc = {
+  text: string
+  title?: string
+}
+
+export type EmbedderPrompt = {
+  document?: (doc: EmbedderDoc) => string
+  query?: (query: string) => string
+}
+
+export type EmbedderModel = {
+  uri: string
+  base?: string // base model to inherit options from (e.g. for ONNX variants)
+  models?: string[] // models based on this configuration
+  prompt?: EmbedderPrompt
+  pooling?: "none" | "mean" | "cls" | "first_token" | "eos" | "last_token" // transformers-only
+}
+
+export type ModelBackend = "transformers" | "llama" | "openai"
+
+export type EmbedderContext = {
+  opts: ResolvedEmbedderOptions
+  root: string
+  status: Progress
+  logger: Logger
+}
+
+export type EmbedderChunk = {
+  seq: number
+  text: string // raw chunk text (e.g. a passage from a document)
+  prompt: string // the actual text sent to the model for embedding (e.g. with instructions or title added)
+  embedding?: number[] // populated after embedding
+}
+
+export type EmbedderOptions = {
+  batchSize?: number // Embed this many texts at a time.
+  model?: EmbedderModel | string
+  maxTokens?: number // Chunk input into pieces of this many tokens or fewer (depending on the model).
+  maxDims?: number // Truncate output to this many dimensions. Only supported by some backends.
+  threads?: number // CPU threads for local inference. 0 = auto. Defaults to half available cores.
+  useGpu?: boolean // Whether to use GPU for local inference if available. Defaults to true.
+  onProgress?: (status: Progress) => void
+}
+
+export type ResolvedEmbedderModel = Merge<EmbedderModel, { prompt: Required<EmbedderPrompt> }>
+export type ResolvedEmbedderOptions = Merge<
+  SetOptional<Required<EmbedderOptions>, "onProgress">,
+  { model: ResolvedEmbedderModel }
+>

package/src/embed/llama.ts

@@ -0,0 +1,111 @@
+import type { Llama, LlamaEmbeddingContext, LlamaModel } from "node-llama-cpp"
+import type { LogLevel } from "../log.ts"
+import type { EmbedderBackend, EmbedderContext, EmbedderDevice } from "./index.ts"
+
+import { LlamaLogLevel } from "node-llama-cpp"
+import { availableParallelism } from "node:os"
+import { join } from "pathe"
+import { parseModelUri } from "./models.ts"
+
+export class LlamaBackend implements EmbedderBackend {
+  static llama?: Llama
+  device: EmbedderDevice
+  maxTokens: number
+  dims: number
+  #contexts: LlamaEmbeddingContext[] = []
+  #model: LlamaModel
+  #poolSize: number
+  #threadsPerCtx: number
+  #ctx: EmbedderContext
+
+  private constructor(llama: Llama, model: LlamaModel, ctx: EmbedderContext) {
+    this.#model = model
+    this.#ctx = ctx
+    this.maxTokens = model.trainContextSize
+    this.dims = model.embeddingVectorSize
+    this.device = llama.gpu ? "gpu" : "cpu"
+
+    const useGpu = llama.gpu && ctx.opts.useGpu
+    const cores = availableParallelism()
+    const poolSize = useGpu
+      ? Math.min(8, Math.max(1, Math.floor(cores / 4)))
+      : Math.min(8, Math.max(1, Math.floor(cores / 4)))
+
+    this.#threadsPerCtx = useGpu ? 0 : Math.max(1, Math.floor(ctx.opts.threads / poolSize))
+    this.#poolSize = poolSize
+    ctx.logger.debug({
+      cores,
+      gpu: llama.gpu,
+      poolSize,
+      threadsPerCtx: this.#threadsPerCtx,
+    })
+  }
+
+  static async load(this: void, ctx: EmbedderContext) {
+    const { model, variant } = parseModelUri(ctx.opts.model.uri)
+    const { getLlama, resolveModelFile } = await import("node-llama-cpp")
+    const modelsDir = join(ctx.root, "models")
+    const uri = `hf:${model}${variant ? `:${variant}` : ""}`
+    const modelPath = await resolveModelFile(uri, modelsDir)
+
+    LlamaBackend.llama ??= await getLlama({
+      gpu: ctx.opts.useGpu ? "auto" : false,
+      logLevel: LlamaLogLevel.error,
+      logger: (level, message) => {
+        const l = level as LogLevel
+        // oxlint-disable-next-line typescript/no-unnecessary-condition
+        ;(ctx.logger[l] ?? ctx.logger.log)(level.toString(), message)
+      },
+    })
+
+    const lm = await LlamaBackend.llama.loadModel({
+      // the below makes GPU super slow
+      // defaultContextFlashAttention: true,
+      modelPath,
+    })
+    return new LlamaBackend(LlamaBackend.llama, lm, ctx)
+  }
+
+  /** Get or create up to `count` embedding contexts */
+  private async acquire(count: number): Promise<LlamaEmbeddingContext[]> {
+    const needed = Math.min(count, this.#poolSize) - this.#contexts.length
+    for (let i = 0; i < needed; i++) {
+      try {
+        this.#ctx.logger.debug(
+          `Creating embedding context ${this.#contexts.length + 1}/${this.#poolSize}...`
+        )
+        this.#contexts.push(
+          // oxlint-disable-next-line no-await-in-loop
+          await this.#model.createEmbeddingContext({
+            contextSize: this.#ctx.opts.maxTokens,
+            threads: this.#threadsPerCtx,
+          })
+        )
+      } catch {
+        this.#ctx.logger.warn(
+          `Failed to create embedding context ${this.#contexts.length + 1}. Adjusting pool size down to ${this.#contexts.length}.`
+        )
+        // adjust pool size down if we fail to create contexts,
+        // which can happen if we run out of VRAM or hit some other resource limit
+        this.#poolSize = this.#contexts.length
+        break
+      }
+    }
+    return this.#contexts
+  }
+
+  async embed(texts: string[]): Promise<number[][]> {
+    const contexts = await this.acquire(texts.length)
+    return Promise.all(
+      texts.map((text, idx) =>
+        contexts[idx % contexts.length]
+          .getEmbeddingFor(text)
+          .then((embedding) => [...embedding.vector])
+      )
+    )
+  }
+
+  toks(input: string) {
+    return this.#model.tokenize(input).length
+  }
+}

package/src/embed/models.ts

@@ -0,0 +1,104 @@
+import type { SetOptional } from "type-fest"
+import type {
+  EmbedderBackend,
+  EmbedderContext,
+  EmbedderModel,
+  ModelBackend,
+  ResolvedEmbedderModel,
+} from "./index.ts"
+
+import { defu } from "defu"
+
+export function parseModelUri(uri: string) {
+  const [backend, model, variant] = uri.split(":")
+  if (!backend || !model)
+    throw new Error(`Invalid model URI: ${uri}. Expected format "provider:model[:variant]"`)
+  return { backend, model, variant } as { model: string; backend: ModelBackend; variant?: string }
+}
+
+export function resolveModel(opts?: string | EmbedderModel): ResolvedEmbedderModel {
+  const uri = typeof opts === "string" ? opts : (opts?.uri ?? DEFAULTS.uri)
+  const options = typeof opts === "string" ? { uri } : (opts ?? { uri })
+  const base = MODELS[options.base ?? MODELS[uri]?.base ?? ""] ?? {}
+  const model = MODELS[uri] ?? {}
+  return defu(options, model, base, DEFAULTS) as ResolvedEmbedderModel
+}
+
+export function loadModel(ctx: EmbedderContext): Promise<EmbedderBackend> {
+  const { backend } = parseModelUri(ctx.opts.model.uri) as { backend: string }
+  if (backend === "transformers") {
+    return import("./transformers.ts").then(({ TransformersBackend }) =>
+      TransformersBackend.load(ctx)
+    )
+  } else if (backend === "llama") {
+    return import("./llama.ts").then(({ LlamaBackend }) => LlamaBackend.load(ctx))
+  } else if (backend === "openai") {
+    return import("./openai.ts").then(({ OpenAIBackend }) => OpenAIBackend.load(ctx))
+  } else {
+    throw new Error(`Unsupported model backend: ${String(backend)}`)
+  }
+}
+
+const DEFAULTS: ResolvedEmbedderModel = {
+  prompt: {
+    document: (doc) => {
+      // If the title isn't found near the start of the text,
+      // prepend it to ensure it's included in the embedding.
+      const title = doc.title?.trim()
+      if (title?.length) {
+        const idx = doc.text.indexOf(title)
+        if (idx === -1 || idx > 10) return `${title}\n\n${doc.text}`
+      }
+      return doc.text
+    },
+    query: (query) => query,
+  },
+  uri: "transformers:Snowflake/snowflake-arctic-embed-s:q8",
+}
+
+const MODELS: Record<string, SetOptional<EmbedderModel, "uri">> = {
+  BAAI: {
+    models: [
+      "transformers:BAAI/bge-large-en-v1.5", // 55.44%, 335M params, 304M active, 1024 dims, 512 context
+      "transformers:BAAI/bge-base-en-v1.5", //  54.75%, 109M params,  86M active,  768 dims, 512 context
+      "transformers:BAAI/bge-small-en-v1.5", // 53.86%,  33M params,  22M active,  512 dims, 512 context
+    ],
+    pooling: "cls",
+    prompt: {
+      query: (query) => `Represent this sentence for searching relevant passages: ${query}`,
+    },
+  },
+  openai: {
+    models: [
+      "openai:text-embedding-3-small",
+      "openai:text-embedding-3-large",
+      "openai:text-embedding-ada-002",
+    ],
+  },
+  snowflake: {
+    models: [
+      "transformers:Snowflake/snowflake-arctic-embed-m-v2.0:q8", // 58.41%, 305M params, 113M active, 768 dims, 8192 context
+      "transformers:Snowflake/snowflake-arctic-embed-s:q8", //      54.85%,  33M params,  22M active, 384 dims,  512 context
+      "transformers:Snowflake/snowflake-arctic-embed-xs:q8", //     52.65%,  23M params,  11M active, 384 dims,  512 context
+      "llama:mradermacher/snowflake-arctic-embed-s-GGUF:Q4_K_M",
+    ],
+    pooling: "cls",
+    prompt: {
+      query: (query) => `Represent this sentence for searching relevant passages: ${query}`,
+    },
+  },
+  "transformers:onnx-community/embeddinggemma-300m-ONNX:q8": {
+    pooling: "mean",
+    // 55.69%, 308M params, 106M active, 768 dims, 2048 context
+    prompt: {
+      document: (doc) => `title: ${doc.title ?? "none"} | ${doc.text}`,
+      query: (query) => `task: search result | query: ${query}`,
+    },
+  },
+}
+
+for (const [base, options] of Object.entries(MODELS)) {
+  for (const model of options.models ?? []) {
+    MODELS[model] = { base, uri: model }
+  }
+}

package/src/embed/openai.ts

@@ -0,0 +1,95 @@
+import type { encode as Tokenize } from "gpt-tokenizer"
+import type { EmbedderBackend, EmbedderContext } from "./index.ts"
+
+import { parseModelUri } from "./models.ts"
+
+const OPENAI_EMBEDDING_URL = "https://api.openai.com/v1/embeddings"
+
+// Context sizes for known OpenAI embedding models
+const MODEL_INFO: Record<string, { contextSize: number; vectorSize: number } | undefined> = {
+  "text-embedding-3-large": { contextSize: 8191, vectorSize: 3072 },
+  "text-embedding-3-small": { contextSize: 8191, vectorSize: 1536 },
+  "text-embedding-ada-002": { contextSize: 8191, vectorSize: 1536 },
+}
+
+export class OpenAIBackend implements EmbedderBackend {
+  device = "api" as const
+  maxTokens: number
+  dims: number
+  #model: string
+  #apiKey: string
+  #tokenizer: typeof Tokenize
+  #ctx: EmbedderContext
+
+  // oxlint-disable-next-line max-params
+  private constructor(
+    model: string,
+    apiKey: string,
+    info: { contextSize: number; vectorSize: number },
+    tokenizer: typeof Tokenize,
+    ctx: EmbedderContext
+  ) {
+    this.#model = model
+    this.#apiKey = apiKey
+    this.maxTokens = info.contextSize
+    this.dims = info.vectorSize
+    this.#tokenizer = tokenizer
+    this.#ctx = ctx
+  }
+
+  static async load(this: void, ctx: EmbedderContext): Promise<OpenAIBackend> {
+    const { model } = parseModelUri(ctx.opts.model.uri)
+    const apiKey = process.env.OPENAI_API_KEY
+    if (!apiKey)
+      throw new Error("Missing `OPENAI_API_KEY` environment variable for OpenAI embeddings.")
+
+    const { encode } = await import(`gpt-tokenizer/model/${model}`)
+    // Probe the model for dimensions if not in our known list
+    let info = MODEL_INFO[model]
+    if (!info) {
+      ctx.status.status = "probing model dimensions..."
+      const backend = new OpenAIBackend(
+        model,
+        apiKey,
+        { contextSize: 8191, vectorSize: 0 },
+        encode,
+        ctx
+      )
+      const result = await backend.embed(["test"])
+      info = { contextSize: 8191, vectorSize: result[0].length }
+    }
+
+    return new OpenAIBackend(model, apiKey, info, encode, ctx)
+  }
+
+  async embed(texts: string[]): Promise<number[][]> {
+    const response = await fetch(OPENAI_EMBEDDING_URL, {
+      body: JSON.stringify({
+        dimensions: this.#ctx.opts.maxDims,
+        input: texts,
+        model: this.#model,
+      }),
+      headers: {
+        Authorization: `Bearer ${this.#apiKey}`,
+        "Content-Type": "application/json",
+      },
+      method: "POST",
+    })
+
+    if (!response.ok) {
+      const error = await response.text()
+      throw new Error(`OpenAI embedding API error (${response.status}): ${error}`)
+    }
+
+    const data = (await response.json()) as {
+      data: { embedding: number[]; index: number }[]
+    }
+
+    // Sort by index to maintain input order
+    return data.data.toSorted((a, b) => a.index - b.index).map((d) => d.embedding)
+  }
+
+  toks(input: string): number {
+    return this.#tokenizer(input).length
+  }
+}

package/src/embed/transformers.ts

@@ -0,0 +1,81 @@
+import type {
+  DataType,
+  FeatureExtractionPipeline,
+  ProgressInfo,
+  Tensor,
+} from "@huggingface/transformers"
+import type { EmbedderBackend, EmbedderContext, EmbedderDevice } from "./index.ts"
+
+import { parseModelUri } from "./models.ts"
+
+export class TransformersBackend implements EmbedderBackend {
+  device: EmbedderDevice = "cpu"
+  maxTokens: number
+  dims: number
+  #pipeline: FeatureExtractionPipeline
+  #ctx: EmbedderContext
+  private normalize?: (tensor: Tensor) => Tensor // optional normalization function, e.g. for L2 normalization after truncation
+
+  private constructor(pipeline: FeatureExtractionPipeline, ctx: EmbedderContext) {
+    this.#pipeline = pipeline
+    this.#ctx = ctx
+    this.maxTokens = pipeline.model.config.max_position_embeddings
+    this.dims = (pipeline.model.config as { hidden_size?: number }).hidden_size ?? 0
+    this.device = pipeline.model.sessions.model?.config?.device ?? "cpu"
+  }
+
+  static async load(this: void, ctx: EmbedderContext) {
+    const parsed = parseModelUri(ctx.opts.model.uri)
+    const { pipeline, layer_norm } = await import("@huggingface/transformers")
+
+    const extractor = await pipeline("feature-extraction", parsed.model, {
+      // device: "webgpu",
+      dtype: (parsed.variant ?? "auto") as DataType,
+      progress_callback: (event) => TransformersBackend.onProgress(ctx, event),
+      session_options: { intraOpNumThreads: ctx.opts.threads },
+    })
+    const backend = new TransformersBackend(extractor, ctx)
+
+    // Matryoshka: layer_norm → truncate
+    const dims = ctx.opts.maxDims
+    if (dims < backend.dims)
+      backend.normalize = (output) =>
+        layer_norm(output, [output.dims[1] ?? 0])
+          // oxlint-disable-next-line unicorn/no-null
+          .slice(null, [0, dims])
+          .normalize(2, -1)
+
+    return backend
+  }
+
+  async embed(texts: string[]): Promise<number[][]> {
+    const output = await this.#pipeline(texts, {
+      normalize: !this.normalize,
+      pooling: this.#ctx.opts.model.pooling,
+    })
+    return (this.normalize?.(output) ?? output).tolist() as number[][]
+  }
+
+  toks(input: string) {
+    return this.#pipeline.tokenizer.tokenize(input).length
+  }
+
+  static onProgress(ctx: EmbedderContext, event: ProgressInfo) {
+    if (event.status === "initiate") {
+      ctx.status.child(event.name).child(event.file).status = event.status
+    } else if (event.status === "download") {
+      ctx.status.child(event.name).child(event.file).status = event.status
+    } else if (event.status === "progress") {
+      ctx.status.child(event.name).child(event.file).set({
+        max: event.total,
+        status: event.status,
+        value: event.loaded,
+      })
+    } else if (event.status === "done") {
+      ctx.status.child(event.name).child(event.file).set({ status: event.status }).stop()
+    } else if (event.status === "ready") {
+      ctx.status.name = `model \`${ctx.opts.model.uri}\` loaded`
+      ctx.status.child(event.task).set({ status: event.status }).stop()
+    }
+  }
+}

package/src/frecency.ts

@@ -0,0 +1,58 @@
+import type { Db, DocRow } from "./db.ts"
+
+// Exponential decay frecency, based on:
+// https://wiki.mozilla.org/User:Jesse/NewFrecency
+// Ported from snacks.nvim picker frecency
+
+const HALF_LIFE = 30 * 24 * 3600 // 30 days in seconds
+const LAMBDA = Math.LN2 / HALF_LIFE // λ = ln(2) / half_life
+const VISIT_VALUE = 1 // default points per visit
+
+export class Frecency {
+  #db: Db
+  #now: number
+
+  constructor(db: Db, now = Date.now() / 1000) {
+    this.#db = db
+    this.#now = now
+  }
+
+  /** Convert a score into a deadline timestamp.
+   *  deadline = now + ln(score) / λ */
+  toDeadline(score: number): number {
+    return this.#now + Math.log(score) / LAMBDA
+  }
+
+  /** Convert a deadline timestamp back into a current score.
+   *  score = e^(λ * (deadline - now)) */
+  toScore(deadline: number): number {
+    return Math.exp(LAMBDA * (deadline - this.#now))
+  }
+
+  /** Get the current frecency score for a doc.
+   *  If no deadline exists, estimates from updatedAt (a single visit at file mtime, decayed to now). */
+  get(doc: DocRow): number {
+    if (doc.deadline) return this.toScore(doc.deadline)
+    // Seed: treat updatedAt as a single past visit, decayed to now
+    // Since ln(1)/λ = 0 for VISIT_VALUE=1, deadline = timestamp
+    const ts = new Date(doc.updated_at).getTime() / 1000
+    return this.toScore(ts)
+  }
+
+  /** Record a visit — adds a decayed value to the doc's score.
+   *  @param opts.value - points to add (default: 1)
+   *  @param opts.ts - timestamp of the visit in seconds (default: now).
+   *    Use file mtime for seeding from external edits. */
+  visit(doc: DocRow, opts?: { value?: number; ts?: number }) {
+    const value = opts?.value ?? VISIT_VALUE
+    const ts = opts?.ts ?? this.#now
+    const visitDeadline = ts + Math.log(value) / LAMBDA
+    const score = this.get(doc) + this.toScore(visitDeadline)
+    this.#db.setDeadline(doc.id, this.toDeadline(score))
+  }
+
+  /** Get frecency scores for multiple docs, sorted descending. */
+  rank(docs: DocRow[]): { doc: DocRow; score: number }[] {
+    return docs.map((doc) => ({ doc, score: this.get(doc) })).toSorted((a, b) => b.score - a.score)
+  }
+}

package/src/fs.ts

@@ -0,0 +1,36 @@
+import { statSync, existsSync } from "node:fs"
+import { stat } from "node:fs/promises"
+import { homedir } from "node:os"
+import { dirname, join, resolve } from "pathe"
+
+export function sstat(path: string) {
+  try {
+    return statSync(path)
+  } catch {}
+}
+
+export async function astat(path: string) {
+  return await stat(path).catch(() => undefined)
+}
+
+export function findUp(root: string, name: string, stop?: string) {
+  let current = resolve(root)
+  // oxlint-disable-next-line typescript/no-unnecessary-condition
+  while (true) {
+    const check = join(current, name)
+    if (sstat(check)?.isFile()) return check
+    if (stop && existsSync(join(current, stop))) return // reached stop directory without finding the file
+    const next = dirname(current)
+    if (next === current) break // reached filesystem root
+    current = next
+  }
+}
+
+// Similar to path.resolve but also expands ~ to the user home directory
+export function normPath(...paths: string[]) {
+  return resolve(...paths.map((p) => p.replace(/^~(?=\/|\\|$)/, homedir())))
+}
+
+export function gitRoot(path: string) {
+  return findUp(path, ".git")
+}