@pify/search 0.1.0

package/src/engine.ts

@@ -0,0 +1,196 @@
+/**
+ * The two engines, behind one shape.
+ *
+ * The fast path is the real thing: `@ff-labs/fff-node`, a Rust index with a
+ * live watcher, typo-resistant matching, git status and frecency built in. It
+ * scans this whole suite in about 80ms and is not something worth
+ * reimplementing.
+ *
+ * The slow path exists because a native binary is a promise you cannot always
+ * keep. An unsupported platform, a locked-down install, a postinstall that
+ * never ran — any of those and a binary-only search extension is an extension
+ * that does nothing. The fallback is pure TypeScript with no dependencies: a
+ * trigram index for content, a fuzzy scorer for paths. It is slower, and it
+ * works everywhere pi does.
+ *
+ * Both answer the same questions, so the tools never learn which one they got.
+ * `/search` says which is live, because a user comparing timings deserves to
+ * know why.
+ */
+
+export interface FileHit {
+  path: string;
+  score?: number;
+  git?: string;
+  size?: number;
+  modifiedMs?: number;
+}
+
+export interface ContentHit {
+  path: string;
+  line: number;
+  text: string;
+  git?: string;
+}
+
+export interface Page<T> {
+  items: T[];
+  total: number;
+  cursor: string | null;
+}
+
+export type GrepMode = "literal" | "regex" | "fuzzy";
+
+export interface FindOptions {
+  limit?: number;
+  cursor?: string;
+}
+
+export interface GrepOptions extends FindOptions {
+  mode?: GrepMode;
+  caseInsensitive?: boolean;
+  /** Only search paths matching this glob. */
+  glob?: string;
+}
+
+export interface SearchEngine {
+  readonly name: "fff" | "builtin";
+  /** Resolve once the first index build has landed. */
+  ready(timeoutMs: number): Promise<boolean>;
+  find(query: string, options?: FindOptions): Promise<Page<FileHit>>;
+  grep(pattern: string, options?: GrepOptions): Promise<Page<ContentHit>>;
+  /** Note that a path was used, so frecency can favour it later. */
+  touch?(path: string): void;
+  dispose(): void;
+  /** How many files the index holds, when the engine can say. */
+  indexed?(): number;
+}
+
+/** fff calls it "plain"; this package calls it what a user would call it. */
+const FFF_MODE: Record<GrepMode, string> = { literal: "plain", regex: "regex", fuzzy: "fuzzy" };
+
+interface FffResult<T> {
+  ok: boolean;
+  value?: T;
+  error?: string;
+}
+
+interface FffPage {
+  items: Array<Record<string, unknown>>;
+  /**
+   * For grep this is the size of THIS page, not a grand total — fff searches
+   * files lazily and cannot know the total without finishing the job. Reading
+   * it as a total is how a result set of twenty looks like the whole answer.
+   */
+  totalMatched?: number;
+  nextCursor?: unknown;
+}
+
+interface FffFinder {
+  waitForScan(ms: number): Promise<unknown>;
+  fileSearch(query: string, options?: Record<string, unknown>): FffResult<FffPage>;
+  grep(pattern: string, options?: Record<string, unknown>): FffResult<FffPage>;
+  destroy(): void;
+}
+
+/**
+ * Adapter over fff. Its two searches paginate differently — files by page
+ * index, content by an opaque cursor it hands back — so this translates both
+ * into the one string cursor the tools see.
+ */
+export function fffEngine(finder: FffFinder): SearchEngine {
+  // fff's grep cursor is an opaque object, and this interface hands back a
+  // string. Keeping them here rather than serialising the object's innards
+  // avoids depending on a shape its author marked internal.
+  const cursors = new Map<string, unknown>();
+  let cursorSeq = 0;
+
+  function tokenFor(cursor: unknown): string | null {
+    if (cursor === null || cursor === undefined) return null;
+    const token = `c${++cursorSeq}`;
+    cursors.set(token, cursor);
+    // A session asks a lot; only the recent handful can still be in play.
+    if (cursors.size > 32) cursors.delete(cursors.keys().next().value as string);
+    return token;
+  }
+
+  return {
+    name: "fff",
+    async ready(timeoutMs) {
+      try {
+        await finder.waitForScan(timeoutMs);
+        return true;
+      } catch {
+        return false;
+      }
+    },
+    async find(query, options = {}) {
+      const limit = options.limit ?? 20;
+      // fff pages files by index, not by offset.
+      const page = Math.max(0, Number.parseInt(options.cursor ?? "0", 10) || 0);
+      const result = finder.fileSearch(query, { pageSize: limit, pageIndex: page });
+      if (!result.ok || !result.value) return { items: [], total: 0, cursor: null };
+      const items = result.value.items.map((raw) => ({
+        path: String(raw.relativePath ?? ""),
+        git: typeof raw.gitStatus === "string" && raw.gitStatus !== "clean" ? raw.gitStatus : undefined,
+        size: typeof raw.size === "number" ? raw.size : undefined,
+        modifiedMs: typeof raw.modified === "number" ? raw.modified * 1000 : undefined,
+        score: typeof raw.totalFrecencyScore === "number" ? raw.totalFrecencyScore : undefined,
+      }));
+      const total = result.value.totalMatched ?? items.length;
+      // A full page means there may be another; fff reports the match count
+      // for the query, so this is a real total rather than a guess.
+      const hasMore = items.length === limit && (page + 1) * limit < total;
+      return { items, total, cursor: hasMore ? String(page + 1) : null };
+    },
+    async grep(pattern, options = {}) {
+      const limit = options.limit ?? 20;
+      const previous = options.cursor ? cursors.get(options.cursor) : undefined;
+      const result = finder.grep(pattern, {
+        mode: FFF_MODE[options.mode ?? "literal"],
+        pageSize: limit,
+        smartCase: options.caseInsensitive !== false,
+        ...(previous ? { cursor: previous } : {}),
+      });
+      if (!result.ok || !result.value) return { items: [], total: 0, cursor: null };
+      const items = result.value.items.map((raw) => ({
+        path: String(raw.relativePath ?? ""),
+        line: typeof raw.lineNumber === "number" ? raw.lineNumber : 0,
+        text: String(raw.lineContent ?? ""),
+        git: typeof raw.gitStatus === "string" && raw.gitStatus !== "clean" ? raw.gitStatus : undefined,
+      }));
+      // Deliberately items.length: fff searches lazily, so the only honest
+      // total is what has actually been found so far.
+      return { items, total: items.length, cursor: tokenFor(result.value.nextCursor) };
+    },
+    dispose() {
+      cursors.clear();
+      try {
+        finder.destroy();
+      } catch {
+        // already gone
+      }
+    },
+  };
+}
+
+/**
+ * Load fff if it is installed and its binary is present for this platform.
+ * Never throws: a missing engine is the ordinary case this package is built to
+ * survive, not an error to report.
+ */
+export async function loadFff(basePath: string): Promise<SearchEngine | null> {
+  if (process.env.PIFY_SEARCH_ENGINE === "builtin") return null;
+  for (const specifier of ["@ff-labs/fff-node", "@ff-labs/fff-bun"]) {
+    try {
+      const mod = (await import(specifier)) as {
+        FileFinder?: { create(options: { basePath: string }): FffResult<FffFinder> };
+      };
+      const created = mod.FileFinder?.create({ basePath });
+      if (created?.ok && created.value) return fffEngine(created.value);
+    } catch {
+      // Not installed, or no binary for this platform — try the next.
+    }
+  }
+  return null;
+}

package/src/format.ts

@@ -0,0 +1,61 @@
+/**
+ * How results read.
+ *
+ * A search result is consumed by a model that will decide what to open next,
+ * so the shape matters: one line per hit, path first, and the cursor stated in
+ * words rather than left in a details field the model may not look at. A
+ * result set that silently stops at twenty looks like a complete answer, which
+ * is how an agent concludes something does not exist.
+ */
+
+import type { ContentHit, FileHit, GrepMode, Page, SearchEngine } from "./engine.ts";
+
+function more(page: Page<unknown>, tool: string): string {
+  if (!page.cursor) return "";
+  const shown = page.items.length;
+  return `\n\n${page.total - shown} more. Continue with ${tool} cursor="${page.cursor}".`;
+}
+
+export function formatFiles(page: Page<FileHit>, query: string): string {
+  if (page.items.length === 0) {
+    return `No file matches "${query}". Try fewer characters — matching is fuzzy, so a fragment of the name works better than a guess at the full path.`;
+  }
+  const lines = page.items.map((hit) => {
+    const marks: string[] = [];
+    if (hit.git) marks.push(hit.git);
+    return `${hit.path}${marks.length > 0 ? `  (${marks.join(", ")})` : ""}`;
+  });
+  return `${page.total} file${page.total === 1 ? "" : "s"} match "${query}", best first:\n${lines.join("\n")}${more(page, "fffind")}`;
+}
+
+export function formatMatches(page: Page<ContentHit>, pattern: string, mode: GrepMode): string {
+  if (page.items.length === 0) {
+    const hint =
+      mode === "literal"
+        ? ' Try mode="fuzzy" if you are unsure of the exact wording, or mode="regex" for a pattern.'
+        : "";
+    return `No match for "${pattern}" (${mode}).${hint}`;
+  }
+  const lines = page.items.map((hit) => `${hit.path}:${hit.line}: ${hit.text.trim().slice(0, 200)}`);
+  return `${page.total} match${page.total === 1 ? "" : "es"} for "${pattern}" (${mode}):\n${lines.join("\n")}${more(page, "ffgrep")}`;
+}
+
+export function formatStatus(engine: SearchEngine | null, root: string): string {
+  if (!engine) {
+    return "No search engine is running. fffind and ffgrep will start one on first use.";
+  }
+  const lines = [
+    `Engine: ${engine.name === "fff" ? "fff (native index, live watcher)" : "builtin (pure TypeScript, no native binary)"}`,
+    `Root:   ${root}`,
+  ];
+  const count = engine.indexed?.();
+  if (typeof count === "number") lines.push(`Files:  ${count} indexed`);
+  if (engine.name === "builtin") {
+    lines.push(
+      "",
+      "The native engine was not available — either @ff-labs/fff-node is not installed or it has no",
+      "binary for this platform. The builtin engine is slower but needs nothing installed.",
+    );
+  }
+  return lines.join("\n");
+}

package/src/frecency.ts

@@ -0,0 +1,83 @@
+/**
+ * How much a file's history is worth now.
+ *
+ * A file you opened twelve times last week should outrank one you have never
+ * touched, but not forever — otherwise last month's task keeps winning today's
+ * search. Frecency is the usual answer: every access decays, and the score is
+ * what is left of all of them.
+ *
+ * The half-life is fff's, and so is its reasoning: an agent session is shorter
+ * and more concentrated than a human's week, so the default here is the fast
+ * decay. Ten-day half-life is for a person browsing; three days is for a tool
+ * that touched forty files this afternoon.
+ *
+ * Pure: the caller supplies now, so the same input always scores the same.
+ */
+
+/** ln(2)/3 — a three-day half-life. */
+export const DECAY_FAST = 0.231;
+/** ln(2)/10 — a ten-day half-life. */
+export const DECAY_SLOW = 0.0693;
+
+const DAY_MS = 86_400_000;
+
+/** Older than this contributes nothing, so the store cannot grow forever. */
+export const MAX_HISTORY_DAYS = 7;
+/** Beyond this, the oldest access is dropped rather than kept and decayed. */
+export const MAX_TIMESTAMPS = 128;
+
+export type History = Record<string, number[]>;
+
+export function parseHistory(raw: string | null): History {
+  if (!raw) return {};
+  try {
+    const data = JSON.parse(raw) as unknown;
+    if (!data || typeof data !== "object" || Array.isArray(data)) return {};
+    const out: History = {};
+    for (const [path, value] of Object.entries(data as Record<string, unknown>)) {
+      if (!Array.isArray(value)) continue;
+      const stamps = value.filter((v): v is number => typeof v === "number" && Number.isFinite(v) && v > 0);
+      if (stamps.length > 0) out[path] = stamps.slice(-MAX_TIMESTAMPS);
+    }
+    return out;
+  } catch {
+    return {};
+  }
+}
+
+/** Record an access, dropping what has aged out rather than keeping it. */
+export function noteAccess(history: History, path: string, now: number): History {
+  const cutoff = now - MAX_HISTORY_DAYS * DAY_MS;
+  const kept = (history[path] ?? []).filter((t) => t >= cutoff);
+  kept.push(now);
+  return { ...history, [path]: kept.slice(-MAX_TIMESTAMPS) };
+}
+
+/**
+ * The decayed weight of every remaining access. Summed rather than averaged:
+ * ten recent touches should beat one, which an average would hide.
+ */
+export function frecencyOf(history: History, path: string, now: number, decay = DECAY_FAST): number {
+  const stamps = history[path];
+  if (!stamps || stamps.length === 0) return 0;
+  let total = 0;
+  for (const at of stamps) {
+    const ageDays = (now - at) / DAY_MS;
+    if (ageDays < 0 || ageDays > MAX_HISTORY_DAYS) continue;
+    total += Math.exp(-decay * ageDays);
+  }
+  // Scaled into the same range as the other bonuses in fuzzy.ts, and capped so
+  // a much-used file cannot outrank an exact filename match on its own.
+  return Math.min(200, Math.round(total * 25));
+}
+
+/** Drop everything that has aged out, so the file on disk stays bounded. */
+export function pruneHistory(history: History, now: number): History {
+  const cutoff = now - MAX_HISTORY_DAYS * DAY_MS;
+  const out: History = {};
+  for (const [path, stamps] of Object.entries(history)) {
+    const kept = stamps.filter((t) => t >= cutoff);
+    if (kept.length > 0) out[path] = kept.slice(-MAX_TIMESTAMPS);
+  }
+  return out;
+}

package/src/fuzzy.ts

@@ -0,0 +1,180 @@
+/**
+ * Fuzzy path matching, and what makes one result better than another.
+ *
+ * `find` answers with everything that matches a glob, in whatever order the
+ * filesystem handed them over. That is the wrong shape for the question people
+ * actually ask, which is "the auth route file — you know the one". Two things
+ * fix it: a match that tolerates the way people type, and an order that puts
+ * the file you probably meant first.
+ *
+ * The scoring model is fff's: a base score for match quality, plus a frecency
+ * boost, a proximity term, and a bonus for matching the filename rather than
+ * some directory halfway up the path. Typo tolerance scales with the query,
+ * because one wrong letter in four characters is a different mistake from one
+ * wrong letter in twenty.
+ *
+ * Pure. No index, no clock, no disk — the caller supplies all three.
+ */
+
+export interface Candidate {
+  /** Path relative to the search root, forward-slashed. */
+  path: string;
+  /** Frecency, already decayed by the caller: 0 when never touched. */
+  frecency?: number;
+  /** Git working-tree state, if known. */
+  git?: "modified" | "staged" | "untracked" | undefined;
+  /** Modified-at, for the recency thresholds. */
+  mtimeMs?: number;
+}
+
+export interface Scored {
+  path: string;
+  score: number;
+  /** Where the query characters landed, for highlighting. */
+  positions: number[];
+}
+
+/** How many typos a query of this length may carry: fff's clamp(len/4, 2, 6). */
+export function maxTypos(query: string): number {
+  return Math.min(6, Math.max(2, Math.floor(query.length / 4)));
+}
+
+const BONUS_EXACT_FILENAME = 300;
+/** `auth` should find `auth.ts` before `authentication.md`: the stem is the name. */
+const BONUS_EXACT_STEM = 200;
+const BONUS_FILENAME_PREFIX = 120;
+const BONUS_IN_FILENAME = 60;
+const BONUS_CONSECUTIVE = 12;
+const BONUS_BOUNDARY = 18;
+const PENALTY_LEADING = 2;
+const PENALTY_TYPO = 40;
+const PENALTY_DEPTH = 3;
+
+/** A discrete boost for how recently the file changed — fff's thresholds. */
+export function recencyBoost(mtimeMs: number | undefined, now: number): number {
+  if (!mtimeMs) return 0;
+  const age = (now - mtimeMs) / 1000;
+  if (age < 120) return 16;
+  if (age < 900) return 8;
+  if (age < 3600) return 4;
+  if (age < 86_400) return 2;
+  if (age < 604_800) return 1;
+  return 0;
+}
+
+/** Work in progress is what you are most likely to be looking for. */
+export function gitBoost(status: Candidate["git"]): number {
+  if (status === "modified") return 24;
+  if (status === "staged") return 20;
+  if (status === "untracked") return 12;
+  return 0;
+}
+
+/**
+ * Subsequence match, preferring later starts so `auth` in `src/auth/x.ts`
+ * anchors on the filename rather than the first `a` in the path. Returns null
+ * when the query cannot be found even with the typo budget spent.
+ */
+function matchPositions(haystack: string, needle: string, budget: number): { positions: number[]; typos: number } | null {
+  const hay = haystack.toLowerCase();
+  const need = needle.toLowerCase();
+  const positions: number[] = [];
+  let typos = 0;
+  let at = 0;
+
+  for (let n = 0; n < need.length; n++) {
+    const ch = need[n]!;
+    const found = hay.indexOf(ch, at);
+    if (found === -1) {
+      // Skipping a query character is the typo: a transposition or a slip.
+      typos++;
+      if (typos > budget) return null;
+      continue;
+    }
+    positions.push(found);
+    at = found + 1;
+  }
+  if (positions.length === 0) return null;
+  return { positions, typos };
+}
+
+function isBoundary(text: string, index: number): boolean {
+  if (index === 0) return true;
+  const prev = text[index - 1]!;
+  return prev === "/" || prev === "_" || prev === "-" || prev === "." || (prev === prev.toLowerCase() && text[index] !== text[index]!.toLowerCase());
+}
+
+/** Score one candidate, or null when the query does not match it at all. */
+export function scoreCandidate(candidate: Candidate, query: string, now: number): Scored | null {
+  const path = candidate.path;
+  if (query === "") {
+    return { path, score: (candidate.frecency ?? 0) + gitBoost(candidate.git) + recencyBoost(candidate.mtimeMs, now), positions: [] };
+  }
+
+  const match = matchPositions(path, query, maxTypos(query));
+  if (!match) return null;
+
+  const slash = path.lastIndexOf("/");
+  const filename = slash === -1 ? path : path.slice(slash + 1);
+  const lowerName = filename.toLowerCase();
+  const lowerQuery = query.toLowerCase();
+
+  const dot = lowerName.lastIndexOf(".");
+  const stem = dot <= 0 ? lowerName : lowerName.slice(0, dot);
+
+  let score = 0;
+  if (lowerName === lowerQuery) score += BONUS_EXACT_FILENAME;
+  else if (stem === lowerQuery) score += BONUS_EXACT_STEM;
+  else if (lowerName.startsWith(lowerQuery)) score += BONUS_FILENAME_PREFIX;
+  else if (lowerName.includes(lowerQuery)) score += BONUS_IN_FILENAME;
+
+  for (let i = 0; i < match.positions.length; i++) {
+    const at = match.positions[i]!;
+    if (i > 0 && at === match.positions[i - 1]! + 1) score += BONUS_CONSECUTIVE;
+    if (isBoundary(path, at)) score += BONUS_BOUNDARY;
+    // A match that starts deep in the string is a weaker match.
+    if (i === 0) score -= Math.min(at, 40) * PENALTY_LEADING;
+  }
+
+  score -= match.typos * PENALTY_TYPO;
+  // A shallow path is more likely to be the one meant than a deep one.
+  score -= (path.split("/").length - 1) * PENALTY_DEPTH;
+  score += candidate.frecency ?? 0;
+  score += gitBoost(candidate.git);
+  score += recencyBoost(candidate.mtimeMs, now);
+
+  return { path, score, positions: match.positions };
+}
+
+export interface Page<T> {
+  items: T[];
+  /** Opaque cursor for the next page, or null when this was the last. */
+  cursor: string | null;
+  total: number;
+}
+
+/**
+ * Rank and cut. The cursor is an offset rather than a token into stored state:
+ * an index that moves under a long-lived cursor would silently skip or repeat
+ * results, and an offset at least fails the same way a human would expect.
+ */
+export function rankAndPage(
+  candidates: readonly Candidate[],
+  query: string,
+  now: number,
+  limit: number,
+  cursor?: string,
+): Page<Scored> {
+  const scored: Scored[] = [];
+  for (const candidate of candidates) {
+    const hit = scoreCandidate(candidate, query, now);
+    if (hit) scored.push(hit);
+  }
+  scored.sort((a, b) => b.score - a.score || a.path.localeCompare(b.path));
+
+  const offset = Number.parseInt(cursor ?? "0", 10);
+  const start = Number.isFinite(offset) && offset > 0 ? offset : 0;
+  const items = scored.slice(start, start + limit);
+  const next = start + items.length;
+  return { items, cursor: next < scored.length ? String(next) : null, total: scored.length };
+}

package/src/match.ts

@@ -0,0 +1,119 @@
+/**
+ * Deciding whether a line really matches, once the index has narrowed the
+ * field.
+ *
+ * The trigram index answers "which files could contain this", never "which
+ * lines do". Every candidate is still read and matched here, so a wrong
+ * candidate costs time and never correctness — which is what makes it safe to
+ * narrow aggressively.
+ *
+ * Three modes, because three different questions get asked: the exact string,
+ * a regex, and "something like this" for when the caller does not know how the
+ * thing is spelled.
+ */
+
+import { maxTypos } from "./fuzzy.ts";
+
+export type Mode = "literal" | "regex" | "fuzzy";
+
+export interface LineMatch {
+  /** 1-based, as every editor and every error message counts them. */
+  line: number;
+  text: string;
+  /** Byte-free column range of the hit, for highlighting. */
+  start: number;
+  end: number;
+}
+
+export interface Matcher {
+  test(line: string): { start: number; end: number } | null;
+}
+
+/** A matcher for one pattern, or null when the pattern itself is broken. */
+export function buildMatcher(pattern: string, mode: Mode, caseInsensitive: boolean): Matcher | null {
+  if (pattern === "") return null;
+
+  if (mode === "literal") {
+    const needle = caseInsensitive ? pattern.toLowerCase() : pattern;
+    return {
+      test(line) {
+        const hay = caseInsensitive ? line.toLowerCase() : line;
+        const at = hay.indexOf(needle);
+        return at === -1 ? null : { start: at, end: at + needle.length };
+      },
+    };
+  }
+
+  if (mode === "regex") {
+    let re: RegExp;
+    try {
+      re = new RegExp(pattern, caseInsensitive ? "i" : "");
+    } catch {
+      return null;
+    }
+    return {
+      test(line) {
+        const m = re.exec(line);
+        return m ? { start: m.index, end: m.index + m[0].length } : null;
+      },
+    };
+  }
+
+  // Fuzzy: the query's characters in order, within a typo budget, and close
+  // enough together to be one word rather than three scattered letters.
+  const budget = maxTypos(pattern);
+  const needle = pattern.toLowerCase();
+  const span = Math.max(needle.length * 3, needle.length + 8);
+  return {
+    test(line) {
+      const hay = line.toLowerCase();
+      for (let start = 0; start < hay.length; start++) {
+        if (hay[start] !== needle[0] && budget === 0) continue;
+        let typos = 0;
+        let at = start;
+        let matched = 0;
+        for (let n = 0; n < needle.length; n++) {
+          const found = hay.indexOf(needle[n]!, at);
+          if (found === -1 || found - start > span) {
+            typos++;
+            if (typos > budget) break;
+            continue;
+          }
+          at = found + 1;
+          matched++;
+        }
+        if (matched > 0 && typos <= budget && matched >= needle.length - budget) {
+          return { start, end: Math.min(at, hay.length) };
+        }
+      }
+      return null;
+    },
+  };
+}
+
+/** Every matching line in one file's content. */
+export function matchLines(content: string, matcher: Matcher, limit: number): LineMatch[] {
+  const out: LineMatch[] = [];
+  const lines = content.split(/\r?\n/);
+  for (let i = 0; i < lines.length && out.length < limit; i++) {
+    const text = lines[i]!;
+    const hit = matcher.test(text);
+    if (hit) out.push({ line: i + 1, text, start: hit.start, end: hit.end });
+  }
+  return out;
+}
+
+/**
+ * Whether a file is worth reading as text at all.
+ *
+ * A NUL byte in the first few kilobytes is the classic signal, and it is
+ * cheaper and more reliable than trusting an extension: a `.dat` may be text
+ * and a `.txt` may not.
+ */
+export function looksBinary(sample: string): boolean {
+  const window = sample.slice(0, 8192);
+  for (let i = 0; i < window.length; i++) {
+    if (window.charCodeAt(i) === 0) return true;
+  }
+  return false;
+}