@pify/search 0.1.0 → 0.3.0

package/native/src/walk.rs

@@ -0,0 +1,114 @@
+//! What belongs in the index.
+//!
+//! An index holding `node_modules` answers slowly and wrongly — the file you
+//! meant is buried under ten thousand you did not. The rules are deliberately
+//! the same list the TypeScript fallback uses, so the two engines index the
+//! same tree and a search cannot find a file under one engine and miss it
+//! under the other.
+
+use std::path::{Path, PathBuf};
+use walkdir::WalkDir;
+
+pub const MAX_INDEXABLE_BYTES: u64 = 2 * 1024 * 1024;
+pub const MAX_SEARCHABLE_BYTES: u64 = 10 * 1024 * 1024;
+
+const SKIP_DIRS: &[&str] = &[
+    ".git", ".hg", ".svn", "node_modules", ".venv", "venv", "__pycache__", ".mypy_cache",
+    ".pytest_cache", ".ruff_cache", "dist", "build", "out", "target", ".next", ".nuxt",
+    ".svelte-kit", ".turbo", ".cache", ".gradle", ".idea", ".vscode-test", "coverage",
+    ".nyc_output", "vendor", "Pods", ".terraform",
+];
+
+const BINARY_EXTENSIONS: &[&str] = &[
+    "png", "jpg", "jpeg", "gif", "bmp", "ico", "webp", "avif", "tiff", "psd", "mp3", "mp4", "wav",
+    "ogg", "flac", "avi", "mov", "mkv", "webm", "zip", "gz", "tgz", "bz2", "xz", "7z", "rar", "jar",
+    "war", "pdf", "doc", "docx", "xls", "xlsx", "ppt", "pptx", "exe", "dll", "so", "dylib", "bin",
+    "o", "a", "lib", "obj", "pdb", "class", "pyc", "pyo", "wasm", "node", "ttf", "otf", "woff",
+    "woff2", "eot", "db", "sqlite", "sqlite3", "pack", "idx",
+];
+
+#[derive(Debug)]
+pub struct Found {
+    pub rel: String,
+    pub absolute: PathBuf,
+    pub size: u64,
+    pub mtime_ms: i64,
+}
+
+pub fn normalize(path: &str) -> String {
+    path.replace('\\', "/").trim_start_matches("./").to_string()
+}
+
+pub fn relative(root: &Path, absolute: &Path) -> String {
+    match absolute.strip_prefix(root) {
+        Ok(rel) => normalize(&rel.to_string_lossy()),
+        Err(_) => normalize(&absolute.to_string_lossy()),
+    }
+}
+
+pub fn mtime_ms(meta: &std::fs::Metadata) -> i64 {
+    meta.modified()
+        .ok()
+        .and_then(|t| t.duration_since(std::time::UNIX_EPOCH).ok())
+        .map(|d| d.as_millis() as i64)
+        .unwrap_or(0)
+}
+
+fn extension_of(path: &str) -> &str {
+    let name = path.rsplit('/').next().unwrap_or(path);
+    match name.rfind('.') {
+        Some(dot) if dot > 0 => &name[dot + 1..],
+        _ => "",
+    }
+}
+
+/// Whether this file's *content* is worth putting in the trigram index.
+pub fn index_content(rel: &str, size: u64) -> bool {
+    if size == 0 || size > MAX_INDEXABLE_BYTES {
+        return false;
+    }
+    let ext = extension_of(rel).to_ascii_lowercase();
+    !BINARY_EXTENSIONS.contains(&ext.as_str())
+}
+
+/// A NUL byte in the first few kilobytes: cheaper and more reliable than
+/// trusting an extension, since a `.dat` may be text and a `.txt` may not.
+pub fn looks_binary(bytes: &[u8]) -> bool {
+    bytes.iter().take(8192).any(|&b| b == 0)
+}
+
+pub fn collect(root: &Path, max_files: usize) -> Vec<Found> {
+    let mut out = Vec::new();
+    let walker = WalkDir::new(root)
+        .max_depth(24)
+        .follow_links(false)
+        .into_iter()
+        .filter_entry(|entry| {
+            if entry.depth() == 0 {
+                return true;
+            }
+            let name = entry.file_name().to_string_lossy();
+            if entry.file_type().is_dir() {
+                return !SKIP_DIRS.contains(&name.as_ref());
+            }
+            name != ".DS_Store"
+        });
+
+    for entry in walker.flatten() {
+        if out.len() >= max_files {
+            break;
+        }
+        if !entry.file_type().is_file() {
+            continue;
+        }
+        let Ok(meta) = entry.metadata() else { continue };
+        let absolute = entry.path().to_path_buf();
+        out.push(Found {
+            rel: relative(root, &absolute),
+            absolute,
+            size: meta.len(),
+            mtime_ms: mtime_ms(&meta),
+        });
+    }
+    out
+}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@pify/search",
-  "version": "0.1.0",
+  "version": "0.3.0",
   "description": "Fuzzy file finding and indexed content search for pi, with no native binary and no daemon",
   "keywords": [
     "pi-package",
@@ -30,6 +30,9 @@
     "extensions",
     "src",
     "skills",
+    "native/src",
+    "native/Cargo.toml",
+    "native/build.rs",
     "README.md",
     "LICENSE"
   ],
@@ -44,7 +47,8 @@
   "scripts": {
     "typecheck": "tsc --noEmit",
     "test": "bun test",
-    "prepublishOnly": "npm run typecheck && npm test"
+    "prepublishOnly": "npm run typecheck && npm test",
+    "build:native": "cargo build --release --manifest-path native/Cargo.toml"
   },
   "peerDependencies": {
     "@earendil-works/pi-ai": "*",

package/src/engine.ts

@@ -1,21 +1,23 @@
 /**
- * The two engines, behind one shape.
+ * Three engines, behind one shape.
  *
- * The fast path is the real thing: `@ff-labs/fff-node`, a Rust index with a
- * live watcher, typo-resistant matching, git status and frecency built in. It
- * scans this whole suite in about 80ms and is not something worth
- * reimplementing.
+ * **native** — this package's own Rust core, compiled per platform. It indexes
+ * this suite (417 files) in about 33ms, and a literal search then reads the 5
+ * files the trigram index says could match rather than all 417.
  *
- * The slow path exists because a native binary is a promise you cannot always
- * keep. An unsupported platform, a locked-down install, a postinstall that
- * never ran — any of those and a binary-only search extension is an extension
- * that does nothing. The fallback is pure TypeScript with no dependencies: a
- * trigram index for content, a fuzzy scorer for paths. It is slower, and it
- * works everywhere pi does.
+ * **fff** — `@ff-labs/fff-node`, used when it is installed and the native core
+ * is not. A mature engine with its own watcher and git integration.
  *
- * Both answer the same questions, so the tools never learn which one they got.
- * `/search` says which is live, because a user comparing timings deserves to
- * know why.
+ * **builtin** — pure TypeScript, no dependencies, no binary. It exists because
+ * a native binary is a promise you cannot always keep: an unsupported
+ * platform, a locked-down install, a postinstall that never ran. A
+ * binary-only search extension in any of those cases is one that silently does
+ * nothing.
+ *
+ * All three answer the same questions, so the tools never learn which one they
+ * got. The scoring constants are shared deliberately between the native core
+ * and the fallback, so losing the binary changes how fast a search is and not
+ * how it is ordered. `/search` says which engine is live.
  */
 
 export interface FileHit {
@@ -54,16 +56,27 @@ export interface GrepOptions extends FindOptions {
 }
 
 export interface SearchEngine {
-  readonly name: "fff" | "builtin";
+  readonly name: "native" | "fff" | "builtin";
   /** Resolve once the first index build has landed. */
   ready(timeoutMs: number): Promise<boolean>;
   find(query: string, options?: FindOptions): Promise<Page<FileHit>>;
   grep(pattern: string, options?: GrepOptions): Promise<Page<ContentHit>>;
   /** Note that a path was used, so frecency can favour it later. */
   touch?(path: string): void;
+  /** Re-read one path after a change, when the engine can. */
+  refresh?(path: string): void;
+  forget?(path: string): void;
   dispose(): void;
   /** How many files the index holds, when the engine can say. */
   indexed?(): number;
+  /**
+   * How the last index build was paid for: files whose contents came back from
+   * the stored index versus files that had to be read. Only an engine that
+   * persists its index can answer, and it is worth surfacing — "reused 0" on a
+   * tree that has not changed is the visible symptom of a cache that is
+   * silently not working.
+   */
+  stats?(): { reused: number; rebuilt: number };
 }
 
 /** fff calls it "plain"; this package calls it what a user would call it. */

package/src/format.ts

@@ -45,11 +45,24 @@ export function formatStatus(engine: SearchEngine | null, root: string): string
     return "No search engine is running. fffind and ffgrep will start one on first use.";
   }
   const lines = [
-    `Engine: ${engine.name === "fff" ? "fff (native index, live watcher)" : "builtin (pure TypeScript, no native binary)"}`,
+    `Engine: ${
+      engine.name === "native"
+        ? "native (this package's Rust core)"
+        : engine.name === "fff"
+          ? "fff (@ff-labs/fff-node)"
+          : "builtin (pure TypeScript, no native binary)"
+    }`,
     `Root:   ${root}`,
   ];
   const count = engine.indexed?.();
   if (typeof count === "number") lines.push(`Files:  ${count} indexed`);
+  const stats = engine.stats?.();
+  if (stats && stats.reused + stats.rebuilt > 0) {
+    lines.push(
+      `Index:  ${stats.reused} reused from cache, ${stats.rebuilt} read from disk` +
+        (stats.reused === 0 ? " (first run for this tree)" : ""),
+    );
+  }
   if (engine.name === "builtin") {
     lines.push(
       "",

package/src/native.ts

@@ -0,0 +1,226 @@
+/**
+ * The native core, when there is one.
+ *
+ * A Rust index built for this package: the same trigram narrowing and the same
+ * scoring constants as the TypeScript fallback, compiled. It builds an index
+ * of this suite — 417 files — in about 40ms, and a literal search then reads 5
+ * of those files instead of all of them.
+ *
+ * The index is stored between sessions, so a second start on an unchanged tree
+ * reloads instead of re-reading it. See `cachePathFor` for where it lives.
+ *
+ * The binary is an optional dependency per platform, in the napi convention.
+ * If none of them installed, this returns null and the caller falls back;
+ * nothing here ever throws for a missing binary, because a missing binary is
+ * the ordinary case this package is designed to survive.
+ */
+
+import { createHash } from "node:crypto";
+import { createRequire } from "node:module";
+import { existsSync } from "node:fs";
+import { homedir, tmpdir } from "node:os";
+import { basename, dirname, join, resolve } from "node:path";
+import { fileURLToPath } from "node:url";
+
+import type { ContentHit, FileHit, GrepOptions, Page, SearchEngine, FindOptions } from "./engine.ts";
+
+interface NativePage<T> {
+  items: T[];
+  total: number;
+  /** Offset of the next page, or -1 when this was the last. */
+  next: number;
+  scanned?: number;
+}
+
+interface NativeIndex {
+  fileCount(): number;
+  indexedCount(): number;
+  /** Files whose contents came from the stored index instead of from disk. */
+  reusedCount(): number;
+  /** Files that had to be read because they were new or had changed. */
+  rebuiltCount(): number;
+  save(): void;
+  touch(path: string): void;
+  refresh(path: string): void;
+  forget(path: string): void;
+  find(query: string, limit: number, offset: number, nowMs: number): NativePage<FileHit>;
+  grep(
+    pattern: string,
+    mode: string,
+    limit: number,
+    offset: number,
+    caseInsensitive: boolean,
+  ): NativePage<ContentHit>;
+}
+
+interface NativeModule {
+  SearchIndex: new (root: string, maxFiles?: number, cachePath?: string) => NativeIndex;
+}
+
+/**
+ * Where a tree's stored index lives.
+ *
+ * Not inside the repository: an index is a derived artifact, it is large, and
+ * writing one into someone's working tree means it shows up in their `git
+ * status` and their diffs. It goes in the platform's cache directory, keyed by
+ * the absolute path of the root so two checkouts of the same project keep
+ * their own.
+ *
+ * `PIFY_SEARCH_CACHE_DIR` relocates it and `PIFY_SEARCH_NO_CACHE=1` turns it
+ * off, which is how the no-cache path stays tested on a machine that has one.
+ */
+export function cachePathFor(root: string): string | undefined {
+  if (process.env.PIFY_SEARCH_NO_CACHE === "1") return undefined;
+
+  const base =
+    process.env.PIFY_SEARCH_CACHE_DIR ??
+    (process.platform === "win32"
+      ? join(process.env.LOCALAPPDATA ?? tmpdir(), "pify-search")
+      : process.platform === "darwin"
+        ? join(homedir(), "Library", "Caches", "pify-search")
+        : join(process.env.XDG_CACHE_HOME ?? join(homedir(), ".cache"), "pify-search"));
+
+  const absolute = resolve(root);
+  // The readable part is for a human looking at the cache directory; the hash
+  // is what actually makes the name unique, since two projects can share a
+  // basename and paths differ only in case on some platforms.
+  const label = (basename(absolute) || "root").replace(/[^A-Za-z0-9._-]/g, "_").slice(0, 40);
+  const digest = createHash("sha256").update(absolute.toLowerCase()).digest("hex").slice(0, 16);
+  return join(base, `${label}-${digest}.idx`);
+}
+
+/** The napi triple for this host, matching how the binaries are published. */
+export function tripleOf(platform: string, arch: string): string | null {
+  const key = `${platform}-${arch}`;
+  const known: Record<string, string> = {
+    "win32-x64": "win32-x64",
+    "win32-arm64": "win32-arm64",
+    "darwin-x64": "darwin-x64",
+    "darwin-arm64": "darwin-arm64",
+    "linux-x64": "linux-x64",
+    "linux-arm64": "linux-arm64",
+  };
+  return known[key] ?? null;
+}
+
+function candidatePaths(triple: string): string[] {
+  const here = dirname(fileURLToPath(import.meta.url));
+  const root = join(here, "..");
+  return [
+    // A locally built binary, which is how the package is developed.
+    join(root, `pify-search.${triple}.node`),
+    join(root, "native", "target", "release", `pify-search.${triple}.node`),
+  ];
+}
+
+/**
+ * Load the native core, or null when this platform has no binary.
+ * `PIFY_SEARCH_ENGINE=builtin` forces the fallback, which is how both paths
+ * get tested on a machine that has the binary.
+ */
+export function loadNative(root: string, maxFiles?: number): SearchEngine | null {
+  if (process.env.PIFY_SEARCH_ENGINE === "builtin" || process.env.PIFY_SEARCH_ENGINE === "fff") {
+    return null;
+  }
+  const triple = tripleOf(process.platform, process.arch);
+  if (!triple) return null;
+
+  const require = createRequire(import.meta.url);
+  let mod: NativeModule | null = null;
+  for (const path of candidatePaths(triple)) {
+    if (!existsSync(path)) continue;
+    try {
+      mod = require(path) as NativeModule;
+      break;
+    } catch {
+      // A binary that will not load is the same as one that is not there.
+    }
+  }
+  if (!mod) {
+    try {
+      mod = require(`@pify/search-${triple}`) as NativeModule;
+    } catch {
+      return null;
+    }
+  }
+
+  let index: NativeIndex;
+  try {
+    index = new mod.SearchIndex(root, maxFiles, cachePathFor(root));
+  } catch {
+    return null;
+  }
+
+  const pageOf = <T>(page: NativePage<T>): Page<T> => ({
+    items: page.items,
+    total: page.total,
+    cursor: page.next >= 0 ? String(page.next) : null,
+  });
+  const offsetOf = (cursor?: string) => {
+    const n = Number.parseInt(cursor ?? "0", 10);
+    return Number.isFinite(n) && n > 0 ? n : 0;
+  };
+
+  return {
+    name: "native",
+    async ready() {
+      // The index is built in the constructor, so by here it is done.
+      return true;
+    },
+    async find(query: string, options: FindOptions = {}) {
+      return pageOf(index.find(query, options.limit ?? 20, offsetOf(options.cursor), Date.now()));
+    },
+    async grep(pattern: string, options: GrepOptions = {}) {
+      return pageOf(
+        index.grep(
+          pattern,
+          options.mode ?? "literal",
+          options.limit ?? 20,
+          offsetOf(options.cursor),
+          options.caseInsensitive !== false,
+        ),
+      );
+    },
+    touch(path: string) {
+      try {
+        index.touch(path);
+      } catch {
+        // Frecency is a ranking nicety, never a reason to fail a search.
+      }
+    },
+    refresh(path: string) {
+      try {
+        index.refresh(path);
+      } catch {
+        // A file we cannot re-read stays as it was.
+      }
+    },
+    forget(path: string) {
+      try {
+        index.forget(path);
+      } catch {
+        // ditto
+      }
+    },
+    indexed() {
+      try {
+        return index.fileCount();
+      } catch {
+        return 0;
+      }
+    },
+    stats() {
+      try {
+        return { reused: index.reusedCount(), rebuilt: index.rebuiltCount() };
+      } catch {
+        return { reused: 0, rebuilt: 0 };
+      }
+    },
+    dispose() {
+      // Deliberately not saving here. Anything `refresh` changed during the
+      // session has a new mtime on disk, so the next start sees the mismatch
+      // and re-reads those files anyway — writing the whole index out at exit
+      // would cost a multi-megabyte write to save a handful of file reads.
+    },
+  };
+}