decorated-pi 0.7.0 → 0.7.1

package/README.md

@@ -48,12 +48,33 @@ Drop‑in replacements for Pi's built‑in tools, with better UX and fewer waste
 
 #### Smarter `@` File Search
 
-| Aspect | Pi native `@` | `decorated-pi` |
+`decorated-pi` replaces pi's built-in `@` autocomplete with a high-speed file finder backed by **[`@ff-labs/fff-node`](https://github.com/dmtrKovalenko/fff)**— a Rust SIMD fuzzy file search engine with in-memory index, frecency ranking, and git status awareness. Pi's native provider shells out to `fd` on every keystroke.
+
+| Aspect | Pi native `@` | `decorated-pi` (FFF) |
 | ------ | :---: | :---: |
-| **Speed** | ❌ re‑scans filesystem on every trigger | ✅ caches once per `@` trigger |
-| **Noise filtering** | ❌ no penalty system, shows hidden files | ✅ tiered penalty auto‑filters clutter |
-| **Default suggestions** | ❌ all files visible on empty query | ✅ only visible project files |
-| **Match precision** | ❌ case‑insensitive simple scoring | ✅ multi‑level case‑sensitive scoring |
+| **Speed** | ❌ walks filesystem via `fd` subprocess on every keystroke | ✅ in‑memory index built once per session, ~0.1 ms / query |
+| **Ranking** | ❌ 4‑bucket case‑sensitive score (exact/starts/contains/path) | ✅ fuzzy match + frecency + git status (boots from git log) |
+| **Noise** | ❌ shows every file in the project, including `.git`, `node_modules`, `dist` | ✅ filters `git‑ignored` files; substring filter on path keeps short queries relevant |
+
+###### Benchmark of `@`
+
+```
+┌─ smart-at benchmark
+├─ generated 500,000 files in 3.9 s
+├─ FFF scan complete in 664 ms
+│  RSS after FFF index: 408 MB  (+330 MB over baseline)
+├─ accuracy (14 queries)
+│                  top‑1   top‑3   top‑5   false‑pos
+│  smart-at         93%     93%     93%    0
+│  native (fd)      86%     93%     93%    0
+└─ corpus cleaned
+
+name                                            hz      mean
+· smart-at                                      8.10    123 ms
+· native @  (fd subprocess)                     3.12    320 ms
+
+Summary: smart-at is 2.59x faster than native @  (fd subprocess)
+```
 
 #### LSP support
 

package/hooks/smart-at.ts

@@ -1,318 +1,168 @@
 /**
  * smart-at — high-speed file search autocomplete.
  *
- * Triggered on session_start by registering an autocomplete provider.
- * Uses git ls-files (in git repos) or fd (elsewhere) for candidates,
- * then fuzzy-matches with path penalties.
+ * Uses @ff-labs/fff-node (FFF) for cross-platform fuzzy file search.
+ * FFF maintains an in-memory index, scores files by fuzzy match +
+ * frecency + git status, and returns ranked results. We create one
+ * FileFinder per session and query it directly for every @ prefix.
  */
 
-import { spawnSync } from "child_process";
-import { existsSync } from "node:fs";
-import { join } from "node:path";
-import type { ExtensionAPI, ExtensionContext } from "@earendil-works/pi-coding-agent";
+import { FileFinder, type MixedItem } from "@ff-labs/fff-node";
+import type { ExtensionContext } from "@earendil-works/pi-coding-agent";
 import type { Module, Skeleton } from "./skeleton.js";
 
-// ─── Types ────────────────────────────────────────────────────────────────
+// Characters that, when preceding an "@", allow us to start a smart-at
+// completion.
+const AT_BOUNDARY = new Set<string>([" ", "\t", "(", "["]);
 
-type PenaltyTier = 0 | 1 | 2 | 3 | 4;
+const AUTOCOMPLETE_LIMIT = 20;
+const FFF_SUPERSET = 100;
+const WIDGET_FOOTER = "\x1b[2mpowered by fff\x1b[0m";
 
-interface FileCandidate {
-  path: string;
-  name: string;
-  isDir: boolean;
-  tier: PenaltyTier;
-  penalty: number;
+interface AutocompleteItem {
+  value: string;
+  label: string;
+  description?: string;
 }
 
-// ─── Hard-exclude & penalty rules ────────────────────────────────────────
-
-const HARD_EXCLUDE_DIRS = new Set(["node_modules", ".git", ".pnpm", ".svn"]);
-
-const BAD_DIRS = new Set(["build", "dist", "coverage", "out", "target"]);
-
-const EXT_PENALTY: Record<string, number> = {
-  o: -150, obj: -150, a: -150, so: -150, dll: -150, exe: -150,
-  wasm: -150, class: -120, pyc: -120,
-  bmp: -100, png: -100, jpg: -100, gif: -100, ico: -100, svg: -80,
-  mp3: -100, wav: -100, mp4: -100, avi: -100,
-  pdf: -100, zip: -100, tar: -100, gz: -100,
-  lock: -80,
-};
-
-// ─── Penalty pre-compute ─────────────────────────────────────────────────
-
-function computePenaltyMeta(filePath: string, isDir: boolean, gitIgnored: boolean): { tier: PenaltyTier; penalty: number } {
-  const parts = filePath.replace(/\/$/, "").split("/");
-  const name = parts[parts.length - 1] || filePath;
-  const ext = (!isDir && name.includes(".")) ? (name.split(".").pop()?.toLowerCase() || "") : "";
-  const depth = parts.length;
-  const dirSegments = isDir ? parts : parts.slice(0, -1);
-
-  let tier: PenaltyTier = 0;
-  let tierPenalty = 0;
-
-  if (gitIgnored) { tier = 1; tierPenalty = -400; }
-  else if (dirSegments.some(d => d.startsWith(".") || d.startsWith("__"))) { tier = 2; tierPenalty = -300; }
-  else if (dirSegments.some(d => BAD_DIRS.has(d))) { tier = 3; tierPenalty = -200; }
-  else if (!isDir && (EXT_PENALTY[ext] ?? 0) < 0) { tier = 4; tierPenalty = EXT_PENALTY[ext]!; }
-
-  const basePenalty = -(depth * 30) - name.length;
-  return { tier, penalty: tierPenalty + basePenalty };
-}
-
-function computePenalty(filePath: string, isDir: boolean, gitIgnored: boolean): number {
-  return computePenaltyMeta(filePath, isDir, gitIgnored).penalty;
-}
-
-// ─── Candidate collection ────────────────────────────────────────────────
-
-const SPAWN_OPTS = { timeout: 5000, encoding: "utf-8" as const, maxBuffer: 10 * 1024 * 1024 };
-
-function isGitWorkTree(cwd: string): boolean {
-  return spawnSync("git", ["rev-parse", "--is-inside-work-tree"], { ...SPAWN_OPTS, cwd }).status === 0;
-}
-
-function commandExists(command: string): boolean {
-  const result = spawnSync(
-    process.platform === "win32" ? "where" : (process.env.SHELL || "sh"),
-    process.platform === "win32" ? [command] : ["-lc", `command -v '${command.replace(/'/g, `'"'"'`)}'`],
-    SPAWN_OPTS,
-  );
-  return result.status === 0;
-}
-
-function collectCandidates(cwd: string): FileCandidate[] {
-  const candidates: FileCandidate[] = [];
-  const isChildOfHardExclude = (p: string): boolean => {
-    const parts = p.replace(/\/$/, "").split("/");
-    for (let i = 0; i < parts.length - 1; i++) {
-      if (HARD_EXCLUDE_DIRS.has(parts[i]!)) return true;
-    }
-    return false;
-  };
-
-  const opts = { ...SPAWN_OPTS, cwd };
-  const isGit = isGitWorkTree(cwd);
-
-  if (isGit) {
-    const r1 = spawnSync("git", ["ls-files", "--cached", "--others", "--exclude-standard"], opts);
-    const visibleFiles = (r1.status === 0 && r1.stdout)
-      ? r1.stdout.trim().split("\n").filter(Boolean) : [];
-
-    const r2 = spawnSync("git", ["ls-files", "--others", "--ignored", "--exclude-standard", "--directory", "--no-empty-directory"], opts);
-    const ignoredDirs = new Set<string>();
-    const ignoredFiles: string[] = [];
-    if (r2.status === 0 && r2.stdout) {
-      for (const raw of r2.stdout.trim().split("\n").filter(Boolean)) {
-        const entry = raw.replace(/^\.\//, "");
-        if (isChildOfHardExclude(entry)) continue;
-        if (entry.endsWith("/")) ignoredDirs.add(entry.replace(/\/$/, ""));
-        else ignoredFiles.push(entry);
-      }
-    }
-
-    const dirSet = new Set<string>();
-    for (const f of visibleFiles) {
-      if (isChildOfHardExclude(f)) continue;
-      const name = f.split("/").pop() || f;
-      const meta = computePenaltyMeta(f, false, false);
-      candidates.push({ path: f, name, isDir: false, tier: meta.tier, penalty: meta.penalty });
-      const parts = f.split("/");
-      let current = "";
-      for (let i = 0; i < parts.length - 1; i++) {
-        current = current ? `${current}/${parts[i]}` : parts[i]!;
-        dirSet.add(current);
-      }
-    }
-    for (const f of ignoredFiles) {
-      const name = f.split("/").pop() || f;
-      const meta = computePenaltyMeta(f, false, true);
-      candidates.push({ path: f, name, isDir: false, tier: meta.tier, penalty: meta.penalty });
-    }
-    for (const d of dirSet) {
-      const name = d.split("/").pop() || d;
-      const meta = computePenaltyMeta(d, true, false);
-      candidates.push({ path: d + "/", name, isDir: true, tier: meta.tier, penalty: meta.penalty });
-    }
-    for (const d of ignoredDirs) {
-      const name = d.split("/").pop() || d;
-      const meta = computePenaltyMeta(d, true, true);
-      candidates.push({ path: d + "/", name, isDir: true, tier: meta.tier, penalty: meta.penalty });
-    }
-    for (const hd of HARD_EXCLUDE_DIRS) {
-      if (!existsSync(join(opts.cwd, hd))) continue;
-      const meta = computePenaltyMeta(hd, true, false);
-      candidates.push({ path: hd + "/", name: hd, isDir: true, tier: meta.tier, penalty: meta.penalty });
-    }
-  } else {
-    const rel = (s: string) => {
-      let r = s.startsWith(cwd + "/") ? s.slice(cwd.length + 1) : s;
-      return r.startsWith("./") ? r.slice(2) : r;
-    };
-    const fdExcludes = [...HARD_EXCLUDE_DIRS].flatMap(d => ["--exclude", d]);
-    const fdOpts = { ...SPAWN_OPTS, cwd: undefined as string | undefined };
-
-    const r1 = spawnSync("fd", ["--type", "f", "--hidden", "--no-ignore", ...fdExcludes, ".", cwd], fdOpts);
-    if (r1.status === 0 && r1.stdout) {
-      for (const raw of r1.stdout.trim().split("\n").filter(Boolean)) {
-        const f = rel(raw);
-        const name = f.split("/").pop() || f;
-        const meta = computePenaltyMeta(f, false, false);
-        candidates.push({ path: f, name, isDir: false, tier: meta.tier, penalty: meta.penalty });
-      }
-    }
-    const r2 = spawnSync("fd", ["--type", "d", "--hidden", "--no-ignore", ...fdExcludes, ".", cwd], fdOpts);
-    if (r2.status === 0 && r2.stdout) {
-      for (const raw of r2.stdout.trim().split("\n").filter(Boolean)) {
-        const d = rel(raw).replace(/\/$/, "");
-        const name = d.split("/").pop() || d;
-        const meta = computePenaltyMeta(d, true, false);
-        candidates.push({ path: d + "/", name, isDir: true, tier: meta.tier, penalty: meta.penalty });
-      }
-    }
-    for (const hd of HARD_EXCLUDE_DIRS) {
-      if (!existsSync(join(cwd, hd))) continue;
-      const meta = computePenaltyMeta(hd, true, false);
-      candidates.push({ path: hd + "/", name: hd, isDir: true, tier: meta.tier, penalty: meta.penalty });
+function atPrefix(text: string): string | null {
+  for (let i = text.length - 1; i >= 0; i--) {
+    if (text[i] !== "@") continue;
+    const b = text[i - 1];
+    if (i === 0 || AT_BOUNDARY.has(b)) {
+      return text.slice(i);
     }
+    return null;
   }
-  return candidates;
+  return null;
 }
 
-// ─── Match scoring ───────────────────────────────────────────────────────
-
-function fuzzyScore(text: string, query: string): number {
-  let qi = 0, firstMatch = -1, lastMatch = -1;
-  for (let ti = 0; ti < text.length && qi < query.length; ti++) {
-    if (text[ti] === query[qi]) {
-      if (firstMatch < 0) firstMatch = ti;
-      lastMatch = ti;
-      qi++;
-    }
-  }
-  if (qi < query.length) return 0;
-  const span = lastMatch - firstMatch + 1;
-  return Math.max(10, 200 - span * 3 - text.length);
+function isGitIgnored(item: MixedItem): boolean {
+  return item.type === "file" && item.item.gitStatus === "ignored";
 }
 
-function computeMatchScore(candidate: FileCandidate, query: string): number {
-  const { path: filePath, name, isDir } = candidate;
-  const stem = name.replace(/\.[^.]+$/, "");
-  const parts = filePath.replace(/\/$/, "").split("/");
-  const inDir = parts.slice(0, -1).some(d => d.includes(query));
-
-  let s = 0;
-  if (stem === query) s = isDir ? 1500 : 1050;
-  else if (name.startsWith(query + ".") || name.startsWith(query + "_")) s = 1000;
-  else if (name.startsWith(query)) s = 900;
-  else if (name.includes(query)) s = 600;
-  else if (filePath.includes(query)) s = 300;
-  else s = fuzzyScore(name, query);
-
-  if (!s) return 0;
-  if (isDir) s += 100;
-  if (inDir) s += 50;
-  return s;
+function toAutocompleteItem(item: MixedItem): AutocompleteItem {
+  const path = item.item.relativePath;
+  const label = item.type === "file"
+    ? item.item.fileName
+    : path.replace(/\/+$/, "").split("/").pop() || path;
+  return {
+    value: "@" + path,
+    label,
+    description: path,
+  };
 }
 
-function smartSearch(candidates: FileCandidate[], query: string): string[] {
-  if (!query) {
-    const visible = candidates.filter(c => c.tier === 0 || c.tier === 3 || c.tier === 4);
-    return visible
-      .sort((a, b) => {
-        if (a.isDir !== b.isDir) return a.isDir ? -1 : 1;
-        return b.penalty - a.penalty || a.path.localeCompare(b.path);
-      })
-      .slice(0, 20)
-      .map(c => c.path);
-  }
-  const tokens = query.split(/\s+/).filter(Boolean);
-  if (tokens.length === 1) {
-    const scored = candidates
-      .map(c => ({ path: c.path, total: computeMatchScore(c, tokens[0]!) + c.penalty, matchScore: computeMatchScore(c, tokens[0]!) }))
-      .filter(x => x.matchScore > 0);
-    return scored
-      .sort((a, b) => b.total - a.total || a.path.localeCompare(b.path))
-      .slice(0, 20)
-      .map(x => x.path);
-  }
-  const seen = new Set<string>();
-  for (const t of tokens) {
-    const scored = candidates
-      .map(c => ({ path: c.path, total: computeMatchScore(c, t) + c.penalty, matchScore: computeMatchScore(c, t) }))
-      .filter(x => x.matchScore > 0)
-      .sort((a, b) => b.total - a.total)
-      .slice(0, 20);
-    for (const { path } of scored) seen.add(path);
-  }
-  return [...seen].slice(0, 20);
+interface BuiltResult {
+  items: AutocompleteItem[];
 }
 
-function atPrefix(text: string): string | null {
-  for (let i = text.length - 1; i >= 0; i--) {
-    if (text[i] !== "@") continue;
-    const b = text[i - 1];
-    if (i === 0 || b === " " || b === "\t" || b === "(" || b === "[") return text.slice(i);
-    return null;
-  }
-  return null;
+/** Take FFF's ranked results, drop git-ignored files, and for non-empty
+ *  queries keep only paths that contain the query as a substring. This
+ *  stops FFF's loose fuzzy matching from ranking every file in a directory
+ *  that happens to share a few letters (e.g. "mm/" for query "mmc"). */
+function buildResult(items: MixedItem[], lowerQuery: string): BuiltResult | null {
+  const filtered = items
+    .filter((it) => !isGitIgnored(it))
+    .filter(
+      (it) =>
+        !lowerQuery || it.item.relativePath.toLowerCase().includes(lowerQuery),
+    )
+    .slice(0, AUTOCOMPLETE_LIMIT)
+    .map(toAutocompleteItem);
+  return filtered.length ? { items: filtered } : null;
 }
 
-export const __smartAtTest = {
-  computePenaltyMeta, computePenalty, fuzzyScore, computeMatchScore, smartSearch, atPrefix,
-};
+export const __smartAtTest = { atPrefix };
+
+/** Active FileFinder for the current session; freed in session_shutdown
+ *  to avoid leaking FFF's native handle + LMDB mmap regions. */
+let currentFinder: FileFinder | null = null;
 
 export const smartAtModule: Module = {
   name: "smart-at",
   hooks: {
     session_start: [
-      (_event: any, ctx: ExtensionContext) => {
+      async (_event: any, ctx: ExtensionContext) => {
         const cwd = String(ctx.cwd || "").trim();
-        let cache: FileCandidate[] | null = null;
+        const created = FileFinder.create({ basePath: cwd || "." });
+        if (!created.ok) {
+          // FFF not available on this platform; silently skip.
+          return;
+        }
+
+        const finder = created.value;
+        currentFinder = finder;
 
-        const getOrBuildCache = () => (cache ??= collectCandidates(cwd));
-        const clearCache = () => { cache = null; };
+        // Start the scan in the background. We don't wait for it here so
+        // session_start returns immediately; the provider queries the
+        // (possibly partial) index and FFF returns whatever it has indexed
+        // so far. Full results appear as the scan progresses.
+        void finder.waitForScan(60_000);
 
         ctx.ui.addAutocompleteProvider((orig: any) => ({
-          getSuggestions: (lines: any, cl: any, cc: any, opts: any) => {
+          getSuggestions: (
+            lines: string[],
+            cl: number,
+            cc: number,
+            opts?: { signal?: AbortSignal },
+          ) => {
             const prefix = atPrefix((lines[cl] || "").slice(0, cc));
             if (!prefix) {
-              clearCache();
               ctx.ui.setWidget("smart-at", undefined);
               return orig.getSuggestions(lines, cl, cc, opts);
             }
-            const candidates = getOrBuildCache();
-            const results = smartSearch(candidates, prefix.slice(1));
-            if (!results.length) {
+
+            if (currentFinder !== finder || finder.isDestroyed) {
               ctx.ui.setWidget("smart-at", undefined);
-              return null;
+              return orig.getSuggestions(lines, cl, cc, opts);
             }
-            ctx.ui.setWidget("smart-at", ["\x1b[2mpowered by decorated-pi\x1b[0m"]);
-            return Promise.resolve({
-              items: results.map((f: string) => ({
-                value: "@" + f,
-                label: f.replace(/\/$/, "").split("/").pop() || f,
-                description: f,
-              })),
-              prefix,
+
+            // Respect upstream cancellation so fast typing doesn't pile up
+            // expensive FFF queries.
+            if (opts?.signal?.aborted) {
+              return orig.getSuggestions(lines, cl, cc, opts);
+            }
+
+            const query = prefix.slice(1);
+            const lowerQuery = query.toLowerCase();
+            const r = finder.mixedSearch(lowerQuery, {
+              pageSize: lowerQuery ? FFF_SUPERSET : AUTOCOMPLETE_LIMIT,
             });
+            if (!r.ok) {
+              ctx.ui.setWidget("smart-at", undefined);
+              return null;
+            }
+
+            const result = buildResult(r.value.items, lowerQuery);
+            if (!result) {
+              ctx.ui.setWidget("smart-at", undefined);
+              return null;
+            }
+
+            ctx.ui.setWidget("smart-at", [WIDGET_FOOTER]);
+            return Promise.resolve({ ...result, prefix });
           },
           applyCompletion: (...args: any[]) => {
-            clearCache();
             ctx.ui.setWidget("smart-at", undefined);
             return orig.applyCompletion.apply(orig, args);
           },
-          shouldTriggerFileCompletion: orig.shouldTriggerFileCompletion?.bind(orig),
+          shouldTriggerFileCompletion:
+            orig.shouldTriggerFileCompletion?.bind(orig),
         }));
       },
     ],
+    session_shutdown: [
+      (_event: any, _ctx: ExtensionContext) => {
+        if (currentFinder && !currentFinder.isDestroyed) {
+          currentFinder.destroy();
+        }
+        currentFinder = null;
+      },
+    ],
   },
 };
 
 export function setupSmartAt(sk: Skeleton): void {
-  sk.declareDependency({
-    label: "fd",
-    check: () => commandExists("fd") || isGitWorkTree(process.cwd()),
-  });
   sk.register(smartAtModule);
 }

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "decorated-pi",
-    "version": "0.7.0",
+    "version": "0.7.1",
     "description": "decorated-pi is a practical enhancement pack for pi coding agent — token-efficient workflow, cache-friendly design, and smarter tools.",
     "keywords": [
         "pi",
@@ -12,7 +12,11 @@
         "lsp",
         "lsp-client",
         "secret-redaction",
-        "codegraph"
+        "codegraph",
+        "fff",
+        "smart-at",
+        "fuzzy-search",
+        "file-finder"
     ],
     "license": "MIT",
     "repository": {
@@ -22,6 +26,7 @@
     "homepage": "https://github.com/lcwecker/decorated-pi#readme",
     "bugs": "https://github.com/lcwecker/decorated-pi/issues",
     "dependencies": {
+        "@ff-labs/fff-node": "^0.9.4",
         "@modelcontextprotocol/sdk": "^1.29.0",
         "file-type": "^21.3.4",
         "openai": "^6.37.0"
@@ -39,13 +44,15 @@
     },
     "devDependencies": {
         "@types/node": "^22.15.3",
+        "@vitest/coverage-v8": "4.1.8",
         "depcheck": "^1.4.7",
         "publint": "^0.3.21",
-        "vitest": "^4.1.6"
+        "vitest": "4.1.8"
     },
     "scripts": {
         "prepack": "depcheck --fail-on-missing",
-        "test": "vitest run",
+        "test": "vitest run --coverage",
+        "bench": "vitest bench --run",
         "typecheck": "tsc --noEmit",
         "prepublishOnly": "npm install --no-audit --no-fund && npm test && npm run typecheck && depcheck --fail-on-missing && publint"
     }

package/tools/patch/core.ts

@@ -1076,12 +1076,18 @@ interface LineRange {
   endLine: number;
 }
 
-/** Build line offset table: offsets[i] = character offset of line i+1 (1-based) */
+/** Build line offset table: offsets[i] = character offset of line i+1 (1-based).
+ *  If the content does not end with a newline, the final line has no
+ *  trailing marker; push an extra offset at content.length so callers
+ *  like lineAtOffset / extractLineRange handle the last line correctly. */
 function buildLineOffsets(content: string): number[] {
   const offsets = [0];
   for (let i = 0; i < content.length; i++) {
     if (content[i] === "\n") offsets.push(i + 1);
   }
+  if (content.length > 0 && content[content.length - 1] !== "\n") {
+    offsets.push(content.length);
+  }
   return offsets;
 }
 

package/tools/patch/index.ts

@@ -215,8 +215,9 @@ export function registerPatchTool(pi: ExtensionAPI): void {
     ].join("\n"),
     promptSnippet: "Edits a file using exact string replacement, with anchor support.",
     promptGuidelines: [
-      "Always prefer modifying files with patch tool over bash commands or python scripts.",
-      "To prevent hallucinations: 1. Keep each edit batch ≤ 5 changes; 2. Process remaining revisions in sequential steps",
+      "The patch tool is the preferred way to modify files — use it over the write tool (full overwrite) or the bash tool (sed/echo).",
+      "When editing an existing file, use patch instead of write to avoid overwriting changes made since the file was last read.",
+      "To prevent hallucinations: 1. Keep each edit batch ≤ 5 changes; 2. Process remaining revisions in sequential steps.",
       "On repeated failures: read the file first to confirm information accuracy.",
     ],
     parameters: PatchSchema,