@ff-labs/pi-fff 0.10.5-dev.774d6bc → 0.10.5-nightly.f565d37

package/README.md

@@ -132,16 +132,28 @@ Mode precedence:
 ## Flags
 
 - `--fff-mode <mode>` — set mode (see above)
-- `--fff-frecency-db <path>` — path to frecency database (also: `FFF_FRECENCY_DB` env)
-- `--fff-history-db <path>` — path to query history database (also: `FFF_HISTORY_DB` env)
+- `--fff-frecency-db <path>` — path to frecency database (also: `FFF_FRECENCY_DB` env). Optional; see [Data](#data) for the default.
+- `--fff-history-db <path>` — path to query history database (also: `FFF_HISTORY_DB` env). Optional; see [Data](#data) for the default.
 - `--fff-enable-root-scan` — allow indexing when launched from `/` (also: `FFF_ENABLE_ROOT_SCAN=1` env). FFF refuses to init at the filesystem root by default.
 - `--fff-enable-home-scan` — index the home directory when launched from `$HOME` (also: `FFF_ENABLE_HOME_SCAN` env). Enabled by default. Disable with `--fff-enable-home-scan=false` or `FFF_ENABLE_HOME_SCAN=0` if your `$HOME` contains huge trees (toolchains, kernel sources, build outputs) that make the background index run for a long time. When launched from `$HOME` with this enabled, pi shows a warning that the whole home tree is being indexed.
 
 ## Data
 
-When database paths are provided, FFF stores:
-- frecency database — file access frequency/recency
-- history database — query-to-file selection history
+FFF uses two LMDB databases:
+- frecency database - file access frequency/recency, used to rank results
+- history database - query-to-file selection history
+
+Each path is resolved independently, in this order:
+
+1. CLI flag — `--fff-frecency-db` / `--fff-history-db`
+2. Env var — `FFF_FRECENCY_DB` / `FFF_HISTORY_DB`
+3. An existing [fff.nvim](https://github.com/dmtrKovalenko/fff.nvim) database, so pi reuses the frecency you built up in your editor:
+   - frecency: `$XDG_CACHE_HOME/nvim/fff_nvim`
+   - history: `$XDG_DATA_HOME/nvim/fff_queries`
+   - `XDG_CACHE_HOME` defaults to `~/.cache` and `XDG_DATA_HOME` to `~/.local/share`; on Windows both fall back under `%LOCALAPPDATA%\nvim-data`. Only directories count — a plain file at those paths is ignored.
+4. pi-local directory, created on demand — `$PI_CODING_AGENT_DIR/fff/{frecency,history}`, defaulting to `~/.pi/agent/fff/{frecency,history}`
+
+The extension only reads these databases; it never records the agent's own searches into your Neovim history. If a database cannot be opened, the finder starts without persistence and pi shows a warning instead of failing.
 
 No project files are uploaded anywhere by this extension. It runs locally and only uses the configured LLM through pi itself.
 

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@ff-labs/pi-fff",
   "public": true,
-  "version": "0.10.5-dev.774d6bc",
+  "version": "0.10.5-nightly.f565d37",
   "description": "pi extension: FFF-powered fuzzy file and content search",
   "type": "module",
   "license": "MIT",
@@ -40,8 +40,8 @@
     "typecheck": "tsc --noEmit"
   },
   "dependencies": {
-    "@ff-labs/fff-bun": "0.10.5-dev.774d6bc",
-    "@ff-labs/fff-node": "0.10.5-dev.774d6bc"
+    "@ff-labs/fff-bun": "0.10.5-nightly.f565d37",
+    "@ff-labs/fff-node": "0.10.5-nightly.f565d37"
   },
   "peerDependencies": {
     "@earendil-works/pi-coding-agent": "*",
@@ -49,6 +49,7 @@
     "@sinclair/typebox": "*"
   },
   "devDependencies": {
+    "@types/bun": "^1.3.8",
     "@types/node": "^22.0.0",
     "typescript": "^5.0.0"
   }

package/src/aux-finders.ts

@@ -1,8 +1,8 @@
 import fs from "node:fs";
 import path from "node:path";
 import type { FileFinderApi } from "@ff-labs/fff-node";
+import type { FilePickerFactory } from "./file-picker";
 import { HOME_DIR } from "./paths";
-import { loadSdk, SCAN_TIMEOUT_MS } from "./sdk";
 
 export const MAX_AUX = 3;
 export const IDLE_TTL_MS = 5 * 60 * 1000;
@@ -16,10 +16,9 @@ interface AuxPicker {
 export interface AuxOpts {
   enableFsRootScanning: boolean;
   enableHomeDirScanning?: boolean;
+  pickers: FilePickerFactory;
   // Called before a newly spawned aux picker starts a scan that covers $HOME.
   onHomeDirScan?: (root: string) => void;
-  frecencyDbPath?: string;
-  historyDbPath?: string;
 }
 
 export class AuxFinderPool {
@@ -101,26 +100,13 @@ export class AuxFinderPool {
       this.opts.onHomeDirScan?.(root);
     }
 
-    const { FileFinder } = await loadSdk();
-    const result = FileFinder.create({
+    const finder = await this.opts.pickers.create({
       basePath: root,
-      frecencyDbPath: this.opts.frecencyDbPath,
-      historyDbPath: this.opts.historyDbPath,
-      aiMode: true,
       enableHomeDirScanning,
       enableFsRootScanning: this.opts.enableFsRootScanning,
     });
 
-    if (!result.ok) {
-      throw new Error(`Failed to create aux file finder for ${root}: ${result.error}`);
-    }
-
-    await result.value.waitForScan(SCAN_TIMEOUT_MS);
-    const entry: AuxPicker = {
-      root,
-      finder: result.value,
-      lastUsed: Date.now(),
-    };
+    const entry: AuxPicker = { root, finder, lastUsed: Date.now() };
     this.entries.push(entry);
     return entry;
   }

package/src/file-picker.ts

@@ -0,0 +1,73 @@
+import type { FileFinderApi, InitOptions, Result } from "@ff-labs/fff-node";
+import { type FileFinderStatic, loadSdk, SCAN_TIMEOUT_MS } from "./sdk";
+
+export interface PickerOptions {
+  basePath: string;
+  enableHomeDirScanning?: boolean;
+  enableFsRootScanning?: boolean;
+}
+
+/** Opens every picker in this pi process — the cwd picker and the aux pickers —
+ * on the same frecency/history databases. */
+export class FilePickerFactory {
+  private dbDisabled = false;
+  private readonly frecencyDbPath: string;
+  private readonly historyDbPath: string;
+  private readonly onDbFailure?: (error: string) => void;
+
+  constructor(opts: {
+    frecencyDbPath: string;
+    historyDbPath: string;
+    onDbFailure?: (error: string) => void;
+  }) {
+    this.frecencyDbPath = opts.frecencyDbPath;
+    this.historyDbPath = opts.historyDbPath;
+    this.onDbFailure = opts.onDbFailure;
+  }
+
+  /** True once the databases were given up on, so pickers open without them. */
+  get databasesDisabled(): boolean {
+    return this.dbDisabled;
+  }
+
+  /** Opens a scanned, ready-to-use picker. Throws if it cannot be created. */
+  async create(options: PickerOptions): Promise<FileFinderApi> {
+    const { FileFinder } = await loadSdk();
+    const result = this.openWithDbFallback(FileFinder, options);
+
+    if (!result.ok) {
+      throw new Error(
+        `Failed to create FFF file picker for ${options.basePath}: ${result.error}`,
+      );
+    }
+
+    // waitForScan() also resolves on timeout, so this bounds startup rather
+    // than guaranteeing a complete index.
+    await result.value.waitForScan(SCAN_TIMEOUT_MS);
+    return result.value;
+  }
+
+  private openWithDbFallback(
+    FileFinder: FileFinderStatic,
+    options: PickerOptions,
+  ): Result<FileFinderApi> {
+    const init: InitOptions = { ...options, aiMode: true };
+    if (this.dbDisabled) return FileFinder.create(init);
+
+    const result = FileFinder.create({
+      ...init,
+      frecencyDbPath: this.frecencyDbPath,
+      historyDbPath: this.historyDbPath,
+    });
+    if (result.ok) return result;
+
+    // A failure here is usually transient (broken lock, corruption) and self-heals
+    // on restart, so drop the databases instead of leaving pi without a picker
+    const dbLess = FileFinder.create(init);
+    if (!dbLess.ok) return result; // db error is the more useful one to report
+
+    this.dbDisabled = true;
+    this.onDbFailure?.(result.error);
+    return dbLess;
+  }
+}

package/src/index.ts

@@ -22,9 +22,9 @@ import type {
 } from "@ff-labs/fff-node";
 import { Type } from "@sinclair/typebox";
 import { AuxFinderPool, routePathConstraint } from "./aux-finders";
+import { FilePickerFactory } from "./file-picker";
+import { isHomeDir, resolveDbPaths } from "./paths";
 import { buildQuery } from "./query";
-import { isHomeDir } from "./paths";
-import { loadSdk, SCAN_TIMEOUT_MS } from "./sdk";
 
 export { SCAN_TIMEOUT_MS } from "./sdk";
 
@@ -162,16 +162,7 @@ export function fffFileAnnotation(item: {
   return "";
 }
 
-// fff-core native definition classifier (byte-level scanner in Rust) is enabled
-// via GrepOptions.classifyDefinitions. Each GrepMatch carries isDefinition for
-// downstream consumers; pi-fff does NOT use it to re-sort.
-//
-// Ordering policy: NO CUSTOM SORTING. The engine already returns items in
-// frecency order (most-accessed files first). pi-fff only groups consecutive
-// matches into per-file blocks and preserves whatever order the engine
-// provided — inside a file we keep matches in source-line order because the
-// engine emits them that way.
-
+// DO NOT ATTEMPT TO RESORT OUTPUT HERE IT ONLY CONFUSES MODELS
 function formatGrepOutput(result: GrepResult): string {
   if (result.items.length === 0) return "No matches found";
 
@@ -179,7 +170,6 @@ function formatGrepOutput(result: GrepResult): string {
   // This preserves native frecency ordering across files without re-sorting.
   const lines: string[] = [];
   let currentFile = "";
-  let shown = 0;
 
   for (const match of result.items) {
     if (match.relativePath !== currentFile) {
@@ -194,7 +184,6 @@ function formatGrepOutput(result: GrepResult): string {
     });
 
     lines.push(` ${match.lineNumber}: ${truncateLine(match.lineContent)}`);
-    shown++;
 
     match.contextAfter?.forEach((line: string, i: number) => {
       const lineNum = match.lineNumber + 1 + i;
@@ -318,15 +307,14 @@ export default function fffExtension(pi: ExtensionAPI) {
 
   const toolNames = resolveToolNames(currentMode);
 
-  // DB path resolution: flag > env > undefined (no persistent DBs)
-  const frecencyDbPath =
-    (pi.getFlag("fff-frecency-db") as string | undefined) ??
-    process.env.FFF_FRECENCY_DB ??
-    undefined;
-  const historyDbPath =
-    (pi.getFlag("fff-history-db") as string | undefined) ??
-    process.env.FFF_HISTORY_DB ??
-    undefined;
+  // DB path resolution: flag > env > existing fff.nvim db > pi-local data dir.
+  const resolvedDbPaths = resolveDbPaths({
+    frecency:
+      (pi.getFlag("fff-frecency-db") as string | undefined) ??
+      process.env.FFF_FRECENCY_DB,
+    history:
+      (pi.getFlag("fff-history-db") as string | undefined) ?? process.env.FFF_HISTORY_DB,
+  });
 
   // flag (boolean) > env ("1"/"true", or "0"/"false") > default.
   function resolveBoolOpt(flagName: string, envName: string, fallback = false): boolean {
@@ -380,12 +368,21 @@ export default function fffExtension(pi: ExtensionAPI) {
     );
   }
 
+  const pickers = new FilePickerFactory({
+    frecencyDbPath: resolvedDbPaths.frecency,
+    historyDbPath: resolvedDbPaths.history,
+    onDbFailure: (error) =>
+      uiCtx?.ui.notify(
+        `(fff): Failed to open frecency/history database (${error}). Continuing without frecency persistence.`,
+        "error",
+      ),
+  });
+
   const auxPool = new AuxFinderPool({
     enableFsRootScanning,
     enableHomeDirScanning,
     onHomeDirScan: warnHomeDirScan,
-    frecencyDbPath,
-    historyDbPath,
+    pickers,
   });
 
   // in case cwd changes we need to figure this out
@@ -402,22 +399,14 @@ export default function fffExtension(pi: ExtensionAPI) {
         finderCwd = null;
       }
 
-      const { FileFinder } = await loadSdk();
-      const result = FileFinder.create({
+      // if the dbs can't be opened the factory falls back to a db-less picker,
+      // e.g. when some other process corrupts the lock
+      mainFinder = await pickers.create({
         basePath: cwd,
-        frecencyDbPath,
-        historyDbPath,
-        aiMode: true,
         enableHomeDirScanning,
         enableFsRootScanning,
       });
-
-      if (!result.ok)
-        throw new Error(`Failed to create FFF file finder: ${result.error}`);
-
-      mainFinder = result.value;
       finderCwd = cwd;
-      await mainFinder.waitForScan(SCAN_TIMEOUT_MS);
       return mainFinder;
     })().finally(() => {
       finderPromise = null;

package/src/paths.ts

@@ -1,9 +1,67 @@
+import fs from "node:fs";
 import os from "node:os";
 import path from "node:path";
 
 // Resolved once per process: os.homedir() hits the env/passwd on every call.
 export const HOME_DIR = path.resolve(os.homedir());
 
+// fff.nvim db dir names (`frecency.db_path` / `history.db_path` in lua/fff/conf.lua).
+const NVIM_FRECENCY_DIR = "fff_nvim";
+const NVIM_HISTORY_DIR = "fff_queries";
+
+export interface DbPaths {
+  frecency: string;
+  history: string;
+}
+
 export function isHomeDir(dir: string): boolean {
   return path.resolve(dir) === HOME_DIR;
 }
+
+// Resolution order: explicit override > existing fff.nvim db > pi-local data dir.
+// Reusing the nvim db lets pi rank files by the frecency the user built in their editor.
+export function resolveDbPaths(overrides: {
+  frecency?: string;
+  history?: string;
+}): DbPaths {
+  return {
+    frecency:
+      overrides.frecency ??
+      existingDir(nvimCacheDir(), NVIM_FRECENCY_DIR) ??
+      path.join(piDataDir(), "fff", "frecency"),
+    history:
+      overrides.history ??
+      existingDir(nvimDataDir(), NVIM_HISTORY_DIR) ??
+      path.join(piDataDir(), "fff", "history"),
+  };
+}
+
+function nvimCacheDir(): string {
+  const xdg = process.env.XDG_CACHE_HOME;
+  if (xdg) return path.join(xdg, "nvim");
+  if (process.platform === "win32" && process.env.LOCALAPPDATA)
+    return path.join(process.env.LOCALAPPDATA, "nvim-data", "cache");
+  return path.join(HOME_DIR, ".cache", "nvim");
+}
+
+function nvimDataDir(): string {
+  const xdg = process.env.XDG_DATA_HOME;
+  if (xdg) return path.join(xdg, "nvim");
+  if (process.platform === "win32" && process.env.LOCALAPPDATA)
+    return path.join(process.env.LOCALAPPDATA, "nvim-data");
+  return path.join(HOME_DIR, ".local", "share", "nvim");
+}
+
+function piDataDir(): string {
+  return process.env.PI_CODING_AGENT_DIR ?? path.join(HOME_DIR, ".pi", "agent");
+}
+
+// LMDB environments are directories, so a stray file at the same path is not a db.
+function existingDir(parent: string, name: string): string | undefined {
+  const candidate = path.join(parent, name);
+  try {
+    return fs.statSync(candidate).isDirectory() ? candidate : undefined;
+  } catch {
+    return undefined;
+  }
+}