@pify/search 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/README.md +66 -0
- package/extensions/search.ts +186 -0
- package/package.json +86 -0
- package/skills/search/SKILL.md +37 -0
- package/src/builtin.ts +228 -0
- package/src/engine.ts +196 -0
- package/src/format.ts +61 -0
- package/src/frecency.ts +83 -0
- package/src/fuzzy.ts +180 -0
- package/src/match.ts +119 -0
- package/src/trigram.ts +184 -0
- package/src/walk.ts +0 -0
package/LICENSE
ADDED
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 pifydev
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
package/README.md
ADDED
|
@@ -0,0 +1,66 @@
|
|
|
1
|
+
# @pify/search
|
|
2
|
+
|
|
3
|
+
Fuzzy file finding and indexed content search for [pi](https://github.com/earendil-works/pi) — fast when a native index is available, and working when it is not.
|
|
4
|
+
|
|
5
|
+
Part of the [Pify suite](https://github.com/pifydev). Install with [`pify install search`](https://github.com/pifydev/cli) or `pi install npm:@pify/search`.
|
|
6
|
+
|
|
7
|
+
## Why
|
|
8
|
+
|
|
9
|
+
pi's `find` and `grep` spawn a process and read the tree on every call. That is the right design for a one-shot command and the wrong one for an agent, which asks over and over inside a single session. An index built once and kept current answers the fiftieth question as fast as the first.
|
|
10
|
+
|
|
11
|
+
The other half is the shape of the question. `find` wants a glob; people want *"the auth route file — you know the one"*. Fuzzy, typo-tolerant matching with results ranked by what you have actually been working on answers that; a glob does not.
|
|
12
|
+
|
|
13
|
+
## Tools
|
|
14
|
+
|
|
15
|
+
### `fffind`
|
|
16
|
+
|
|
17
|
+
| Parameter | Type | Notes |
|
|
18
|
+
|---|---|---|
|
|
19
|
+
| `query` | string | Part of a name or path; typos tolerated |
|
|
20
|
+
| `limit` | number, optional | Per page, default 20 |
|
|
21
|
+
| `cursor` | string, optional | From a previous call |
|
|
22
|
+
|
|
23
|
+
`worktre entr` finds `worktree/src/enter.ts`. Results are ranked: an exact filename beats a matching stem, which beats a prefix, which beats a directory that merely contains the query — and recently edited or git-modified files rise, because that is what you are probably looking for.
|
|
24
|
+
|
|
25
|
+
### `ffgrep`
|
|
26
|
+
|
|
27
|
+
| Parameter | Type | Notes |
|
|
28
|
+
|---|---|---|
|
|
29
|
+
| `pattern` | string | What to search for |
|
|
30
|
+
| `mode` | `literal` \| `regex` \| `fuzzy` | Default `literal` |
|
|
31
|
+
| `caseSensitive` | boolean, optional | Default false |
|
|
32
|
+
| `limit` / `cursor` | | Paging, default 20 per page |
|
|
33
|
+
|
|
34
|
+
Three modes because three different questions get asked: the exact string, a shape, and *"something like this"* for when you do not know how it is spelled.
|
|
35
|
+
|
|
36
|
+
## Two engines, one interface
|
|
37
|
+
|
|
38
|
+
**Fast path** — [`@ff-labs/fff-node`](https://github.com/dmtrKovalenko/fff), a Rust index with a live file watcher, typo-resistant matching, git status and frecency. It scans this whole suite in about 80ms. Installed as an *optional* dependency: if the platform binary lands, it is used.
|
|
39
|
+
|
|
40
|
+
**Fallback** — pure TypeScript, no dependencies. A trigram index for content, a fuzzy scorer for paths, an `fs.watch` subscription to stay current. Slower, and it runs wherever pi runs.
|
|
41
|
+
|
|
42
|
+
The fallback is the point. A native binary is a promise you cannot always keep: an unsupported platform, a locked-down install, a blocked postinstall — any of those, and a binary-only search extension is one that silently does nothing. `/search` says which engine is live.
|
|
43
|
+
|
|
44
|
+
Both are checked against each other on a real tree (`test/live/engines.mjs`, 21/21): the same files found, the same literal matches, the same refusal to search `node_modules`, and cursors that advance rather than repeat.
|
|
45
|
+
|
|
46
|
+
## How the content index works
|
|
47
|
+
|
|
48
|
+
Every overlapping three-byte window of every text file is a *trigram*, packed into one number and mapped to the files containing it. A search extracts the trigrams its pattern must contain and intersects those posting lists, so only files that could match are ever read. (The design is [tgrep](https://github.com/microsoft/tgrep)'s, which reports up to 52× over ripgrep on very large trees.)
|
|
49
|
+
|
|
50
|
+
The index only ever **narrows**; every surviving candidate is still matched for real, so a wrong candidate costs time and never correctness. The rule that makes it safe: a pattern with nothing indexable — `\d+`, a fuzzy query, one branch of an alternation that could match anywhere — reports "no candidate set is safe" and everything is read. Confusing *that* with "nothing matched" is how an index starts silently hiding results, so the two are different values throughout.
|
|
51
|
+
|
|
52
|
+
## Ranking
|
|
53
|
+
|
|
54
|
+
Frecency decays on a three-day half-life — an agent session is shorter and more concentrated than a human's week, so yesterday's file should not outrank today's. Every `read`, `edit` or `write` in the session counts as an access. History is capped at seven days and 128 timestamps per file, so the store cannot grow without bound.
|
|
55
|
+
|
|
56
|
+
## pi's own tools are left alone
|
|
57
|
+
|
|
58
|
+
This package adds two tools; it does not replace `find`, `grep` or `multi_grep`. Replacing them would put every search in the session behind whichever engine happened to load, and a fallback that is slower than the thing it replaced is not an improvement anyone asked for. Use `fffind`/`ffgrep` when a search is worth an index; the built-ins are still there when it is not.
|
|
59
|
+
|
|
60
|
+
## Command
|
|
61
|
+
|
|
62
|
+
`/search` — which engine is running, the indexed root, and how many files it holds.
|
|
63
|
+
|
|
64
|
+
## License
|
|
65
|
+
|
|
66
|
+
MIT © [Pify maintainers](https://github.com/pifydev)
|
|
@@ -0,0 +1,186 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* @pify/search — fuzzy file finding and indexed content search for pi.
|
|
3
|
+
*
|
|
4
|
+
* pi's `find` and `grep` spawn a process and read the tree on every call.
|
|
5
|
+
* That is the right design for a one-shot command and the wrong one for an
|
|
6
|
+
* agent, which asks over and over inside a single session. An index built once
|
|
7
|
+
* and kept current answers the second question and the fiftieth from memory.
|
|
8
|
+
*
|
|
9
|
+
* Two engines behind one interface. The fast path is `@ff-labs/fff-node`, a
|
|
10
|
+
* Rust index with a live watcher, typo-resistant matching, git status and
|
|
11
|
+
* frecency — it scans this suite in about 80ms. The fallback is pure
|
|
12
|
+
* TypeScript with no dependencies, because a native binary is a promise you
|
|
13
|
+
* cannot always keep: an unsupported platform or a blocked postinstall would
|
|
14
|
+
* otherwise leave a search extension that does nothing.
|
|
15
|
+
*
|
|
16
|
+
* pi's own tools are left alone. Replacing `grep` and `find` would put every
|
|
17
|
+
* search in the session behind whichever engine loaded, and a fallback that is
|
|
18
|
+
* slower than the thing it replaced is not an improvement anyone asked for.
|
|
19
|
+
*/
|
|
20
|
+
import {
|
|
21
|
+
getAgentDir,
|
|
22
|
+
type ExtensionAPI,
|
|
23
|
+
type ExtensionContext,
|
|
24
|
+
} from "@earendil-works/pi-coding-agent";
|
|
25
|
+
import { StringEnum } from "@earendil-works/pi-ai";
|
|
26
|
+
import { Type } from "typebox";
|
|
27
|
+
|
|
28
|
+
import { createHash } from "node:crypto";
|
|
29
|
+
import { mkdirSync, readFileSync, writeFileSync } from "node:fs";
|
|
30
|
+
import { dirname, join } from "node:path";
|
|
31
|
+
|
|
32
|
+
import { loadFff, type SearchEngine } from "../src/engine.ts";
|
|
33
|
+
import { builtinEngine } from "../src/builtin.ts";
|
|
34
|
+
import { parseHistory, pruneHistory, type History } from "../src/frecency.ts";
|
|
35
|
+
import { formatFiles, formatMatches, formatStatus } from "../src/format.ts";
|
|
36
|
+
|
|
37
|
+
const READY_TIMEOUT_MS = 20_000;
|
|
38
|
+
|
|
39
|
+
export default function searchExtension(pi: ExtensionAPI) {
|
|
40
|
+
let engine: SearchEngine | null = null;
|
|
41
|
+
let starting: Promise<SearchEngine | null> | null = null;
|
|
42
|
+
let root = "";
|
|
43
|
+
let historyFile: string | null = null;
|
|
44
|
+
|
|
45
|
+
function historyPath(cwd: string): string {
|
|
46
|
+
const key = createHash("sha256").update(cwd.toLowerCase()).digest("hex").slice(0, 12);
|
|
47
|
+
return join(getAgentDir(), "pify-search", `${key}.json`);
|
|
48
|
+
}
|
|
49
|
+
|
|
50
|
+
function loadHistory(): History {
|
|
51
|
+
if (!historyFile) return {};
|
|
52
|
+
try {
|
|
53
|
+
return pruneHistory(parseHistory(readFileSync(historyFile, "utf8")), Date.now());
|
|
54
|
+
} catch {
|
|
55
|
+
return {};
|
|
56
|
+
}
|
|
57
|
+
}
|
|
58
|
+
|
|
59
|
+
function saveHistory(history: History): void {
|
|
60
|
+
if (!historyFile) return;
|
|
61
|
+
try {
|
|
62
|
+
mkdirSync(dirname(historyFile), { recursive: true });
|
|
63
|
+
writeFileSync(historyFile, `${JSON.stringify(pruneHistory(history, Date.now()))}\n`);
|
|
64
|
+
} catch {
|
|
65
|
+
// A history that cannot be written costs ranking, never a search.
|
|
66
|
+
}
|
|
67
|
+
}
|
|
68
|
+
|
|
69
|
+
/** Start once; every caller awaits the same start. */
|
|
70
|
+
async function ensureEngine(ctx: ExtensionContext): Promise<SearchEngine | null> {
|
|
71
|
+
if (engine) return engine;
|
|
72
|
+
if (!starting) {
|
|
73
|
+
root = ctx.cwd;
|
|
74
|
+
historyFile = historyPath(ctx.cwd);
|
|
75
|
+
starting = (async () => {
|
|
76
|
+
const fast = await loadFff(root);
|
|
77
|
+
const chosen =
|
|
78
|
+
fast ?? builtinEngine(root, { history: loadHistory(), onHistoryChange: saveHistory });
|
|
79
|
+
await chosen.ready(READY_TIMEOUT_MS);
|
|
80
|
+
engine = chosen;
|
|
81
|
+
return chosen;
|
|
82
|
+
})();
|
|
83
|
+
}
|
|
84
|
+
return starting;
|
|
85
|
+
}
|
|
86
|
+
|
|
87
|
+
// ── Tools ────────────────────────────────────────────────────────────
|
|
88
|
+
|
|
89
|
+
pi.registerTool({
|
|
90
|
+
name: "fffind",
|
|
91
|
+
label: "Find files",
|
|
92
|
+
description:
|
|
93
|
+
"Find files by name or path with fuzzy, typo-tolerant matching, ranked so the file you meant " +
|
|
94
|
+
"comes first — recently edited and git-modified files rank higher. Prefer this over find/ls " +
|
|
95
|
+
"when you know roughly what the file is called but not exactly where it is. Returns a cursor " +
|
|
96
|
+
"for the next page when there are more results.",
|
|
97
|
+
parameters: Type.Object({
|
|
98
|
+
query: Type.String({ description: "Part of the filename or path; typos are tolerated" }),
|
|
99
|
+
limit: Type.Optional(Type.Number({ description: "Results per page (default 20)" })),
|
|
100
|
+
cursor: Type.Optional(Type.String({ description: "Cursor from a previous call" })),
|
|
101
|
+
}),
|
|
102
|
+
async execute(_id, params: { query: string; limit?: number; cursor?: string }, _signal, _onUpdate, ctx) {
|
|
103
|
+
const active = await ensureEngine(ctx as ExtensionContext);
|
|
104
|
+
if (!active) throw new Error("No search engine available.");
|
|
105
|
+
const page = await active.find(params.query.trim(), {
|
|
106
|
+
limit: Math.min(100, Math.max(1, params.limit ?? 20)),
|
|
107
|
+
...(params.cursor ? { cursor: params.cursor } : {}),
|
|
108
|
+
});
|
|
109
|
+
return {
|
|
110
|
+
content: [{ type: "text", text: formatFiles(page, params.query) }],
|
|
111
|
+
details: { total: page.total, cursor: page.cursor, engine: active.name },
|
|
112
|
+
};
|
|
113
|
+
},
|
|
114
|
+
});
|
|
115
|
+
|
|
116
|
+
pi.registerTool({
|
|
117
|
+
name: "ffgrep",
|
|
118
|
+
label: "Search contents",
|
|
119
|
+
description:
|
|
120
|
+
"Search file contents from an in-memory index rather than re-reading the tree. mode=literal " +
|
|
121
|
+
"(default) for an exact string, regex for a pattern, fuzzy when you are unsure of the exact " +
|
|
122
|
+
"wording. Prefer this over grep for repeated searches in one session. Returns a cursor for " +
|
|
123
|
+
"the next page when there are more matches.",
|
|
124
|
+
parameters: Type.Object({
|
|
125
|
+
pattern: Type.String({ description: "What to search for" }),
|
|
126
|
+
mode: Type.Optional(StringEnum(["literal", "regex", "fuzzy"] as const)),
|
|
127
|
+
caseSensitive: Type.Optional(Type.Boolean({ description: "Default false" })),
|
|
128
|
+
limit: Type.Optional(Type.Number({ description: "Matches per page (default 20)" })),
|
|
129
|
+
cursor: Type.Optional(Type.String({ description: "Cursor from a previous call" })),
|
|
130
|
+
}),
|
|
131
|
+
async execute(
|
|
132
|
+
_id,
|
|
133
|
+
params: { pattern: string; mode?: "literal" | "regex" | "fuzzy"; caseSensitive?: boolean; limit?: number; cursor?: string },
|
|
134
|
+
_signal,
|
|
135
|
+
_onUpdate,
|
|
136
|
+
ctx,
|
|
137
|
+
) {
|
|
138
|
+
const active = await ensureEngine(ctx as ExtensionContext);
|
|
139
|
+
if (!active) throw new Error("No search engine available.");
|
|
140
|
+
const page = await active.grep(params.pattern, {
|
|
141
|
+
mode: params.mode ?? "literal",
|
|
142
|
+
caseInsensitive: params.caseSensitive !== true,
|
|
143
|
+
limit: Math.min(100, Math.max(1, params.limit ?? 20)),
|
|
144
|
+
...(params.cursor ? { cursor: params.cursor } : {}),
|
|
145
|
+
});
|
|
146
|
+
return {
|
|
147
|
+
content: [{ type: "text", text: formatMatches(page, params.pattern, params.mode ?? "literal") }],
|
|
148
|
+
details: { total: page.total, cursor: page.cursor, engine: active.name },
|
|
149
|
+
};
|
|
150
|
+
},
|
|
151
|
+
});
|
|
152
|
+
|
|
153
|
+
// ── Frecency: what the session actually touched ──────────────────────
|
|
154
|
+
|
|
155
|
+
pi.on("tool_call", async (event) => {
|
|
156
|
+
const name = (event as { toolName?: string }).toolName;
|
|
157
|
+
if (name !== "read" && name !== "edit" && name !== "write") return undefined;
|
|
158
|
+
const path = (event as { input?: { path?: unknown } }).input?.path;
|
|
159
|
+
if (typeof path === "string" && engine?.touch) engine.touch(path);
|
|
160
|
+
return undefined;
|
|
161
|
+
});
|
|
162
|
+
|
|
163
|
+
// ── Lifecycle ────────────────────────────────────────────────────────
|
|
164
|
+
|
|
165
|
+
pi.on("session_start", async (_event, ctx) => {
|
|
166
|
+
// Indexing starts now rather than at the first search, so the first
|
|
167
|
+
// question is as fast as the fiftieth. Deliberately not awaited: a large
|
|
168
|
+
// tree must not hold up the session.
|
|
169
|
+
void ensureEngine(ctx);
|
|
170
|
+
});
|
|
171
|
+
|
|
172
|
+
pi.on("session_shutdown", async () => {
|
|
173
|
+
engine?.dispose();
|
|
174
|
+
engine = null;
|
|
175
|
+
starting = null;
|
|
176
|
+
});
|
|
177
|
+
|
|
178
|
+
pi.registerCommand("search", {
|
|
179
|
+
description: "Which search engine is running, and what it has indexed",
|
|
180
|
+
handler: async (_args, ctx) => {
|
|
181
|
+
if (!ctx.hasUI) return;
|
|
182
|
+
const active = await ensureEngine(ctx);
|
|
183
|
+
ctx.ui.notify(formatStatus(active, root), "info");
|
|
184
|
+
},
|
|
185
|
+
});
|
|
186
|
+
}
|
package/package.json
ADDED
|
@@ -0,0 +1,86 @@
|
|
|
1
|
+
{
|
|
2
|
+
"name": "@pify/search",
|
|
3
|
+
"version": "0.1.0",
|
|
4
|
+
"description": "Fuzzy file finding and indexed content search for pi, with no native binary and no daemon",
|
|
5
|
+
"keywords": [
|
|
6
|
+
"pi-package",
|
|
7
|
+
"pi-extension",
|
|
8
|
+
"pi",
|
|
9
|
+
"pify",
|
|
10
|
+
"search",
|
|
11
|
+
"grep",
|
|
12
|
+
"fuzzy",
|
|
13
|
+
"trigram"
|
|
14
|
+
],
|
|
15
|
+
"homepage": "https://github.com/pifydev/search#readme",
|
|
16
|
+
"bugs": {
|
|
17
|
+
"url": "https://github.com/pifydev/search/issues"
|
|
18
|
+
},
|
|
19
|
+
"repository": {
|
|
20
|
+
"type": "git",
|
|
21
|
+
"url": "git+https://github.com/pifydev/search.git"
|
|
22
|
+
},
|
|
23
|
+
"license": "MIT",
|
|
24
|
+
"author": "Pify maintainers",
|
|
25
|
+
"type": "module",
|
|
26
|
+
"engines": {
|
|
27
|
+
"node": ">=22.19.0"
|
|
28
|
+
},
|
|
29
|
+
"files": [
|
|
30
|
+
"extensions",
|
|
31
|
+
"src",
|
|
32
|
+
"skills",
|
|
33
|
+
"README.md",
|
|
34
|
+
"LICENSE"
|
|
35
|
+
],
|
|
36
|
+
"pi": {
|
|
37
|
+
"extensions": [
|
|
38
|
+
"./extensions/search.ts"
|
|
39
|
+
],
|
|
40
|
+
"skills": [
|
|
41
|
+
"./skills"
|
|
42
|
+
]
|
|
43
|
+
},
|
|
44
|
+
"scripts": {
|
|
45
|
+
"typecheck": "tsc --noEmit",
|
|
46
|
+
"test": "bun test",
|
|
47
|
+
"prepublishOnly": "npm run typecheck && npm test"
|
|
48
|
+
},
|
|
49
|
+
"peerDependencies": {
|
|
50
|
+
"@earendil-works/pi-ai": "*",
|
|
51
|
+
"@earendil-works/pi-coding-agent": "*",
|
|
52
|
+
"@earendil-works/pi-tui": "*",
|
|
53
|
+
"typebox": "*"
|
|
54
|
+
},
|
|
55
|
+
"peerDependenciesMeta": {
|
|
56
|
+
"@earendil-works/pi-ai": {
|
|
57
|
+
"optional": true
|
|
58
|
+
},
|
|
59
|
+
"@earendil-works/pi-coding-agent": {
|
|
60
|
+
"optional": true
|
|
61
|
+
},
|
|
62
|
+
"@earendil-works/pi-tui": {
|
|
63
|
+
"optional": true
|
|
64
|
+
},
|
|
65
|
+
"typebox": {
|
|
66
|
+
"optional": true
|
|
67
|
+
}
|
|
68
|
+
},
|
|
69
|
+
"devDependencies": {
|
|
70
|
+
"@earendil-works/pi-ai": "^0.85.1",
|
|
71
|
+
"@earendil-works/pi-coding-agent": "^0.85.1",
|
|
72
|
+
"@earendil-works/pi-tui": "^0.85.1",
|
|
73
|
+
"@types/node": "^22.10.2",
|
|
74
|
+
"typebox": "^1.1.38",
|
|
75
|
+
"typescript": "^5.7.2"
|
|
76
|
+
},
|
|
77
|
+
"publishConfig": {
|
|
78
|
+
"access": "public"
|
|
79
|
+
},
|
|
80
|
+
"dependencies": {
|
|
81
|
+
"@ff-labs/fff-node": "^0.10.6"
|
|
82
|
+
},
|
|
83
|
+
"optionalDependencies": {
|
|
84
|
+
"@ff-labs/fff-node": "^0.10.6"
|
|
85
|
+
}
|
|
86
|
+
}
|
|
@@ -0,0 +1,37 @@
|
|
|
1
|
+
---
|
|
2
|
+
name: search
|
|
3
|
+
description: Use when looking for a file whose name you only half remember, or searching code for a string, symbol, or pattern — especially when you will search more than once in a session
|
|
4
|
+
---
|
|
5
|
+
|
|
6
|
+
# Searching this project
|
|
7
|
+
|
|
8
|
+
`@pify/search` keeps an index in memory, so the second search in a session is
|
|
9
|
+
as cheap as the first. Prefer it over `find` and `grep`, which re-read the tree
|
|
10
|
+
every time they run.
|
|
11
|
+
|
|
12
|
+
## Finding a file
|
|
13
|
+
|
|
14
|
+
`fffind query="auth route"` — fuzzy and typo-tolerant. Give a fragment of the
|
|
15
|
+
name rather than a guess at the full path: `worktre entr` finds
|
|
16
|
+
`worktree/src/enter.ts`. Results are ranked, so the first one is usually the
|
|
17
|
+
one you meant — recently edited and git-modified files rank higher.
|
|
18
|
+
|
|
19
|
+
## Searching contents
|
|
20
|
+
|
|
21
|
+
`ffgrep pattern="..."` with one of three modes:
|
|
22
|
+
|
|
23
|
+
- `literal` (default) — an exact string. Use it when you know the spelling.
|
|
24
|
+
- `regex` — a pattern. Use it for shapes, like `function\s+handle\w+`.
|
|
25
|
+
- `fuzzy` — when you are unsure of the wording. Use it after a literal search
|
|
26
|
+
comes back empty, before concluding the thing does not exist.
|
|
27
|
+
|
|
28
|
+
## Paging
|
|
29
|
+
|
|
30
|
+
Both tools return a `cursor` when there is more. An empty first page means no
|
|
31
|
+
match; a page with a cursor does **not** mean the whole answer — fetch the next
|
|
32
|
+
page before concluding anything about how often something appears.
|
|
33
|
+
|
|
34
|
+
## When not to use this
|
|
35
|
+
|
|
36
|
+
Reading a file you already know the path of: use `read`. Listing one directory:
|
|
37
|
+
use `ls`. This is for finding things.
|
package/src/builtin.ts
ADDED
|
@@ -0,0 +1,228 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The engine that needs nothing installed.
|
|
3
|
+
*
|
|
4
|
+
* Everything here is the pure modules wired to a disk: walk the tree once,
|
|
5
|
+
* hold the paths in memory, build a trigram index over the text files, and
|
|
6
|
+
* keep both current from a watcher. It is slower than the Rust one and it runs
|
|
7
|
+
* wherever pi runs, which is the whole point — a search extension that only
|
|
8
|
+
* works when a native binary installed is an extension that sometimes does
|
|
9
|
+
* nothing at all.
|
|
10
|
+
*/
|
|
11
|
+
|
|
12
|
+
import { readFileSync, readdirSync, statSync, watch, type FSWatcher } from "node:fs";
|
|
13
|
+
import { join, relative } from "node:path";
|
|
14
|
+
|
|
15
|
+
import type { ContentHit, FileHit, GrepOptions, Page, SearchEngine, FindOptions } from "./engine.ts";
|
|
16
|
+
import { rankAndPage, type Candidate } from "./fuzzy.ts";
|
|
17
|
+
import { buildMatcher, looksBinary, matchLines } from "./match.ts";
|
|
18
|
+
import { TrigramIndex, planForLiteral, planForPatterns, planForRegex } from "./trigram.ts";
|
|
19
|
+
import {
|
|
20
|
+
MAX_SEARCHABLE_BYTES,
|
|
21
|
+
includeFile,
|
|
22
|
+
indexContent,
|
|
23
|
+
normalizePath,
|
|
24
|
+
skipDirectory,
|
|
25
|
+
} from "./walk.ts";
|
|
26
|
+
import { frecencyOf, noteAccess, type History } from "./frecency.ts";
|
|
27
|
+
|
|
28
|
+
interface Entry {
|
|
29
|
+
id: number;
|
|
30
|
+
path: string;
|
|
31
|
+
absolute: string;
|
|
32
|
+
size: number;
|
|
33
|
+
mtimeMs: number;
|
|
34
|
+
}
|
|
35
|
+
|
|
36
|
+
export interface BuiltinOptions {
|
|
37
|
+
/** Frecency history, owned by the caller so it can be persisted. */
|
|
38
|
+
history?: History;
|
|
39
|
+
onHistoryChange?: (history: History) => void;
|
|
40
|
+
/** Hard cap, so a wrong root cannot eat the session. */
|
|
41
|
+
maxFiles?: number;
|
|
42
|
+
}
|
|
43
|
+
|
|
44
|
+
export function builtinEngine(root: string, options: BuiltinOptions = {}): SearchEngine {
|
|
45
|
+
const entries = new Map<number, Entry>();
|
|
46
|
+
const byPath = new Map<string, number>();
|
|
47
|
+
const index = new TrigramIndex();
|
|
48
|
+
const watchers: FSWatcher[] = [];
|
|
49
|
+
const maxFiles = options.maxFiles ?? 50_000;
|
|
50
|
+
let history: History = options.history ?? {};
|
|
51
|
+
let nextId = 1;
|
|
52
|
+
let scanned = false;
|
|
53
|
+
|
|
54
|
+
function add(absolute: string, size: number, mtimeMs: number): void {
|
|
55
|
+
const path = normalizePath(relative(root, absolute));
|
|
56
|
+
if (!path || !includeFile(path)) return;
|
|
57
|
+
const existing = byPath.get(path);
|
|
58
|
+
const id = existing ?? nextId++;
|
|
59
|
+
entries.set(id, { id, path, absolute, size, mtimeMs });
|
|
60
|
+
byPath.set(path, id);
|
|
61
|
+
|
|
62
|
+
if (!indexContent(path, size)) {
|
|
63
|
+
index.remove(id);
|
|
64
|
+
return;
|
|
65
|
+
}
|
|
66
|
+
try {
|
|
67
|
+
const content = readFileSync(absolute, "utf8");
|
|
68
|
+
// A NUL byte is cheaper and more reliable than trusting an extension.
|
|
69
|
+
if (looksBinary(content)) index.remove(id);
|
|
70
|
+
else index.add(id, content);
|
|
71
|
+
} catch {
|
|
72
|
+
index.remove(id);
|
|
73
|
+
}
|
|
74
|
+
}
|
|
75
|
+
|
|
76
|
+
function drop(absolute: string): void {
|
|
77
|
+
const path = normalizePath(relative(root, absolute));
|
|
78
|
+
const id = byPath.get(path);
|
|
79
|
+
if (id === undefined) return;
|
|
80
|
+
byPath.delete(path);
|
|
81
|
+
entries.delete(id);
|
|
82
|
+
index.remove(id);
|
|
83
|
+
}
|
|
84
|
+
|
|
85
|
+
function scan(dir: string, depth = 0): void {
|
|
86
|
+
if (entries.size >= maxFiles || depth > 24) return;
|
|
87
|
+
let listing: string[];
|
|
88
|
+
try {
|
|
89
|
+
listing = readdirSync(dir);
|
|
90
|
+
} catch {
|
|
91
|
+
return;
|
|
92
|
+
}
|
|
93
|
+
for (const name of listing) {
|
|
94
|
+
if (entries.size >= maxFiles) return;
|
|
95
|
+
const absolute = join(dir, name);
|
|
96
|
+
let stats;
|
|
97
|
+
try {
|
|
98
|
+
stats = statSync(absolute);
|
|
99
|
+
} catch {
|
|
100
|
+
continue;
|
|
101
|
+
}
|
|
102
|
+
if (stats.isDirectory()) {
|
|
103
|
+
if (skipDirectory(name)) continue;
|
|
104
|
+
scan(absolute, depth + 1);
|
|
105
|
+
} else if (stats.isFile()) {
|
|
106
|
+
add(absolute, stats.size, stats.mtimeMs);
|
|
107
|
+
}
|
|
108
|
+
}
|
|
109
|
+
}
|
|
110
|
+
|
|
111
|
+
function startWatching(): void {
|
|
112
|
+
try {
|
|
113
|
+
const watcher = watch(root, { recursive: true }, (_event, filename) => {
|
|
114
|
+
if (!filename) return;
|
|
115
|
+
const absolute = join(root, String(filename));
|
|
116
|
+
const parts = normalizePath(String(filename)).split("/");
|
|
117
|
+
if (parts.some((part) => skipDirectory(part))) return;
|
|
118
|
+
try {
|
|
119
|
+
const stats = statSync(absolute);
|
|
120
|
+
if (stats.isFile()) add(absolute, stats.size, stats.mtimeMs);
|
|
121
|
+
} catch {
|
|
122
|
+
// Gone: a delete, a rename away, or something we may not read.
|
|
123
|
+
drop(absolute);
|
|
124
|
+
}
|
|
125
|
+
});
|
|
126
|
+
watchers.push(watcher);
|
|
127
|
+
} catch {
|
|
128
|
+
// Recursive watching is not available everywhere; the index is then
|
|
129
|
+
// simply as fresh as the last scan, which is still useful.
|
|
130
|
+
}
|
|
131
|
+
}
|
|
132
|
+
|
|
133
|
+
function candidates(now: number): Candidate[] {
|
|
134
|
+
return [...entries.values()].map((entry) => ({
|
|
135
|
+
path: entry.path,
|
|
136
|
+
frecency: frecencyOf(history, entry.path, now),
|
|
137
|
+
mtimeMs: entry.mtimeMs,
|
|
138
|
+
}));
|
|
139
|
+
}
|
|
140
|
+
|
|
141
|
+
return {
|
|
142
|
+
name: "builtin",
|
|
143
|
+
async ready() {
|
|
144
|
+
if (!scanned) {
|
|
145
|
+
scan(root);
|
|
146
|
+
startWatching();
|
|
147
|
+
scanned = true;
|
|
148
|
+
}
|
|
149
|
+
return true;
|
|
150
|
+
},
|
|
151
|
+
async find(query: string, opts: FindOptions = {}): Promise<Page<FileHit>> {
|
|
152
|
+
const now = Date.now();
|
|
153
|
+
const page = rankAndPage(candidates(now), query, now, opts.limit ?? 20, opts.cursor);
|
|
154
|
+
return {
|
|
155
|
+
items: page.items.map((hit) => {
|
|
156
|
+
const id = byPath.get(hit.path);
|
|
157
|
+
const entry = id === undefined ? undefined : entries.get(id);
|
|
158
|
+
return { path: hit.path, score: hit.score, size: entry?.size, modifiedMs: entry?.mtimeMs };
|
|
159
|
+
}),
|
|
160
|
+
total: page.total,
|
|
161
|
+
cursor: page.cursor,
|
|
162
|
+
};
|
|
163
|
+
},
|
|
164
|
+
async grep(pattern: string, opts: GrepOptions = {}): Promise<Page<ContentHit>> {
|
|
165
|
+
const mode = opts.mode ?? "literal";
|
|
166
|
+
const caseInsensitive = opts.caseInsensitive ?? true;
|
|
167
|
+
const matcher = buildMatcher(pattern, mode, caseInsensitive);
|
|
168
|
+
if (!matcher) return { items: [], total: 0, cursor: null };
|
|
169
|
+
|
|
170
|
+
// Fuzzy matching has no required substring, so the index cannot narrow
|
|
171
|
+
// it; saying so honestly beats narrowing wrongly and losing matches.
|
|
172
|
+
const plan =
|
|
173
|
+
mode === "fuzzy"
|
|
174
|
+
? { kind: "all" as const }
|
|
175
|
+
: planForPatterns([mode === "regex" ? planForRegex(pattern, caseInsensitive) : planForLiteral(pattern, caseInsensitive)]);
|
|
176
|
+
const narrowed = index.candidates(plan);
|
|
177
|
+
|
|
178
|
+
const hits: ContentHit[] = [];
|
|
179
|
+
const limit = opts.limit ?? 20;
|
|
180
|
+
const offset = Number.parseInt(opts.cursor ?? "0", 10);
|
|
181
|
+
const start = Number.isFinite(offset) && offset > 0 ? offset : 0;
|
|
182
|
+
// One page beyond what is asked for, so `total` can say whether there is
|
|
183
|
+
// more without reading the entire tree to find out.
|
|
184
|
+
const budget = start + limit + 1;
|
|
185
|
+
|
|
186
|
+
const searchIds = narrowed === null ? [...entries.keys()] : [...narrowed];
|
|
187
|
+
for (const id of searchIds) {
|
|
188
|
+
if (hits.length >= budget) break;
|
|
189
|
+
const entry = entries.get(id);
|
|
190
|
+
if (!entry || entry.size > MAX_SEARCHABLE_BYTES) continue;
|
|
191
|
+
if (opts.glob && !entry.path.includes(opts.glob.replace(/\*/g, ""))) continue;
|
|
192
|
+
let content: string;
|
|
193
|
+
try {
|
|
194
|
+
content = readFileSync(entry.absolute, "utf8");
|
|
195
|
+
} catch {
|
|
196
|
+
continue;
|
|
197
|
+
}
|
|
198
|
+
if (looksBinary(content)) continue;
|
|
199
|
+
for (const line of matchLines(content, matcher, budget - hits.length)) {
|
|
200
|
+
hits.push({ path: entry.path, line: line.line, text: line.text });
|
|
201
|
+
}
|
|
202
|
+
}
|
|
203
|
+
|
|
204
|
+
hits.sort((a, b) => a.path.localeCompare(b.path) || a.line - b.line);
|
|
205
|
+
const items = hits.slice(start, start + limit);
|
|
206
|
+
return { items, total: hits.length, cursor: start + items.length < hits.length ? String(start + items.length) : null };
|
|
207
|
+
},
|
|
208
|
+
touch(path: string) {
|
|
209
|
+
history = noteAccess(history, normalizePath(path), Date.now());
|
|
210
|
+
options.onHistoryChange?.(history);
|
|
211
|
+
},
|
|
212
|
+
indexed() {
|
|
213
|
+
return entries.size;
|
|
214
|
+
},
|
|
215
|
+
dispose() {
|
|
216
|
+
for (const watcher of watchers) {
|
|
217
|
+
try {
|
|
218
|
+
watcher.close();
|
|
219
|
+
} catch {
|
|
220
|
+
// already closed
|
|
221
|
+
}
|
|
222
|
+
}
|
|
223
|
+
watchers.length = 0;
|
|
224
|
+
entries.clear();
|
|
225
|
+
byPath.clear();
|
|
226
|
+
},
|
|
227
|
+
};
|
|
228
|
+
}
|