@sixb/agent 0.2.4 → 0.2.6
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +2 -2
- package/dist/agent-prompt.d.ts +13 -3
- package/dist/agent-prompt.d.ts.map +1 -1
- package/dist/artifacts/layout-check/layout-check.d.ts +21 -24
- package/dist/artifacts/layout-check/layout-check.d.ts.map +1 -1
- package/dist/artifacts/publish-tool.d.ts.map +1 -1
- package/dist/background-commands.d.ts +2 -2
- package/dist/context-compaction.d.ts.map +1 -1
- package/dist/conversation-messages.d.ts +1 -1
- package/dist/conversation-messages.d.ts.map +1 -1
- package/dist/index.d.ts +0 -1
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +11082 -9281
- package/dist/index.js.map +37 -33
- package/dist/login-shell.d.ts +18 -2
- package/dist/login-shell.d.ts.map +1 -1
- package/dist/model-adapters.d.ts +5 -2
- package/dist/model-adapters.d.ts.map +1 -1
- package/dist/model-lines.d.ts +1 -0
- package/dist/model-lines.d.ts.map +1 -1
- package/dist/run-agent-turn.d.ts +3 -14
- package/dist/run-agent-turn.d.ts.map +1 -1
- package/dist/run-conversation-turn.d.ts.map +1 -1
- package/dist/run-environment.d.ts.map +1 -1
- package/dist/run-sandbox.d.ts +11 -1
- package/dist/run-sandbox.d.ts.map +1 -1
- package/dist/run-subagent.d.ts.map +1 -1
- package/dist/sandbox-api-context.d.ts.map +1 -1
- package/dist/search-script.d.ts +7 -0
- package/dist/search-script.d.ts.map +1 -0
- package/dist/skills/built-in.d.ts.map +1 -1
- package/dist/subagent-tools.d.ts +11 -1
- package/dist/subagent-tools.d.ts.map +1 -1
- package/dist/tools/bash.d.ts +1 -1
- package/dist/tools/bash.d.ts.map +1 -1
- package/dist/tools/glob.d.ts +41 -0
- package/dist/tools/glob.d.ts.map +1 -0
- package/dist/tools/grep.d.ts +64 -0
- package/dist/tools/grep.d.ts.map +1 -0
- package/dist/tools/model-spec.d.ts.map +1 -1
- package/dist/tools/sandbox-search.d.ts +26 -0
- package/dist/tools/sandbox-search.d.ts.map +1 -0
- package/dist/tools/search/command.d.ts +15 -0
- package/dist/tools/search/command.d.ts.map +1 -0
- package/dist/tools/search/gitignore.d.ts +28 -0
- package/dist/tools/search/gitignore.d.ts.map +1 -0
- package/dist/tools/search/glob.d.ts +15 -0
- package/dist/tools/search/glob.d.ts.map +1 -0
- package/dist/tools/search/main.d.ts +2 -0
- package/dist/tools/search/main.d.ts.map +1 -0
- package/dist/tools/search/match.d.ts +30 -0
- package/dist/tools/search/match.d.ts.map +1 -0
- package/dist/tools/search/pattern.d.ts +19 -0
- package/dist/tools/search/pattern.d.ts.map +1 -0
- package/dist/tools/search/read.d.ts +62 -0
- package/dist/tools/search/read.d.ts.map +1 -0
- package/dist/tools/search/request.d.ts +80 -0
- package/dist/tools/search/request.d.ts.map +1 -0
- package/dist/tools/search/search.d.ts +4 -0
- package/dist/tools/search/search.d.ts.map +1 -0
- package/dist/tools/search/walk.d.ts +43 -0
- package/dist/tools/search/walk.d.ts.map +1 -0
- package/dist/tools/tool-outputs.d.ts.map +1 -1
- package/dist/turn-inbox.d.ts +75 -0
- package/dist/turn-inbox.d.ts.map +1 -0
- package/package.json +7 -5
- package/src/agent-prompt.ts +63 -28
- package/src/agent-runtime/preflight.ts +1 -0
- package/src/artifacts/layout-check/check.cjs +176 -95
- package/src/artifacts/layout-check/layout-check.ts +118 -76
- package/src/artifacts/publish-tool.ts +48 -99
- package/src/background-commands.ts +8 -6
- package/src/context-compaction.ts +1 -0
- package/src/conversation-messages.ts +11 -4
- package/src/generated/search.mjs +985 -0
- package/src/index.ts +0 -1
- package/src/login-shell.ts +30 -3
- package/src/model-adapters.ts +23 -4
- package/src/model-lines.ts +1 -1
- package/src/run-agent-turn.ts +85 -129
- package/src/run-conversation-turn.ts +15 -6
- package/src/run-environment.ts +42 -11
- package/src/run-sandbox.ts +60 -1
- package/src/run-subagent.ts +9 -2
- package/src/sandbox-api-context.ts +10 -1
- package/src/search-script-assets.d.ts +5 -0
- package/src/search-script.ts +9 -0
- package/src/skills/built-in.ts +16 -0
- package/src/skills/sixb-artifacts/SKILL.md +7 -7
- package/src/skills/sixb-artifacts/canvas.md +3 -2
- package/src/skills/sixb-artifacts/document.md +92 -5
- package/src/skills/sixb-artifacts/examples/document/service-proposal/design.css +114 -0
- package/src/skills/sixb-artifacts/examples/document/service-proposal/document.html +159 -0
- package/src/skills/sixb-artifacts/examples/document/service-proposal/images/fieldwise.svg +1 -0
- package/src/skills/sixb-artifacts/examples/document/service-proposal/images/harbor-pine.svg +1 -0
- package/src/skills/sixb-artifacts/queries.md +2 -1
- package/src/subagent-tools.ts +23 -5
- package/src/tools/bash.ts +9 -10
- package/src/tools/glob.ts +124 -0
- package/src/tools/grep.ts +190 -0
- package/src/tools/model-spec.ts +4 -0
- package/src/tools/sandbox-search.ts +219 -0
- package/src/tools/search/command.ts +116 -0
- package/src/tools/search/gitignore.ts +101 -0
- package/src/tools/search/glob.ts +90 -0
- package/src/tools/search/main.ts +9 -0
- package/src/tools/search/match.ts +278 -0
- package/src/tools/search/pattern.ts +178 -0
- package/src/tools/search/read.ts +237 -0
- package/src/tools/search/request.ts +101 -0
- package/src/tools/search/search.ts +297 -0
- package/src/tools/search/walk.ts +145 -0
- package/src/tools/tool-outputs.ts +7 -2
- package/src/turn-inbox.ts +283 -0
- package/dist/artifacts/board-previews.d.ts +0 -41
- package/dist/artifacts/board-previews.d.ts.map +0 -1
- package/src/artifacts/board-previews.ts +0 -218
|
@@ -0,0 +1,237 @@
|
|
|
1
|
+
import { Buffer } from "node:buffer"
|
|
2
|
+
import { type FileHandle, open } from "node:fs/promises"
|
|
3
|
+
import { SEARCH_LIMITS } from "./request"
|
|
4
|
+
|
|
5
|
+
export type FileText =
|
|
6
|
+
| { readonly text: string }
|
|
7
|
+
| { readonly skipped: "binary" | "too_large" | "unreadable" }
|
|
8
|
+
|
|
9
|
+
/** A read that knows what it will hold before it holds it. */
|
|
10
|
+
export interface PendingRead<R> {
|
|
11
|
+
/** The bytes the read holds once started, until its result is taken. */
|
|
12
|
+
readonly bytes: number
|
|
13
|
+
/** Never rejects. */
|
|
14
|
+
read(): Promise<R>
|
|
15
|
+
/** Lets go of a read never started. Never rejects. */
|
|
16
|
+
close(): Promise<void>
|
|
17
|
+
}
|
|
18
|
+
|
|
19
|
+
/**
|
|
20
|
+
* A file opened to be read: one open, and its size from that descriptor. Its text comes from
|
|
21
|
+
* {@link PendingRead.read}, unless it is too large or unreadable, which is known already.
|
|
22
|
+
*/
|
|
23
|
+
export async function openText(path: string, strict: boolean): Promise<PendingRead<FileText>> {
|
|
24
|
+
let file: FileHandle | undefined
|
|
25
|
+
|
|
26
|
+
try {
|
|
27
|
+
file = await open(path, "r")
|
|
28
|
+
const { size } = await file.stat()
|
|
29
|
+
|
|
30
|
+
if (size > SEARCH_LIMITS.fileBytes) {
|
|
31
|
+
await closeQuietly(file)
|
|
32
|
+
return settled({ skipped: "too_large" })
|
|
33
|
+
}
|
|
34
|
+
|
|
35
|
+
const opened = file
|
|
36
|
+
return {
|
|
37
|
+
bytes: size,
|
|
38
|
+
read: () => readText(opened, size, strict),
|
|
39
|
+
close: () => closeQuietly(opened),
|
|
40
|
+
}
|
|
41
|
+
} catch {
|
|
42
|
+
if (file) await closeQuietly(file)
|
|
43
|
+
return settled({ skipped: "unreadable" })
|
|
44
|
+
}
|
|
45
|
+
}
|
|
46
|
+
|
|
47
|
+
function settled(text: FileText): PendingRead<FileText> {
|
|
48
|
+
return { bytes: 0, read: async () => text, close: async () => {} }
|
|
49
|
+
}
|
|
50
|
+
|
|
51
|
+
/**
|
|
52
|
+
* The text of a file opened at `size` bytes, read through its descriptor and closed. A file
|
|
53
|
+
* over {@link READ_AT_ONCE_BYTES} is read in two: its first {@link SEARCH_LIMITS.binaryProbeBytes},
|
|
54
|
+
* then the rest unless they show it is binary, so a large binary file is never read whole. A file
|
|
55
|
+
* found by the walk is read as UTF-8 leniently, so a mostly ASCII file in another encoding still
|
|
56
|
+
* matches; a file named as the path must be UTF-8 text. A leading byte order mark is dropped, so
|
|
57
|
+
* `^` finds the first line. Never rejects.
|
|
58
|
+
*/
|
|
59
|
+
async function readText(file: FileHandle, size: number, strict: boolean): Promise<FileText> {
|
|
60
|
+
try {
|
|
61
|
+
const first = size <= READ_AT_ONCE_BYTES ? size : PROBE_BYTES
|
|
62
|
+
let bytes = await readBytes(file, Buffer.allocUnsafe(first), 0)
|
|
63
|
+
if (bytes.subarray(0, PROBE_BYTES).includes(0)) return { skipped: "binary" }
|
|
64
|
+
|
|
65
|
+
if (bytes.length === first && first < size) {
|
|
66
|
+
const whole = Buffer.allocUnsafe(size)
|
|
67
|
+
bytes.copy(whole)
|
|
68
|
+
bytes = await readBytes(file, whole, first)
|
|
69
|
+
}
|
|
70
|
+
|
|
71
|
+
let text: string
|
|
72
|
+
if (strict) {
|
|
73
|
+
try {
|
|
74
|
+
text = new TextDecoder("utf-8", { fatal: true, ignoreBOM: true }).decode(bytes)
|
|
75
|
+
} catch {
|
|
76
|
+
return { skipped: "binary" }
|
|
77
|
+
}
|
|
78
|
+
} else {
|
|
79
|
+
text = bytes.toString("utf8")
|
|
80
|
+
}
|
|
81
|
+
|
|
82
|
+
return { text: text.startsWith("\uFEFF") ? text.slice(1) : text }
|
|
83
|
+
} catch {
|
|
84
|
+
return { skipped: "unreadable" }
|
|
85
|
+
} finally {
|
|
86
|
+
await closeQuietly(file)
|
|
87
|
+
}
|
|
88
|
+
}
|
|
89
|
+
|
|
90
|
+
const PROBE_BYTES = SEARCH_LIMITS.binaryProbeBytes
|
|
91
|
+
|
|
92
|
+
/** Up to this size, a file is read in one call: most source files, and two calls cost more. */
|
|
93
|
+
const READ_AT_ONCE_BYTES = 64 * 1024
|
|
94
|
+
|
|
95
|
+
/**
|
|
96
|
+
* Fills `bytes` from `from` on, and returns what was read: the file as it was when opened, if it
|
|
97
|
+
* grows meanwhile, or less if it shrank.
|
|
98
|
+
*/
|
|
99
|
+
async function readBytes(file: FileHandle, bytes: Buffer, from: number): Promise<Buffer> {
|
|
100
|
+
let filled = from
|
|
101
|
+
|
|
102
|
+
while (filled < bytes.length) {
|
|
103
|
+
const { bytesRead } = await file.read(bytes, filled, bytes.length - filled, filled)
|
|
104
|
+
if (bytesRead === 0) break
|
|
105
|
+
filled += bytesRead
|
|
106
|
+
}
|
|
107
|
+
|
|
108
|
+
return bytes.subarray(0, filled)
|
|
109
|
+
}
|
|
110
|
+
|
|
111
|
+
async function closeQuietly(file: FileHandle): Promise<void> {
|
|
112
|
+
await file.close().catch(() => {})
|
|
113
|
+
}
|
|
114
|
+
|
|
115
|
+
/**
|
|
116
|
+
* How far reads run ahead of the file searched. Reading is most of a search's time, and mostly
|
|
117
|
+
* waiting: reads in flight overlap that wait. At most {@link READ_AHEAD} files are open, and a read
|
|
118
|
+
* starts only while the reads started hold at most {@link READ_AHEAD_BYTES} of file with it, but
|
|
119
|
+
* for the next file in line, which always starts: one file of 10 MiB past the bound at most. A
|
|
120
|
+
* binary file's bytes are let go once its first bytes show it is one.
|
|
121
|
+
*
|
|
122
|
+
* Memory: the texts read ahead take up to twice their bytes once decoded (a string with one
|
|
123
|
+
* character beyond Latin-1 takes two bytes for each): 52 MiB at worst, plus the file searched and
|
|
124
|
+
* the copies its search makes. 16 MiB reads a source tree as fast as 32 did, and holds half as
|
|
125
|
+
* much over a folder of large files.
|
|
126
|
+
*/
|
|
127
|
+
export const READ_AHEAD = 32
|
|
128
|
+
export const READ_AHEAD_BYTES = 16 * 1024 * 1024
|
|
129
|
+
|
|
130
|
+
interface Slot<T, R> {
|
|
131
|
+
readonly item: T
|
|
132
|
+
/** Its {@link PendingRead}, once opened. */
|
|
133
|
+
pending?: PendingRead<R>
|
|
134
|
+
/** Settles once {@link pending} is known. */
|
|
135
|
+
opening?: Promise<void>
|
|
136
|
+
/** The bytes it holds against {@link READ_AHEAD_BYTES}. */
|
|
137
|
+
holds: number
|
|
138
|
+
result?: Promise<R>
|
|
139
|
+
}
|
|
140
|
+
|
|
141
|
+
/**
|
|
142
|
+
* Files read ahead of the one searched, handed back in the order they come, so a search finds the
|
|
143
|
+
* same lines, in the same order, as reading one file at a time. Files are opened as they come,
|
|
144
|
+
* and read as soon as they are open and their bytes fit. A search that stops reads no further than
|
|
145
|
+
* {@link READ_AHEAD} files past where it stopped.
|
|
146
|
+
*/
|
|
147
|
+
export class ReadAhead<T, R> {
|
|
148
|
+
readonly #slots: Slot<T, R>[] = []
|
|
149
|
+
#held = 0
|
|
150
|
+
#exhausted = false
|
|
151
|
+
#stopped = false
|
|
152
|
+
|
|
153
|
+
constructor(
|
|
154
|
+
private readonly items: Iterator<T>,
|
|
155
|
+
/** Never rejects. */
|
|
156
|
+
private readonly open: (item: T) => Promise<PendingRead<R>>,
|
|
157
|
+
/** What a read that holds nothing once done looks like, a binary file. */
|
|
158
|
+
private readonly holdsNothing: (result: R) => boolean = () => false
|
|
159
|
+
) {}
|
|
160
|
+
|
|
161
|
+
async next(): Promise<{ readonly item: T; readonly read: R } | undefined> {
|
|
162
|
+
this.#fill()
|
|
163
|
+
const slot = this.#slots[0]
|
|
164
|
+
if (!slot) return undefined
|
|
165
|
+
|
|
166
|
+
await slot.opening
|
|
167
|
+
this.#start()
|
|
168
|
+
const read = await (slot.result as Promise<R>)
|
|
169
|
+
|
|
170
|
+
this.#slots.shift()
|
|
171
|
+
this.#held -= slot.holds
|
|
172
|
+
this.#fill()
|
|
173
|
+
this.#start()
|
|
174
|
+
|
|
175
|
+
return { item: slot.item, read }
|
|
176
|
+
}
|
|
177
|
+
|
|
178
|
+
/**
|
|
179
|
+
* Starts no other read, waits for those in flight and closes the files never read: no file
|
|
180
|
+
* stays open past the search.
|
|
181
|
+
*/
|
|
182
|
+
async stop(): Promise<void> {
|
|
183
|
+
this.#exhausted = true
|
|
184
|
+
this.#stopped = true
|
|
185
|
+
|
|
186
|
+
await Promise.all(
|
|
187
|
+
this.#slots.splice(0).map(async (slot) => {
|
|
188
|
+
await slot.opening
|
|
189
|
+
if (slot.result) await slot.result
|
|
190
|
+
else await slot.pending?.close()
|
|
191
|
+
})
|
|
192
|
+
)
|
|
193
|
+
}
|
|
194
|
+
|
|
195
|
+
#fill(): void {
|
|
196
|
+
while (!this.#exhausted && this.#slots.length < READ_AHEAD) {
|
|
197
|
+
const step = this.items.next()
|
|
198
|
+
if (step.done) {
|
|
199
|
+
this.#exhausted = true
|
|
200
|
+
return
|
|
201
|
+
}
|
|
202
|
+
|
|
203
|
+
const slot: Slot<T, R> = { item: step.value, holds: 0 }
|
|
204
|
+
this.#slots.push(slot)
|
|
205
|
+
slot.opening = this.open(step.value).then((pending) => {
|
|
206
|
+
slot.pending = pending
|
|
207
|
+
this.#start()
|
|
208
|
+
})
|
|
209
|
+
}
|
|
210
|
+
}
|
|
211
|
+
|
|
212
|
+
/**
|
|
213
|
+
* Starts the reads of the files opened, as long as their bytes fit. The first file in line always
|
|
214
|
+
* starts: the search waits on it, and no read behind it can make room.
|
|
215
|
+
*/
|
|
216
|
+
#start(): void {
|
|
217
|
+
if (this.#stopped) return
|
|
218
|
+
|
|
219
|
+
for (let index = 0; index < this.#slots.length; index += 1) {
|
|
220
|
+
const slot = this.#slots[index] as Slot<T, R>
|
|
221
|
+
const pending = slot.pending
|
|
222
|
+
if (slot.result || !pending) continue
|
|
223
|
+
if (index > 0 && this.#held + pending.bytes > READ_AHEAD_BYTES) continue
|
|
224
|
+
|
|
225
|
+
slot.holds = pending.bytes
|
|
226
|
+
this.#held += slot.holds
|
|
227
|
+
slot.result = pending.read().then((result) => {
|
|
228
|
+
if (this.holdsNothing(result) && this.#slots.includes(slot)) {
|
|
229
|
+
this.#held -= slot.holds
|
|
230
|
+
slot.holds = 0
|
|
231
|
+
this.#start()
|
|
232
|
+
}
|
|
233
|
+
return result
|
|
234
|
+
})
|
|
235
|
+
}
|
|
236
|
+
}
|
|
237
|
+
}
|
|
@@ -0,0 +1,101 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* What `search.mjs` takes and prints: the contract between the `glob` and `grep` tools and the
|
|
3
|
+
* engine the run installs.
|
|
4
|
+
*/
|
|
5
|
+
|
|
6
|
+
/** How much a search returns, and which files `grep` reads. */
|
|
7
|
+
export const SEARCH_LIMITS = {
|
|
8
|
+
files: 100,
|
|
9
|
+
lines: 100,
|
|
10
|
+
linesPerFile: 20,
|
|
11
|
+
lineCharacters: 500,
|
|
12
|
+
/** How much of a cut line shows before its first match. */
|
|
13
|
+
matchLead: 40,
|
|
14
|
+
/**
|
|
15
|
+
* How much of a line `grep` matches. A backtracking regex can take seconds on one long line of a
|
|
16
|
+
* minified file; past this, a match is missed rather than the search stalled.
|
|
17
|
+
*/
|
|
18
|
+
matchCharacters: 4_000,
|
|
19
|
+
/** What `grep`'s matches may add to its model text, in UTF-8 bytes, before it is truncated. */
|
|
20
|
+
textBytes: 24 * 1024,
|
|
21
|
+
fileBytes: 10 * 1024 * 1024,
|
|
22
|
+
/** A NUL byte in this many first bytes makes a file binary. */
|
|
23
|
+
binaryProbeBytes: 8 * 1024,
|
|
24
|
+
} as const
|
|
25
|
+
|
|
26
|
+
interface SearchScopeRequest {
|
|
27
|
+
/** Relative to the working directory, or absolute. Default: the working directory. */
|
|
28
|
+
readonly path?: string
|
|
29
|
+
/**
|
|
30
|
+
* The run's context directory: the second root a search may start from. A search that starts
|
|
31
|
+
* outside it never enters it.
|
|
32
|
+
*/
|
|
33
|
+
readonly contextDirectory?: string
|
|
34
|
+
/** How long the walk may take before it stops with what it found. */
|
|
35
|
+
readonly timeoutMs: number
|
|
36
|
+
}
|
|
37
|
+
|
|
38
|
+
export interface GlobSearchRequest extends SearchScopeRequest {
|
|
39
|
+
readonly tool: "glob"
|
|
40
|
+
readonly pattern: string
|
|
41
|
+
}
|
|
42
|
+
|
|
43
|
+
export interface GrepSearchRequest extends SearchScopeRequest {
|
|
44
|
+
readonly tool: "grep"
|
|
45
|
+
/** A JavaScript regular expression, or the text itself with `literal`. */
|
|
46
|
+
readonly pattern: string
|
|
47
|
+
/** A `glob` pattern the files searched must match. */
|
|
48
|
+
readonly include?: string
|
|
49
|
+
readonly literal?: boolean
|
|
50
|
+
}
|
|
51
|
+
|
|
52
|
+
export type SearchRequest = GlobSearchRequest | GrepSearchRequest
|
|
53
|
+
|
|
54
|
+
/**
|
|
55
|
+
* Paths are relative to the working directory when they are under it, absolute otherwise.
|
|
56
|
+
* `incomplete` says the walk stopped at its deadline.
|
|
57
|
+
*/
|
|
58
|
+
export interface GlobSearchResult {
|
|
59
|
+
readonly files: string[]
|
|
60
|
+
readonly truncated: boolean
|
|
61
|
+
readonly incomplete?: true
|
|
62
|
+
}
|
|
63
|
+
|
|
64
|
+
export interface GrepMatch {
|
|
65
|
+
readonly path: string
|
|
66
|
+
/** One-based. */
|
|
67
|
+
readonly line: number
|
|
68
|
+
readonly text: string
|
|
69
|
+
}
|
|
70
|
+
|
|
71
|
+
export interface GrepSearchResult {
|
|
72
|
+
readonly matches: GrepMatch[]
|
|
73
|
+
/** How many files the matches are in. */
|
|
74
|
+
readonly files: number
|
|
75
|
+
/** Matching lines counted past what a file shows, by path. */
|
|
76
|
+
readonly more?: Record<string, number>
|
|
77
|
+
/** Another file matches past the cap. */
|
|
78
|
+
readonly truncated: boolean
|
|
79
|
+
readonly incomplete?: true
|
|
80
|
+
}
|
|
81
|
+
|
|
82
|
+
export type SearchErrorCode =
|
|
83
|
+
| "path_missing"
|
|
84
|
+
| "path_outside"
|
|
85
|
+
| "path_is_file"
|
|
86
|
+
| "path_not_searchable"
|
|
87
|
+
| "invalid_pattern"
|
|
88
|
+
| "invalid_include"
|
|
89
|
+
| "file_binary"
|
|
90
|
+
| "file_too_large"
|
|
91
|
+
|
|
92
|
+
/** A search that cannot run as asked: the caller says why in its own words. */
|
|
93
|
+
export class SearchError extends Error {
|
|
94
|
+
constructor(
|
|
95
|
+
readonly code: SearchErrorCode,
|
|
96
|
+
message: string
|
|
97
|
+
) {
|
|
98
|
+
super(message)
|
|
99
|
+
this.name = "SearchError"
|
|
100
|
+
}
|
|
101
|
+
}
|
|
@@ -0,0 +1,297 @@
|
|
|
1
|
+
import { Buffer } from "node:buffer"
|
|
2
|
+
import { lstatSync, realpathSync, statSync } from "node:fs"
|
|
3
|
+
import { basename, dirname, relative, resolve } from "node:path"
|
|
4
|
+
import type { IgnoreFile } from "./gitignore"
|
|
5
|
+
import { compileGlob, type GlobMatcher } from "./glob"
|
|
6
|
+
import { cutLine, lines, textMatcher } from "./match"
|
|
7
|
+
import { PatternError } from "./pattern"
|
|
8
|
+
import { type FileText, openText, ReadAhead } from "./read"
|
|
9
|
+
import {
|
|
10
|
+
type GlobSearchRequest,
|
|
11
|
+
type GlobSearchResult,
|
|
12
|
+
type GrepMatch,
|
|
13
|
+
type GrepSearchRequest,
|
|
14
|
+
type GrepSearchResult,
|
|
15
|
+
SEARCH_LIMITS,
|
|
16
|
+
SearchError,
|
|
17
|
+
type SearchErrorCode,
|
|
18
|
+
type SearchRequest,
|
|
19
|
+
} from "./request"
|
|
20
|
+
import { childPath, FileWalk, readIgnoreFile, type WalkedFile } from "./walk"
|
|
21
|
+
|
|
22
|
+
export function globFiles(request: GlobSearchRequest, workingDirectory: string): GlobSearchResult {
|
|
23
|
+
const scope = resolveScope(request, workingDirectory)
|
|
24
|
+
const matcher = compileGlobOrFail(request.pattern, "invalid_pattern")
|
|
25
|
+
if (scope.file) throw new SearchError("path_is_file", "The path is a file.")
|
|
26
|
+
|
|
27
|
+
const walk = scope.walk(matcher, matcher.namesHidden)
|
|
28
|
+
const files: string[] = []
|
|
29
|
+
let truncated = false
|
|
30
|
+
|
|
31
|
+
for (const file of walk.files()) {
|
|
32
|
+
if (!matcher.matches(file.relative)) continue
|
|
33
|
+
if (files.length === SEARCH_LIMITS.files) {
|
|
34
|
+
truncated = true
|
|
35
|
+
break
|
|
36
|
+
}
|
|
37
|
+
files.push(scope.display(file.path))
|
|
38
|
+
}
|
|
39
|
+
|
|
40
|
+
return { files, truncated, ...(walk.incomplete ? { incomplete: true as const } : {}) }
|
|
41
|
+
}
|
|
42
|
+
|
|
43
|
+
export async function grepFiles(
|
|
44
|
+
request: GrepSearchRequest,
|
|
45
|
+
workingDirectory: string
|
|
46
|
+
): Promise<GrepSearchResult> {
|
|
47
|
+
const scope = resolveScope(request, workingDirectory)
|
|
48
|
+
const matcher = textMatcher(request.pattern, request.literal === true)
|
|
49
|
+
const include =
|
|
50
|
+
request.include === undefined
|
|
51
|
+
? undefined
|
|
52
|
+
: compileGlobOrFail(request.include, "invalid_include")
|
|
53
|
+
|
|
54
|
+
// A file named as the path is searched alone, whatever include says.
|
|
55
|
+
const walk = scope.file ? undefined : scope.walk(include, true)
|
|
56
|
+
const searched = function* (): Generator<WalkedFile> {
|
|
57
|
+
if (!walk) {
|
|
58
|
+
yield { path: scope.target, relative: basename(scope.target), symlink: false }
|
|
59
|
+
return
|
|
60
|
+
}
|
|
61
|
+
|
|
62
|
+
for (const file of walk.files()) {
|
|
63
|
+
if (include && !include.matches(file.relative)) continue
|
|
64
|
+
if (file.symlink && !scope.holds(file.path)) continue
|
|
65
|
+
yield file
|
|
66
|
+
}
|
|
67
|
+
}
|
|
68
|
+
// Files are read ahead of the one searched, and come back in the walk's order.
|
|
69
|
+
const reads = new ReadAhead(
|
|
70
|
+
searched(),
|
|
71
|
+
(file: WalkedFile) => openText(file.path, walk === undefined),
|
|
72
|
+
(read: FileText) => !("text" in read)
|
|
73
|
+
)
|
|
74
|
+
|
|
75
|
+
const matches: GrepMatch[] = []
|
|
76
|
+
const more: Record<string, number> = {}
|
|
77
|
+
let files = 0
|
|
78
|
+
let textBytes = 0
|
|
79
|
+
let textFull = false
|
|
80
|
+
let truncated = false
|
|
81
|
+
let stopped = false
|
|
82
|
+
|
|
83
|
+
try {
|
|
84
|
+
for (let next = await reads.next(); next; next = await reads.next()) {
|
|
85
|
+
const { item: file, read } = next
|
|
86
|
+
|
|
87
|
+
if (!("text" in read)) {
|
|
88
|
+
// A file named as the path says why it was not searched, rather than "No matches".
|
|
89
|
+
if (walk === undefined && read.skipped !== "unreadable") throw unsearchedError(read.skipped)
|
|
90
|
+
continue
|
|
91
|
+
}
|
|
92
|
+
|
|
93
|
+
// Most files hold no match: one test of the whole text spares splitting them into lines.
|
|
94
|
+
if (!matcher.mayMatch(read.text)) continue
|
|
95
|
+
|
|
96
|
+
const path = scope.display(file.path)
|
|
97
|
+
let shown = 0
|
|
98
|
+
let counted = 0
|
|
99
|
+
|
|
100
|
+
for (const [index, line] of lines(read.text).entries()) {
|
|
101
|
+
// One file can hold enough lines, or slow enough ones, to outlast the deadline alone.
|
|
102
|
+
if (Date.now() >= scope.deadline) {
|
|
103
|
+
stopped = true
|
|
104
|
+
break
|
|
105
|
+
}
|
|
106
|
+
const match = matcher.line(line)
|
|
107
|
+
if (match === undefined) continue
|
|
108
|
+
|
|
109
|
+
const text = cutLine(line, match)
|
|
110
|
+
const bytes =
|
|
111
|
+
Buffer.byteLength(` ${index + 1}: ${text}\n`) +
|
|
112
|
+
(shown === 0 ? Buffer.byteLength(`${path}\n\n`) : 0)
|
|
113
|
+
if (textBytes + bytes > SEARCH_LIMITS.textBytes) textFull = true
|
|
114
|
+
const full = textFull || matches.length === SEARCH_LIMITS.lines
|
|
115
|
+
|
|
116
|
+
if (shown === 0 && full) {
|
|
117
|
+
truncated = true
|
|
118
|
+
break
|
|
119
|
+
}
|
|
120
|
+
if (shown > 0 && (shown === SEARCH_LIMITS.linesPerFile || full)) {
|
|
121
|
+
counted += 1
|
|
122
|
+
continue
|
|
123
|
+
}
|
|
124
|
+
|
|
125
|
+
matches.push({ path, line: index + 1, text })
|
|
126
|
+
textBytes += bytes
|
|
127
|
+
shown += 1
|
|
128
|
+
}
|
|
129
|
+
|
|
130
|
+
if (shown > 0) files += 1
|
|
131
|
+
if (counted > 0) more[path] = counted
|
|
132
|
+
if (truncated || stopped) break
|
|
133
|
+
}
|
|
134
|
+
} finally {
|
|
135
|
+
await reads.stop()
|
|
136
|
+
}
|
|
137
|
+
|
|
138
|
+
return {
|
|
139
|
+
matches,
|
|
140
|
+
files,
|
|
141
|
+
...(Object.keys(more).length > 0 ? { more } : {}),
|
|
142
|
+
truncated,
|
|
143
|
+
...(walk?.incomplete || stopped ? { incomplete: true as const } : {}),
|
|
144
|
+
}
|
|
145
|
+
}
|
|
146
|
+
|
|
147
|
+
interface SearchScope {
|
|
148
|
+
/** The real path searched. */
|
|
149
|
+
readonly target: string
|
|
150
|
+
/** The target is a file. */
|
|
151
|
+
readonly file: boolean
|
|
152
|
+
/** The time, in epoch milliseconds, at which the search stops with what it found. */
|
|
153
|
+
readonly deadline: number
|
|
154
|
+
/** Whether a path's target lies in the run's roots. */
|
|
155
|
+
holds(path: string): boolean
|
|
156
|
+
display(path: string): string
|
|
157
|
+
walk(matcher: GlobMatcher | undefined, hidden: boolean): FileWalk
|
|
158
|
+
}
|
|
159
|
+
|
|
160
|
+
/**
|
|
161
|
+
* Where a search starts, confined to the working directory and the run's context directory once
|
|
162
|
+
* links are resolved, as `read` confines a path.
|
|
163
|
+
*/
|
|
164
|
+
function resolveScope(request: SearchRequest, workingDirectory: string): SearchScope {
|
|
165
|
+
const deadline = Date.now() + request.timeoutMs
|
|
166
|
+
const working = realpathSync(workingDirectory)
|
|
167
|
+
const context =
|
|
168
|
+
request.contextDirectory === undefined
|
|
169
|
+
? undefined
|
|
170
|
+
: realPath(resolve(working, request.contextDirectory))
|
|
171
|
+
|
|
172
|
+
const target = realPath(resolve(working, request.path ?? "."))
|
|
173
|
+
if (target === undefined) throw new SearchError("path_missing", "The path does not exist.")
|
|
174
|
+
|
|
175
|
+
// The context directory may sit inside the working directory: the closer root is the one used.
|
|
176
|
+
const root =
|
|
177
|
+
context !== undefined && isWithin(target, context)
|
|
178
|
+
? context
|
|
179
|
+
: isWithin(target, working)
|
|
180
|
+
? working
|
|
181
|
+
: undefined
|
|
182
|
+
if (root === undefined) {
|
|
183
|
+
throw new SearchError("path_outside", "The path is outside the run's folders.")
|
|
184
|
+
}
|
|
185
|
+
|
|
186
|
+
const stats = statSync(target)
|
|
187
|
+
if (!stats.isFile() && !stats.isDirectory()) {
|
|
188
|
+
throw new SearchError("path_not_searchable", "The path is not a file or a folder.")
|
|
189
|
+
}
|
|
190
|
+
|
|
191
|
+
const holds = (path: string): boolean => {
|
|
192
|
+
const real = realPath(path)
|
|
193
|
+
return (
|
|
194
|
+
real !== undefined &&
|
|
195
|
+
(isWithin(real, working) || (context !== undefined && isWithin(real, context)))
|
|
196
|
+
)
|
|
197
|
+
}
|
|
198
|
+
|
|
199
|
+
const display = (path: string): string =>
|
|
200
|
+
isWithin(path, working) ? relative(working, path) || "." : path
|
|
201
|
+
|
|
202
|
+
const walk = (matcher: GlobMatcher | undefined, hidden: boolean): FileWalk => {
|
|
203
|
+
const namesNodeModules = relative(root, target).split("/").includes("node_modules")
|
|
204
|
+
return new FileWalk({
|
|
205
|
+
root: target,
|
|
206
|
+
ignoreFiles: [...repositoryIgnoreFiles(root), ...ancestorIgnoreFiles(root, target)],
|
|
207
|
+
hidden,
|
|
208
|
+
nodeModules: namesNodeModules || matcher?.namesNodeModules === true,
|
|
209
|
+
skipFolders: new Set(context !== undefined && !isWithin(target, context) ? [context] : []),
|
|
210
|
+
...(matcher ? { mayContain: (folder: string) => matcher.mayContain(folder) } : {}),
|
|
211
|
+
deadline,
|
|
212
|
+
})
|
|
213
|
+
}
|
|
214
|
+
|
|
215
|
+
return { target, file: stats.isFile(), deadline, holds, display, walk }
|
|
216
|
+
}
|
|
217
|
+
|
|
218
|
+
/** How far above a root the git repository it lies in is looked for. */
|
|
219
|
+
const REPOSITORY_LEVELS = 20
|
|
220
|
+
|
|
221
|
+
/**
|
|
222
|
+
* The ignore rules of the git repository a root lies in, from above the root: its
|
|
223
|
+
* `.git/info/exclude`, then the `.gitignore` of each folder from the repository's top down to the
|
|
224
|
+
* folder above the root, as git applies them. Only those files are read; nothing outside the
|
|
225
|
+
* roots is walked. A root outside any repository gets none.
|
|
226
|
+
*/
|
|
227
|
+
function repositoryIgnoreFiles(root: string): IgnoreFile[] {
|
|
228
|
+
const above: string[] = []
|
|
229
|
+
let folder = root
|
|
230
|
+
|
|
231
|
+
for (let level = 0; level <= REPOSITORY_LEVELS; level += 1) {
|
|
232
|
+
if (holdsGit(folder)) {
|
|
233
|
+
const exclude = readIgnoreFile(folder, ".git/info/exclude")
|
|
234
|
+
return [
|
|
235
|
+
...(exclude ? [exclude] : []),
|
|
236
|
+
...above.flatMap((parent) => readIgnoreFile(parent) ?? []),
|
|
237
|
+
]
|
|
238
|
+
}
|
|
239
|
+
|
|
240
|
+
const parent = dirname(folder)
|
|
241
|
+
if (parent === folder) return []
|
|
242
|
+
folder = parent
|
|
243
|
+
above.unshift(folder)
|
|
244
|
+
}
|
|
245
|
+
|
|
246
|
+
return []
|
|
247
|
+
}
|
|
248
|
+
|
|
249
|
+
/** A folder holding `.git`: a repository's top, or a worktree's, where `.git` is a file. */
|
|
250
|
+
function holdsGit(folder: string): boolean {
|
|
251
|
+
try {
|
|
252
|
+
lstatSync(childPath(folder, ".git"))
|
|
253
|
+
return true
|
|
254
|
+
} catch {
|
|
255
|
+
return false
|
|
256
|
+
}
|
|
257
|
+
}
|
|
258
|
+
|
|
259
|
+
/** The `.gitignore` files from the root down to the folder above the target, outermost first. */
|
|
260
|
+
function ancestorIgnoreFiles(root: string, target: string): IgnoreFile[] {
|
|
261
|
+
if (target === root) return []
|
|
262
|
+
|
|
263
|
+
const folders: string[] = []
|
|
264
|
+
for (let folder = dirname(target); isWithin(folder, root); folder = dirname(folder)) {
|
|
265
|
+
folders.unshift(folder)
|
|
266
|
+
if (folder === root) break
|
|
267
|
+
}
|
|
268
|
+
|
|
269
|
+
return folders.flatMap((folder) => readIgnoreFile(folder) ?? [])
|
|
270
|
+
}
|
|
271
|
+
|
|
272
|
+
function compileGlobOrFail(pattern: string, code: SearchErrorCode): GlobMatcher {
|
|
273
|
+
try {
|
|
274
|
+
return compileGlob(pattern)
|
|
275
|
+
} catch (error) {
|
|
276
|
+
if (error instanceof PatternError) throw new SearchError(code, error.message)
|
|
277
|
+
throw error
|
|
278
|
+
}
|
|
279
|
+
}
|
|
280
|
+
|
|
281
|
+
function unsearchedError(reason: "binary" | "too_large"): SearchError {
|
|
282
|
+
return reason === "binary"
|
|
283
|
+
? new SearchError("file_binary", "The file is binary, or not UTF-8 text.")
|
|
284
|
+
: new SearchError("file_too_large", "The file is over 10 MiB.")
|
|
285
|
+
}
|
|
286
|
+
|
|
287
|
+
function realPath(path: string): string | undefined {
|
|
288
|
+
try {
|
|
289
|
+
return realpathSync(path)
|
|
290
|
+
} catch {
|
|
291
|
+
return undefined
|
|
292
|
+
}
|
|
293
|
+
}
|
|
294
|
+
|
|
295
|
+
function isWithin(path: string, root: string): boolean {
|
|
296
|
+
return path === root || path.startsWith(root.endsWith("/") ? root : `${root}/`)
|
|
297
|
+
}
|