@sixb/agent 0.2.4 → 0.2.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +2 -2
- package/dist/agent-prompt.d.ts.map +1 -1
- package/dist/artifacts/layout-check/layout-check.d.ts +21 -24
- package/dist/artifacts/layout-check/layout-check.d.ts.map +1 -1
- package/dist/artifacts/publish-tool.d.ts.map +1 -1
- package/dist/index.d.ts +0 -1
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +10590 -8888
- package/dist/index.js.map +30 -26
- package/dist/run-agent-turn.d.ts +3 -14
- package/dist/run-agent-turn.d.ts.map +1 -1
- package/dist/run-environment.d.ts.map +1 -1
- package/dist/run-subagent.d.ts.map +1 -1
- package/dist/sandbox-api-context.d.ts.map +1 -1
- package/dist/search-script.d.ts +7 -0
- package/dist/search-script.d.ts.map +1 -0
- package/dist/skills/built-in.d.ts.map +1 -1
- package/dist/tools/glob.d.ts +41 -0
- package/dist/tools/glob.d.ts.map +1 -0
- package/dist/tools/grep.d.ts +64 -0
- package/dist/tools/grep.d.ts.map +1 -0
- package/dist/tools/model-spec.d.ts.map +1 -1
- package/dist/tools/sandbox-search.d.ts +26 -0
- package/dist/tools/sandbox-search.d.ts.map +1 -0
- package/dist/tools/search/command.d.ts +15 -0
- package/dist/tools/search/command.d.ts.map +1 -0
- package/dist/tools/search/gitignore.d.ts +28 -0
- package/dist/tools/search/gitignore.d.ts.map +1 -0
- package/dist/tools/search/glob.d.ts +15 -0
- package/dist/tools/search/glob.d.ts.map +1 -0
- package/dist/tools/search/main.d.ts +2 -0
- package/dist/tools/search/main.d.ts.map +1 -0
- package/dist/tools/search/match.d.ts +30 -0
- package/dist/tools/search/match.d.ts.map +1 -0
- package/dist/tools/search/pattern.d.ts +19 -0
- package/dist/tools/search/pattern.d.ts.map +1 -0
- package/dist/tools/search/read.d.ts +62 -0
- package/dist/tools/search/read.d.ts.map +1 -0
- package/dist/tools/search/request.d.ts +80 -0
- package/dist/tools/search/request.d.ts.map +1 -0
- package/dist/tools/search/search.d.ts +4 -0
- package/dist/tools/search/search.d.ts.map +1 -0
- package/dist/tools/search/walk.d.ts +43 -0
- package/dist/tools/search/walk.d.ts.map +1 -0
- package/dist/tools/tool-outputs.d.ts.map +1 -1
- package/dist/turn-inbox.d.ts +75 -0
- package/dist/turn-inbox.d.ts.map +1 -0
- package/package.json +6 -4
- package/src/agent-prompt.ts +1 -0
- package/src/agent-runtime/preflight.ts +1 -0
- package/src/artifacts/layout-check/check.cjs +176 -95
- package/src/artifacts/layout-check/layout-check.ts +118 -76
- package/src/artifacts/publish-tool.ts +48 -99
- package/src/generated/search.mjs +985 -0
- package/src/index.ts +0 -1
- package/src/run-agent-turn.ts +85 -129
- package/src/run-conversation-turn.ts +4 -4
- package/src/run-environment.ts +4 -0
- package/src/run-subagent.ts +1 -1
- package/src/sandbox-api-context.ts +3 -0
- package/src/search-script-assets.d.ts +5 -0
- package/src/search-script.ts +9 -0
- package/src/skills/built-in.ts +16 -0
- package/src/skills/sixb-artifacts/SKILL.md +7 -7
- package/src/skills/sixb-artifacts/canvas.md +3 -2
- package/src/skills/sixb-artifacts/document.md +92 -5
- package/src/skills/sixb-artifacts/examples/document/service-proposal/design.css +114 -0
- package/src/skills/sixb-artifacts/examples/document/service-proposal/document.html +159 -0
- package/src/skills/sixb-artifacts/examples/document/service-proposal/images/fieldwise.svg +1 -0
- package/src/skills/sixb-artifacts/examples/document/service-proposal/images/harbor-pine.svg +1 -0
- package/src/skills/sixb-artifacts/queries.md +2 -1
- package/src/tools/glob.ts +124 -0
- package/src/tools/grep.ts +190 -0
- package/src/tools/model-spec.ts +4 -0
- package/src/tools/sandbox-search.ts +219 -0
- package/src/tools/search/command.ts +116 -0
- package/src/tools/search/gitignore.ts +101 -0
- package/src/tools/search/glob.ts +90 -0
- package/src/tools/search/main.ts +9 -0
- package/src/tools/search/match.ts +278 -0
- package/src/tools/search/pattern.ts +178 -0
- package/src/tools/search/read.ts +237 -0
- package/src/tools/search/request.ts +101 -0
- package/src/tools/search/search.ts +297 -0
- package/src/tools/search/walk.ts +145 -0
- package/src/tools/tool-outputs.ts +7 -2
- package/src/turn-inbox.ts +283 -0
- package/dist/artifacts/board-previews.d.ts +0 -41
- package/dist/artifacts/board-previews.d.ts.map +0 -1
- package/src/artifacts/board-previews.ts +0 -218
|
@@ -0,0 +1,278 @@
|
|
|
1
|
+
import { SEARCH_LIMITS, SearchError } from "./request"
|
|
2
|
+
|
|
3
|
+
/** Where a line first matched `grep`'s pattern, in UTF-16 units. */
|
|
4
|
+
export interface LineMatch {
|
|
5
|
+
readonly index: number
|
|
6
|
+
readonly length: number
|
|
7
|
+
}
|
|
8
|
+
|
|
9
|
+
export interface TextMatcher {
|
|
10
|
+
/**
|
|
11
|
+
* Where a line first matches, over its first {@link SEARCH_LIMITS.matchCharacters} characters.
|
|
12
|
+
*/
|
|
13
|
+
line(line: string): LineMatch | undefined
|
|
14
|
+
/**
|
|
15
|
+
* Whether any line of a file may match: false only when none does, so a file that cannot match
|
|
16
|
+
* is never split into lines. It may say true of a file with no matching line.
|
|
17
|
+
*/
|
|
18
|
+
mayMatch(text: string): boolean
|
|
19
|
+
}
|
|
20
|
+
|
|
21
|
+
/**
|
|
22
|
+
* `grep`'s pattern, matched line by line. Case is smart: ignored when the pattern has no uppercase
|
|
23
|
+
* letter. Literal text is a plain substring search, which no line can slow down.
|
|
24
|
+
*/
|
|
25
|
+
export function textMatcher(pattern: string, literal: boolean): TextMatcher {
|
|
26
|
+
return literal ? literalMatcher(pattern) : regexMatcher(pattern)
|
|
27
|
+
}
|
|
28
|
+
|
|
29
|
+
function literalMatcher(pattern: string): TextMatcher {
|
|
30
|
+
const ignoreCase = !/\p{Lu}/u.test(pattern)
|
|
31
|
+
const needle = ignoreCase ? pattern.toLowerCase() : pattern
|
|
32
|
+
|
|
33
|
+
return {
|
|
34
|
+
line(line) {
|
|
35
|
+
const matched = linePrefix(line, SEARCH_LIMITS.matchCharacters)
|
|
36
|
+
const index = (ignoreCase ? matched.toLowerCase() : matched).indexOf(needle)
|
|
37
|
+
|
|
38
|
+
// Lowercasing lengthens a few characters (İ): the index is then near the match, not on it.
|
|
39
|
+
return index < 0 ? undefined : { index: Math.min(index, line.length), length: needle.length }
|
|
40
|
+
},
|
|
41
|
+
|
|
42
|
+
// A line's text is part of the file's. Lowercasing is the same character by character, but
|
|
43
|
+
// for Σ, which reads what follows it: only where a line is cut at 4,000 characters does that
|
|
44
|
+
// differ from the file's text, and a file with such a line is searched line by line.
|
|
45
|
+
mayMatch(text) {
|
|
46
|
+
if (!ignoreCase) return text.includes(needle)
|
|
47
|
+
if (hasLineOver(text, SEARCH_LIMITS.matchCharacters)) return true
|
|
48
|
+
|
|
49
|
+
return text.toLowerCase().includes(needle)
|
|
50
|
+
},
|
|
51
|
+
}
|
|
52
|
+
}
|
|
53
|
+
|
|
54
|
+
function regexMatcher(pattern: string): TextMatcher {
|
|
55
|
+
const regex = lineRegex(pattern)
|
|
56
|
+
const whole = wholeTextRegex(pattern, regex)
|
|
57
|
+
|
|
58
|
+
return {
|
|
59
|
+
line(line) {
|
|
60
|
+
const found = regex.exec(linePrefix(line, SEARCH_LIMITS.matchCharacters))
|
|
61
|
+
return found ? { index: found.index, length: found[0].length } : undefined
|
|
62
|
+
},
|
|
63
|
+
|
|
64
|
+
mayMatch(text) {
|
|
65
|
+
if (!whole || !linesAreWhole(text)) return true
|
|
66
|
+
return whole.test(text)
|
|
67
|
+
},
|
|
68
|
+
}
|
|
69
|
+
}
|
|
70
|
+
|
|
71
|
+
/**
|
|
72
|
+
* Unicode mode first, so `\p{…}` works; without it when that refuses what plain mode accepts.
|
|
73
|
+
* Case is ignored when the pattern writes no uppercase letter and no property escape: `\p{Lu}`
|
|
74
|
+
* ignoring case would match every cased letter.
|
|
75
|
+
*/
|
|
76
|
+
function lineRegex(pattern: string): RegExp {
|
|
77
|
+
const written = pattern.replace(/\\(?:[pP]\{[^}]*\}|.)/g, "")
|
|
78
|
+
const flags = /\p{Lu}/u.test(written) || /\\[pP]\{/.test(pattern) ? "" : "i"
|
|
79
|
+
|
|
80
|
+
try {
|
|
81
|
+
return new RegExp(pattern, `${flags}u`)
|
|
82
|
+
} catch {}
|
|
83
|
+
|
|
84
|
+
try {
|
|
85
|
+
return new RegExp(pattern, flags)
|
|
86
|
+
} catch (error) {
|
|
87
|
+
const message = error instanceof Error ? error.message : String(error)
|
|
88
|
+
throw new SearchError(
|
|
89
|
+
"invalid_pattern",
|
|
90
|
+
message.replace(/^Invalid regular expression: (?:\/.*\/[a-z]*: )?/, "")
|
|
91
|
+
)
|
|
92
|
+
}
|
|
93
|
+
}
|
|
94
|
+
|
|
95
|
+
/**
|
|
96
|
+
* The line regex with `m`, to test a whole file at once; nothing for a pattern written with a
|
|
97
|
+
* lookaround or a modifier group, or that can match a line break. The test stands in for the lines
|
|
98
|
+
* only on a file whose lines are whole ({@link linesAreWhole}).
|
|
99
|
+
*
|
|
100
|
+
* Why a line's match is then a match in the whole text: the line is a stretch of it, and the
|
|
101
|
+
* same characters match the same way. The pattern consumes no line break, so a match never
|
|
102
|
+
* leaves the line: what can still differ is what its assertions read.
|
|
103
|
+
*
|
|
104
|
+
* - `^` and `$`: with `m`, they hold where a line starts or ends, `\n` and `\r\n` alike. With no
|
|
105
|
+
* lone `\r` and no U+2028 or U+2029 in the file, those are exactly a line's edges: in the line
|
|
106
|
+
* and in the whole text, they hold at the same places.
|
|
107
|
+
* - `\b` and `\B`: past a line's edge, the whole text holds `\n` or `\r`, which is no word
|
|
108
|
+
* character, as nothing is: they read the same at a line's edges.
|
|
109
|
+
* - Backreferences repeat what their group matched in the line.
|
|
110
|
+
* - A lookaround could say no where an assertion says yes, and is rare in a search: a pattern
|
|
111
|
+
* with one is searched line by line, so nothing above rests on how one reads. A modifier group
|
|
112
|
+
* changes the flags themselves: `(?-m:^)` turns `m` back off.
|
|
113
|
+
*
|
|
114
|
+
* A match in the whole text that no line holds, across lines, is never reported: it only costs a
|
|
115
|
+
* file split into lines.
|
|
116
|
+
*
|
|
117
|
+
* Why the test stays fast: a line bounds what each try can read, and with it how far a regex
|
|
118
|
+
* backtracks. A pattern that can match a line break backtracks over the whole file instead —
|
|
119
|
+
* `[^x]*y` takes seconds on 100 KB — which is the other reason it is searched line by line.
|
|
120
|
+
*/
|
|
121
|
+
function wholeTextRegex(pattern: string, line: RegExp): RegExp | undefined {
|
|
122
|
+
if (!readsOneLine(pattern)) return undefined
|
|
123
|
+
return new RegExp(line.source, `${line.flags}m`)
|
|
124
|
+
}
|
|
125
|
+
|
|
126
|
+
/**
|
|
127
|
+
* Whether the line breaks `m` sees are the lines `grep` splits, each matched whole: no line is
|
|
128
|
+
* cut at {@link SEARCH_LIMITS.matchCharacters}, and no lone `\r`, U+2028 or U+2029, which end a
|
|
129
|
+
* line for `^` and `$` but stay inside one for {@link lines}.
|
|
130
|
+
*/
|
|
131
|
+
function linesAreWhole(text: string): boolean {
|
|
132
|
+
return !hasLineOver(text, SEARCH_LIMITS.matchCharacters) && !hasLoneBreak(text)
|
|
133
|
+
}
|
|
134
|
+
|
|
135
|
+
/** A `\r` not followed by `\n`, U+2028 or U+2029. Searched for as text, faster than a regex. */
|
|
136
|
+
function hasLoneBreak(text: string): boolean {
|
|
137
|
+
if (text.includes("\u2028") || text.includes("\u2029")) return true
|
|
138
|
+
|
|
139
|
+
for (let index = text.indexOf("\r"); index >= 0; index = text.indexOf("\r", index + 1)) {
|
|
140
|
+
if (text.charCodeAt(index + 1) !== LINE_FEED) return true
|
|
141
|
+
}
|
|
142
|
+
|
|
143
|
+
return false
|
|
144
|
+
}
|
|
145
|
+
|
|
146
|
+
const LINE_FEED = 0x0a
|
|
147
|
+
|
|
148
|
+
/** Escapes that never match a line break, outside a class and inside one. */
|
|
149
|
+
const ESCAPES = new Set(["b", "B", "d", "w", "S", "k"])
|
|
150
|
+
const CLASS_ESCAPES = new Set(["d", "w", "S"])
|
|
151
|
+
|
|
152
|
+
/**
|
|
153
|
+
* Whether a pattern, as written, has no lookaround, no modifier group, and cannot match `\n`,
|
|
154
|
+
* `\r`, U+2028 or U+2029. What it cannot rule out says no:
|
|
155
|
+
*
|
|
156
|
+
* - `(?=`, `(?!`, `(?<=`, `(?<!`, and `(?i:`, `(?m:`, `(?s:`, `(?-…:` outside a class.
|
|
157
|
+
* - `.` never matches a line break, without the `s` flag, nor do `\d`, `\w` and `\S`.
|
|
158
|
+
* - A negated class does, unless it names them: `[^…]` says no.
|
|
159
|
+
* - `\s`, `\D`, `\W`, `\n`, `\r`, `\t`, `\v`, `\f`, `\0`, `\x…`, `\u…`, `\c…`, `\p{…}` and digit
|
|
160
|
+
* escapes (an octal `\12` is `\n` without the `u` flag) all may, alone or as a range's end.
|
|
161
|
+
* - So may a control character or a line separator written as itself, escaped or not, and a
|
|
162
|
+
* range that spans one.
|
|
163
|
+
* - `\b` and `\B` are assertions, and `\k<name>` repeats a group's text; in a class, `\b` is a
|
|
164
|
+
* backspace, a control character.
|
|
165
|
+
*/
|
|
166
|
+
function readsOneLine(pattern: string): boolean {
|
|
167
|
+
let inClass = false
|
|
168
|
+
/** The class's last character, a range's start. */
|
|
169
|
+
let previous: number | undefined
|
|
170
|
+
|
|
171
|
+
for (let index = 0; index < pattern.length; index += 1) {
|
|
172
|
+
let code = pattern.charCodeAt(index)
|
|
173
|
+
if (isLineBreakOrControl(code)) return false
|
|
174
|
+
|
|
175
|
+
if (code === BACKSLASH) {
|
|
176
|
+
index += 1
|
|
177
|
+
const escaped = pattern[index] ?? ""
|
|
178
|
+
if (/[A-Za-z0-9]/.test(escaped)) {
|
|
179
|
+
if (!(inClass ? CLASS_ESCAPES : ESCAPES).has(escaped)) return false
|
|
180
|
+
previous = undefined
|
|
181
|
+
continue
|
|
182
|
+
}
|
|
183
|
+
|
|
184
|
+
// An escaped symbol is the symbol itself.
|
|
185
|
+
code = pattern.charCodeAt(index)
|
|
186
|
+
if (isLineBreakOrControl(code)) return false
|
|
187
|
+
} else if (!inClass && code === OPEN_PAREN) {
|
|
188
|
+
if (LOOKAROUND_OR_MODIFIER.test(pattern.slice(index, index + 4))) return false
|
|
189
|
+
} else if (!inClass && code === OPEN_BRACKET) {
|
|
190
|
+
if (pattern[index + 1] === "^") return false
|
|
191
|
+
inClass = true
|
|
192
|
+
previous = undefined
|
|
193
|
+
continue
|
|
194
|
+
} else if (inClass && code === CLOSE_BRACKET) {
|
|
195
|
+
inClass = false
|
|
196
|
+
continue
|
|
197
|
+
} else if (inClass && code === DASH && previous !== undefined && index + 1 < pattern.length) {
|
|
198
|
+
const end =
|
|
199
|
+
pattern[index + 1] === "\\" ? pattern.charCodeAt(index + 2) : pattern.charCodeAt(index + 1)
|
|
200
|
+
if (previous <= 0x2029 && end >= 0x2028) return false
|
|
201
|
+
}
|
|
202
|
+
|
|
203
|
+
previous = inClass ? code : undefined
|
|
204
|
+
}
|
|
205
|
+
|
|
206
|
+
return true
|
|
207
|
+
}
|
|
208
|
+
|
|
209
|
+
/** At a `(`: a lookaround, or a group that changes the `i`, `m` or `s` flag. */
|
|
210
|
+
const LOOKAROUND_OR_MODIFIER = /^\(\?(?:[=!]|<[=!]|[ims-])/
|
|
211
|
+
|
|
212
|
+
const OPEN_PAREN = 0x28
|
|
213
|
+
const BACKSLASH = 0x5c
|
|
214
|
+
const OPEN_BRACKET = 0x5b
|
|
215
|
+
const CLOSE_BRACKET = 0x5d
|
|
216
|
+
const DASH = 0x2d
|
|
217
|
+
|
|
218
|
+
function isLineBreakOrControl(code: number): boolean {
|
|
219
|
+
return code < 0x20 || code === 0x2028 || code === 0x2029
|
|
220
|
+
}
|
|
221
|
+
|
|
222
|
+
/** Whether a line of the text, its `\r` included, is longer than `length` UTF-16 units. */
|
|
223
|
+
function hasLineOver(text: string, length: number): boolean {
|
|
224
|
+
if (text.length <= length) return false
|
|
225
|
+
|
|
226
|
+
for (let start = 0; start <= text.length; ) {
|
|
227
|
+
const newline = text.indexOf("\n", start)
|
|
228
|
+
const end = newline < 0 ? text.length : newline
|
|
229
|
+
if (end - start > length) return true
|
|
230
|
+
start = end + 1
|
|
231
|
+
}
|
|
232
|
+
|
|
233
|
+
return false
|
|
234
|
+
}
|
|
235
|
+
|
|
236
|
+
/** A file's lines, without their `\n` or `\r\n`. */
|
|
237
|
+
export function lines(text: string): string[] {
|
|
238
|
+
const split = text.split("\n")
|
|
239
|
+
if (split.at(-1) === "") split.pop()
|
|
240
|
+
return split.map((line) => (line.endsWith("\r") ? line.slice(0, -1) : line))
|
|
241
|
+
}
|
|
242
|
+
|
|
243
|
+
/**
|
|
244
|
+
* A line as `grep` shows it: whole when it fits, else cut to {@link SEARCH_LIMITS.lineCharacters}
|
|
245
|
+
* characters around its first match, so a match far into a minified line is still seen. A cut
|
|
246
|
+
* never splits a character written as two units.
|
|
247
|
+
*/
|
|
248
|
+
export function cutLine(line: string, match: LineMatch): string {
|
|
249
|
+
const budget = SEARCH_LIMITS.lineCharacters
|
|
250
|
+
if (line.length <= budget) return line
|
|
251
|
+
|
|
252
|
+
// The window opens a little before the match, or ends with the line, the leading `…` in the
|
|
253
|
+
// budget; a match within the line's first characters keeps its start.
|
|
254
|
+
const room = budget - 1
|
|
255
|
+
const start =
|
|
256
|
+
match.index + match.length <= budget
|
|
257
|
+
? 0
|
|
258
|
+
: Math.max(0, Math.min(match.index - SEARCH_LIMITS.matchLead, line.length - room))
|
|
259
|
+
if (start === 0) return `${linePrefix(line, budget)}…`
|
|
260
|
+
|
|
261
|
+
const from = isLowSurrogate(line.charCodeAt(start)) ? start + 1 : start
|
|
262
|
+
// One unit past the room, so a character the room would split is seen and left out.
|
|
263
|
+
const window = linePrefix(line.slice(from, from + room + 1), room)
|
|
264
|
+
|
|
265
|
+
return `…${window}${from + window.length < line.length ? "…" : ""}`
|
|
266
|
+
}
|
|
267
|
+
|
|
268
|
+
function isLowSurrogate(code: number): boolean {
|
|
269
|
+
return code >= 0xdc00 && code <= 0xdfff
|
|
270
|
+
}
|
|
271
|
+
|
|
272
|
+
/** A line's first `length` UTF-16 units, never half of a character written as two. */
|
|
273
|
+
function linePrefix(line: string, length: number): string {
|
|
274
|
+
if (line.length <= length) return line
|
|
275
|
+
|
|
276
|
+
const code = line.charCodeAt(length - 1)
|
|
277
|
+
return line.slice(0, code >= 0xd800 && code <= 0xdbff ? length - 1 : length)
|
|
278
|
+
}
|
|
@@ -0,0 +1,178 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Path patterns, as `glob` and `.gitignore` write them: `*` and `?` within a name, `[abc]`,
|
|
3
|
+
* `[!abc]` and `[^abc]`, `**` as a whole segment for any number of folders, `\` to escape.
|
|
4
|
+
* `{a,b}` belongs to `glob` alone and is expanded before compiling ({@link expandBraces}).
|
|
5
|
+
*/
|
|
6
|
+
|
|
7
|
+
/** More alternatives than any real pattern names: past it, `{…}` is a mistake. */
|
|
8
|
+
const MAX_ALTERNATIVES = 256
|
|
9
|
+
|
|
10
|
+
export class PatternError extends Error {}
|
|
11
|
+
|
|
12
|
+
/** A pattern's regular expression source, for a whole path relative to where it applies. */
|
|
13
|
+
export function pathPatternSource(pattern: string): string {
|
|
14
|
+
const segments = pattern.split("/")
|
|
15
|
+
let source = ""
|
|
16
|
+
|
|
17
|
+
segments.forEach((segment, index) => {
|
|
18
|
+
const last = index === segments.length - 1
|
|
19
|
+
|
|
20
|
+
if (segment === "**") {
|
|
21
|
+
// `a/**` is everything inside a; `**/b` and `a/**/b` cross any number of folders.
|
|
22
|
+
source += last ? ".*" : "(?:[^/]*/)*"
|
|
23
|
+
return
|
|
24
|
+
}
|
|
25
|
+
|
|
26
|
+
source += segmentSource(segment)
|
|
27
|
+
if (!last) source += "/"
|
|
28
|
+
})
|
|
29
|
+
|
|
30
|
+
return source
|
|
31
|
+
}
|
|
32
|
+
|
|
33
|
+
/** One name's source: never crosses a `/`. */
|
|
34
|
+
export function segmentSource(segment: string): string {
|
|
35
|
+
let source = ""
|
|
36
|
+
|
|
37
|
+
for (let index = 0; index < segment.length; index += 1) {
|
|
38
|
+
const char = segment[index] as string
|
|
39
|
+
|
|
40
|
+
if (char === "\\") {
|
|
41
|
+
index += 1
|
|
42
|
+
source += escapeRegExp(segment[index] ?? "\\")
|
|
43
|
+
} else if (char === "*") {
|
|
44
|
+
while (segment[index + 1] === "*") index += 1
|
|
45
|
+
source += "[^/]*"
|
|
46
|
+
} else if (char === "?") {
|
|
47
|
+
source += "[^/]"
|
|
48
|
+
} else if (char === "[") {
|
|
49
|
+
const parsed = characterClass(segment, index)
|
|
50
|
+
if (parsed) {
|
|
51
|
+
source += parsed.source
|
|
52
|
+
index = parsed.end
|
|
53
|
+
} else {
|
|
54
|
+
source += "\\["
|
|
55
|
+
}
|
|
56
|
+
} else {
|
|
57
|
+
source += escapeRegExp(char)
|
|
58
|
+
}
|
|
59
|
+
}
|
|
60
|
+
|
|
61
|
+
return source
|
|
62
|
+
}
|
|
63
|
+
|
|
64
|
+
/** `[...]` from `start`, or nothing when it never closes and is a literal `[`. */
|
|
65
|
+
function characterClass(
|
|
66
|
+
segment: string,
|
|
67
|
+
start: number
|
|
68
|
+
): { readonly source: string; readonly end: number } | undefined {
|
|
69
|
+
let index = start + 1
|
|
70
|
+
const negated = segment[index] === "!" || segment[index] === "^"
|
|
71
|
+
if (negated) index += 1
|
|
72
|
+
|
|
73
|
+
let body = ""
|
|
74
|
+
// A `]` right after the opening is part of the class.
|
|
75
|
+
if (segment[index] === "]") {
|
|
76
|
+
body += "\\]"
|
|
77
|
+
index += 1
|
|
78
|
+
}
|
|
79
|
+
|
|
80
|
+
for (; index < segment.length; index += 1) {
|
|
81
|
+
const char = segment[index] as string
|
|
82
|
+
|
|
83
|
+
if (char === "]") {
|
|
84
|
+
return { source: negated ? `[^/${body}]` : `[${body}]`, end: index }
|
|
85
|
+
}
|
|
86
|
+
if (char === "\\" && index + 1 < segment.length) {
|
|
87
|
+
index += 1
|
|
88
|
+
body += escapeClassChar(segment[index] as string)
|
|
89
|
+
} else {
|
|
90
|
+
body += char === "-" ? "-" : escapeClassChar(char)
|
|
91
|
+
}
|
|
92
|
+
}
|
|
93
|
+
|
|
94
|
+
return undefined
|
|
95
|
+
}
|
|
96
|
+
|
|
97
|
+
/**
|
|
98
|
+
* Expand `{a,b}` into its alternatives, nested ones included. A brace without a comma, or that
|
|
99
|
+
* never closes, is a literal brace.
|
|
100
|
+
*/
|
|
101
|
+
export function expandBraces(pattern: string): string[] {
|
|
102
|
+
const expanded: string[] = []
|
|
103
|
+
expandInto(pattern, expanded)
|
|
104
|
+
return expanded
|
|
105
|
+
}
|
|
106
|
+
|
|
107
|
+
function expandInto(pattern: string, into: string[]): void {
|
|
108
|
+
for (let index = 0; index < pattern.length; index += 1) {
|
|
109
|
+
const char = pattern[index]
|
|
110
|
+
|
|
111
|
+
if (char === "\\") {
|
|
112
|
+
index += 1
|
|
113
|
+
continue
|
|
114
|
+
}
|
|
115
|
+
if (char !== "{") continue
|
|
116
|
+
|
|
117
|
+
const group = braceGroup(pattern, index)
|
|
118
|
+
if (!group) continue
|
|
119
|
+
|
|
120
|
+
const prefix = pattern.slice(0, index)
|
|
121
|
+
const suffix = pattern.slice(group.end + 1)
|
|
122
|
+
for (const option of group.options) expandInto(`${prefix}${option}${suffix}`, into)
|
|
123
|
+
return
|
|
124
|
+
}
|
|
125
|
+
|
|
126
|
+
if (into.length >= MAX_ALTERNATIVES) {
|
|
127
|
+
throw new PatternError(`it names more than ${MAX_ALTERNATIVES} alternatives with {…}`)
|
|
128
|
+
}
|
|
129
|
+
into.push(pattern)
|
|
130
|
+
}
|
|
131
|
+
|
|
132
|
+
/** The `{…}` opening at `start`: its options split on top-level commas, and its closing index. */
|
|
133
|
+
function braceGroup(
|
|
134
|
+
pattern: string,
|
|
135
|
+
start: number
|
|
136
|
+
): { readonly options: string[]; readonly end: number } | undefined {
|
|
137
|
+
const options: string[] = []
|
|
138
|
+
let depth = 0
|
|
139
|
+
let optionStart = start + 1
|
|
140
|
+
|
|
141
|
+
for (let index = start + 1; index < pattern.length; index += 1) {
|
|
142
|
+
const char = pattern[index]
|
|
143
|
+
|
|
144
|
+
if (char === "\\") {
|
|
145
|
+
index += 1
|
|
146
|
+
} else if (char === "{") {
|
|
147
|
+
depth += 1
|
|
148
|
+
} else if (char === "}" && depth > 0) {
|
|
149
|
+
depth -= 1
|
|
150
|
+
} else if (char === "," && depth === 0) {
|
|
151
|
+
options.push(pattern.slice(optionStart, index))
|
|
152
|
+
optionStart = index + 1
|
|
153
|
+
} else if (char === "}") {
|
|
154
|
+
if (options.length === 0) return undefined
|
|
155
|
+
options.push(pattern.slice(optionStart, index))
|
|
156
|
+
return { options, end: index }
|
|
157
|
+
}
|
|
158
|
+
}
|
|
159
|
+
|
|
160
|
+
return undefined
|
|
161
|
+
}
|
|
162
|
+
|
|
163
|
+
/** Compile a regular expression source, or say why the pattern is not one. */
|
|
164
|
+
export function compilePattern(source: string, flags = ""): RegExp {
|
|
165
|
+
try {
|
|
166
|
+
return new RegExp(`^${source}$`, flags)
|
|
167
|
+
} catch (error) {
|
|
168
|
+
throw new PatternError(error instanceof Error ? error.message : String(error))
|
|
169
|
+
}
|
|
170
|
+
}
|
|
171
|
+
|
|
172
|
+
function escapeRegExp(text: string): string {
|
|
173
|
+
return text.replace(/[.*+?^${}()|[\]\\/]/g, "\\$&")
|
|
174
|
+
}
|
|
175
|
+
|
|
176
|
+
function escapeClassChar(char: string): string {
|
|
177
|
+
return /[\\\]^[-]/.test(char) ? `\\${char}` : char
|
|
178
|
+
}
|
|
@@ -0,0 +1,237 @@
|
|
|
1
|
+
import { Buffer } from "node:buffer"
|
|
2
|
+
import { type FileHandle, open } from "node:fs/promises"
|
|
3
|
+
import { SEARCH_LIMITS } from "./request"
|
|
4
|
+
|
|
5
|
+
export type FileText =
|
|
6
|
+
| { readonly text: string }
|
|
7
|
+
| { readonly skipped: "binary" | "too_large" | "unreadable" }
|
|
8
|
+
|
|
9
|
+
/** A read that knows what it will hold before it holds it. */
|
|
10
|
+
export interface PendingRead<R> {
|
|
11
|
+
/** The bytes the read holds once started, until its result is taken. */
|
|
12
|
+
readonly bytes: number
|
|
13
|
+
/** Never rejects. */
|
|
14
|
+
read(): Promise<R>
|
|
15
|
+
/** Lets go of a read never started. Never rejects. */
|
|
16
|
+
close(): Promise<void>
|
|
17
|
+
}
|
|
18
|
+
|
|
19
|
+
/**
|
|
20
|
+
* A file opened to be read: one open, and its size from that descriptor. Its text comes from
|
|
21
|
+
* {@link PendingRead.read}, unless it is too large or unreadable, which is known already.
|
|
22
|
+
*/
|
|
23
|
+
export async function openText(path: string, strict: boolean): Promise<PendingRead<FileText>> {
|
|
24
|
+
let file: FileHandle | undefined
|
|
25
|
+
|
|
26
|
+
try {
|
|
27
|
+
file = await open(path, "r")
|
|
28
|
+
const { size } = await file.stat()
|
|
29
|
+
|
|
30
|
+
if (size > SEARCH_LIMITS.fileBytes) {
|
|
31
|
+
await closeQuietly(file)
|
|
32
|
+
return settled({ skipped: "too_large" })
|
|
33
|
+
}
|
|
34
|
+
|
|
35
|
+
const opened = file
|
|
36
|
+
return {
|
|
37
|
+
bytes: size,
|
|
38
|
+
read: () => readText(opened, size, strict),
|
|
39
|
+
close: () => closeQuietly(opened),
|
|
40
|
+
}
|
|
41
|
+
} catch {
|
|
42
|
+
if (file) await closeQuietly(file)
|
|
43
|
+
return settled({ skipped: "unreadable" })
|
|
44
|
+
}
|
|
45
|
+
}
|
|
46
|
+
|
|
47
|
+
function settled(text: FileText): PendingRead<FileText> {
|
|
48
|
+
return { bytes: 0, read: async () => text, close: async () => {} }
|
|
49
|
+
}
|
|
50
|
+
|
|
51
|
+
/**
|
|
52
|
+
* The text of a file opened at `size` bytes, read through its descriptor and closed. A file
|
|
53
|
+
* over {@link READ_AT_ONCE_BYTES} is read in two: its first {@link SEARCH_LIMITS.binaryProbeBytes},
|
|
54
|
+
* then the rest unless they show it is binary, so a large binary file is never read whole. A file
|
|
55
|
+
* found by the walk is read as UTF-8 leniently, so a mostly ASCII file in another encoding still
|
|
56
|
+
* matches; a file named as the path must be UTF-8 text. A leading byte order mark is dropped, so
|
|
57
|
+
* `^` finds the first line. Never rejects.
|
|
58
|
+
*/
|
|
59
|
+
async function readText(file: FileHandle, size: number, strict: boolean): Promise<FileText> {
|
|
60
|
+
try {
|
|
61
|
+
const first = size <= READ_AT_ONCE_BYTES ? size : PROBE_BYTES
|
|
62
|
+
let bytes = await readBytes(file, Buffer.allocUnsafe(first), 0)
|
|
63
|
+
if (bytes.subarray(0, PROBE_BYTES).includes(0)) return { skipped: "binary" }
|
|
64
|
+
|
|
65
|
+
if (bytes.length === first && first < size) {
|
|
66
|
+
const whole = Buffer.allocUnsafe(size)
|
|
67
|
+
bytes.copy(whole)
|
|
68
|
+
bytes = await readBytes(file, whole, first)
|
|
69
|
+
}
|
|
70
|
+
|
|
71
|
+
let text: string
|
|
72
|
+
if (strict) {
|
|
73
|
+
try {
|
|
74
|
+
text = new TextDecoder("utf-8", { fatal: true, ignoreBOM: true }).decode(bytes)
|
|
75
|
+
} catch {
|
|
76
|
+
return { skipped: "binary" }
|
|
77
|
+
}
|
|
78
|
+
} else {
|
|
79
|
+
text = bytes.toString("utf8")
|
|
80
|
+
}
|
|
81
|
+
|
|
82
|
+
return { text: text.startsWith("\uFEFF") ? text.slice(1) : text }
|
|
83
|
+
} catch {
|
|
84
|
+
return { skipped: "unreadable" }
|
|
85
|
+
} finally {
|
|
86
|
+
await closeQuietly(file)
|
|
87
|
+
}
|
|
88
|
+
}
|
|
89
|
+
|
|
90
|
+
const PROBE_BYTES = SEARCH_LIMITS.binaryProbeBytes
|
|
91
|
+
|
|
92
|
+
/** Up to this size, a file is read in one call: most source files, and two calls cost more. */
|
|
93
|
+
const READ_AT_ONCE_BYTES = 64 * 1024
|
|
94
|
+
|
|
95
|
+
/**
|
|
96
|
+
* Fills `bytes` from `from` on, and returns what was read: the file as it was when opened, if it
|
|
97
|
+
* grows meanwhile, or less if it shrank.
|
|
98
|
+
*/
|
|
99
|
+
async function readBytes(file: FileHandle, bytes: Buffer, from: number): Promise<Buffer> {
|
|
100
|
+
let filled = from
|
|
101
|
+
|
|
102
|
+
while (filled < bytes.length) {
|
|
103
|
+
const { bytesRead } = await file.read(bytes, filled, bytes.length - filled, filled)
|
|
104
|
+
if (bytesRead === 0) break
|
|
105
|
+
filled += bytesRead
|
|
106
|
+
}
|
|
107
|
+
|
|
108
|
+
return bytes.subarray(0, filled)
|
|
109
|
+
}
|
|
110
|
+
|
|
111
|
+
async function closeQuietly(file: FileHandle): Promise<void> {
|
|
112
|
+
await file.close().catch(() => {})
|
|
113
|
+
}
|
|
114
|
+
|
|
115
|
+
/**
|
|
116
|
+
* How far reads run ahead of the file searched. Reading is most of a search's time, and mostly
|
|
117
|
+
* waiting: reads in flight overlap that wait. At most {@link READ_AHEAD} files are open, and a read
|
|
118
|
+
* starts only while the reads started hold at most {@link READ_AHEAD_BYTES} of file with it, but
|
|
119
|
+
* for the next file in line, which always starts: one file of 10 MiB past the bound at most. A
|
|
120
|
+
* binary file's bytes are let go once its first bytes show it is one.
|
|
121
|
+
*
|
|
122
|
+
* Memory: the texts read ahead take up to twice their bytes once decoded (a string with one
|
|
123
|
+
* character beyond Latin-1 takes two bytes for each): 52 MiB at worst, plus the file searched and
|
|
124
|
+
* the copies its search makes. 16 MiB reads a source tree as fast as 32 did, and holds half as
|
|
125
|
+
* much over a folder of large files.
|
|
126
|
+
*/
|
|
127
|
+
export const READ_AHEAD = 32
|
|
128
|
+
export const READ_AHEAD_BYTES = 16 * 1024 * 1024
|
|
129
|
+
|
|
130
|
+
interface Slot<T, R> {
|
|
131
|
+
readonly item: T
|
|
132
|
+
/** Its {@link PendingRead}, once opened. */
|
|
133
|
+
pending?: PendingRead<R>
|
|
134
|
+
/** Settles once {@link pending} is known. */
|
|
135
|
+
opening?: Promise<void>
|
|
136
|
+
/** The bytes it holds against {@link READ_AHEAD_BYTES}. */
|
|
137
|
+
holds: number
|
|
138
|
+
result?: Promise<R>
|
|
139
|
+
}
|
|
140
|
+
|
|
141
|
+
/**
|
|
142
|
+
* Files read ahead of the one searched, handed back in the order they come, so a search finds the
|
|
143
|
+
* same lines, in the same order, as reading one file at a time. Files are opened as they come,
|
|
144
|
+
* and read as soon as they are open and their bytes fit. A search that stops reads no further than
|
|
145
|
+
* {@link READ_AHEAD} files past where it stopped.
|
|
146
|
+
*/
|
|
147
|
+
export class ReadAhead<T, R> {
|
|
148
|
+
readonly #slots: Slot<T, R>[] = []
|
|
149
|
+
#held = 0
|
|
150
|
+
#exhausted = false
|
|
151
|
+
#stopped = false
|
|
152
|
+
|
|
153
|
+
constructor(
|
|
154
|
+
private readonly items: Iterator<T>,
|
|
155
|
+
/** Never rejects. */
|
|
156
|
+
private readonly open: (item: T) => Promise<PendingRead<R>>,
|
|
157
|
+
/** What a read that holds nothing once done looks like, a binary file. */
|
|
158
|
+
private readonly holdsNothing: (result: R) => boolean = () => false
|
|
159
|
+
) {}
|
|
160
|
+
|
|
161
|
+
async next(): Promise<{ readonly item: T; readonly read: R } | undefined> {
|
|
162
|
+
this.#fill()
|
|
163
|
+
const slot = this.#slots[0]
|
|
164
|
+
if (!slot) return undefined
|
|
165
|
+
|
|
166
|
+
await slot.opening
|
|
167
|
+
this.#start()
|
|
168
|
+
const read = await (slot.result as Promise<R>)
|
|
169
|
+
|
|
170
|
+
this.#slots.shift()
|
|
171
|
+
this.#held -= slot.holds
|
|
172
|
+
this.#fill()
|
|
173
|
+
this.#start()
|
|
174
|
+
|
|
175
|
+
return { item: slot.item, read }
|
|
176
|
+
}
|
|
177
|
+
|
|
178
|
+
/**
|
|
179
|
+
* Starts no other read, waits for those in flight and closes the files never read: no file
|
|
180
|
+
* stays open past the search.
|
|
181
|
+
*/
|
|
182
|
+
async stop(): Promise<void> {
|
|
183
|
+
this.#exhausted = true
|
|
184
|
+
this.#stopped = true
|
|
185
|
+
|
|
186
|
+
await Promise.all(
|
|
187
|
+
this.#slots.splice(0).map(async (slot) => {
|
|
188
|
+
await slot.opening
|
|
189
|
+
if (slot.result) await slot.result
|
|
190
|
+
else await slot.pending?.close()
|
|
191
|
+
})
|
|
192
|
+
)
|
|
193
|
+
}
|
|
194
|
+
|
|
195
|
+
#fill(): void {
|
|
196
|
+
while (!this.#exhausted && this.#slots.length < READ_AHEAD) {
|
|
197
|
+
const step = this.items.next()
|
|
198
|
+
if (step.done) {
|
|
199
|
+
this.#exhausted = true
|
|
200
|
+
return
|
|
201
|
+
}
|
|
202
|
+
|
|
203
|
+
const slot: Slot<T, R> = { item: step.value, holds: 0 }
|
|
204
|
+
this.#slots.push(slot)
|
|
205
|
+
slot.opening = this.open(step.value).then((pending) => {
|
|
206
|
+
slot.pending = pending
|
|
207
|
+
this.#start()
|
|
208
|
+
})
|
|
209
|
+
}
|
|
210
|
+
}
|
|
211
|
+
|
|
212
|
+
/**
|
|
213
|
+
* Starts the reads of the files opened, as long as their bytes fit. The first file in line always
|
|
214
|
+
* starts: the search waits on it, and no read behind it can make room.
|
|
215
|
+
*/
|
|
216
|
+
#start(): void {
|
|
217
|
+
if (this.#stopped) return
|
|
218
|
+
|
|
219
|
+
for (let index = 0; index < this.#slots.length; index += 1) {
|
|
220
|
+
const slot = this.#slots[index] as Slot<T, R>
|
|
221
|
+
const pending = slot.pending
|
|
222
|
+
if (slot.result || !pending) continue
|
|
223
|
+
if (index > 0 && this.#held + pending.bytes > READ_AHEAD_BYTES) continue
|
|
224
|
+
|
|
225
|
+
slot.holds = pending.bytes
|
|
226
|
+
this.#held += slot.holds
|
|
227
|
+
slot.result = pending.read().then((result) => {
|
|
228
|
+
if (this.holdsNothing(result) && this.#slots.includes(slot)) {
|
|
229
|
+
this.#held -= slot.holds
|
|
230
|
+
slot.holds = 0
|
|
231
|
+
this.#start()
|
|
232
|
+
}
|
|
233
|
+
return result
|
|
234
|
+
})
|
|
235
|
+
}
|
|
236
|
+
}
|
|
237
|
+
}
|