@sixb/agent 0.2.4 → 0.2.5

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (90) hide show
  1. package/README.md +2 -2
  2. package/dist/agent-prompt.d.ts.map +1 -1
  3. package/dist/artifacts/layout-check/layout-check.d.ts +21 -24
  4. package/dist/artifacts/layout-check/layout-check.d.ts.map +1 -1
  5. package/dist/artifacts/publish-tool.d.ts.map +1 -1
  6. package/dist/index.d.ts +0 -1
  7. package/dist/index.d.ts.map +1 -1
  8. package/dist/index.js +10590 -8888
  9. package/dist/index.js.map +30 -26
  10. package/dist/run-agent-turn.d.ts +3 -14
  11. package/dist/run-agent-turn.d.ts.map +1 -1
  12. package/dist/run-environment.d.ts.map +1 -1
  13. package/dist/run-subagent.d.ts.map +1 -1
  14. package/dist/sandbox-api-context.d.ts.map +1 -1
  15. package/dist/search-script.d.ts +7 -0
  16. package/dist/search-script.d.ts.map +1 -0
  17. package/dist/skills/built-in.d.ts.map +1 -1
  18. package/dist/tools/glob.d.ts +41 -0
  19. package/dist/tools/glob.d.ts.map +1 -0
  20. package/dist/tools/grep.d.ts +64 -0
  21. package/dist/tools/grep.d.ts.map +1 -0
  22. package/dist/tools/model-spec.d.ts.map +1 -1
  23. package/dist/tools/sandbox-search.d.ts +26 -0
  24. package/dist/tools/sandbox-search.d.ts.map +1 -0
  25. package/dist/tools/search/command.d.ts +15 -0
  26. package/dist/tools/search/command.d.ts.map +1 -0
  27. package/dist/tools/search/gitignore.d.ts +28 -0
  28. package/dist/tools/search/gitignore.d.ts.map +1 -0
  29. package/dist/tools/search/glob.d.ts +15 -0
  30. package/dist/tools/search/glob.d.ts.map +1 -0
  31. package/dist/tools/search/main.d.ts +2 -0
  32. package/dist/tools/search/main.d.ts.map +1 -0
  33. package/dist/tools/search/match.d.ts +30 -0
  34. package/dist/tools/search/match.d.ts.map +1 -0
  35. package/dist/tools/search/pattern.d.ts +19 -0
  36. package/dist/tools/search/pattern.d.ts.map +1 -0
  37. package/dist/tools/search/read.d.ts +62 -0
  38. package/dist/tools/search/read.d.ts.map +1 -0
  39. package/dist/tools/search/request.d.ts +80 -0
  40. package/dist/tools/search/request.d.ts.map +1 -0
  41. package/dist/tools/search/search.d.ts +4 -0
  42. package/dist/tools/search/search.d.ts.map +1 -0
  43. package/dist/tools/search/walk.d.ts +43 -0
  44. package/dist/tools/search/walk.d.ts.map +1 -0
  45. package/dist/tools/tool-outputs.d.ts.map +1 -1
  46. package/dist/turn-inbox.d.ts +75 -0
  47. package/dist/turn-inbox.d.ts.map +1 -0
  48. package/package.json +6 -4
  49. package/src/agent-prompt.ts +1 -0
  50. package/src/agent-runtime/preflight.ts +1 -0
  51. package/src/artifacts/layout-check/check.cjs +176 -95
  52. package/src/artifacts/layout-check/layout-check.ts +118 -76
  53. package/src/artifacts/publish-tool.ts +48 -99
  54. package/src/generated/search.mjs +985 -0
  55. package/src/index.ts +0 -1
  56. package/src/run-agent-turn.ts +85 -129
  57. package/src/run-conversation-turn.ts +4 -4
  58. package/src/run-environment.ts +4 -0
  59. package/src/run-subagent.ts +1 -1
  60. package/src/sandbox-api-context.ts +3 -0
  61. package/src/search-script-assets.d.ts +5 -0
  62. package/src/search-script.ts +9 -0
  63. package/src/skills/built-in.ts +16 -0
  64. package/src/skills/sixb-artifacts/SKILL.md +7 -7
  65. package/src/skills/sixb-artifacts/canvas.md +3 -2
  66. package/src/skills/sixb-artifacts/document.md +92 -5
  67. package/src/skills/sixb-artifacts/examples/document/service-proposal/design.css +114 -0
  68. package/src/skills/sixb-artifacts/examples/document/service-proposal/document.html +159 -0
  69. package/src/skills/sixb-artifacts/examples/document/service-proposal/images/fieldwise.svg +1 -0
  70. package/src/skills/sixb-artifacts/examples/document/service-proposal/images/harbor-pine.svg +1 -0
  71. package/src/skills/sixb-artifacts/queries.md +2 -1
  72. package/src/tools/glob.ts +124 -0
  73. package/src/tools/grep.ts +190 -0
  74. package/src/tools/model-spec.ts +4 -0
  75. package/src/tools/sandbox-search.ts +219 -0
  76. package/src/tools/search/command.ts +116 -0
  77. package/src/tools/search/gitignore.ts +101 -0
  78. package/src/tools/search/glob.ts +90 -0
  79. package/src/tools/search/main.ts +9 -0
  80. package/src/tools/search/match.ts +278 -0
  81. package/src/tools/search/pattern.ts +178 -0
  82. package/src/tools/search/read.ts +237 -0
  83. package/src/tools/search/request.ts +101 -0
  84. package/src/tools/search/search.ts +297 -0
  85. package/src/tools/search/walk.ts +145 -0
  86. package/src/tools/tool-outputs.ts +7 -2
  87. package/src/turn-inbox.ts +283 -0
  88. package/dist/artifacts/board-previews.d.ts +0 -41
  89. package/dist/artifacts/board-previews.d.ts.map +0 -1
  90. package/src/artifacts/board-previews.ts +0 -218
@@ -0,0 +1,278 @@
1
+ import { SEARCH_LIMITS, SearchError } from "./request"
2
+
3
+ /** Where a line first matched `grep`'s pattern, in UTF-16 units. */
4
+ export interface LineMatch {
5
+ readonly index: number
6
+ readonly length: number
7
+ }
8
+
9
+ export interface TextMatcher {
10
+ /**
11
+ * Where a line first matches, over its first {@link SEARCH_LIMITS.matchCharacters} characters.
12
+ */
13
+ line(line: string): LineMatch | undefined
14
+ /**
15
+ * Whether any line of a file may match: false only when none does, so a file that cannot match
16
+ * is never split into lines. It may say true of a file with no matching line.
17
+ */
18
+ mayMatch(text: string): boolean
19
+ }
20
+
21
+ /**
22
+ * `grep`'s pattern, matched line by line. Case is smart: ignored when the pattern has no uppercase
23
+ * letter. Literal text is a plain substring search, which no line can slow down.
24
+ */
25
+ export function textMatcher(pattern: string, literal: boolean): TextMatcher {
26
+ return literal ? literalMatcher(pattern) : regexMatcher(pattern)
27
+ }
28
+
29
+ function literalMatcher(pattern: string): TextMatcher {
30
+ const ignoreCase = !/\p{Lu}/u.test(pattern)
31
+ const needle = ignoreCase ? pattern.toLowerCase() : pattern
32
+
33
+ return {
34
+ line(line) {
35
+ const matched = linePrefix(line, SEARCH_LIMITS.matchCharacters)
36
+ const index = (ignoreCase ? matched.toLowerCase() : matched).indexOf(needle)
37
+
38
+ // Lowercasing lengthens a few characters (İ): the index is then near the match, not on it.
39
+ return index < 0 ? undefined : { index: Math.min(index, line.length), length: needle.length }
40
+ },
41
+
42
+ // A line's text is part of the file's. Lowercasing is the same character by character, but
43
+ // for Σ, which reads what follows it: only where a line is cut at 4,000 characters does that
44
+ // differ from the file's text, and a file with such a line is searched line by line.
45
+ mayMatch(text) {
46
+ if (!ignoreCase) return text.includes(needle)
47
+ if (hasLineOver(text, SEARCH_LIMITS.matchCharacters)) return true
48
+
49
+ return text.toLowerCase().includes(needle)
50
+ },
51
+ }
52
+ }
53
+
54
+ function regexMatcher(pattern: string): TextMatcher {
55
+ const regex = lineRegex(pattern)
56
+ const whole = wholeTextRegex(pattern, regex)
57
+
58
+ return {
59
+ line(line) {
60
+ const found = regex.exec(linePrefix(line, SEARCH_LIMITS.matchCharacters))
61
+ return found ? { index: found.index, length: found[0].length } : undefined
62
+ },
63
+
64
+ mayMatch(text) {
65
+ if (!whole || !linesAreWhole(text)) return true
66
+ return whole.test(text)
67
+ },
68
+ }
69
+ }
70
+
71
+ /**
72
+ * Unicode mode first, so `\p{…}` works; without it when that refuses what plain mode accepts.
73
+ * Case is ignored when the pattern writes no uppercase letter and no property escape: `\p{Lu}`
74
+ * ignoring case would match every cased letter.
75
+ */
76
+ function lineRegex(pattern: string): RegExp {
77
+ const written = pattern.replace(/\\(?:[pP]\{[^}]*\}|.)/g, "")
78
+ const flags = /\p{Lu}/u.test(written) || /\\[pP]\{/.test(pattern) ? "" : "i"
79
+
80
+ try {
81
+ return new RegExp(pattern, `${flags}u`)
82
+ } catch {}
83
+
84
+ try {
85
+ return new RegExp(pattern, flags)
86
+ } catch (error) {
87
+ const message = error instanceof Error ? error.message : String(error)
88
+ throw new SearchError(
89
+ "invalid_pattern",
90
+ message.replace(/^Invalid regular expression: (?:\/.*\/[a-z]*: )?/, "")
91
+ )
92
+ }
93
+ }
94
+
95
+ /**
96
+ * The line regex with `m`, to test a whole file at once; nothing for a pattern written with a
97
+ * lookaround or a modifier group, or that can match a line break. The test stands in for the lines
98
+ * only on a file whose lines are whole ({@link linesAreWhole}).
99
+ *
100
+ * Why a line's match is then a match in the whole text: the line is a stretch of it, and the
101
+ * same characters match the same way. The pattern consumes no line break, so a match never
102
+ * leaves the line: what can still differ is what its assertions read.
103
+ *
104
+ * - `^` and `$`: with `m`, they hold where a line starts or ends, `\n` and `\r\n` alike. With no
105
+ * lone `\r` and no U+2028 or U+2029 in the file, those are exactly a line's edges: in the line
106
+ * and in the whole text, they hold at the same places.
107
+ * - `\b` and `\B`: past a line's edge, the whole text holds `\n` or `\r`, which is no word
108
+ * character, as nothing is: they read the same at a line's edges.
109
+ * - Backreferences repeat what their group matched in the line.
110
+ * - A lookaround could say no where an assertion says yes, and is rare in a search: a pattern
111
+ * with one is searched line by line, so nothing above rests on how one reads. A modifier group
112
+ * changes the flags themselves: `(?-m:^)` turns `m` back off.
113
+ *
114
+ * A match in the whole text that no line holds, across lines, is never reported: it only costs a
115
+ * file split into lines.
116
+ *
117
+ * Why the test stays fast: a line bounds what each try can read, and with it how far a regex
118
+ * backtracks. A pattern that can match a line break backtracks over the whole file instead —
119
+ * `[^x]*y` takes seconds on 100 KB — which is the other reason it is searched line by line.
120
+ */
121
+ function wholeTextRegex(pattern: string, line: RegExp): RegExp | undefined {
122
+ if (!readsOneLine(pattern)) return undefined
123
+ return new RegExp(line.source, `${line.flags}m`)
124
+ }
125
+
126
+ /**
127
+ * Whether the line breaks `m` sees are the lines `grep` splits, each matched whole: no line is
128
+ * cut at {@link SEARCH_LIMITS.matchCharacters}, and no lone `\r`, U+2028 or U+2029, which end a
129
+ * line for `^` and `$` but stay inside one for {@link lines}.
130
+ */
131
+ function linesAreWhole(text: string): boolean {
132
+ return !hasLineOver(text, SEARCH_LIMITS.matchCharacters) && !hasLoneBreak(text)
133
+ }
134
+
135
+ /** A `\r` not followed by `\n`, U+2028 or U+2029. Searched for as text, faster than a regex. */
136
+ function hasLoneBreak(text: string): boolean {
137
+ if (text.includes("\u2028") || text.includes("\u2029")) return true
138
+
139
+ for (let index = text.indexOf("\r"); index >= 0; index = text.indexOf("\r", index + 1)) {
140
+ if (text.charCodeAt(index + 1) !== LINE_FEED) return true
141
+ }
142
+
143
+ return false
144
+ }
145
+
146
+ const LINE_FEED = 0x0a
147
+
148
+ /** Escapes that never match a line break, outside a class and inside one. */
149
+ const ESCAPES = new Set(["b", "B", "d", "w", "S", "k"])
150
+ const CLASS_ESCAPES = new Set(["d", "w", "S"])
151
+
152
+ /**
153
+ * Whether a pattern, as written, has no lookaround, no modifier group, and cannot match `\n`,
154
+ * `\r`, U+2028 or U+2029. What it cannot rule out says no:
155
+ *
156
+ * - `(?=`, `(?!`, `(?<=`, `(?<!`, and `(?i:`, `(?m:`, `(?s:`, `(?-…:` outside a class.
157
+ * - `.` never matches a line break, without the `s` flag, nor do `\d`, `\w` and `\S`.
158
+ * - A negated class does, unless it names them: `[^…]` says no.
159
+ * - `\s`, `\D`, `\W`, `\n`, `\r`, `\t`, `\v`, `\f`, `\0`, `\x…`, `\u…`, `\c…`, `\p{…}` and digit
160
+ * escapes (an octal `\12` is `\n` without the `u` flag) all may, alone or as a range's end.
161
+ * - So may a control character or a line separator written as itself, escaped or not, and a
162
+ * range that spans one.
163
+ * - `\b` and `\B` are assertions, and `\k<name>` repeats a group's text; in a class, `\b` is a
164
+ * backspace, a control character.
165
+ */
166
+ function readsOneLine(pattern: string): boolean {
167
+ let inClass = false
168
+ /** The class's last character, a range's start. */
169
+ let previous: number | undefined
170
+
171
+ for (let index = 0; index < pattern.length; index += 1) {
172
+ let code = pattern.charCodeAt(index)
173
+ if (isLineBreakOrControl(code)) return false
174
+
175
+ if (code === BACKSLASH) {
176
+ index += 1
177
+ const escaped = pattern[index] ?? ""
178
+ if (/[A-Za-z0-9]/.test(escaped)) {
179
+ if (!(inClass ? CLASS_ESCAPES : ESCAPES).has(escaped)) return false
180
+ previous = undefined
181
+ continue
182
+ }
183
+
184
+ // An escaped symbol is the symbol itself.
185
+ code = pattern.charCodeAt(index)
186
+ if (isLineBreakOrControl(code)) return false
187
+ } else if (!inClass && code === OPEN_PAREN) {
188
+ if (LOOKAROUND_OR_MODIFIER.test(pattern.slice(index, index + 4))) return false
189
+ } else if (!inClass && code === OPEN_BRACKET) {
190
+ if (pattern[index + 1] === "^") return false
191
+ inClass = true
192
+ previous = undefined
193
+ continue
194
+ } else if (inClass && code === CLOSE_BRACKET) {
195
+ inClass = false
196
+ continue
197
+ } else if (inClass && code === DASH && previous !== undefined && index + 1 < pattern.length) {
198
+ const end =
199
+ pattern[index + 1] === "\\" ? pattern.charCodeAt(index + 2) : pattern.charCodeAt(index + 1)
200
+ if (previous <= 0x2029 && end >= 0x2028) return false
201
+ }
202
+
203
+ previous = inClass ? code : undefined
204
+ }
205
+
206
+ return true
207
+ }
208
+
209
+ /** At a `(`: a lookaround, or a group that changes the `i`, `m` or `s` flag. */
210
+ const LOOKAROUND_OR_MODIFIER = /^\(\?(?:[=!]|<[=!]|[ims-])/
211
+
212
+ const OPEN_PAREN = 0x28
213
+ const BACKSLASH = 0x5c
214
+ const OPEN_BRACKET = 0x5b
215
+ const CLOSE_BRACKET = 0x5d
216
+ const DASH = 0x2d
217
+
218
+ function isLineBreakOrControl(code: number): boolean {
219
+ return code < 0x20 || code === 0x2028 || code === 0x2029
220
+ }
221
+
222
+ /** Whether a line of the text, its `\r` included, is longer than `length` UTF-16 units. */
223
+ function hasLineOver(text: string, length: number): boolean {
224
+ if (text.length <= length) return false
225
+
226
+ for (let start = 0; start <= text.length; ) {
227
+ const newline = text.indexOf("\n", start)
228
+ const end = newline < 0 ? text.length : newline
229
+ if (end - start > length) return true
230
+ start = end + 1
231
+ }
232
+
233
+ return false
234
+ }
235
+
236
+ /** A file's lines, without their `\n` or `\r\n`. */
237
+ export function lines(text: string): string[] {
238
+ const split = text.split("\n")
239
+ if (split.at(-1) === "") split.pop()
240
+ return split.map((line) => (line.endsWith("\r") ? line.slice(0, -1) : line))
241
+ }
242
+
243
+ /**
244
+ * A line as `grep` shows it: whole when it fits, else cut to {@link SEARCH_LIMITS.lineCharacters}
245
+ * characters around its first match, so a match far into a minified line is still seen. A cut
246
+ * never splits a character written as two units.
247
+ */
248
+ export function cutLine(line: string, match: LineMatch): string {
249
+ const budget = SEARCH_LIMITS.lineCharacters
250
+ if (line.length <= budget) return line
251
+
252
+ // The window opens a little before the match, or ends with the line, the leading `…` in the
253
+ // budget; a match within the line's first characters keeps its start.
254
+ const room = budget - 1
255
+ const start =
256
+ match.index + match.length <= budget
257
+ ? 0
258
+ : Math.max(0, Math.min(match.index - SEARCH_LIMITS.matchLead, line.length - room))
259
+ if (start === 0) return `${linePrefix(line, budget)}…`
260
+
261
+ const from = isLowSurrogate(line.charCodeAt(start)) ? start + 1 : start
262
+ // One unit past the room, so a character the room would split is seen and left out.
263
+ const window = linePrefix(line.slice(from, from + room + 1), room)
264
+
265
+ return `…${window}${from + window.length < line.length ? "…" : ""}`
266
+ }
267
+
268
+ function isLowSurrogate(code: number): boolean {
269
+ return code >= 0xdc00 && code <= 0xdfff
270
+ }
271
+
272
+ /** A line's first `length` UTF-16 units, never half of a character written as two. */
273
+ function linePrefix(line: string, length: number): string {
274
+ if (line.length <= length) return line
275
+
276
+ const code = line.charCodeAt(length - 1)
277
+ return line.slice(0, code >= 0xd800 && code <= 0xdbff ? length - 1 : length)
278
+ }
@@ -0,0 +1,178 @@
1
+ /**
2
+ * Path patterns, as `glob` and `.gitignore` write them: `*` and `?` within a name, `[abc]`,
3
+ * `[!abc]` and `[^abc]`, `**` as a whole segment for any number of folders, `\` to escape.
4
+ * `{a,b}` belongs to `glob` alone and is expanded before compiling ({@link expandBraces}).
5
+ */
6
+
7
+ /** More alternatives than any real pattern names: past it, `{…}` is a mistake. */
8
+ const MAX_ALTERNATIVES = 256
9
+
10
+ export class PatternError extends Error {}
11
+
12
+ /** A pattern's regular expression source, for a whole path relative to where it applies. */
13
+ export function pathPatternSource(pattern: string): string {
14
+ const segments = pattern.split("/")
15
+ let source = ""
16
+
17
+ segments.forEach((segment, index) => {
18
+ const last = index === segments.length - 1
19
+
20
+ if (segment === "**") {
21
+ // `a/**` is everything inside a; `**/b` and `a/**/b` cross any number of folders.
22
+ source += last ? ".*" : "(?:[^/]*/)*"
23
+ return
24
+ }
25
+
26
+ source += segmentSource(segment)
27
+ if (!last) source += "/"
28
+ })
29
+
30
+ return source
31
+ }
32
+
33
+ /** One name's source: never crosses a `/`. */
34
+ export function segmentSource(segment: string): string {
35
+ let source = ""
36
+
37
+ for (let index = 0; index < segment.length; index += 1) {
38
+ const char = segment[index] as string
39
+
40
+ if (char === "\\") {
41
+ index += 1
42
+ source += escapeRegExp(segment[index] ?? "\\")
43
+ } else if (char === "*") {
44
+ while (segment[index + 1] === "*") index += 1
45
+ source += "[^/]*"
46
+ } else if (char === "?") {
47
+ source += "[^/]"
48
+ } else if (char === "[") {
49
+ const parsed = characterClass(segment, index)
50
+ if (parsed) {
51
+ source += parsed.source
52
+ index = parsed.end
53
+ } else {
54
+ source += "\\["
55
+ }
56
+ } else {
57
+ source += escapeRegExp(char)
58
+ }
59
+ }
60
+
61
+ return source
62
+ }
63
+
64
+ /** `[...]` from `start`, or nothing when it never closes and is a literal `[`. */
65
+ function characterClass(
66
+ segment: string,
67
+ start: number
68
+ ): { readonly source: string; readonly end: number } | undefined {
69
+ let index = start + 1
70
+ const negated = segment[index] === "!" || segment[index] === "^"
71
+ if (negated) index += 1
72
+
73
+ let body = ""
74
+ // A `]` right after the opening is part of the class.
75
+ if (segment[index] === "]") {
76
+ body += "\\]"
77
+ index += 1
78
+ }
79
+
80
+ for (; index < segment.length; index += 1) {
81
+ const char = segment[index] as string
82
+
83
+ if (char === "]") {
84
+ return { source: negated ? `[^/${body}]` : `[${body}]`, end: index }
85
+ }
86
+ if (char === "\\" && index + 1 < segment.length) {
87
+ index += 1
88
+ body += escapeClassChar(segment[index] as string)
89
+ } else {
90
+ body += char === "-" ? "-" : escapeClassChar(char)
91
+ }
92
+ }
93
+
94
+ return undefined
95
+ }
96
+
97
+ /**
98
+ * Expand `{a,b}` into its alternatives, nested ones included. A brace without a comma, or that
99
+ * never closes, is a literal brace.
100
+ */
101
+ export function expandBraces(pattern: string): string[] {
102
+ const expanded: string[] = []
103
+ expandInto(pattern, expanded)
104
+ return expanded
105
+ }
106
+
107
+ function expandInto(pattern: string, into: string[]): void {
108
+ for (let index = 0; index < pattern.length; index += 1) {
109
+ const char = pattern[index]
110
+
111
+ if (char === "\\") {
112
+ index += 1
113
+ continue
114
+ }
115
+ if (char !== "{") continue
116
+
117
+ const group = braceGroup(pattern, index)
118
+ if (!group) continue
119
+
120
+ const prefix = pattern.slice(0, index)
121
+ const suffix = pattern.slice(group.end + 1)
122
+ for (const option of group.options) expandInto(`${prefix}${option}${suffix}`, into)
123
+ return
124
+ }
125
+
126
+ if (into.length >= MAX_ALTERNATIVES) {
127
+ throw new PatternError(`it names more than ${MAX_ALTERNATIVES} alternatives with {…}`)
128
+ }
129
+ into.push(pattern)
130
+ }
131
+
132
+ /** The `{…}` opening at `start`: its options split on top-level commas, and its closing index. */
133
+ function braceGroup(
134
+ pattern: string,
135
+ start: number
136
+ ): { readonly options: string[]; readonly end: number } | undefined {
137
+ const options: string[] = []
138
+ let depth = 0
139
+ let optionStart = start + 1
140
+
141
+ for (let index = start + 1; index < pattern.length; index += 1) {
142
+ const char = pattern[index]
143
+
144
+ if (char === "\\") {
145
+ index += 1
146
+ } else if (char === "{") {
147
+ depth += 1
148
+ } else if (char === "}" && depth > 0) {
149
+ depth -= 1
150
+ } else if (char === "," && depth === 0) {
151
+ options.push(pattern.slice(optionStart, index))
152
+ optionStart = index + 1
153
+ } else if (char === "}") {
154
+ if (options.length === 0) return undefined
155
+ options.push(pattern.slice(optionStart, index))
156
+ return { options, end: index }
157
+ }
158
+ }
159
+
160
+ return undefined
161
+ }
162
+
163
+ /** Compile a regular expression source, or say why the pattern is not one. */
164
+ export function compilePattern(source: string, flags = ""): RegExp {
165
+ try {
166
+ return new RegExp(`^${source}$`, flags)
167
+ } catch (error) {
168
+ throw new PatternError(error instanceof Error ? error.message : String(error))
169
+ }
170
+ }
171
+
172
+ function escapeRegExp(text: string): string {
173
+ return text.replace(/[.*+?^${}()|[\]\\/]/g, "\\$&")
174
+ }
175
+
176
+ function escapeClassChar(char: string): string {
177
+ return /[\\\]^[-]/.test(char) ? `\\${char}` : char
178
+ }
@@ -0,0 +1,237 @@
1
+ import { Buffer } from "node:buffer"
2
+ import { type FileHandle, open } from "node:fs/promises"
3
+ import { SEARCH_LIMITS } from "./request"
4
+
5
+ export type FileText =
6
+ | { readonly text: string }
7
+ | { readonly skipped: "binary" | "too_large" | "unreadable" }
8
+
9
+ /** A read that knows what it will hold before it holds it. */
10
+ export interface PendingRead<R> {
11
+ /** The bytes the read holds once started, until its result is taken. */
12
+ readonly bytes: number
13
+ /** Never rejects. */
14
+ read(): Promise<R>
15
+ /** Lets go of a read never started. Never rejects. */
16
+ close(): Promise<void>
17
+ }
18
+
19
+ /**
20
+ * A file opened to be read: one open, and its size from that descriptor. Its text comes from
21
+ * {@link PendingRead.read}, unless it is too large or unreadable, which is known already.
22
+ */
23
+ export async function openText(path: string, strict: boolean): Promise<PendingRead<FileText>> {
24
+ let file: FileHandle | undefined
25
+
26
+ try {
27
+ file = await open(path, "r")
28
+ const { size } = await file.stat()
29
+
30
+ if (size > SEARCH_LIMITS.fileBytes) {
31
+ await closeQuietly(file)
32
+ return settled({ skipped: "too_large" })
33
+ }
34
+
35
+ const opened = file
36
+ return {
37
+ bytes: size,
38
+ read: () => readText(opened, size, strict),
39
+ close: () => closeQuietly(opened),
40
+ }
41
+ } catch {
42
+ if (file) await closeQuietly(file)
43
+ return settled({ skipped: "unreadable" })
44
+ }
45
+ }
46
+
47
+ function settled(text: FileText): PendingRead<FileText> {
48
+ return { bytes: 0, read: async () => text, close: async () => {} }
49
+ }
50
+
51
+ /**
52
+ * The text of a file opened at `size` bytes, read through its descriptor and closed. A file
53
+ * over {@link READ_AT_ONCE_BYTES} is read in two: its first {@link SEARCH_LIMITS.binaryProbeBytes},
54
+ * then the rest unless they show it is binary, so a large binary file is never read whole. A file
55
+ * found by the walk is read as UTF-8 leniently, so a mostly ASCII file in another encoding still
56
+ * matches; a file named as the path must be UTF-8 text. A leading byte order mark is dropped, so
57
+ * `^` finds the first line. Never rejects.
58
+ */
59
+ async function readText(file: FileHandle, size: number, strict: boolean): Promise<FileText> {
60
+ try {
61
+ const first = size <= READ_AT_ONCE_BYTES ? size : PROBE_BYTES
62
+ let bytes = await readBytes(file, Buffer.allocUnsafe(first), 0)
63
+ if (bytes.subarray(0, PROBE_BYTES).includes(0)) return { skipped: "binary" }
64
+
65
+ if (bytes.length === first && first < size) {
66
+ const whole = Buffer.allocUnsafe(size)
67
+ bytes.copy(whole)
68
+ bytes = await readBytes(file, whole, first)
69
+ }
70
+
71
+ let text: string
72
+ if (strict) {
73
+ try {
74
+ text = new TextDecoder("utf-8", { fatal: true, ignoreBOM: true }).decode(bytes)
75
+ } catch {
76
+ return { skipped: "binary" }
77
+ }
78
+ } else {
79
+ text = bytes.toString("utf8")
80
+ }
81
+
82
+ return { text: text.startsWith("\uFEFF") ? text.slice(1) : text }
83
+ } catch {
84
+ return { skipped: "unreadable" }
85
+ } finally {
86
+ await closeQuietly(file)
87
+ }
88
+ }
89
+
90
+ const PROBE_BYTES = SEARCH_LIMITS.binaryProbeBytes
91
+
92
+ /** Up to this size, a file is read in one call: most source files, and two calls cost more. */
93
+ const READ_AT_ONCE_BYTES = 64 * 1024
94
+
95
+ /**
96
+ * Fills `bytes` from `from` on, and returns what was read: the file as it was when opened, if it
97
+ * grows meanwhile, or less if it shrank.
98
+ */
99
+ async function readBytes(file: FileHandle, bytes: Buffer, from: number): Promise<Buffer> {
100
+ let filled = from
101
+
102
+ while (filled < bytes.length) {
103
+ const { bytesRead } = await file.read(bytes, filled, bytes.length - filled, filled)
104
+ if (bytesRead === 0) break
105
+ filled += bytesRead
106
+ }
107
+
108
+ return bytes.subarray(0, filled)
109
+ }
110
+
111
+ async function closeQuietly(file: FileHandle): Promise<void> {
112
+ await file.close().catch(() => {})
113
+ }
114
+
115
+ /**
116
+ * How far reads run ahead of the file searched. Reading is most of a search's time, and mostly
117
+ * waiting: reads in flight overlap that wait. At most {@link READ_AHEAD} files are open, and a read
118
+ * starts only while the reads started hold at most {@link READ_AHEAD_BYTES} of file with it, but
119
+ * for the next file in line, which always starts: one file of 10 MiB past the bound at most. A
120
+ * binary file's bytes are let go once its first bytes show it is one.
121
+ *
122
+ * Memory: the texts read ahead take up to twice their bytes once decoded (a string with one
123
+ * character beyond Latin-1 takes two bytes for each): 52 MiB at worst, plus the file searched and
124
+ * the copies its search makes. 16 MiB reads a source tree as fast as 32 did, and holds half as
125
+ * much over a folder of large files.
126
+ */
127
+ export const READ_AHEAD = 32
128
+ export const READ_AHEAD_BYTES = 16 * 1024 * 1024
129
+
130
+ interface Slot<T, R> {
131
+ readonly item: T
132
+ /** Its {@link PendingRead}, once opened. */
133
+ pending?: PendingRead<R>
134
+ /** Settles once {@link pending} is known. */
135
+ opening?: Promise<void>
136
+ /** The bytes it holds against {@link READ_AHEAD_BYTES}. */
137
+ holds: number
138
+ result?: Promise<R>
139
+ }
140
+
141
+ /**
142
+ * Files read ahead of the one searched, handed back in the order they come, so a search finds the
143
+ * same lines, in the same order, as reading one file at a time. Files are opened as they come,
144
+ * and read as soon as they are open and their bytes fit. A search that stops reads no further than
145
+ * {@link READ_AHEAD} files past where it stopped.
146
+ */
147
+ export class ReadAhead<T, R> {
148
+ readonly #slots: Slot<T, R>[] = []
149
+ #held = 0
150
+ #exhausted = false
151
+ #stopped = false
152
+
153
+ constructor(
154
+ private readonly items: Iterator<T>,
155
+ /** Never rejects. */
156
+ private readonly open: (item: T) => Promise<PendingRead<R>>,
157
+ /** What a read that holds nothing once done looks like, a binary file. */
158
+ private readonly holdsNothing: (result: R) => boolean = () => false
159
+ ) {}
160
+
161
+ async next(): Promise<{ readonly item: T; readonly read: R } | undefined> {
162
+ this.#fill()
163
+ const slot = this.#slots[0]
164
+ if (!slot) return undefined
165
+
166
+ await slot.opening
167
+ this.#start()
168
+ const read = await (slot.result as Promise<R>)
169
+
170
+ this.#slots.shift()
171
+ this.#held -= slot.holds
172
+ this.#fill()
173
+ this.#start()
174
+
175
+ return { item: slot.item, read }
176
+ }
177
+
178
+ /**
179
+ * Starts no other read, waits for those in flight and closes the files never read: no file
180
+ * stays open past the search.
181
+ */
182
+ async stop(): Promise<void> {
183
+ this.#exhausted = true
184
+ this.#stopped = true
185
+
186
+ await Promise.all(
187
+ this.#slots.splice(0).map(async (slot) => {
188
+ await slot.opening
189
+ if (slot.result) await slot.result
190
+ else await slot.pending?.close()
191
+ })
192
+ )
193
+ }
194
+
195
+ #fill(): void {
196
+ while (!this.#exhausted && this.#slots.length < READ_AHEAD) {
197
+ const step = this.items.next()
198
+ if (step.done) {
199
+ this.#exhausted = true
200
+ return
201
+ }
202
+
203
+ const slot: Slot<T, R> = { item: step.value, holds: 0 }
204
+ this.#slots.push(slot)
205
+ slot.opening = this.open(step.value).then((pending) => {
206
+ slot.pending = pending
207
+ this.#start()
208
+ })
209
+ }
210
+ }
211
+
212
+ /**
213
+ * Starts the reads of the files opened, as long as their bytes fit. The first file in line always
214
+ * starts: the search waits on it, and no read behind it can make room.
215
+ */
216
+ #start(): void {
217
+ if (this.#stopped) return
218
+
219
+ for (let index = 0; index < this.#slots.length; index += 1) {
220
+ const slot = this.#slots[index] as Slot<T, R>
221
+ const pending = slot.pending
222
+ if (slot.result || !pending) continue
223
+ if (index > 0 && this.#held + pending.bytes > READ_AHEAD_BYTES) continue
224
+
225
+ slot.holds = pending.bytes
226
+ this.#held += slot.holds
227
+ slot.result = pending.read().then((result) => {
228
+ if (this.holdsNothing(result) && this.#slots.includes(slot)) {
229
+ this.#held -= slot.holds
230
+ slot.holds = 0
231
+ this.#start()
232
+ }
233
+ return result
234
+ })
235
+ }
236
+ }
237
+ }