@sixb/agent 0.2.4 → 0.2.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +2 -2
- package/dist/agent-prompt.d.ts.map +1 -1
- package/dist/artifacts/layout-check/layout-check.d.ts +21 -24
- package/dist/artifacts/layout-check/layout-check.d.ts.map +1 -1
- package/dist/artifacts/publish-tool.d.ts.map +1 -1
- package/dist/index.d.ts +0 -1
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +10590 -8888
- package/dist/index.js.map +30 -26
- package/dist/run-agent-turn.d.ts +3 -14
- package/dist/run-agent-turn.d.ts.map +1 -1
- package/dist/run-environment.d.ts.map +1 -1
- package/dist/run-subagent.d.ts.map +1 -1
- package/dist/sandbox-api-context.d.ts.map +1 -1
- package/dist/search-script.d.ts +7 -0
- package/dist/search-script.d.ts.map +1 -0
- package/dist/skills/built-in.d.ts.map +1 -1
- package/dist/tools/glob.d.ts +41 -0
- package/dist/tools/glob.d.ts.map +1 -0
- package/dist/tools/grep.d.ts +64 -0
- package/dist/tools/grep.d.ts.map +1 -0
- package/dist/tools/model-spec.d.ts.map +1 -1
- package/dist/tools/sandbox-search.d.ts +26 -0
- package/dist/tools/sandbox-search.d.ts.map +1 -0
- package/dist/tools/search/command.d.ts +15 -0
- package/dist/tools/search/command.d.ts.map +1 -0
- package/dist/tools/search/gitignore.d.ts +28 -0
- package/dist/tools/search/gitignore.d.ts.map +1 -0
- package/dist/tools/search/glob.d.ts +15 -0
- package/dist/tools/search/glob.d.ts.map +1 -0
- package/dist/tools/search/main.d.ts +2 -0
- package/dist/tools/search/main.d.ts.map +1 -0
- package/dist/tools/search/match.d.ts +30 -0
- package/dist/tools/search/match.d.ts.map +1 -0
- package/dist/tools/search/pattern.d.ts +19 -0
- package/dist/tools/search/pattern.d.ts.map +1 -0
- package/dist/tools/search/read.d.ts +62 -0
- package/dist/tools/search/read.d.ts.map +1 -0
- package/dist/tools/search/request.d.ts +80 -0
- package/dist/tools/search/request.d.ts.map +1 -0
- package/dist/tools/search/search.d.ts +4 -0
- package/dist/tools/search/search.d.ts.map +1 -0
- package/dist/tools/search/walk.d.ts +43 -0
- package/dist/tools/search/walk.d.ts.map +1 -0
- package/dist/tools/tool-outputs.d.ts.map +1 -1
- package/dist/turn-inbox.d.ts +75 -0
- package/dist/turn-inbox.d.ts.map +1 -0
- package/package.json +6 -4
- package/src/agent-prompt.ts +1 -0
- package/src/agent-runtime/preflight.ts +1 -0
- package/src/artifacts/layout-check/check.cjs +176 -95
- package/src/artifacts/layout-check/layout-check.ts +118 -76
- package/src/artifacts/publish-tool.ts +48 -99
- package/src/generated/search.mjs +985 -0
- package/src/index.ts +0 -1
- package/src/run-agent-turn.ts +85 -129
- package/src/run-conversation-turn.ts +4 -4
- package/src/run-environment.ts +4 -0
- package/src/run-subagent.ts +1 -1
- package/src/sandbox-api-context.ts +3 -0
- package/src/search-script-assets.d.ts +5 -0
- package/src/search-script.ts +9 -0
- package/src/skills/built-in.ts +16 -0
- package/src/skills/sixb-artifacts/SKILL.md +7 -7
- package/src/skills/sixb-artifacts/canvas.md +3 -2
- package/src/skills/sixb-artifacts/document.md +92 -5
- package/src/skills/sixb-artifacts/examples/document/service-proposal/design.css +114 -0
- package/src/skills/sixb-artifacts/examples/document/service-proposal/document.html +159 -0
- package/src/skills/sixb-artifacts/examples/document/service-proposal/images/fieldwise.svg +1 -0
- package/src/skills/sixb-artifacts/examples/document/service-proposal/images/harbor-pine.svg +1 -0
- package/src/skills/sixb-artifacts/queries.md +2 -1
- package/src/tools/glob.ts +124 -0
- package/src/tools/grep.ts +190 -0
- package/src/tools/model-spec.ts +4 -0
- package/src/tools/sandbox-search.ts +219 -0
- package/src/tools/search/command.ts +116 -0
- package/src/tools/search/gitignore.ts +101 -0
- package/src/tools/search/glob.ts +90 -0
- package/src/tools/search/main.ts +9 -0
- package/src/tools/search/match.ts +278 -0
- package/src/tools/search/pattern.ts +178 -0
- package/src/tools/search/read.ts +237 -0
- package/src/tools/search/request.ts +101 -0
- package/src/tools/search/search.ts +297 -0
- package/src/tools/search/walk.ts +145 -0
- package/src/tools/tool-outputs.ts +7 -2
- package/src/turn-inbox.ts +283 -0
- package/dist/artifacts/board-previews.d.ts +0 -41
- package/dist/artifacts/board-previews.d.ts.map +0 -1
- package/src/artifacts/board-previews.ts +0 -218
|
@@ -0,0 +1,101 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* What `search.mjs` takes and prints: the contract between the `glob` and `grep` tools and the
|
|
3
|
+
* engine the run installs.
|
|
4
|
+
*/
|
|
5
|
+
|
|
6
|
+
/** How much a search returns, and which files `grep` reads. */
|
|
7
|
+
export const SEARCH_LIMITS = {
|
|
8
|
+
files: 100,
|
|
9
|
+
lines: 100,
|
|
10
|
+
linesPerFile: 20,
|
|
11
|
+
lineCharacters: 500,
|
|
12
|
+
/** How much of a cut line shows before its first match. */
|
|
13
|
+
matchLead: 40,
|
|
14
|
+
/**
|
|
15
|
+
* How much of a line `grep` matches. A backtracking regex can take seconds on one long line of a
|
|
16
|
+
* minified file; past this, a match is missed rather than the search stalled.
|
|
17
|
+
*/
|
|
18
|
+
matchCharacters: 4_000,
|
|
19
|
+
/** What `grep`'s matches may add to its model text, in UTF-8 bytes, before it is truncated. */
|
|
20
|
+
textBytes: 24 * 1024,
|
|
21
|
+
fileBytes: 10 * 1024 * 1024,
|
|
22
|
+
/** A NUL byte in this many first bytes makes a file binary. */
|
|
23
|
+
binaryProbeBytes: 8 * 1024,
|
|
24
|
+
} as const
|
|
25
|
+
|
|
26
|
+
interface SearchScopeRequest {
|
|
27
|
+
/** Relative to the working directory, or absolute. Default: the working directory. */
|
|
28
|
+
readonly path?: string
|
|
29
|
+
/**
|
|
30
|
+
* The run's context directory: the second root a search may start from. A search that starts
|
|
31
|
+
* outside it never enters it.
|
|
32
|
+
*/
|
|
33
|
+
readonly contextDirectory?: string
|
|
34
|
+
/** How long the walk may take before it stops with what it found. */
|
|
35
|
+
readonly timeoutMs: number
|
|
36
|
+
}
|
|
37
|
+
|
|
38
|
+
export interface GlobSearchRequest extends SearchScopeRequest {
|
|
39
|
+
readonly tool: "glob"
|
|
40
|
+
readonly pattern: string
|
|
41
|
+
}
|
|
42
|
+
|
|
43
|
+
export interface GrepSearchRequest extends SearchScopeRequest {
|
|
44
|
+
readonly tool: "grep"
|
|
45
|
+
/** A JavaScript regular expression, or the text itself with `literal`. */
|
|
46
|
+
readonly pattern: string
|
|
47
|
+
/** A `glob` pattern the files searched must match. */
|
|
48
|
+
readonly include?: string
|
|
49
|
+
readonly literal?: boolean
|
|
50
|
+
}
|
|
51
|
+
|
|
52
|
+
export type SearchRequest = GlobSearchRequest | GrepSearchRequest
|
|
53
|
+
|
|
54
|
+
/**
|
|
55
|
+
* Paths are relative to the working directory when they are under it, absolute otherwise.
|
|
56
|
+
* `incomplete` says the walk stopped at its deadline.
|
|
57
|
+
*/
|
|
58
|
+
export interface GlobSearchResult {
|
|
59
|
+
readonly files: string[]
|
|
60
|
+
readonly truncated: boolean
|
|
61
|
+
readonly incomplete?: true
|
|
62
|
+
}
|
|
63
|
+
|
|
64
|
+
export interface GrepMatch {
|
|
65
|
+
readonly path: string
|
|
66
|
+
/** One-based. */
|
|
67
|
+
readonly line: number
|
|
68
|
+
readonly text: string
|
|
69
|
+
}
|
|
70
|
+
|
|
71
|
+
export interface GrepSearchResult {
|
|
72
|
+
readonly matches: GrepMatch[]
|
|
73
|
+
/** How many files the matches are in. */
|
|
74
|
+
readonly files: number
|
|
75
|
+
/** Matching lines counted past what a file shows, by path. */
|
|
76
|
+
readonly more?: Record<string, number>
|
|
77
|
+
/** Another file matches past the cap. */
|
|
78
|
+
readonly truncated: boolean
|
|
79
|
+
readonly incomplete?: true
|
|
80
|
+
}
|
|
81
|
+
|
|
82
|
+
export type SearchErrorCode =
|
|
83
|
+
| "path_missing"
|
|
84
|
+
| "path_outside"
|
|
85
|
+
| "path_is_file"
|
|
86
|
+
| "path_not_searchable"
|
|
87
|
+
| "invalid_pattern"
|
|
88
|
+
| "invalid_include"
|
|
89
|
+
| "file_binary"
|
|
90
|
+
| "file_too_large"
|
|
91
|
+
|
|
92
|
+
/** A search that cannot run as asked: the caller says why in its own words. */
|
|
93
|
+
export class SearchError extends Error {
|
|
94
|
+
constructor(
|
|
95
|
+
readonly code: SearchErrorCode,
|
|
96
|
+
message: string
|
|
97
|
+
) {
|
|
98
|
+
super(message)
|
|
99
|
+
this.name = "SearchError"
|
|
100
|
+
}
|
|
101
|
+
}
|
|
@@ -0,0 +1,297 @@
|
|
|
1
|
+
import { Buffer } from "node:buffer"
|
|
2
|
+
import { lstatSync, realpathSync, statSync } from "node:fs"
|
|
3
|
+
import { basename, dirname, relative, resolve } from "node:path"
|
|
4
|
+
import type { IgnoreFile } from "./gitignore"
|
|
5
|
+
import { compileGlob, type GlobMatcher } from "./glob"
|
|
6
|
+
import { cutLine, lines, textMatcher } from "./match"
|
|
7
|
+
import { PatternError } from "./pattern"
|
|
8
|
+
import { type FileText, openText, ReadAhead } from "./read"
|
|
9
|
+
import {
|
|
10
|
+
type GlobSearchRequest,
|
|
11
|
+
type GlobSearchResult,
|
|
12
|
+
type GrepMatch,
|
|
13
|
+
type GrepSearchRequest,
|
|
14
|
+
type GrepSearchResult,
|
|
15
|
+
SEARCH_LIMITS,
|
|
16
|
+
SearchError,
|
|
17
|
+
type SearchErrorCode,
|
|
18
|
+
type SearchRequest,
|
|
19
|
+
} from "./request"
|
|
20
|
+
import { childPath, FileWalk, readIgnoreFile, type WalkedFile } from "./walk"
|
|
21
|
+
|
|
22
|
+
export function globFiles(request: GlobSearchRequest, workingDirectory: string): GlobSearchResult {
|
|
23
|
+
const scope = resolveScope(request, workingDirectory)
|
|
24
|
+
const matcher = compileGlobOrFail(request.pattern, "invalid_pattern")
|
|
25
|
+
if (scope.file) throw new SearchError("path_is_file", "The path is a file.")
|
|
26
|
+
|
|
27
|
+
const walk = scope.walk(matcher, matcher.namesHidden)
|
|
28
|
+
const files: string[] = []
|
|
29
|
+
let truncated = false
|
|
30
|
+
|
|
31
|
+
for (const file of walk.files()) {
|
|
32
|
+
if (!matcher.matches(file.relative)) continue
|
|
33
|
+
if (files.length === SEARCH_LIMITS.files) {
|
|
34
|
+
truncated = true
|
|
35
|
+
break
|
|
36
|
+
}
|
|
37
|
+
files.push(scope.display(file.path))
|
|
38
|
+
}
|
|
39
|
+
|
|
40
|
+
return { files, truncated, ...(walk.incomplete ? { incomplete: true as const } : {}) }
|
|
41
|
+
}
|
|
42
|
+
|
|
43
|
+
export async function grepFiles(
|
|
44
|
+
request: GrepSearchRequest,
|
|
45
|
+
workingDirectory: string
|
|
46
|
+
): Promise<GrepSearchResult> {
|
|
47
|
+
const scope = resolveScope(request, workingDirectory)
|
|
48
|
+
const matcher = textMatcher(request.pattern, request.literal === true)
|
|
49
|
+
const include =
|
|
50
|
+
request.include === undefined
|
|
51
|
+
? undefined
|
|
52
|
+
: compileGlobOrFail(request.include, "invalid_include")
|
|
53
|
+
|
|
54
|
+
// A file named as the path is searched alone, whatever include says.
|
|
55
|
+
const walk = scope.file ? undefined : scope.walk(include, true)
|
|
56
|
+
const searched = function* (): Generator<WalkedFile> {
|
|
57
|
+
if (!walk) {
|
|
58
|
+
yield { path: scope.target, relative: basename(scope.target), symlink: false }
|
|
59
|
+
return
|
|
60
|
+
}
|
|
61
|
+
|
|
62
|
+
for (const file of walk.files()) {
|
|
63
|
+
if (include && !include.matches(file.relative)) continue
|
|
64
|
+
if (file.symlink && !scope.holds(file.path)) continue
|
|
65
|
+
yield file
|
|
66
|
+
}
|
|
67
|
+
}
|
|
68
|
+
// Files are read ahead of the one searched, and come back in the walk's order.
|
|
69
|
+
const reads = new ReadAhead(
|
|
70
|
+
searched(),
|
|
71
|
+
(file: WalkedFile) => openText(file.path, walk === undefined),
|
|
72
|
+
(read: FileText) => !("text" in read)
|
|
73
|
+
)
|
|
74
|
+
|
|
75
|
+
const matches: GrepMatch[] = []
|
|
76
|
+
const more: Record<string, number> = {}
|
|
77
|
+
let files = 0
|
|
78
|
+
let textBytes = 0
|
|
79
|
+
let textFull = false
|
|
80
|
+
let truncated = false
|
|
81
|
+
let stopped = false
|
|
82
|
+
|
|
83
|
+
try {
|
|
84
|
+
for (let next = await reads.next(); next; next = await reads.next()) {
|
|
85
|
+
const { item: file, read } = next
|
|
86
|
+
|
|
87
|
+
if (!("text" in read)) {
|
|
88
|
+
// A file named as the path says why it was not searched, rather than "No matches".
|
|
89
|
+
if (walk === undefined && read.skipped !== "unreadable") throw unsearchedError(read.skipped)
|
|
90
|
+
continue
|
|
91
|
+
}
|
|
92
|
+
|
|
93
|
+
// Most files hold no match: one test of the whole text spares splitting them into lines.
|
|
94
|
+
if (!matcher.mayMatch(read.text)) continue
|
|
95
|
+
|
|
96
|
+
const path = scope.display(file.path)
|
|
97
|
+
let shown = 0
|
|
98
|
+
let counted = 0
|
|
99
|
+
|
|
100
|
+
for (const [index, line] of lines(read.text).entries()) {
|
|
101
|
+
// One file can hold enough lines, or slow enough ones, to outlast the deadline alone.
|
|
102
|
+
if (Date.now() >= scope.deadline) {
|
|
103
|
+
stopped = true
|
|
104
|
+
break
|
|
105
|
+
}
|
|
106
|
+
const match = matcher.line(line)
|
|
107
|
+
if (match === undefined) continue
|
|
108
|
+
|
|
109
|
+
const text = cutLine(line, match)
|
|
110
|
+
const bytes =
|
|
111
|
+
Buffer.byteLength(` ${index + 1}: ${text}\n`) +
|
|
112
|
+
(shown === 0 ? Buffer.byteLength(`${path}\n\n`) : 0)
|
|
113
|
+
if (textBytes + bytes > SEARCH_LIMITS.textBytes) textFull = true
|
|
114
|
+
const full = textFull || matches.length === SEARCH_LIMITS.lines
|
|
115
|
+
|
|
116
|
+
if (shown === 0 && full) {
|
|
117
|
+
truncated = true
|
|
118
|
+
break
|
|
119
|
+
}
|
|
120
|
+
if (shown > 0 && (shown === SEARCH_LIMITS.linesPerFile || full)) {
|
|
121
|
+
counted += 1
|
|
122
|
+
continue
|
|
123
|
+
}
|
|
124
|
+
|
|
125
|
+
matches.push({ path, line: index + 1, text })
|
|
126
|
+
textBytes += bytes
|
|
127
|
+
shown += 1
|
|
128
|
+
}
|
|
129
|
+
|
|
130
|
+
if (shown > 0) files += 1
|
|
131
|
+
if (counted > 0) more[path] = counted
|
|
132
|
+
if (truncated || stopped) break
|
|
133
|
+
}
|
|
134
|
+
} finally {
|
|
135
|
+
await reads.stop()
|
|
136
|
+
}
|
|
137
|
+
|
|
138
|
+
return {
|
|
139
|
+
matches,
|
|
140
|
+
files,
|
|
141
|
+
...(Object.keys(more).length > 0 ? { more } : {}),
|
|
142
|
+
truncated,
|
|
143
|
+
...(walk?.incomplete || stopped ? { incomplete: true as const } : {}),
|
|
144
|
+
}
|
|
145
|
+
}
|
|
146
|
+
|
|
147
|
+
interface SearchScope {
|
|
148
|
+
/** The real path searched. */
|
|
149
|
+
readonly target: string
|
|
150
|
+
/** The target is a file. */
|
|
151
|
+
readonly file: boolean
|
|
152
|
+
/** The time, in epoch milliseconds, at which the search stops with what it found. */
|
|
153
|
+
readonly deadline: number
|
|
154
|
+
/** Whether a path's target lies in the run's roots. */
|
|
155
|
+
holds(path: string): boolean
|
|
156
|
+
display(path: string): string
|
|
157
|
+
walk(matcher: GlobMatcher | undefined, hidden: boolean): FileWalk
|
|
158
|
+
}
|
|
159
|
+
|
|
160
|
+
/**
|
|
161
|
+
* Where a search starts, confined to the working directory and the run's context directory once
|
|
162
|
+
* links are resolved, as `read` confines a path.
|
|
163
|
+
*/
|
|
164
|
+
function resolveScope(request: SearchRequest, workingDirectory: string): SearchScope {
|
|
165
|
+
const deadline = Date.now() + request.timeoutMs
|
|
166
|
+
const working = realpathSync(workingDirectory)
|
|
167
|
+
const context =
|
|
168
|
+
request.contextDirectory === undefined
|
|
169
|
+
? undefined
|
|
170
|
+
: realPath(resolve(working, request.contextDirectory))
|
|
171
|
+
|
|
172
|
+
const target = realPath(resolve(working, request.path ?? "."))
|
|
173
|
+
if (target === undefined) throw new SearchError("path_missing", "The path does not exist.")
|
|
174
|
+
|
|
175
|
+
// The context directory may sit inside the working directory: the closer root is the one used.
|
|
176
|
+
const root =
|
|
177
|
+
context !== undefined && isWithin(target, context)
|
|
178
|
+
? context
|
|
179
|
+
: isWithin(target, working)
|
|
180
|
+
? working
|
|
181
|
+
: undefined
|
|
182
|
+
if (root === undefined) {
|
|
183
|
+
throw new SearchError("path_outside", "The path is outside the run's folders.")
|
|
184
|
+
}
|
|
185
|
+
|
|
186
|
+
const stats = statSync(target)
|
|
187
|
+
if (!stats.isFile() && !stats.isDirectory()) {
|
|
188
|
+
throw new SearchError("path_not_searchable", "The path is not a file or a folder.")
|
|
189
|
+
}
|
|
190
|
+
|
|
191
|
+
const holds = (path: string): boolean => {
|
|
192
|
+
const real = realPath(path)
|
|
193
|
+
return (
|
|
194
|
+
real !== undefined &&
|
|
195
|
+
(isWithin(real, working) || (context !== undefined && isWithin(real, context)))
|
|
196
|
+
)
|
|
197
|
+
}
|
|
198
|
+
|
|
199
|
+
const display = (path: string): string =>
|
|
200
|
+
isWithin(path, working) ? relative(working, path) || "." : path
|
|
201
|
+
|
|
202
|
+
const walk = (matcher: GlobMatcher | undefined, hidden: boolean): FileWalk => {
|
|
203
|
+
const namesNodeModules = relative(root, target).split("/").includes("node_modules")
|
|
204
|
+
return new FileWalk({
|
|
205
|
+
root: target,
|
|
206
|
+
ignoreFiles: [...repositoryIgnoreFiles(root), ...ancestorIgnoreFiles(root, target)],
|
|
207
|
+
hidden,
|
|
208
|
+
nodeModules: namesNodeModules || matcher?.namesNodeModules === true,
|
|
209
|
+
skipFolders: new Set(context !== undefined && !isWithin(target, context) ? [context] : []),
|
|
210
|
+
...(matcher ? { mayContain: (folder: string) => matcher.mayContain(folder) } : {}),
|
|
211
|
+
deadline,
|
|
212
|
+
})
|
|
213
|
+
}
|
|
214
|
+
|
|
215
|
+
return { target, file: stats.isFile(), deadline, holds, display, walk }
|
|
216
|
+
}
|
|
217
|
+
|
|
218
|
+
/** How far above a root the git repository it lies in is looked for. */
|
|
219
|
+
const REPOSITORY_LEVELS = 20
|
|
220
|
+
|
|
221
|
+
/**
|
|
222
|
+
* The ignore rules of the git repository a root lies in, from above the root: its
|
|
223
|
+
* `.git/info/exclude`, then the `.gitignore` of each folder from the repository's top down to the
|
|
224
|
+
* folder above the root, as git applies them. Only those files are read; nothing outside the
|
|
225
|
+
* roots is walked. A root outside any repository gets none.
|
|
226
|
+
*/
|
|
227
|
+
function repositoryIgnoreFiles(root: string): IgnoreFile[] {
|
|
228
|
+
const above: string[] = []
|
|
229
|
+
let folder = root
|
|
230
|
+
|
|
231
|
+
for (let level = 0; level <= REPOSITORY_LEVELS; level += 1) {
|
|
232
|
+
if (holdsGit(folder)) {
|
|
233
|
+
const exclude = readIgnoreFile(folder, ".git/info/exclude")
|
|
234
|
+
return [
|
|
235
|
+
...(exclude ? [exclude] : []),
|
|
236
|
+
...above.flatMap((parent) => readIgnoreFile(parent) ?? []),
|
|
237
|
+
]
|
|
238
|
+
}
|
|
239
|
+
|
|
240
|
+
const parent = dirname(folder)
|
|
241
|
+
if (parent === folder) return []
|
|
242
|
+
folder = parent
|
|
243
|
+
above.unshift(folder)
|
|
244
|
+
}
|
|
245
|
+
|
|
246
|
+
return []
|
|
247
|
+
}
|
|
248
|
+
|
|
249
|
+
/** A folder holding `.git`: a repository's top, or a worktree's, where `.git` is a file. */
|
|
250
|
+
function holdsGit(folder: string): boolean {
|
|
251
|
+
try {
|
|
252
|
+
lstatSync(childPath(folder, ".git"))
|
|
253
|
+
return true
|
|
254
|
+
} catch {
|
|
255
|
+
return false
|
|
256
|
+
}
|
|
257
|
+
}
|
|
258
|
+
|
|
259
|
+
/** The `.gitignore` files from the root down to the folder above the target, outermost first. */
|
|
260
|
+
function ancestorIgnoreFiles(root: string, target: string): IgnoreFile[] {
|
|
261
|
+
if (target === root) return []
|
|
262
|
+
|
|
263
|
+
const folders: string[] = []
|
|
264
|
+
for (let folder = dirname(target); isWithin(folder, root); folder = dirname(folder)) {
|
|
265
|
+
folders.unshift(folder)
|
|
266
|
+
if (folder === root) break
|
|
267
|
+
}
|
|
268
|
+
|
|
269
|
+
return folders.flatMap((folder) => readIgnoreFile(folder) ?? [])
|
|
270
|
+
}
|
|
271
|
+
|
|
272
|
+
function compileGlobOrFail(pattern: string, code: SearchErrorCode): GlobMatcher {
|
|
273
|
+
try {
|
|
274
|
+
return compileGlob(pattern)
|
|
275
|
+
} catch (error) {
|
|
276
|
+
if (error instanceof PatternError) throw new SearchError(code, error.message)
|
|
277
|
+
throw error
|
|
278
|
+
}
|
|
279
|
+
}
|
|
280
|
+
|
|
281
|
+
function unsearchedError(reason: "binary" | "too_large"): SearchError {
|
|
282
|
+
return reason === "binary"
|
|
283
|
+
? new SearchError("file_binary", "The file is binary, or not UTF-8 text.")
|
|
284
|
+
: new SearchError("file_too_large", "The file is over 10 MiB.")
|
|
285
|
+
}
|
|
286
|
+
|
|
287
|
+
function realPath(path: string): string | undefined {
|
|
288
|
+
try {
|
|
289
|
+
return realpathSync(path)
|
|
290
|
+
} catch {
|
|
291
|
+
return undefined
|
|
292
|
+
}
|
|
293
|
+
}
|
|
294
|
+
|
|
295
|
+
function isWithin(path: string, root: string): boolean {
|
|
296
|
+
return path === root || path.startsWith(root.endsWith("/") ? root : `${root}/`)
|
|
297
|
+
}
|
|
@@ -0,0 +1,145 @@
|
|
|
1
|
+
import { Buffer } from "node:buffer"
|
|
2
|
+
import { type Dirent, lstatSync, readdirSync, readFileSync, statSync } from "node:fs"
|
|
3
|
+
import { type IgnoreFile, isIgnored, parseGitignore } from "./gitignore"
|
|
4
|
+
|
|
5
|
+
export interface WalkOptions {
|
|
6
|
+
/** The folder to walk: a real, absolute path. */
|
|
7
|
+
readonly root: string
|
|
8
|
+
/** The `.gitignore` files of the folders above the root that apply, outermost first. */
|
|
9
|
+
readonly ignoreFiles: readonly IgnoreFile[]
|
|
10
|
+
/** Enter folders and list files whose name starts with `.`. */
|
|
11
|
+
readonly hidden: boolean
|
|
12
|
+
readonly nodeModules: boolean
|
|
13
|
+
/** Real, absolute paths of folders never entered: the run's context directory. */
|
|
14
|
+
readonly skipFolders: ReadonlySet<string>
|
|
15
|
+
/** Whether a folder, relative to the root, may hold what the search looks for. */
|
|
16
|
+
readonly mayContain?: (relativeFolder: string) => boolean
|
|
17
|
+
/** The time, in epoch milliseconds, at which the walk stops. */
|
|
18
|
+
readonly deadline: number
|
|
19
|
+
}
|
|
20
|
+
|
|
21
|
+
export interface WalkedFile {
|
|
22
|
+
readonly path: string
|
|
23
|
+
/** From the root, with `/`. */
|
|
24
|
+
readonly relative: string
|
|
25
|
+
/** A link to a file: listed, but its target may be anywhere. */
|
|
26
|
+
readonly symlink: boolean
|
|
27
|
+
}
|
|
28
|
+
|
|
29
|
+
/**
|
|
30
|
+
* The files under a folder, in byte order of their paths, so that the first results of a capped
|
|
31
|
+
* search are the same on every run. `.git`, hidden entries and `node_modules` unless asked for,
|
|
32
|
+
* what `.gitignore` files exclude, and unreadable entries are skipped. Links to folders are not
|
|
33
|
+
* followed. Past its deadline, the walk stops and says it is {@link incomplete}.
|
|
34
|
+
*/
|
|
35
|
+
export class FileWalk {
|
|
36
|
+
#incomplete = false
|
|
37
|
+
|
|
38
|
+
constructor(private readonly options: WalkOptions) {}
|
|
39
|
+
|
|
40
|
+
get incomplete(): boolean {
|
|
41
|
+
return this.#incomplete
|
|
42
|
+
}
|
|
43
|
+
|
|
44
|
+
*files(): Generator<WalkedFile> {
|
|
45
|
+
yield* this.#folder(this.options.root, "", this.options.ignoreFiles)
|
|
46
|
+
}
|
|
47
|
+
|
|
48
|
+
*#folder(
|
|
49
|
+
folder: string,
|
|
50
|
+
relativeFolder: string,
|
|
51
|
+
ignoreFiles: readonly IgnoreFile[]
|
|
52
|
+
): Generator<WalkedFile> {
|
|
53
|
+
if (this.#pastDeadline()) return
|
|
54
|
+
|
|
55
|
+
let entries: Dirent[]
|
|
56
|
+
try {
|
|
57
|
+
entries = readdirSync(folder, { withFileTypes: true })
|
|
58
|
+
} catch {
|
|
59
|
+
return
|
|
60
|
+
}
|
|
61
|
+
|
|
62
|
+
const own = entries.some((entry) => entry.name === ".gitignore")
|
|
63
|
+
? readIgnoreFile(folder)
|
|
64
|
+
: undefined
|
|
65
|
+
const applying = own ? [...ignoreFiles, own] : ignoreFiles
|
|
66
|
+
|
|
67
|
+
for (const entry of sortedEntries(entries)) {
|
|
68
|
+
if (this.#incomplete) return
|
|
69
|
+
|
|
70
|
+
const { name } = entry
|
|
71
|
+
if (name === ".git") continue
|
|
72
|
+
if (!this.options.hidden && name.startsWith(".")) continue
|
|
73
|
+
|
|
74
|
+
const path = childPath(folder, name)
|
|
75
|
+
const relative = relativeFolder ? `${relativeFolder}/${name}` : name
|
|
76
|
+
|
|
77
|
+
if (entry.isDirectory()) {
|
|
78
|
+
if (name === "node_modules" && !this.options.nodeModules) continue
|
|
79
|
+
if (this.options.skipFolders.has(path)) continue
|
|
80
|
+
if (isIgnored(applying, path, true)) continue
|
|
81
|
+
if (this.options.mayContain && !this.options.mayContain(relative)) continue
|
|
82
|
+
|
|
83
|
+
yield* this.#folder(path, relative, applying)
|
|
84
|
+
continue
|
|
85
|
+
}
|
|
86
|
+
|
|
87
|
+
const symlink = entry.isSymbolicLink()
|
|
88
|
+
if (!entry.isFile() && !(symlink && isLinkToFile(path))) continue
|
|
89
|
+
if (isIgnored(applying, path, false)) continue
|
|
90
|
+
if (this.#pastDeadline()) return
|
|
91
|
+
|
|
92
|
+
yield { path, relative, symlink }
|
|
93
|
+
}
|
|
94
|
+
}
|
|
95
|
+
|
|
96
|
+
#pastDeadline(): boolean {
|
|
97
|
+
if (Date.now() >= this.options.deadline) this.#incomplete = true
|
|
98
|
+
return this.#incomplete
|
|
99
|
+
}
|
|
100
|
+
}
|
|
101
|
+
|
|
102
|
+
/** Past this size, an ignore file is not read. */
|
|
103
|
+
const IGNORE_FILE_BYTES = 1024 * 1024
|
|
104
|
+
|
|
105
|
+
/**
|
|
106
|
+
* A folder's `.gitignore`, or another ignore file applying from it, when it is a regular file it
|
|
107
|
+
* can read. A link is not followed, as git does not, and a device or a FIFO is never opened.
|
|
108
|
+
*/
|
|
109
|
+
export function readIgnoreFile(folder: string, name = ".gitignore"): IgnoreFile | undefined {
|
|
110
|
+
try {
|
|
111
|
+
const path = childPath(folder, name)
|
|
112
|
+
const stats = lstatSync(path)
|
|
113
|
+
if (!stats.isFile() || stats.size > IGNORE_FILE_BYTES) return undefined
|
|
114
|
+
|
|
115
|
+
return { folder, rules: parseGitignore(readFileSync(path, "utf8")) }
|
|
116
|
+
} catch {
|
|
117
|
+
return undefined
|
|
118
|
+
}
|
|
119
|
+
}
|
|
120
|
+
|
|
121
|
+
/**
|
|
122
|
+
* Entries in the order their paths sort in, byte by byte: a folder as its name and a `/`, so that
|
|
123
|
+
* `a.txt` comes before `a/b.txt` as it does in a sorted list of paths.
|
|
124
|
+
*/
|
|
125
|
+
function sortedEntries(entries: readonly Dirent[]): Dirent[] {
|
|
126
|
+
return entries
|
|
127
|
+
.map((entry) => ({
|
|
128
|
+
entry,
|
|
129
|
+
key: Buffer.from(entry.isDirectory() ? `${entry.name}/` : entry.name),
|
|
130
|
+
}))
|
|
131
|
+
.sort((left, right) => Buffer.compare(left.key, right.key))
|
|
132
|
+
.map(({ entry }) => entry)
|
|
133
|
+
}
|
|
134
|
+
|
|
135
|
+
export function childPath(folder: string, name: string): string {
|
|
136
|
+
return folder.endsWith("/") ? `${folder}${name}` : `${folder}/${name}`
|
|
137
|
+
}
|
|
138
|
+
|
|
139
|
+
function isLinkToFile(path: string): boolean {
|
|
140
|
+
try {
|
|
141
|
+
return statSync(path).isFile()
|
|
142
|
+
} catch {
|
|
143
|
+
return false
|
|
144
|
+
}
|
|
145
|
+
}
|
|
@@ -22,6 +22,8 @@ import type { AgentMessageRecord } from "@sixb/core/storage"
|
|
|
22
22
|
import { listSandboxFiles } from "../sandbox-files"
|
|
23
23
|
import { type AgentSandboxHandle, contextPath } from "../sandbox-handle"
|
|
24
24
|
import { BASH_TOOL_SPEC, keptCommandOutput } from "./bash"
|
|
25
|
+
import { GLOB_TOOL_SPEC } from "./glob"
|
|
26
|
+
import { GREP_TOOL_SPEC } from "./grep"
|
|
25
27
|
import { READ_TOOL_SPEC } from "./read"
|
|
26
28
|
import { AgentFileBudget } from "./tool-files"
|
|
27
29
|
import { LEGACY_VIEW_FILE_TOOL } from "./view-file"
|
|
@@ -65,11 +67,14 @@ const LISTED_OUTPUTS_MAX_FILES = 1_000
|
|
|
65
67
|
const LIST_TIMEOUT_MS = 30_000
|
|
66
68
|
|
|
67
69
|
/**
|
|
68
|
-
* Never saved here: the readers page through files themselves, and `
|
|
69
|
-
*
|
|
70
|
+
* Never saved here: the readers page through files themselves, `glob` and `grep` cap what they
|
|
71
|
+
* return and show it as text a saved JSON would not be, and `web_fetch` saves its page itself
|
|
72
|
+
* ({@link ToolOutputFiles.store}).
|
|
70
73
|
*/
|
|
71
74
|
const UNSAVED_TOOLS: ReadonlySet<string> = new Set([
|
|
72
75
|
READ_TOOL_SPEC.name,
|
|
76
|
+
GLOB_TOOL_SPEC.name,
|
|
77
|
+
GREP_TOOL_SPEC.name,
|
|
73
78
|
LEGACY_VIEW_FILE_TOOL,
|
|
74
79
|
WEB_FETCH_TOOL_SPEC.name,
|
|
75
80
|
])
|