dsh-remote-agent 0.4.3 → 0.6.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +69 -0
- package/README.en.md +49 -1
- package/README.md +40 -1
- package/SECURITY.md +24 -0
- package/lib/hub/node-registry.js +73 -0
- package/lib/hub/registry-ingest.js +18 -0
- package/lib/hub/tools.js +261 -4
- package/lib/node/fs-service.js +504 -0
- package/lib/node/index.js +44 -2
- package/lib/node/task-engine.js +51 -4
- package/lib/node/transport.js +2 -0
- package/lib/protocol.js +13 -0
- package/package.json +1 -1
- package/scripts/check-doc-drift.mjs +18 -3
- package/scripts/diag-inject.mjs +3 -3
|
@@ -0,0 +1,504 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* 远程文件工具集(提案 #25)的副机本地实现 —— 四个只读操作:
|
|
3
|
+
* ls / read / grep / tree
|
|
4
|
+
*
|
|
5
|
+
* 安全模型(与任务链路同一条底线,缺一不可):
|
|
6
|
+
* - **只读**。整个模块没有任何写路径 —— fs-service 拿不到"写"这个动词。
|
|
7
|
+
* - **roots 边界**:每个入口路径先 realpathSync.native(展开 8.3 短名与符号
|
|
8
|
+
* 链接,与 exec.js resolveWorkspace 同款口径),再与 roots 前缀比对。
|
|
9
|
+
* - **denyPaths 生效**:isDenied 与 artifacts.js / task-engine 同一条规则,
|
|
10
|
+
* 被禁路径在枚举与读取两层都被拦(枚举层拦是为了连名字都不上报)。
|
|
11
|
+
* - **符号链接不跟随**:tree/grep 遍历时 withFileTypes 发现的链接只列条目
|
|
12
|
+
* 本身、绝不深入 —— 否则 roots 内一个指向 C:\Users 的链接就是逃逸口。
|
|
13
|
+
*
|
|
14
|
+
* 上限(防御 50GB 级文件夹被一次请求拖死副机):
|
|
15
|
+
* - read:文本模式仅支持 ≤ READABLE_FILE_MAX (8MB) 的文件按行分段;二进制
|
|
16
|
+
* 只回 sha256/size/头部预览;单次响应文本量 ≤ READ_TEXT_MAX (256KB)。
|
|
17
|
+
* - grep:文件数 ≤ GREP_MAX_FILES (2000)、单文件 ≤ GREP_FILE_MAX (2MB)、
|
|
18
|
+
* 命中 ≤ maxMatches (200)、总时长 ≤ GREP_TIME_MS (10s)。
|
|
19
|
+
* - tree:深度 ≤ depth、总条目 ≤ maxEntries (2000)。
|
|
20
|
+
*/
|
|
21
|
+
|
|
22
|
+
import fs from 'node:fs'
|
|
23
|
+
import path from 'node:path'
|
|
24
|
+
import { createHash } from 'node:crypto'
|
|
25
|
+
import { isDenied } from './exec.js'
|
|
26
|
+
import { normalizePath, extname } from '../glob.js'
|
|
27
|
+
import { ERROR_CODE } from '../protocol.js'
|
|
28
|
+
|
|
29
|
+
/** 文本模式 read 的文件大小上限:超过就拒绝(建议 grep 定位后再读片段)。 */
|
|
30
|
+
export const READABLE_FILE_MAX = 8 * 1024 * 1024
|
|
31
|
+
/** 单次 read 响应携带的文本量上限(字节)。 */
|
|
32
|
+
export const READ_TEXT_MAX = 256 * 1024
|
|
33
|
+
/** grep 递归扫描的文件数上限。 */
|
|
34
|
+
export const GREP_MAX_FILES = 2000
|
|
35
|
+
/** grep 单文件大小上限:更大的文件直接跳过(防止二进制/巨型日志拖死)。 */
|
|
36
|
+
export const GREP_FILE_MAX = 2 * 1024 * 1024
|
|
37
|
+
/** grep 默认总时长上限(ms):超时停下并如实报告 truncated。 */
|
|
38
|
+
export const GREP_TIME_MS = 10 * 1000
|
|
39
|
+
/** grep 默认命中上限。 */
|
|
40
|
+
export const GREP_MAX_MATCHES = 200
|
|
41
|
+
/** tree 默认最大深度与条目数。 */
|
|
42
|
+
export const TREE_DEFAULT_DEPTH = 3
|
|
43
|
+
export const TREE_MAX_ENTRIES = 2000
|
|
44
|
+
/** ls 单页条目上限。 */
|
|
45
|
+
export const LS_PAGE_MAX = 2000
|
|
46
|
+
/** 二进制嗅探窗口:前 64 字节出现 NUL 即视为二进制。 */
|
|
47
|
+
const SNIFF_BYTES = 64
|
|
48
|
+
|
|
49
|
+
/** 复用任务链路的错误形态(code 会被 hub 工具层原样透出)。 */
|
|
50
|
+
export class FsError extends Error {
|
|
51
|
+
constructor(message, code) {
|
|
52
|
+
super(message)
|
|
53
|
+
this.name = 'FsError'
|
|
54
|
+
this.code = code
|
|
55
|
+
}
|
|
56
|
+
}
|
|
57
|
+
|
|
58
|
+
/**
|
|
59
|
+
* 解析"可以是文件也可以是目录"的路径:realpath + roots 前缀 + denyPaths。
|
|
60
|
+
* 与 resolveWorkspace 的差异只在两点:不要求是目录;不存在时直接拒
|
|
61
|
+
* (读一个不存在的文件没有"任务会创建它"的借口)。
|
|
62
|
+
*/
|
|
63
|
+
function resolveAny(requested, cfg) {
|
|
64
|
+
const { roots, caseInsensitive } = cfg
|
|
65
|
+
if (typeof requested !== 'string' || requested === '') {
|
|
66
|
+
throw new FsError('path is required', ERROR_CODE.WORKSPACE_DENIED)
|
|
67
|
+
}
|
|
68
|
+
if (!path.isAbsolute(requested)) {
|
|
69
|
+
throw new FsError(`path must be absolute: ${requested}`, ERROR_CODE.WORKSPACE_DENIED)
|
|
70
|
+
}
|
|
71
|
+
let resolved
|
|
72
|
+
try {
|
|
73
|
+
resolved = fs.realpathSync.native ? fs.realpathSync.native(requested) : fs.realpathSync(requested)
|
|
74
|
+
} catch {
|
|
75
|
+
throw new FsError(`path not found: ${requested}`, ERROR_CODE.ARTIFACT_MISSING)
|
|
76
|
+
}
|
|
77
|
+
const allowed = (roots ?? []).map((r) => {
|
|
78
|
+
let base = r
|
|
79
|
+
try {
|
|
80
|
+
base = fs.realpathSync.native ? fs.realpathSync.native(r) : fs.realpathSync(r)
|
|
81
|
+
} catch {
|
|
82
|
+
// 与 resolveWorkspace 同口径:不可达的根保持原样,后面照样拒一切子路径
|
|
83
|
+
}
|
|
84
|
+
return normalizePath(base, { caseInsensitive })
|
|
85
|
+
})
|
|
86
|
+
const norm = normalizePath(resolved, { caseInsensitive })
|
|
87
|
+
const ok = allowed.some((r) => norm === r || norm.startsWith(`${r}/`))
|
|
88
|
+
if (!ok) {
|
|
89
|
+
throw new FsError(
|
|
90
|
+
`path outside allowed roots: ${resolved}`,
|
|
91
|
+
ERROR_CODE.WORKSPACE_DENIED,
|
|
92
|
+
)
|
|
93
|
+
}
|
|
94
|
+
if (isDenied(resolved, cfg.denyPaths ?? [], { caseInsensitive })) {
|
|
95
|
+
throw new FsError(`path denied by denyPaths: ${resolved}`, ERROR_CODE.WORKSPACE_DENIED)
|
|
96
|
+
}
|
|
97
|
+
return resolved
|
|
98
|
+
}
|
|
99
|
+
|
|
100
|
+
/** 枚举层拒绝:条目级 denyPaths(连名字都不上报给 hub)。 */
|
|
101
|
+
function entryDenied(absPath, cfg) {
|
|
102
|
+
return isDenied(absPath, cfg.denyPaths ?? [], { caseInsensitive: cfg.caseInsensitive })
|
|
103
|
+
}
|
|
104
|
+
|
|
105
|
+
/** 目录条目排序:目录优先,同级按名字(locale 感知,大小写不敏感)。 */
|
|
106
|
+
function sortEntries(list) {
|
|
107
|
+
return list.sort((a, b) => {
|
|
108
|
+
if (a.type !== b.type) return a.type === 'dir' ? -1 : 1
|
|
109
|
+
return a.name.localeCompare(b.name, undefined, { sensitivity: 'base', numeric: true })
|
|
110
|
+
})
|
|
111
|
+
}
|
|
112
|
+
|
|
113
|
+
/** 文件是否疑似二进制:前 SNIFF_BYTES 字节出现 NUL。 */
|
|
114
|
+
function looksBinary(fd) {
|
|
115
|
+
const buf = Buffer.alloc(SNIFF_BYTES)
|
|
116
|
+
// ⚠ fs.readSync 直接返回 bytesRead 数字(不是 {bytesRead, buffer} 对象 ——
|
|
117
|
+
// 那是 fs.read 的 promise 版)。解构会得到 undefined,subarray(0, undefined)
|
|
118
|
+
// 变成整个缓冲,而 alloc 未写区域全零 → **任何文件都被误判二进制**。
|
|
119
|
+
const bytesRead = fs.readSync(fd, buf, 0, SNIFF_BYTES, 0)
|
|
120
|
+
return buf.subarray(0, bytesRead).includes(0)
|
|
121
|
+
}
|
|
122
|
+
|
|
123
|
+
/**
|
|
124
|
+
* 建一个 fs-service 实例。roots/denyPaths 来自副机配置(与任务链路同源),
|
|
125
|
+
* caseInsensitive 跟随 Windows 语义。
|
|
126
|
+
*/
|
|
127
|
+
export function createFsService(cfg) {
|
|
128
|
+
const { roots, caseInsensitive = false } = cfg
|
|
129
|
+
const conf = { roots, caseInsensitive, denyPaths: cfg.denyPaths ?? [] }
|
|
130
|
+
|
|
131
|
+
return {
|
|
132
|
+
/** 列目录(单层,分页)。 */
|
|
133
|
+
ls({ path: p, limit = 500, offset = 0 } = {}) {
|
|
134
|
+
const dir = resolveAny(p, conf)
|
|
135
|
+
let st
|
|
136
|
+
try {
|
|
137
|
+
st = fs.statSync(dir)
|
|
138
|
+
} catch {
|
|
139
|
+
throw new FsError(`path not found: ${dir}`, ERROR_CODE.ARTIFACT_MISSING)
|
|
140
|
+
}
|
|
141
|
+
if (!st.isDirectory()) {
|
|
142
|
+
throw new FsError(`not a directory: ${dir}`, ERROR_CODE.WORKSPACE_DENIED)
|
|
143
|
+
}
|
|
144
|
+
const limitN = Math.min(Math.max(Number(limit) || 500, 1), LS_PAGE_MAX)
|
|
145
|
+
const offsetN = Math.max(Number(offset) || 0, 0)
|
|
146
|
+
const all = []
|
|
147
|
+
for (const ent of fs.readdirSync(dir, { withFileTypes: true })) {
|
|
148
|
+
const abs = path.join(dir, ent.name)
|
|
149
|
+
if (entryDenied(abs, conf)) continue
|
|
150
|
+
let type = 'file'
|
|
151
|
+
let size = 0
|
|
152
|
+
let mtimeMs = 0
|
|
153
|
+
if (ent.isSymbolicLink()) {
|
|
154
|
+
// 链接只报条目本身,不深入也不解引用 —— 防止 roots 内链接逃逸
|
|
155
|
+
type = 'link'
|
|
156
|
+
} else if (ent.isDirectory()) {
|
|
157
|
+
type = 'dir'
|
|
158
|
+
} else if (ent.isFile()) {
|
|
159
|
+
try {
|
|
160
|
+
const s = fs.statSync(abs)
|
|
161
|
+
size = s.size
|
|
162
|
+
mtimeMs = s.mtimeMs
|
|
163
|
+
} catch {
|
|
164
|
+
// 竞态(文件刚被删):仍上报名字,大小/时间留空
|
|
165
|
+
}
|
|
166
|
+
} else {
|
|
167
|
+
continue // socket/fifo 等非常规条目不上报
|
|
168
|
+
}
|
|
169
|
+
all.push({ name: ent.name, type, size, mtimeMs })
|
|
170
|
+
}
|
|
171
|
+
sortEntries(all)
|
|
172
|
+
return {
|
|
173
|
+
path: dir,
|
|
174
|
+
total: all.length,
|
|
175
|
+
offset: offsetN,
|
|
176
|
+
truncated: offsetN + limitN < all.length,
|
|
177
|
+
entries: all.slice(offsetN, offsetN + limitN),
|
|
178
|
+
}
|
|
179
|
+
},
|
|
180
|
+
|
|
181
|
+
/**
|
|
182
|
+
* 读文件。文本模式:≤ READABLE_FILE_MAX 的文件按行分段(offset/limit),
|
|
183
|
+
* 单次文本量 ≤ READ_TEXT_MAX;二进制模式:只回 sha256/size/头部预览。
|
|
184
|
+
*/
|
|
185
|
+
read({ path: p, offset = 0, limit = 500 } = {}) {
|
|
186
|
+
const file = resolveAny(p, conf)
|
|
187
|
+
let st
|
|
188
|
+
try {
|
|
189
|
+
st = fs.statSync(file)
|
|
190
|
+
} catch {
|
|
191
|
+
throw new FsError(`path not found: ${file}`, ERROR_CODE.ARTIFACT_MISSING)
|
|
192
|
+
}
|
|
193
|
+
if (!st.isFile()) {
|
|
194
|
+
throw new FsError(`not a regular file: ${file}`, ERROR_CODE.WORKSPACE_DENIED)
|
|
195
|
+
}
|
|
196
|
+
let fd
|
|
197
|
+
try {
|
|
198
|
+
fd = fs.openSync(file, 'r')
|
|
199
|
+
if (looksBinary(fd)) {
|
|
200
|
+
// 二进制:hash 全文件 + 头部预览(hex),绝不回原始字节
|
|
201
|
+
const buf = Buffer.alloc(Math.min(st.size, SNIFF_BYTES))
|
|
202
|
+
fs.readSync(fd, buf, 0, buf.length, 0)
|
|
203
|
+
const hash = createHash('sha256')
|
|
204
|
+
const chunk = Buffer.alloc(1024 * 1024)
|
|
205
|
+
for (;;) {
|
|
206
|
+
const n = fs.readSync(fd, chunk, 0, chunk.length, null)
|
|
207
|
+
if (n <= 0) break
|
|
208
|
+
hash.update(chunk.subarray(0, n))
|
|
209
|
+
}
|
|
210
|
+
return {
|
|
211
|
+
path: file,
|
|
212
|
+
binary: true,
|
|
213
|
+
size: st.size,
|
|
214
|
+
sha256: hash.digest('hex'),
|
|
215
|
+
ext: extname(normalizePath(file, { caseInsensitive })),
|
|
216
|
+
head: buf.toString('hex'),
|
|
217
|
+
}
|
|
218
|
+
}
|
|
219
|
+
if (st.size > READABLE_FILE_MAX) {
|
|
220
|
+
throw new FsError(
|
|
221
|
+
`file too large for text read (${st.size} > ${READABLE_FILE_MAX}); use grep to locate then read a narrower path`,
|
|
222
|
+
ERROR_CODE.ARTIFACT_TOO_LARGE,
|
|
223
|
+
)
|
|
224
|
+
}
|
|
225
|
+
const raw = fs.readFileSync(file)
|
|
226
|
+
const text = raw.toString('utf8')
|
|
227
|
+
const lines = text.split('\n')
|
|
228
|
+
// 尾部空行是 split 的伪影:文件以 \n 结尾时多出一个空串
|
|
229
|
+
if (lines.length > 0 && lines[lines.length - 1] === '') lines.pop()
|
|
230
|
+
const offsetN = Math.max(Number(offset) || 0, 0)
|
|
231
|
+
const limitN = Math.min(Math.max(Number(limit) || 500, 1), 5000)
|
|
232
|
+
const slice = []
|
|
233
|
+
let bytes = 0
|
|
234
|
+
let truncated = false
|
|
235
|
+
for (let i = offsetN; i < lines.length; i++) {
|
|
236
|
+
const line = lines[i]
|
|
237
|
+
// 单行超长截断:一行 1MB 的压缩 JSON 会独自顶爆响应
|
|
238
|
+
const clipped = line.length > 2000 ? `${line.slice(0, 2000)}…[line truncated]` : line
|
|
239
|
+
const cost = Buffer.byteLength(clipped, 'utf8') + 1
|
|
240
|
+
if (bytes + cost > READ_TEXT_MAX) {
|
|
241
|
+
truncated = true
|
|
242
|
+
break
|
|
243
|
+
}
|
|
244
|
+
slice.push(clipped)
|
|
245
|
+
bytes += cost
|
|
246
|
+
if (slice.length >= limitN) {
|
|
247
|
+
truncated = i < lines.length - 1
|
|
248
|
+
break
|
|
249
|
+
}
|
|
250
|
+
}
|
|
251
|
+
return {
|
|
252
|
+
path: file,
|
|
253
|
+
binary: false,
|
|
254
|
+
size: st.size,
|
|
255
|
+
totalLines: lines.length,
|
|
256
|
+
offset: offsetN,
|
|
257
|
+
truncated: truncated || offsetN + slice.length < lines.length,
|
|
258
|
+
lines: slice,
|
|
259
|
+
}
|
|
260
|
+
} finally {
|
|
261
|
+
try {
|
|
262
|
+
if (fd !== undefined) fs.closeSync(fd)
|
|
263
|
+
} catch {
|
|
264
|
+
// 关闭失败不影响结果
|
|
265
|
+
}
|
|
266
|
+
}
|
|
267
|
+
},
|
|
268
|
+
|
|
269
|
+
/**
|
|
270
|
+
* 副机本地全文搜索:递归扫 roots 内目录,文本文件逐行匹配。
|
|
271
|
+
* 纯 JS(Windows 没有外部 grep 可 spawn);上限见模块头注释。
|
|
272
|
+
*/
|
|
273
|
+
grep({ path: p, pattern, regex = false, ignoreCase = true, glob = '', maxMatches = GREP_MAX_MATCHES } = {}) {
|
|
274
|
+
if (typeof pattern !== 'string' || pattern === '') {
|
|
275
|
+
throw new FsError('pattern is required', ERROR_CODE.WORKSPACE_DENIED)
|
|
276
|
+
}
|
|
277
|
+
let re
|
|
278
|
+
try {
|
|
279
|
+
re = regex
|
|
280
|
+
? new RegExp(pattern, ignoreCase ? 'i' : '')
|
|
281
|
+
: new RegExp(pattern.replace(/[.*+?^${}()|[\]\\]/g, '\\$&'), ignoreCase ? 'i' : '')
|
|
282
|
+
} catch (err) {
|
|
283
|
+
throw new FsError(`invalid pattern: ${err.message}`, ERROR_CODE.WORKSPACE_DENIED)
|
|
284
|
+
}
|
|
285
|
+
const dir = resolveAny(p, conf)
|
|
286
|
+
let st
|
|
287
|
+
try {
|
|
288
|
+
st = fs.statSync(dir)
|
|
289
|
+
} catch {
|
|
290
|
+
throw new FsError(`path not found: ${dir}`, ERROR_CODE.ARTIFACT_MISSING)
|
|
291
|
+
}
|
|
292
|
+
const maxN = Math.min(Math.max(Number(maxMatches) || GREP_MAX_MATCHES, 1), GREP_MAX_MATCHES)
|
|
293
|
+
const matches = []
|
|
294
|
+
let filesScanned = 0
|
|
295
|
+
let filesSkipped = 0
|
|
296
|
+
let truncatedByTime = false
|
|
297
|
+
let truncatedByLimit = false
|
|
298
|
+
const started = Date.now()
|
|
299
|
+
|
|
300
|
+
const isGlobMatch = (name) => {
|
|
301
|
+
if (!glob) return true
|
|
302
|
+
// 极简 glob:* 任意(不含 /)、? 单字符 —— 覆盖 *.txt / src* 之类
|
|
303
|
+
const esc = glob.replace(/[.+^${}()|[\]\\]/g, '\\$&').replace(/\*/g, '[^/]*').replace(/\?/g, '[^/]')
|
|
304
|
+
return new RegExp(`^${esc}$`, caseInsensitive ? 'i' : '').test(name)
|
|
305
|
+
}
|
|
306
|
+
|
|
307
|
+
const scan = (dirAbs) => {
|
|
308
|
+
if (matches.length >= maxN) {
|
|
309
|
+
truncatedByLimit = true
|
|
310
|
+
return
|
|
311
|
+
}
|
|
312
|
+
if (Date.now() - started > GREP_TIME_MS) {
|
|
313
|
+
truncatedByTime = true
|
|
314
|
+
return
|
|
315
|
+
}
|
|
316
|
+
let ents
|
|
317
|
+
try {
|
|
318
|
+
ents = fs.readdirSync(dirAbs, { withFileTypes: true })
|
|
319
|
+
} catch {
|
|
320
|
+
filesSkipped++
|
|
321
|
+
return
|
|
322
|
+
}
|
|
323
|
+
for (const ent of ents) {
|
|
324
|
+
if (matches.length >= maxN) {
|
|
325
|
+
truncatedByLimit = true
|
|
326
|
+
return
|
|
327
|
+
}
|
|
328
|
+
if (Date.now() - started > GREP_TIME_MS) {
|
|
329
|
+
truncatedByTime = true
|
|
330
|
+
return
|
|
331
|
+
}
|
|
332
|
+
const abs = path.join(dirAbs, ent.name)
|
|
333
|
+
if (entryDenied(abs, conf)) continue
|
|
334
|
+
if (ent.isDirectory()) {
|
|
335
|
+
scan(abs)
|
|
336
|
+
continue
|
|
337
|
+
}
|
|
338
|
+
// 链接与非常规文件跳过:不跟随、不读
|
|
339
|
+
if (!ent.isFile()) {
|
|
340
|
+
filesSkipped++
|
|
341
|
+
continue
|
|
342
|
+
}
|
|
343
|
+
if (!isGlobMatch(ent.name)) continue
|
|
344
|
+
let fst
|
|
345
|
+
try {
|
|
346
|
+
fst = fs.statSync(abs)
|
|
347
|
+
} catch {
|
|
348
|
+
filesSkipped++
|
|
349
|
+
continue
|
|
350
|
+
}
|
|
351
|
+
if (fst.size > GREP_FILE_MAX || fst.size === 0) {
|
|
352
|
+
filesSkipped++
|
|
353
|
+
continue
|
|
354
|
+
}
|
|
355
|
+
let fd
|
|
356
|
+
try {
|
|
357
|
+
fd = fs.openSync(abs, 'r')
|
|
358
|
+
if (looksBinary(fd)) {
|
|
359
|
+
filesSkipped++
|
|
360
|
+
continue
|
|
361
|
+
}
|
|
362
|
+
const content = fs.readFileSync(abs)
|
|
363
|
+
filesScanned++
|
|
364
|
+
const text = content.toString('utf8')
|
|
365
|
+
const lines = text.split('\n')
|
|
366
|
+
for (let i = 0; i < lines.length && matches.length < maxN; i++) {
|
|
367
|
+
if (re.test(lines[i])) {
|
|
368
|
+
matches.push({
|
|
369
|
+
file: abs,
|
|
370
|
+
line: i + 1,
|
|
371
|
+
text: lines[i].length > 500 ? `${lines[i].slice(0, 500)}…` : lines[i],
|
|
372
|
+
})
|
|
373
|
+
// 达到全局上限即如实报告截断 —— 即使恰好在最后一个文件结束,
|
|
374
|
+
// 未报出的命中大概率还存在(用户视角"还有更多"才是事实)
|
|
375
|
+
if (matches.length >= maxN) truncatedByLimit = true
|
|
376
|
+
}
|
|
377
|
+
}
|
|
378
|
+
} catch {
|
|
379
|
+
filesSkipped++
|
|
380
|
+
} finally {
|
|
381
|
+
try {
|
|
382
|
+
if (fd !== undefined) fs.closeSync(fd)
|
|
383
|
+
} catch {
|
|
384
|
+
// 忽略关闭失败
|
|
385
|
+
}
|
|
386
|
+
}
|
|
387
|
+
}
|
|
388
|
+
}
|
|
389
|
+
|
|
390
|
+
if (st.isDirectory()) {
|
|
391
|
+
scan(dir)
|
|
392
|
+
} else if (st.isFile()) {
|
|
393
|
+
// 单文件 grep:直接匹配该文件
|
|
394
|
+
let fd
|
|
395
|
+
try {
|
|
396
|
+
fd = fs.openSync(dir, 'r')
|
|
397
|
+
if (!looksBinary(fd)) {
|
|
398
|
+
filesScanned++
|
|
399
|
+
const lines = fs.readFileSync(dir).toString('utf8').split('\n')
|
|
400
|
+
for (let i = 0; i < lines.length && matches.length < maxN; i++) {
|
|
401
|
+
if (re.test(lines[i])) {
|
|
402
|
+
matches.push({ file: dir, line: i + 1, text: lines[i].slice(0, 500) })
|
|
403
|
+
}
|
|
404
|
+
}
|
|
405
|
+
}
|
|
406
|
+
} catch {
|
|
407
|
+
filesSkipped++
|
|
408
|
+
} finally {
|
|
409
|
+
try {
|
|
410
|
+
if (fd !== undefined) fs.closeSync(fd)
|
|
411
|
+
} catch {
|
|
412
|
+
// 忽略
|
|
413
|
+
}
|
|
414
|
+
}
|
|
415
|
+
} else {
|
|
416
|
+
throw new FsError(`not a directory or file: ${dir}`, ERROR_CODE.WORKSPACE_DENIED)
|
|
417
|
+
}
|
|
418
|
+
|
|
419
|
+
return {
|
|
420
|
+
path: dir,
|
|
421
|
+
pattern,
|
|
422
|
+
filesScanned,
|
|
423
|
+
filesSkipped,
|
|
424
|
+
matches,
|
|
425
|
+
matchCount: matches.length,
|
|
426
|
+
truncated: truncatedByTime || truncatedByLimit,
|
|
427
|
+
truncatedBy: truncatedByTime ? 'time' : truncatedByLimit ? 'maxMatches' : undefined,
|
|
428
|
+
}
|
|
429
|
+
},
|
|
430
|
+
|
|
431
|
+
/** 目录树摘要:扫描在副机跑完只回结构(50GB 也不搬家)。 */
|
|
432
|
+
tree({ path: p, depth = TREE_DEFAULT_DEPTH, maxEntries = TREE_MAX_ENTRIES } = {}) {
|
|
433
|
+
const rootDir = resolveAny(p, conf)
|
|
434
|
+
let st
|
|
435
|
+
try {
|
|
436
|
+
st = fs.statSync(rootDir)
|
|
437
|
+
} catch {
|
|
438
|
+
throw new FsError(`path not found: ${rootDir}`, ERROR_CODE.ARTIFACT_MISSING)
|
|
439
|
+
}
|
|
440
|
+
if (!st.isDirectory()) {
|
|
441
|
+
throw new FsError(`not a directory: ${rootDir}`, ERROR_CODE.WORKSPACE_DENIED)
|
|
442
|
+
}
|
|
443
|
+
const depthN = Math.min(Math.max(Number(depth) || TREE_DEFAULT_DEPTH, 1), 6)
|
|
444
|
+
const maxN = Math.min(Math.max(Number(maxEntries) || TREE_MAX_ENTRIES, 1), TREE_MAX_ENTRIES)
|
|
445
|
+
let used = 0
|
|
446
|
+
let truncated = false
|
|
447
|
+
|
|
448
|
+
const build = (dirAbs, level) => {
|
|
449
|
+
if (level > depthN || used >= maxN) {
|
|
450
|
+
if (used >= maxN) truncated = true
|
|
451
|
+
return []
|
|
452
|
+
}
|
|
453
|
+
let ents
|
|
454
|
+
try {
|
|
455
|
+
ents = fs.readdirSync(dirAbs, { withFileTypes: true })
|
|
456
|
+
} catch {
|
|
457
|
+
return [{ name: '(unreadable)', type: 'error' }]
|
|
458
|
+
}
|
|
459
|
+
const items = []
|
|
460
|
+
const dirs = []
|
|
461
|
+
const files = []
|
|
462
|
+
for (const ent of ents) {
|
|
463
|
+
const abs = path.join(dirAbs, ent.name)
|
|
464
|
+
if (entryDenied(abs, conf)) continue
|
|
465
|
+
if (ent.isDirectory()) dirs.push(ent.name)
|
|
466
|
+
else if (ent.isFile()) files.push(ent.name)
|
|
467
|
+
// 链接只报名字标注 link,不深入(与 ls 同口径)
|
|
468
|
+
else if (ent.isSymbolicLink()) files.push(ent.name)
|
|
469
|
+
}
|
|
470
|
+
const collator = (a, b) => a.localeCompare(b, undefined, { sensitivity: 'base', numeric: true })
|
|
471
|
+
dirs.sort(collator)
|
|
472
|
+
files.sort(collator)
|
|
473
|
+
for (const name of dirs) {
|
|
474
|
+
if (used >= maxN) {
|
|
475
|
+
truncated = true
|
|
476
|
+
break
|
|
477
|
+
}
|
|
478
|
+
used++
|
|
479
|
+
const node = { name, type: 'dir', children: build(path.join(dirAbs, name), level + 1) }
|
|
480
|
+
items.push(node)
|
|
481
|
+
}
|
|
482
|
+
for (const name of files) {
|
|
483
|
+
if (used >= maxN) {
|
|
484
|
+
truncated = true
|
|
485
|
+
break
|
|
486
|
+
}
|
|
487
|
+
used++
|
|
488
|
+
items.push({ name, type: 'file' })
|
|
489
|
+
}
|
|
490
|
+
return items
|
|
491
|
+
}
|
|
492
|
+
|
|
493
|
+
const items = build(rootDir, 1)
|
|
494
|
+
return {
|
|
495
|
+
path: rootDir,
|
|
496
|
+
depth: depthN,
|
|
497
|
+
entries: used,
|
|
498
|
+
maxEntries: maxN,
|
|
499
|
+
truncated: truncated || used >= maxN,
|
|
500
|
+
items,
|
|
501
|
+
}
|
|
502
|
+
},
|
|
503
|
+
}
|
|
504
|
+
}
|
package/lib/node/index.js
CHANGED
|
@@ -51,6 +51,9 @@ import { friendlyConnectError } from '../setup.js'
|
|
|
51
51
|
import { scanRepos } from './repos.js'
|
|
52
52
|
import { TaskEngine } from './task-engine.js'
|
|
53
53
|
import { TaskStore } from './task-store.js'
|
|
54
|
+
// 远程文件工具集(提案 #25):ls/read/grep/tree 只读实现,与任务链路共用
|
|
55
|
+
// roots / denyPaths / caseInsensitive 三个配置(同一个安全边界,两处入口)。
|
|
56
|
+
import { createFsService } from './fs-service.js'
|
|
54
57
|
import { NodeTransport, TRANSPORT_MODE } from './transport.js'
|
|
55
58
|
|
|
56
59
|
const PLUGIN_NAME = 'dsh-remote-agent'
|
|
@@ -452,6 +455,10 @@ function startWorker(ctx, config) {
|
|
|
452
455
|
stateDir,
|
|
453
456
|
]
|
|
454
457
|
|
|
458
|
+
// 远程文件工具集(提案 #25):与任务链路同源的 roots/denyPaths —— 同一个
|
|
459
|
+
// 安全边界,两处入口。只读实现,见 lib/node/fs-service.js 头注释。
|
|
460
|
+
const fsService = createFsService({ roots, denyPaths, caseInsensitive })
|
|
461
|
+
|
|
455
462
|
/* 3) 任务仓库 ------------------------------------------------------- */
|
|
456
463
|
let store
|
|
457
464
|
try {
|
|
@@ -554,7 +561,11 @@ function startWorker(ctx, config) {
|
|
|
554
561
|
: RESOURCE_INTERVAL_MS
|
|
555
562
|
const resourceTimer = resourceIntervalMs > 0
|
|
556
563
|
? setInterval(() => {
|
|
557
|
-
|
|
564
|
+
// 队列健康快照(提案 #26-3)随资源帧搭车:engine 在下方第 7 节才
|
|
565
|
+
// 装配,但 interval 回调最早也在装配完成后才触发,引用安全。
|
|
566
|
+
// engine 尚未就绪(极端:interval 在装配中途先触发)时给空快照。
|
|
567
|
+
const queue = engine ? engine.queueStats() : { queued: 0, running: 0, maxConcurrent: 0, oldestQueuedMs: 0, starved: false }
|
|
568
|
+
send(NODE_TO_HUB.RESOURCE, { nodeId, ...sampler.sample(), queue })
|
|
558
569
|
}, resourceIntervalMs)
|
|
559
570
|
: undefined
|
|
560
571
|
if (resourceTimer?.unref) resourceTimer.unref()
|
|
@@ -601,7 +612,8 @@ function startWorker(ctx, config) {
|
|
|
601
612
|
caseInsensitive,
|
|
602
613
|
send,
|
|
603
614
|
logger,
|
|
604
|
-
|
|
615
|
+
// 默认 4(提案 #26-2):2 太脆,两条慢任务就能锁死整台机器。
|
|
616
|
+
maxConcurrent: Number.isFinite(cfg.maxConcurrent) ? Math.max(1, cfg.maxConcurrent) : 4,
|
|
605
617
|
})
|
|
606
618
|
|
|
607
619
|
// TaskEngine.pump() 用 `void this.execute(taskId)` 起任务 —— 一旦 execute reject,
|
|
@@ -782,6 +794,11 @@ function startWorker(ctx, config) {
|
|
|
782
794
|
// transport 自己已回 pong;这里保留一条兜底,防止上层换实现后 ping 悬空
|
|
783
795
|
send(NODE_TO_HUB.PONG, { echo: frame.id, nodeId, time: Date.now() }, { replyTo: frame.id })
|
|
784
796
|
return
|
|
797
|
+
case HUB_TO_NODE.FS_REQ:
|
|
798
|
+
// 远程文件工具集(提案 #25):异步执行,不挡后续帧;结果回 fs/res,
|
|
799
|
+
// reqId 就是请求信封 id(hub 按 replyTo 对 pending 表)。
|
|
800
|
+
handleFsReq(frame, d)
|
|
801
|
+
return
|
|
785
802
|
case HUB_TO_NODE.ERROR:
|
|
786
803
|
logger.warn('hub reported error', { code: d.code, message: d.message, replyTo: d.replyTo })
|
|
787
804
|
return
|
|
@@ -790,6 +807,31 @@ function startWorker(ctx, config) {
|
|
|
790
807
|
}
|
|
791
808
|
}
|
|
792
809
|
|
|
810
|
+
/**
|
|
811
|
+
* 执行一条远程文件请求(提案 #25)。所有错误——包括被 FsError 表达的
|
|
812
|
+
* 业务拒绝(越界 / denyPaths / 超限)——都折进 fs/res 的 ok:false 载荷,
|
|
813
|
+
* 让 hub 工具层拿到结构化的 code 而不是一条 ERROR 帧。
|
|
814
|
+
*/
|
|
815
|
+
function handleFsReq(frame, d) {
|
|
816
|
+
const op = typeof d.op === 'string' ? d.op : ''
|
|
817
|
+
const ops = { ls: 'ls', read: 'read', grep: 'grep', tree: 'tree' }
|
|
818
|
+
const fn = ops[op]
|
|
819
|
+
if (!fn) {
|
|
820
|
+
send(NODE_TO_HUB.FS_RES, { replyTo: frame.id, ok: false, code: ERROR_CODE.WORKSPACE_DENIED, message: `unknown fs op: ${op || '(empty)'}` })
|
|
821
|
+
return
|
|
822
|
+
}
|
|
823
|
+
const started = Date.now()
|
|
824
|
+
swallow(Promise.resolve().then(() => {
|
|
825
|
+
const result = fsService[fn](d)
|
|
826
|
+
logger.debug('fs op done', { op, ms: Date.now() - started, nodeId })
|
|
827
|
+
send(NODE_TO_HUB.FS_RES, { replyTo: frame.id, ok: true, result })
|
|
828
|
+
}).catch((err) => {
|
|
829
|
+
const code = typeof err?.code === 'string' ? err.code : ERROR_CODE.INTERNAL
|
|
830
|
+
logger.warn('fs op failed', { op, ms: Date.now() - started, code, error: msgOf(err) })
|
|
831
|
+
send(NODE_TO_HUB.FS_RES, { replyTo: frame.id, ok: false, code, message: msgOf(err) })
|
|
832
|
+
}))
|
|
833
|
+
}
|
|
834
|
+
|
|
793
835
|
/* 9) 事件接线 ------------------------------------------------------- */
|
|
794
836
|
const listenerDisposers = []
|
|
795
837
|
const on = (evt, fn) => {
|
package/lib/node/task-engine.js
CHANGED
|
@@ -26,6 +26,23 @@ import { isDenied, resolveWorkspace, runCommand } from './exec.js'
|
|
|
26
26
|
/** 摘要里最多保留多少条 changed 记录。 */
|
|
27
27
|
const MAX_CHANGED = 500
|
|
28
28
|
|
|
29
|
+
/**
|
|
30
|
+
* 默认任务超时(提案 #26-1):**exec 与 agent 分开**。
|
|
31
|
+
*
|
|
32
|
+
* 一条 exec 卡住 = 整台副机 1 小时不可用是 #26 实测的事故形态(两条慢 exec
|
|
33
|
+
* 占满 maxConcurrent,队列永动不了)。exec 是跑命令,绝大多数 1 分钟内结束;
|
|
34
|
+
* 10 分钟足够暴露"命令本身有问题"(路径不存在/权限/巨目录)。agent 会话
|
|
35
|
+
* 可能真的要跑很久,保持 1 小时。
|
|
36
|
+
*
|
|
37
|
+
* tools.js 的 remote_task 默认值必须与这里**同源**(直接 import 本常量),
|
|
38
|
+
* 否则会出现"工具说 10 分钟、副机兜底 1 小时"的分裂。
|
|
39
|
+
*/
|
|
40
|
+
export const EXEC_DEFAULT_TIMEOUT_MS = 10 * 60 * 1000
|
|
41
|
+
export const AGENT_DEFAULT_TIMEOUT_MS = 60 * 60 * 1000
|
|
42
|
+
|
|
43
|
+
/** 队列饥饿判定阈值(提案 #26-3):queued>0 且并发占满且最久排队超过此时长。 */
|
|
44
|
+
export const QUEUE_STARVED_AFTER_MS = 2 * 60 * 1000
|
|
45
|
+
|
|
29
46
|
export class TaskEngine {
|
|
30
47
|
/**
|
|
31
48
|
* @param {object} o
|
|
@@ -53,8 +70,10 @@ export class TaskEngine {
|
|
|
53
70
|
this.logger = o.logger
|
|
54
71
|
/** taskId → { controller, startedAt } */
|
|
55
72
|
this.running = new Map()
|
|
56
|
-
/**
|
|
57
|
-
|
|
73
|
+
/** 任务并发上限,防止副机被一次派单压垮。
|
|
74
|
+
* 默认 4(提案 #26-2):2 太脆 —— 两条慢任务就能锁死整台机器;
|
|
75
|
+
* 4 让"两条慢 + 两条快"正常共存。配置项保留,按内存调整。 */
|
|
76
|
+
this.maxConcurrent = o.maxConcurrent ?? 4
|
|
58
77
|
this.queue = []
|
|
59
78
|
/** 卸载闸门:stop() 置 true 后不再接受/启动新任务(见 stop() 注释)。 */
|
|
60
79
|
this.stopped = false
|
|
@@ -319,7 +338,9 @@ export class TaskEngine {
|
|
|
319
338
|
try {
|
|
320
339
|
res = await runCommand(command, {
|
|
321
340
|
cwd: rec.workspace,
|
|
322
|
-
|
|
341
|
+
// exec 默认 10 分钟(提案 #26-1,与 tools.js 同源常量);任务显式
|
|
342
|
+
// 传 timeoutMs 时以任务为准(长遍历大目录等场景仍可显式放宽)。
|
|
343
|
+
timeoutMs: rec.constraints?.timeoutMs ?? EXEC_DEFAULT_TIMEOUT_MS,
|
|
323
344
|
env: rec.constraints?.extraEnv ?? {},
|
|
324
345
|
signal: controller.signal,
|
|
325
346
|
onData: (chunk, stream) => {
|
|
@@ -369,7 +390,7 @@ export class TaskEngine {
|
|
|
369
390
|
// 落盘为 null 表示未指定 → 传 undefined,让 runner 走"副机默认"路径。
|
|
370
391
|
// in-process 后端会把它叠加到副机默认选择上;headless 后端会明确拒绝。
|
|
371
392
|
model: rec.model ?? undefined,
|
|
372
|
-
timeoutMs: rec.constraints?.timeoutMs ??
|
|
393
|
+
timeoutMs: rec.constraints?.timeoutMs ?? AGENT_DEFAULT_TIMEOUT_MS,
|
|
373
394
|
signal: controller.signal,
|
|
374
395
|
onData: (chunk, stream) => {
|
|
375
396
|
for (const line of chunk.split(/\r?\n/)) {
|
|
@@ -491,6 +512,32 @@ export class TaskEngine {
|
|
|
491
512
|
return map
|
|
492
513
|
}
|
|
493
514
|
|
|
515
|
+
/**
|
|
516
|
+
* 队列健康快照(提案 #26-3):随资源遥测帧上报 hub。
|
|
517
|
+
*
|
|
518
|
+
* 背景(#26 实测事故):两条慢 exec 占满 maxConcurrent 后,pump() 的
|
|
519
|
+
* while 条件永假,后续任务全部 queued 且**在面板上不可见**——用户只知道
|
|
520
|
+
* "卡了",不知道"卡在哪"。这个快照让 hub/面板能说出一句话:
|
|
521
|
+
* "队列被 N 条运行中任务阻塞,最久已等 M 分钟"。
|
|
522
|
+
*
|
|
523
|
+
* starved 判定:有排队 + 并发占满 + 最久排队超过 QUEUE_STARVED_AFTER_MS。
|
|
524
|
+
* 纯读操作,绝不影响任务链路(与资源遥测同一条尽力而为不变式)。
|
|
525
|
+
*/
|
|
526
|
+
queueStats(now = Date.now()) {
|
|
527
|
+
const queuedAt = this.queue
|
|
528
|
+
.map((id) => this.store.get(id)?.createdAt)
|
|
529
|
+
.filter((t) => typeof t === 'number' && t > 0)
|
|
530
|
+
const oldestQueuedMs = queuedAt.length > 0 ? Math.max(0, now - Math.min(...queuedAt)) : 0
|
|
531
|
+
const saturated = this.running.size >= this.maxConcurrent
|
|
532
|
+
return {
|
|
533
|
+
queued: this.queue.length,
|
|
534
|
+
running: this.running.size,
|
|
535
|
+
maxConcurrent: this.maxConcurrent,
|
|
536
|
+
oldestQueuedMs,
|
|
537
|
+
starved: this.queue.length > 0 && saturated && oldestQueuedMs > QUEUE_STARVED_AFTER_MS,
|
|
538
|
+
}
|
|
539
|
+
}
|
|
540
|
+
|
|
494
541
|
/** 重连后向 hub 上报全部任务状态。 */
|
|
495
542
|
syncAll() {
|
|
496
543
|
this.send('task/sync', {
|
package/lib/node/transport.js
CHANGED