dsh-scrape 0.0.1 → 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/README.md +157 -9
- package/cordis.patch.yml +22 -0
- package/dsh-plugin.naming.json +28 -0
- package/lib/cli.js +251 -0
- package/lib/entities.mjs +65 -0
- package/lib/errors.mjs +137 -0
- package/lib/extract.mjs +192 -0
- package/lib/format.mjs +137 -0
- package/lib/html.mjs +822 -0
- package/lib/index.mjs +15 -0
- package/lib/limit.mjs +120 -0
- package/lib/markdown.mjs +161 -0
- package/lib/net.mjs +283 -0
- package/lib/robots.mjs +171 -0
- package/lib/scrape.mjs +560 -0
- package/package.json +45 -9
- package/plugin.mjs +197 -0
- package/scripts/check.mjs +170 -0
- package/scripts/verify-install.mjs +143 -0
package/plugin.mjs
ADDED
|
@@ -0,0 +1,197 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* dsh-scrape 的 host 半:往 DSH 里注册模型可调用的工具 `dsh_scrape`。
|
|
3
|
+
*
|
|
4
|
+
* 契约说明(照本机实测的宿主 0.2.2-rc.x):
|
|
5
|
+
* - 只 import `node:` 内置与包内相对路径,**不 import `@deepseek-ai/*`**(否则会带第二份 cordis);
|
|
6
|
+
* - 工具定义用「裸 JSON Schema ToolDefinition」注册,不需要 schemastery/zod,也没有编译步骤;
|
|
7
|
+
* - `apply(ctx, config)` 的 config 来自 cordis.patch.yml,可调默认超时/字节上限/限速,不写死在代码里。
|
|
8
|
+
*/
|
|
9
|
+
|
|
10
|
+
import { scrape, scrapeMany, VERSION, TOOL_NAME, DEFAULTS } from './lib/scrape.mjs'
|
|
11
|
+
import { describeError } from './lib/errors.mjs'
|
|
12
|
+
import { renderText } from './lib/format.mjs'
|
|
13
|
+
|
|
14
|
+
/** cordis 插件名(= 包名;三处同名:包名 / patch 行 id / 这里)。 */
|
|
15
|
+
export const name = 'dsh-scrape'
|
|
16
|
+
|
|
17
|
+
/** 需要宿主提供的服务:工具注册表。 */
|
|
18
|
+
export const inject = ['tools']
|
|
19
|
+
|
|
20
|
+
/** 默认值;可被 cordis.patch.yml 的 config 覆盖。 */
|
|
21
|
+
const CONFIG_DEFAULTS = {
|
|
22
|
+
timeoutMs: DEFAULTS.timeoutMs,
|
|
23
|
+
maxBytes: DEFAULTS.maxBytes,
|
|
24
|
+
maxUrls: DEFAULTS.maxUrls,
|
|
25
|
+
minIntervalMs: DEFAULTS.minIntervalMs,
|
|
26
|
+
maxChars: DEFAULTS.maxChars,
|
|
27
|
+
minChars: DEFAULTS.minChars,
|
|
28
|
+
robots: DEFAULTS.robots,
|
|
29
|
+
lineRefs: DEFAULTS.lineRefs,
|
|
30
|
+
}
|
|
31
|
+
|
|
32
|
+
/** 工具参数(宿主支持的 JSON Schema 子集)。 */
|
|
33
|
+
const PARAMETERS = {
|
|
34
|
+
type: 'object',
|
|
35
|
+
additionalProperties: false,
|
|
36
|
+
properties: {
|
|
37
|
+
url: {
|
|
38
|
+
type: 'string',
|
|
39
|
+
description: '要抓的网页地址(http/https)。一次一页时用它。',
|
|
40
|
+
},
|
|
41
|
+
urls: {
|
|
42
|
+
type: 'array',
|
|
43
|
+
items: { type: 'string' },
|
|
44
|
+
description: '批量抓多个地址(最多 maxUrls 个;同域自动串行限速)。与 url 二选一。',
|
|
45
|
+
},
|
|
46
|
+
maxBytes: {
|
|
47
|
+
type: 'integer',
|
|
48
|
+
description: `响应体字节上限,默认 ${DEFAULTS.maxBytes};超过会截断并在 warnings 里说明。`,
|
|
49
|
+
},
|
|
50
|
+
robots: {
|
|
51
|
+
type: 'boolean',
|
|
52
|
+
description: '是否遵守 robots.txt,默认 true。设 false 会抓取但仍记一条警告。',
|
|
53
|
+
},
|
|
54
|
+
timeoutMs: {
|
|
55
|
+
type: 'integer',
|
|
56
|
+
description: `单跳超时(毫秒),默认 ${DEFAULTS.timeoutMs}。`,
|
|
57
|
+
},
|
|
58
|
+
lineRefs: {
|
|
59
|
+
type: 'boolean',
|
|
60
|
+
description: "Markdown 里是否加 '<!-- L42 -->' 行号锚点,默认 true(便于引用与回溯)。",
|
|
61
|
+
},
|
|
62
|
+
maxChars: {
|
|
63
|
+
type: 'integer',
|
|
64
|
+
description: `返回的 Markdown 字符上限,默认 ${DEFAULTS.maxChars};超出会截断并标注。`,
|
|
65
|
+
},
|
|
66
|
+
minChars: {
|
|
67
|
+
type: 'integer',
|
|
68
|
+
description: `「可信正文」的字数门槛,默认 ${DEFAULTS.minChars};低于它会降低 confidence 并给 thin_content 警告。`,
|
|
69
|
+
},
|
|
70
|
+
},
|
|
71
|
+
required: [],
|
|
72
|
+
}
|
|
73
|
+
|
|
74
|
+
/** 返回值的 schema:只声明关键字段,保持开放以便演进。 */
|
|
75
|
+
const OUTPUT_SCHEMA = {
|
|
76
|
+
type: 'object',
|
|
77
|
+
properties: {
|
|
78
|
+
tool: { type: 'string' },
|
|
79
|
+
version: { type: 'string' },
|
|
80
|
+
kind: { type: 'string' },
|
|
81
|
+
title: { type: 'string' },
|
|
82
|
+
markdown: { type: 'string' },
|
|
83
|
+
blocked: { description: '为 "robots" 时表示按 robots.txt 拒绝抓取' },
|
|
84
|
+
provenance: {
|
|
85
|
+
type: 'object',
|
|
86
|
+
properties: {
|
|
87
|
+
// 可为 null 的字段不声明 type:宿主只支持单一 type 字符串,声明了反而会把 null 判成非法
|
|
88
|
+
requestedUrl: { description: '调用方给的原始地址;本地 HTML 输入时为 null' },
|
|
89
|
+
finalUrl: { description: '跟随重定向后的最终地址' },
|
|
90
|
+
fetchedAt: { type: 'string', description: '抓取时刻(ISO 8601)' },
|
|
91
|
+
httpStatus: { description: 'HTTP 状态码;本地 HTML 输入时为 null' },
|
|
92
|
+
bytes: { type: 'integer', description: '响应体字节数' },
|
|
93
|
+
charset: { type: 'string', description: '实际用于解码的字符集' },
|
|
94
|
+
truncated: { type: 'boolean', description: '是否因 maxBytes 被截断' },
|
|
95
|
+
redirects: { type: 'array', items: { type: 'object' }, description: '每一跳重定向' },
|
|
96
|
+
robots: { description: 'robots.txt 的结论;robots:false 时为 null' },
|
|
97
|
+
},
|
|
98
|
+
},
|
|
99
|
+
removed: { type: 'object' },
|
|
100
|
+
content: {
|
|
101
|
+
type: 'object',
|
|
102
|
+
properties: {
|
|
103
|
+
chars: { type: 'integer' },
|
|
104
|
+
blocks: { type: 'integer' },
|
|
105
|
+
confidence: { type: 'number' },
|
|
106
|
+
method: { type: 'string' },
|
|
107
|
+
spaSuspected: { type: 'boolean' },
|
|
108
|
+
},
|
|
109
|
+
},
|
|
110
|
+
warnings: { type: 'array', items: { type: 'object' } },
|
|
111
|
+
notes: { type: 'array', items: { type: 'string' } },
|
|
112
|
+
limitations: { type: 'array', items: { type: 'string' } },
|
|
113
|
+
pages: { type: 'array', items: { type: 'object' } },
|
|
114
|
+
failures: { type: 'array', items: { type: 'object' } },
|
|
115
|
+
},
|
|
116
|
+
required: ['tool', 'version', 'kind'],
|
|
117
|
+
}
|
|
118
|
+
|
|
119
|
+
/**
|
|
120
|
+
* 组装工具定义(抽成函数,方便自检与单测直接拿到定义)。
|
|
121
|
+
* @param {object} [config] 插件配置
|
|
122
|
+
* @returns {object} ToolDefinition
|
|
123
|
+
*/
|
|
124
|
+
export function createToolDefinition(config = {}) {
|
|
125
|
+
const settings = { ...CONFIG_DEFAULTS, ...config }
|
|
126
|
+
|
|
127
|
+
return {
|
|
128
|
+
name: TOOL_NAME,
|
|
129
|
+
description:
|
|
130
|
+
'把网页变成干净、可引用、带行号的 Markdown:遵守 robots.txt、同域限速、跟随重定向、自动认字符集(GBK/Big5 不乱码)、按行去噪(script/style/nav/footer/cookie 条/分享按钮),并用密度法挑正文、给 confidence。输出 markdown 与 provenance(最终 URL / 抓取时间 / HTTP 状态 / 字节数),每条内容带 `<!-- L42 -->` 源行号锚点,便于引用回溯。**不执行 JavaScript**:疑似 SPA 会显式警告而不是返回空白。',
|
|
131
|
+
parameters: PARAMETERS,
|
|
132
|
+
output: {
|
|
133
|
+
schema: OUTPUT_SCHEMA,
|
|
134
|
+
render(_args, value) {
|
|
135
|
+
return [{ type: 'text', text: renderText(value, { detail: 'brief' }) }]
|
|
136
|
+
},
|
|
137
|
+
presentationMeta(_args, value) {
|
|
138
|
+
return {
|
|
139
|
+
chars: value.content?.chars ?? 0,
|
|
140
|
+
confidence: value.content?.confidence ?? 0,
|
|
141
|
+
spa: Boolean(value.content?.spaSuspected),
|
|
142
|
+
blocked: value.blocked ?? null,
|
|
143
|
+
warnings: value.warnings?.length ?? 0,
|
|
144
|
+
}
|
|
145
|
+
},
|
|
146
|
+
},
|
|
147
|
+
presentCall(args) {
|
|
148
|
+
const targetList = Array.isArray(args.urls) && args.urls.length > 0 ? `${args.urls.length} 个网址` : args.url ?? '(未给地址)'
|
|
149
|
+
return { card: 'generic', title: `抓取 ${targetList}`, kind: 'search', rawInput: args }
|
|
150
|
+
},
|
|
151
|
+
presentResult(args, result) {
|
|
152
|
+
if (result.isError || !result.meta) return undefined
|
|
153
|
+
const meta = result.meta
|
|
154
|
+
const status = meta.blocked ? `robots 禁止` : `${meta.chars} 字 · 置信度 ${meta.confidence}${meta.spa ? ' · 疑似 SPA' : ''}`
|
|
155
|
+
return {
|
|
156
|
+
card: 'generic',
|
|
157
|
+
title: `抓取 ${args.url ?? `${(args.urls ?? []).length} 个网址`} → ${status}`,
|
|
158
|
+
content: result.content,
|
|
159
|
+
}
|
|
160
|
+
},
|
|
161
|
+
async execute(args, exec) {
|
|
162
|
+
const options = {
|
|
163
|
+
timeoutMs: args.timeoutMs ?? settings.timeoutMs,
|
|
164
|
+
maxBytes: args.maxBytes ?? settings.maxBytes,
|
|
165
|
+
maxChars: args.maxChars ?? settings.maxChars,
|
|
166
|
+
minChars: args.minChars ?? settings.minChars,
|
|
167
|
+
maxUrls: settings.maxUrls,
|
|
168
|
+
minIntervalMs: settings.minIntervalMs,
|
|
169
|
+
robots: args.robots ?? settings.robots,
|
|
170
|
+
lineRefs: args.lineRefs ?? settings.lineRefs,
|
|
171
|
+
signal: exec?.signal,
|
|
172
|
+
}
|
|
173
|
+
|
|
174
|
+
try {
|
|
175
|
+
if (Array.isArray(args.urls) && args.urls.length > 0 && !args.url) {
|
|
176
|
+
return await scrapeMany(args.urls, options)
|
|
177
|
+
}
|
|
178
|
+
return await scrape({ url: args.url, urls: args.urls }, options)
|
|
179
|
+
} catch (error) {
|
|
180
|
+
const info = describeError(error)
|
|
181
|
+
// 抛出即可:工具管道会变成 isError,并把这段人话给模型看
|
|
182
|
+
throw new Error(`[${info.code}] ${info.message}${info.hint ? `\n→ ${info.hint}` : ''}`)
|
|
183
|
+
}
|
|
184
|
+
},
|
|
185
|
+
}
|
|
186
|
+
}
|
|
187
|
+
|
|
188
|
+
/**
|
|
189
|
+
* 插件激活:注册工具。
|
|
190
|
+
* @param {object} ctx cordis 上下文(需已注入 tools)
|
|
191
|
+
* @param {object} [config] 插件配置
|
|
192
|
+
* @returns {void}
|
|
193
|
+
*/
|
|
194
|
+
export function apply(ctx, config = {}) {
|
|
195
|
+
ctx.tools.register(createToolDefinition(config))
|
|
196
|
+
ctx.logger?.info?.(`dsh-scrape ${VERSION}: 已注册工具 ${TOOL_NAME}`)
|
|
197
|
+
}
|
|
@@ -0,0 +1,170 @@
|
|
|
1
|
+
#!/usr/bin/env node
|
|
2
|
+
/**
|
|
3
|
+
* 装机前自检(零依赖,`npm run check` 会调它):
|
|
4
|
+
* 1) 包结构:name/version/type/main/exports/files/dsh.bundle 齐不齐
|
|
5
|
+
* 2) 语法:所有 .js/.mjs 过一遍 node --check
|
|
6
|
+
* 3) 命名四处同源:package.json / cordis.patch.yml / dsh-plugin.naming.json / 代码导出名
|
|
7
|
+
* 4) 真加载:用假 ctx 跑 apply(),确认注册出来的工具名是 dsh_scrape
|
|
8
|
+
* 5) 真 schema 校验:本机装了 DSH 就用宿主自己的 assertSupportedJsonSchema 验 schema
|
|
9
|
+
* 6) 红线:不许 import @deepseek-ai/*、不许有运行时依赖
|
|
10
|
+
* 7) README:必须有安装命令与版本号
|
|
11
|
+
*
|
|
12
|
+
* 只读:不写任何文件、不碰 ~/.dsh、不发网络请求。
|
|
13
|
+
*/
|
|
14
|
+
|
|
15
|
+
import { execFileSync } from 'node:child_process'
|
|
16
|
+
import { existsSync, readFileSync, readdirSync } from 'node:fs'
|
|
17
|
+
import { dirname, join, relative } from 'node:path'
|
|
18
|
+
import { fileURLToPath, pathToFileURL } from 'node:url'
|
|
19
|
+
|
|
20
|
+
import * as plugin from '../plugin.mjs'
|
|
21
|
+
|
|
22
|
+
const root = join(dirname(fileURLToPath(import.meta.url)), '..')
|
|
23
|
+
const problems = []
|
|
24
|
+
const notes = []
|
|
25
|
+
|
|
26
|
+
/**
|
|
27
|
+
* 记一条失败。
|
|
28
|
+
* @param {string} message 描述
|
|
29
|
+
* @returns {void}
|
|
30
|
+
*/
|
|
31
|
+
const fail = (message) => problems.push(message)
|
|
32
|
+
|
|
33
|
+
/**
|
|
34
|
+
* 记一条通过/提示。
|
|
35
|
+
* @param {string} message 描述
|
|
36
|
+
* @returns {void}
|
|
37
|
+
*/
|
|
38
|
+
const note = (message) => notes.push(message)
|
|
39
|
+
|
|
40
|
+
const pkg = JSON.parse(readFileSync(join(root, 'package.json'), 'utf8'))
|
|
41
|
+
const naming = JSON.parse(readFileSync(join(root, 'dsh-plugin.naming.json'), 'utf8'))
|
|
42
|
+
const patch = readFileSync(join(root, 'cordis.patch.yml'), 'utf8')
|
|
43
|
+
|
|
44
|
+
// 1) 包结构
|
|
45
|
+
for (const field of ['name', 'version', 'description', 'license', 'type', 'main']) {
|
|
46
|
+
if (!pkg[field]) fail(`package.json 缺字段 ${field}`)
|
|
47
|
+
}
|
|
48
|
+
if (pkg.type !== 'module') fail('package.json 的 type 必须是 module')
|
|
49
|
+
if (pkg.version !== '0.1.0' && !process.argv.includes('--any-version')) {
|
|
50
|
+
fail(`版本应为 0.1.0,实际 ${pkg.version}`)
|
|
51
|
+
}
|
|
52
|
+
if (!pkg.dsh?.bundle?.patch) fail('package.json 缺 dsh.bundle.patch')
|
|
53
|
+
if (!pkg.files?.includes('lib/')) fail('files 白名单必须包含 lib/')
|
|
54
|
+
if (pkg.dependencies && Object.keys(pkg.dependencies).length > 0) fail('不允许有运行时依赖')
|
|
55
|
+
if (!existsSync(join(root, pkg.main))) fail(`main 指向的文件不存在:${pkg.main}`)
|
|
56
|
+
if (!existsSync(join(root, pkg.bin?.['dsh-scrape'] ?? 'lib/cli.js'))) fail('bin 指向的 CLI 不存在')
|
|
57
|
+
|
|
58
|
+
// 2) 语法
|
|
59
|
+
const targets = ['plugin.mjs', join('lib', 'cli.js'), ...listModules()]
|
|
60
|
+
for (const target of targets) {
|
|
61
|
+
try {
|
|
62
|
+
execFileSync(process.execPath, ['--check', join(root, target)], { stdio: 'pipe' })
|
|
63
|
+
} catch (error) {
|
|
64
|
+
fail(`语法错误 ${target}: ${String(error.stderr ?? error.message).split('\n')[0]}`)
|
|
65
|
+
}
|
|
66
|
+
}
|
|
67
|
+
|
|
68
|
+
// 3) 命名四处同源
|
|
69
|
+
if (!patch.includes(`id: ${pkg.name}`)) fail('cordis.patch.yml 的 id 必须等于包名')
|
|
70
|
+
if (!patch.includes(`name: ${pkg.name}`)) fail('cordis.patch.yml 的 name 必须等于包名')
|
|
71
|
+
if (naming.plugin?.packageName !== pkg.name) fail('naming.json 的 packageName 与 package.json 不一致')
|
|
72
|
+
if (naming.names?.pluginNames?.[0] !== plugin.name) fail('naming.json 的 pluginNames 与代码导出名不一致')
|
|
73
|
+
if (naming.names?.loaderIds?.[0] !== pkg.name) fail('naming.json 的 loaderIds 与包名不一致')
|
|
74
|
+
|
|
75
|
+
// 4) 真加载
|
|
76
|
+
const registered = []
|
|
77
|
+
const fakeCtx = { tools: { register: (definition) => { registered.push(definition); return () => {} } }, logger: { info() {} } }
|
|
78
|
+
try {
|
|
79
|
+
plugin.apply(fakeCtx, {})
|
|
80
|
+
} catch (error) {
|
|
81
|
+
fail(`apply() 抛异常:${error instanceof Error ? error.message : String(error)}`)
|
|
82
|
+
}
|
|
83
|
+
if (registered.length !== 1) fail(`apply() 应注册 1 个工具,实际 ${registered.length}`)
|
|
84
|
+
const tool = registered[0]
|
|
85
|
+
if (!tool) fail('没有注册到任何工具')
|
|
86
|
+
else {
|
|
87
|
+
if (tool.name !== 'dsh_scrape') fail(`工具名应为 dsh_scrape,实际 ${tool.name}`)
|
|
88
|
+
if (naming.names?.tools?.[0] !== tool.name) fail('naming.json 的 tools 与实际注册的工具名不一致')
|
|
89
|
+
if (tool.output === undefined || typeof tool.output.render !== 'function') fail('工具缺 output.render')
|
|
90
|
+
if (typeof tool.execute !== 'function') fail('工具缺 execute')
|
|
91
|
+
if (tool.parameters?.type !== 'object') fail('工具 parameters 必须是 object 根')
|
|
92
|
+
for (const key of ['url', 'urls', 'maxBytes', 'robots']) {
|
|
93
|
+
if (!tool.parameters?.properties?.[key]) fail(`工具参数缺 ${key}`)
|
|
94
|
+
}
|
|
95
|
+
}
|
|
96
|
+
|
|
97
|
+
// 5) 用宿主自己的 schema 校验(装了 DSH 才做)
|
|
98
|
+
const hostRoot = join(process.env.HOME ?? '', '.local/lib/node_modules/@deepseek-ai/dsh')
|
|
99
|
+
const hostTools = join(hostRoot, 'node_modules/@deepseek-ai/dsh-tools/lib/index.js')
|
|
100
|
+
if (existsSync(hostTools)) {
|
|
101
|
+
try {
|
|
102
|
+
const host = await import(pathToFileURL(hostTools).href)
|
|
103
|
+
host.assertSupportedJsonSchema(tool.parameters)
|
|
104
|
+
host.assertSupportedJsonSchema(tool.output.schema)
|
|
105
|
+
note('宿主 assertSupportedJsonSchema:parameters 与 output.schema 均合法')
|
|
106
|
+
} catch (error) {
|
|
107
|
+
fail(`宿主 schema 校验不通过:${error instanceof Error ? error.message : String(error)}`)
|
|
108
|
+
}
|
|
109
|
+
} else {
|
|
110
|
+
note('本机找不到 dsh-tools,跳过宿主 schema 校验')
|
|
111
|
+
}
|
|
112
|
+
|
|
113
|
+
// 5b) 用宿主的 yaml 解析器把 cordis.patch.yml 解出来,确认形状就是 loader 要的 insert 数组
|
|
114
|
+
const hostYaml = join(hostRoot, 'node_modules/yaml/dist/index.js')
|
|
115
|
+
if (existsSync(hostYaml)) {
|
|
116
|
+
try {
|
|
117
|
+
const yaml = await import(pathToFileURL(hostYaml).href)
|
|
118
|
+
const layers = yaml.parse(patch)
|
|
119
|
+
const row = Array.isArray(layers) ? layers[0]?.insert?.[0] : undefined
|
|
120
|
+
if (!row) fail('cordis.patch.yml 形状不对:应为 [{ insert: [{ id, name, config }] }]')
|
|
121
|
+
else {
|
|
122
|
+
if (row.id !== pkg.name) fail(`patch 行 id 应为 ${pkg.name},实际 ${row.id}`)
|
|
123
|
+
if (row.name !== pkg.name) fail(`patch 行 name 应为 ${pkg.name},实际 ${row.name}`)
|
|
124
|
+
if (typeof row.config !== 'object' || row.config === null) fail('patch 行缺 config')
|
|
125
|
+
note('宿主 yaml 解析 cordis.patch.yml:insert 行 id/name/config 均正确')
|
|
126
|
+
}
|
|
127
|
+
} catch (error) {
|
|
128
|
+
fail(`cordis.patch.yml 解析失败:${error instanceof Error ? error.message : String(error)}`)
|
|
129
|
+
}
|
|
130
|
+
} else {
|
|
131
|
+
note('本机找不到 yaml 解析器,跳过 patch 校验')
|
|
132
|
+
}
|
|
133
|
+
|
|
134
|
+
// 6) 红线:不许 import 外部包
|
|
135
|
+
for (const file of ['plugin.mjs', join('lib', 'cli.js'), ...listModules()]) {
|
|
136
|
+
const source = readFileSync(join(root, file), 'utf8')
|
|
137
|
+
if (/from\s+['"]@deepseek-ai\//.test(source)) fail(`${file} 不许 import @deepseek-ai/*(会带第二份 cordis)`)
|
|
138
|
+
for (const match of source.matchAll(/from\s+['"]([^'"]+)['"]/g)) {
|
|
139
|
+
const specifier = match[1]
|
|
140
|
+
if (!specifier.startsWith('node:') && !specifier.startsWith('./') && !specifier.startsWith('../')) {
|
|
141
|
+
fail(`${file} 引入了外部依赖 ${specifier}`)
|
|
142
|
+
}
|
|
143
|
+
}
|
|
144
|
+
}
|
|
145
|
+
|
|
146
|
+
// 7) README 硬要求
|
|
147
|
+
const readme = readFileSync(join(root, 'README.md'), 'utf8')
|
|
148
|
+
if (!/dsh plugin --profile \w+ add dsh-scrape/.test(readme)) fail('README 缺安装命令')
|
|
149
|
+
if (!readme.includes(pkg.version)) fail(`README 里没有提到版本 ${pkg.version}`)
|
|
150
|
+
if (!readme.includes('dsh_scrape')) fail('README 里没有提到工具名 dsh_scrape')
|
|
151
|
+
|
|
152
|
+
/**
|
|
153
|
+
* 列出 lib 目录下所有 .mjs / .js。
|
|
154
|
+
* @returns {string[]} 相对路径
|
|
155
|
+
*/
|
|
156
|
+
function listModules() {
|
|
157
|
+
return readdirSync(join(root, 'lib'))
|
|
158
|
+
.filter((file) => file.endsWith('.mjs') || file.endsWith('.js'))
|
|
159
|
+
.map((file) => join('lib', file))
|
|
160
|
+
}
|
|
161
|
+
|
|
162
|
+
for (const message of notes) process.stdout.write(`✓ ${message}\n`)
|
|
163
|
+
if (problems.length > 0) {
|
|
164
|
+
process.stderr.write(`\n✗ 自检未通过(${problems.length} 项):\n`)
|
|
165
|
+
for (const problem of problems) process.stderr.write(` - ${problem}\n`)
|
|
166
|
+
process.exitCode = 1
|
|
167
|
+
} else {
|
|
168
|
+
process.stdout.write(`\n✓ dsh-scrape ${pkg.version} 自检通过(${targets.length} 个文件、工具 ${tool?.name ?? '(无)'})\n`)
|
|
169
|
+
process.stdout.write(` 包根:${relative(process.cwd(), root) || '.'}\n`)
|
|
170
|
+
}
|
|
@@ -0,0 +1,143 @@
|
|
|
1
|
+
#!/usr/bin/env node
|
|
2
|
+
/**
|
|
3
|
+
* 装验:用**宿主自己的** cordis + SystemPrompt + ToolsService,把 **npm pack 出来的 tarball**
|
|
4
|
+
* 装进一个隔离目录并真注册、真执行、真渲染。
|
|
5
|
+
*
|
|
6
|
+
* 为什么不用 `dsh plugin add`:本机 Node 23 下 dsh CLI 静默空转(缺 `import.meta.main`),
|
|
7
|
+
* 走 CLI 只会得到「什么都没发生的成功」。所以这里直接走宿主的安装代码路径:
|
|
8
|
+
* 1) `npm pack` → 解包到 <tmp>/node_modules/dsh-scrape(离线的、和发布物一致的那份)
|
|
9
|
+
* 2) 用宿主 yaml 解析 cordis.patch.yml,取出 config(= 装上去以后宿主真正会用的配置)
|
|
10
|
+
* 3) new Context() → 真 SystemPrompt → 真 ToolsService → 我们的 apply(ctx, config)
|
|
11
|
+
* 4) 断言 dsh_scrape 出现在注册表、schema 通过宿主校验、execute 能抓能转、render 有内容
|
|
12
|
+
* 全程 `DSH_HOME` 指向临时目录,**不碰 ~/.dsh、不改任何配置**。
|
|
13
|
+
*
|
|
14
|
+
* 用法:
|
|
15
|
+
* node scripts/verify-install.mjs # 自动找本机宿主安装
|
|
16
|
+
* DSH_INSTALL=/path/to/@deepseek-ai/dsh node scripts/verify-install.mjs
|
|
17
|
+
*
|
|
18
|
+
* 退出码:0 通过 | 1 失败 | 3 本机找不到宿主(跳过,不算失败)
|
|
19
|
+
*/
|
|
20
|
+
|
|
21
|
+
import { pathToFileURL } from 'node:url'
|
|
22
|
+
import { execFileSync } from 'node:child_process'
|
|
23
|
+
import { createServer } from 'node:http'
|
|
24
|
+
import { existsSync, mkdirSync, mkdtempSync, readFileSync, rmSync } from 'node:fs'
|
|
25
|
+
import { homedir, tmpdir } from 'node:os'
|
|
26
|
+
import { dirname, join } from 'node:path'
|
|
27
|
+
import { fileURLToPath } from 'node:url'
|
|
28
|
+
|
|
29
|
+
const packageRoot = join(dirname(fileURLToPath(import.meta.url)), '..')
|
|
30
|
+
const problems = []
|
|
31
|
+
|
|
32
|
+
/**
|
|
33
|
+
* 记一条失败。
|
|
34
|
+
* @param {string} message 描述
|
|
35
|
+
* @returns {void}
|
|
36
|
+
*/
|
|
37
|
+
const fail = (message) => problems.push(message)
|
|
38
|
+
|
|
39
|
+
/**
|
|
40
|
+
* 找到本机 DSH 安装目录(含 node_modules/@deepseek-ai/* 的那一层)。
|
|
41
|
+
* @returns {string|null} 宿主安装根
|
|
42
|
+
*/
|
|
43
|
+
function findHostRoot() {
|
|
44
|
+
const candidates = [
|
|
45
|
+
process.env.DSH_INSTALL,
|
|
46
|
+
join(homedir(), '.local/lib/node_modules/@deepseek-ai/dsh'),
|
|
47
|
+
'/usr/local/lib/node_modules/@deepseek-ai/dsh',
|
|
48
|
+
'/opt/homebrew/lib/node_modules/@deepseek-ai/dsh',
|
|
49
|
+
].filter(Boolean)
|
|
50
|
+
for (const candidate of candidates) {
|
|
51
|
+
if (existsSync(join(candidate, 'node_modules/@deepseek-ai/dsh-tools/lib/index.js'))) return candidate
|
|
52
|
+
}
|
|
53
|
+
return null
|
|
54
|
+
}
|
|
55
|
+
|
|
56
|
+
const hostRoot = findHostRoot()
|
|
57
|
+
if (!hostRoot) {
|
|
58
|
+
process.stderr.write('! 本机找不到 DSH 安装(试过 DSH_INSTALL / ~/.local/lib/node_modules/@deepseek-ai/dsh),跳过装验\n')
|
|
59
|
+
process.exit(3)
|
|
60
|
+
}
|
|
61
|
+
|
|
62
|
+
const mods = join(hostRoot, 'node_modules/@deepseek-ai')
|
|
63
|
+
const work = mkdtempSync(join(tmpdir(), 'dsh-scrape-install-'))
|
|
64
|
+
const isolatedHome = join(work, 'home')
|
|
65
|
+
mkdirSync(isolatedHome, { recursive: true })
|
|
66
|
+
process.env.DSH_HOME = isolatedHome
|
|
67
|
+
|
|
68
|
+
try {
|
|
69
|
+
// 1) 打包 + 解包(装的是发布物,不是工作区源码)
|
|
70
|
+
const packed = JSON.parse(execFileSync('npm', ['pack', '--json', '--pack-destination', work], { cwd: packageRoot, encoding: 'utf8' }))
|
|
71
|
+
const tarball = packed[0].filename
|
|
72
|
+
mkdirSync(join(work, 'node_modules'), { recursive: true })
|
|
73
|
+
execFileSync('tar', ['-xzf', join(work, tarball), '-C', work], { stdio: 'pipe' })
|
|
74
|
+
execFileSync('mv', [join(work, 'package'), join(work, 'node_modules', 'dsh-scrape')], { stdio: 'pipe' })
|
|
75
|
+
const installed = join(work, 'node_modules', 'dsh-scrape')
|
|
76
|
+
|
|
77
|
+
// 2) 宿主模块 + patch config
|
|
78
|
+
const { Context } = await import(pathToFileURL(join(mods, 'cordis/lib/index.js')).href)
|
|
79
|
+
const { SystemPrompt } = await import(pathToFileURL(join(mods, 'dsh-system-prompt/lib/index.js')).href)
|
|
80
|
+
const { ToolsService, assertSupportedJsonSchema } = await import(pathToFileURL(join(mods, 'dsh-tools/lib/index.js')).href)
|
|
81
|
+
const yaml = await import(pathToFileURL(join(hostRoot, 'node_modules/yaml/dist/index.js')).href)
|
|
82
|
+
|
|
83
|
+
const patch = yaml.parse(readFileSync(join(installed, 'cordis.patch.yml'), 'utf8'))
|
|
84
|
+
const row = Array.isArray(patch) ? patch[0]?.insert?.[0] : undefined
|
|
85
|
+
if (!row) fail('cordis.patch.yml 形状不对:应为 [{ insert: [{ id, name, config }] }]')
|
|
86
|
+
const plugin = await import(pathToFileURL(join(installed, 'plugin.mjs')).href)
|
|
87
|
+
const pkg = JSON.parse(readFileSync(join(installed, 'package.json'), 'utf8'))
|
|
88
|
+
if (row?.id !== pkg.name) fail(`patch 行 id (${row?.id}) 与包名 (${pkg.name}) 不一致`)
|
|
89
|
+
|
|
90
|
+
// 3) 真注册表
|
|
91
|
+
const ctx = new Context()
|
|
92
|
+
await ctx.plugin(SystemPrompt)
|
|
93
|
+
await ctx.plugin(ToolsService)
|
|
94
|
+
if (!ctx.tools) fail('宿主 ToolsService 没有激活(inject: systemPrompt)')
|
|
95
|
+
|
|
96
|
+
plugin.apply(ctx, row?.config ?? {})
|
|
97
|
+
const definition = ctx.tools.get('dsh_scrape')
|
|
98
|
+
if (!definition) fail('宿主注册表里没有 dsh_scrape')
|
|
99
|
+
else {
|
|
100
|
+
assertSupportedJsonSchema(definition.parameters)
|
|
101
|
+
assertSupportedJsonSchema(definition.output.schema)
|
|
102
|
+
const schemas = ctx.tools.schemas().map((schema) => schema.name)
|
|
103
|
+
if (!schemas.includes('dsh_scrape')) fail(`schemas() 里没有 dsh_scrape:${schemas.join(',')}`)
|
|
104
|
+
}
|
|
105
|
+
|
|
106
|
+
// 4) 真执行一次(本地服务,不联网)
|
|
107
|
+
const server = createServer((req, res) => {
|
|
108
|
+
if (req.url === '/robots.txt') { res.statusCode = 404; res.end('no robots'); return }
|
|
109
|
+
res.setHeader('content-type', 'text/html; charset=utf-8')
|
|
110
|
+
res.end(`<html><head><title>装验页面</title></head><body><main><article><h1>装验页面</h1>
|
|
111
|
+
<p>${'这一段用来证明打包产物装进宿主注册表后真的能抓、能转、能带行号。'.repeat(8)}</p>
|
|
112
|
+
<table><tr><th>项</th><th>值</th></tr><tr><td>引擎</td><td>dsh-scrape</td></tr></table>
|
|
113
|
+
</article></main><footer>页脚噪音</footer></body></html>`)
|
|
114
|
+
})
|
|
115
|
+
await new Promise((resolve) => server.listen(0, '127.0.0.1', resolve))
|
|
116
|
+
const origin = `http://127.0.0.1:${server.address().port}`
|
|
117
|
+
try {
|
|
118
|
+
const result = await definition.execute({ url: `${origin}/post` }, {})
|
|
119
|
+
if (result.provenance.httpStatus !== 200) fail(`HTTP 状态不是 200:${result.provenance.httpStatus}`)
|
|
120
|
+
if (!result.markdown.includes('<!-- L')) fail('Markdown 里没有行号锚点')
|
|
121
|
+
if (!(result.removed.footer > 0)) fail('页脚没有被去噪')
|
|
122
|
+
if (!(result.content.confidence > 0.5)) fail(`置信度过低:${result.content.confidence}`)
|
|
123
|
+
const rendered = definition.output.render({ url: `${origin}/post` }, result)
|
|
124
|
+
if (rendered?.[0]?.text?.length < 200) fail('render 输出太短,模型看不到正文')
|
|
125
|
+
process.stdout.write(` 抓取:HTTP ${result.provenance.httpStatus} · ${result.content.chars} 字 · 置信度 ${result.content.confidence} · 去噪 ${result.removed.total} 字\n`)
|
|
126
|
+
process.stdout.write(` 渲染:${rendered[0].type} ${rendered[0].text.length} 字符\n`)
|
|
127
|
+
} finally {
|
|
128
|
+
server.close()
|
|
129
|
+
}
|
|
130
|
+
} catch (error) {
|
|
131
|
+
fail(`装验抛异常:${error instanceof Error ? error.message : String(error)}`)
|
|
132
|
+
} finally {
|
|
133
|
+
rmSync(work, { recursive: true, force: true })
|
|
134
|
+
}
|
|
135
|
+
|
|
136
|
+
if (problems.length > 0) {
|
|
137
|
+
process.stderr.write(`\n✗ 装验未通过(${problems.length} 项):\n`)
|
|
138
|
+
for (const problem of problems) process.stderr.write(` - ${problem}\n`)
|
|
139
|
+
process.exitCode = 1
|
|
140
|
+
} else {
|
|
141
|
+
process.stdout.write('\n✓ 装验通过:宿主 cordis + SystemPrompt + ToolsService 真注册、真执行、真渲染(DSH_HOME 隔离)\n')
|
|
142
|
+
process.stdout.write(` 宿主:${hostRoot}\n`)
|
|
143
|
+
}
|