dsh-plugin-discover 0.0.0-stage → 1.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +31 -0
- package/LICENSE +26 -0
- package/README.md +209 -3
- package/README.zh.md +208 -0
- package/client.js +477 -0
- package/cordis.patch.yml +3 -0
- package/icon.svg +14 -0
- package/index.js +347 -0
- package/lib/bm25.js +190 -0
- package/lib/build-worker.js +32 -0
- package/lib/build.js +145 -0
- package/lib/cli.js +495 -0
- package/lib/compat.js +157 -0
- package/lib/filters.js +105 -0
- package/lib/model-client.js +163 -0
- package/lib/normalize.js +369 -0
- package/lib/paths.js +49 -0
- package/lib/quality.js +61 -0
- package/lib/readme-clean.js +128 -0
- package/lib/readme-source.js +380 -0
- package/lib/report.js +244 -0
- package/lib/runtime.js +247 -0
- package/lib/search.js +447 -0
- package/lib/store.js +184 -0
- package/lib/tokenize.js +141 -0
- package/lib/vector-store.js +142 -0
- package/lib/vector.js +592 -0
- package/locale/en.json +6 -0
- package/locale/zh.json +6 -0
- package/package.json +67 -6
package/lib/build.js
ADDED
|
@@ -0,0 +1,145 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Build one immutable index directory and publish it by switching `current`.
|
|
3
|
+
* The build downloads only the two public directories, writes records and a
|
|
4
|
+
* manifest, and never touches the previous published index.
|
|
5
|
+
*/
|
|
6
|
+
import { mkdirSync, readFileSync, rmSync, writeFileSync } from 'node:fs'
|
|
7
|
+
import { join } from 'node:path'
|
|
8
|
+
import { fileURLToPath } from 'node:url'
|
|
9
|
+
import {
|
|
10
|
+
acquireLock, cleanupOldIndexes, createIndexDir, publishCurrent,
|
|
11
|
+
} from './store.js'
|
|
12
|
+
import { buildRecords, CATALOG_URL, DSH_SO_URL } from './normalize.js'
|
|
13
|
+
import { detectRuntimeVersion } from './compat.js'
|
|
14
|
+
|
|
15
|
+
const PACKAGE_VERSION = (() => {
|
|
16
|
+
try {
|
|
17
|
+
const file = fileURLToPath(new URL('../package.json', import.meta.url))
|
|
18
|
+
return JSON.parse(readFileSync(file, 'utf8')).version ?? '0.0.0'
|
|
19
|
+
} catch {
|
|
20
|
+
return '0.0.0'
|
|
21
|
+
}
|
|
22
|
+
})()
|
|
23
|
+
|
|
24
|
+
async function fetchJson(url, { timeoutMs = 120_000 } = {}) {
|
|
25
|
+
const response = await fetch(url, {
|
|
26
|
+
headers: { 'user-agent': `dsh-plugin-discover/${PACKAGE_VERSION}` },
|
|
27
|
+
signal: AbortSignal.timeout(timeoutMs),
|
|
28
|
+
})
|
|
29
|
+
if (!response.ok) throw new Error(`${url} returned HTTP ${response.status}`)
|
|
30
|
+
return await response.json()
|
|
31
|
+
}
|
|
32
|
+
|
|
33
|
+
function validateDshSo(value) {
|
|
34
|
+
if (!value || typeof value !== 'object' || !Array.isArray(value.plugins)) {
|
|
35
|
+
throw new Error('dsh.so index has no plugins array')
|
|
36
|
+
}
|
|
37
|
+
if (typeof value.generatedAt !== 'string' && typeof value.generated !== 'string') {
|
|
38
|
+
throw new Error('dsh.so index has no generatedAt')
|
|
39
|
+
}
|
|
40
|
+
return value
|
|
41
|
+
}
|
|
42
|
+
|
|
43
|
+
function validateCatalog(value) {
|
|
44
|
+
if (!value || typeof value !== 'object' || !Array.isArray(value.plugins)) return null
|
|
45
|
+
if (value.format !== 1) return null
|
|
46
|
+
return value
|
|
47
|
+
}
|
|
48
|
+
|
|
49
|
+
function writeText(path, content) {
|
|
50
|
+
writeFileSync(path, content, 'utf8')
|
|
51
|
+
}
|
|
52
|
+
|
|
53
|
+
/**
|
|
54
|
+
* Build the index.
|
|
55
|
+
* @param {{ indexDir: string, onProgress?: (message: string) => void, now?: Date }} options
|
|
56
|
+
*/
|
|
57
|
+
export async function buildIndex({ indexDir, onProgress = () => {}, now = new Date() }) {
|
|
58
|
+
mkdirSync(indexDir, { recursive: true })
|
|
59
|
+
const lock = acquireLock(indexDir, { now: Date.now() })
|
|
60
|
+
if (!lock.ok) {
|
|
61
|
+
return {
|
|
62
|
+
ok: false,
|
|
63
|
+
busy: true,
|
|
64
|
+
lock: lock.lock,
|
|
65
|
+
message: '另一个建库进程正在运行',
|
|
66
|
+
}
|
|
67
|
+
}
|
|
68
|
+
let created = null
|
|
69
|
+
try {
|
|
70
|
+
onProgress('[1/4] 下载 dsh.so 索引…')
|
|
71
|
+
const dshSo = validateDshSo(await fetchJson(DSH_SO_URL, { timeoutMs: 180_000 }))
|
|
72
|
+
onProgress(`[2/4] dsh.so ${dshSo.plugins.length} 条;下载 dsh-plugin-catalog…`)
|
|
73
|
+
|
|
74
|
+
let catalog = null
|
|
75
|
+
let catalogError = null
|
|
76
|
+
try {
|
|
77
|
+
catalog = validateCatalog(await fetchJson(CATALOG_URL, { timeoutMs: 90_000 }))
|
|
78
|
+
if (!catalog) catalogError = 'catalog 格式不是 format 1'
|
|
79
|
+
} catch (error) {
|
|
80
|
+
catalogError = error instanceof Error ? error.message : String(error)
|
|
81
|
+
}
|
|
82
|
+
if (catalogError) onProgress(`[2/4] catalog 不可用,继续用 dsh.so:${catalogError}`)
|
|
83
|
+
|
|
84
|
+
onProgress('[3/4] 归一化、去重、合并…')
|
|
85
|
+
const built = buildRecords(dshSo, catalog)
|
|
86
|
+
created = createIndexDir(indexDir, now)
|
|
87
|
+
const recordsFile = join(created.dir, 'records.json')
|
|
88
|
+
writeText(recordsFile, JSON.stringify(built.records))
|
|
89
|
+
const generatedAt = built.sources.dshSo.generatedAt
|
|
90
|
+
const manifest = {
|
|
91
|
+
format: 1,
|
|
92
|
+
package: 'dsh-plugin-discover',
|
|
93
|
+
version: PACKAGE_VERSION,
|
|
94
|
+
builtAt: new Date().toISOString(),
|
|
95
|
+
generatedAt,
|
|
96
|
+
runtimeVersion: detectRuntimeVersion(),
|
|
97
|
+
sources: built.sources,
|
|
98
|
+
stats: built.stats,
|
|
99
|
+
attribution: 'Powered by dsh.so — DSH Developer Hub',
|
|
100
|
+
license: 'CC-BY-4.0 (index data)',
|
|
101
|
+
catalogError,
|
|
102
|
+
}
|
|
103
|
+
writeText(join(created.dir, 'manifest.json'), `${JSON.stringify(manifest, null, 2)}\n`)
|
|
104
|
+
writeText(join(created.dir, 'README.txt'), [
|
|
105
|
+
'dsh-plugin-discover local index',
|
|
106
|
+
'',
|
|
107
|
+
'Index data: dsh.so — DSH Developer Hub (https://www.dsh.so), CC-BY-4.0.',
|
|
108
|
+
'Catalog data: dsh-plugin-catalog, CC0-1.0.',
|
|
109
|
+
'Keep the generatedAt timestamp when redistributing the data.',
|
|
110
|
+
'This directory contains no model weights and no executables.',
|
|
111
|
+
'',
|
|
112
|
+
].join('\n'))
|
|
113
|
+
onProgress(`[4/4] 写入 ${built.stats.cards} 张卡片,切换 current…`)
|
|
114
|
+
publishCurrent(indexDir, created.name)
|
|
115
|
+
const removed = cleanupOldIndexes(indexDir, created.name)
|
|
116
|
+
onProgress(`完成:${built.stats.cards} 张卡片,数据截至 ${String(generatedAt).slice(0, 10)}`)
|
|
117
|
+
return {
|
|
118
|
+
ok: true,
|
|
119
|
+
dir: created.dir,
|
|
120
|
+
name: created.name,
|
|
121
|
+
manifest,
|
|
122
|
+
stats: built.stats,
|
|
123
|
+
removed,
|
|
124
|
+
}
|
|
125
|
+
} catch (error) {
|
|
126
|
+
if (created) {
|
|
127
|
+
try { rmSync(created.dir, { recursive: true, force: true }) } catch {}
|
|
128
|
+
}
|
|
129
|
+
throw error
|
|
130
|
+
} finally {
|
|
131
|
+
lock.release()
|
|
132
|
+
}
|
|
133
|
+
}
|
|
134
|
+
|
|
135
|
+
/** Freshness policy: whether an existing index should be refreshed automatically. */
|
|
136
|
+
export function isStale(manifest, policy = 'daily', now = Date.now()) {
|
|
137
|
+
if (policy === 'off') return false
|
|
138
|
+
if (!manifest) return policy !== 'off'
|
|
139
|
+
const builtAt = Date.parse(manifest.builtAt ?? '')
|
|
140
|
+
if (!Number.isFinite(builtAt)) return true
|
|
141
|
+
const age = now - builtAt
|
|
142
|
+
if (policy === 'manual') return false
|
|
143
|
+
if (policy === 'weekly') return age > 7 * 86_400_000
|
|
144
|
+
return age > 86_400_000
|
|
145
|
+
}
|
package/lib/cli.js
ADDED
|
@@ -0,0 +1,495 @@
|
|
|
1
|
+
#!/usr/bin/env node
|
|
2
|
+
/**
|
|
3
|
+
* Command line entry for building, inspecting, and querying the local index.
|
|
4
|
+
* This is a standalone package bin; DSH has no plugin-contributed `dsh`
|
|
5
|
+
* subcommand, so `dsh plugin add` installs the package and this bin runs
|
|
6
|
+
* beside it.
|
|
7
|
+
*/
|
|
8
|
+
import { parseArgs } from 'node:util'
|
|
9
|
+
import { createHash } from 'node:crypto'
|
|
10
|
+
import { createReadStream, existsSync, mkdirSync, readFileSync, statSync, writeFileSync } from 'node:fs'
|
|
11
|
+
import { spawnSync } from 'node:child_process'
|
|
12
|
+
import { createRequire } from 'node:module'
|
|
13
|
+
import { homedir } from 'node:os'
|
|
14
|
+
import { dirname, join } from 'node:path'
|
|
15
|
+
import { defaultIndexDir } from './paths.js'
|
|
16
|
+
import { buildIndex } from './build.js'
|
|
17
|
+
import {
|
|
18
|
+
isLockStale, readBuildLog, readLock, readManifest, readRecords,
|
|
19
|
+
} from './store.js'
|
|
20
|
+
import { createSearcher, explain, search } from './search.js'
|
|
21
|
+
import { textReport } from './report.js'
|
|
22
|
+
import { PRESETS } from './filters.js'
|
|
23
|
+
import { detectRuntimeVersion } from './compat.js'
|
|
24
|
+
|
|
25
|
+
const USAGE = `dsh-plugin-discover — local DSH plugin index
|
|
26
|
+
|
|
27
|
+
Usage:
|
|
28
|
+
dsh-plugin-discover build [--index-dir <dir>] [--json] [--quiet]
|
|
29
|
+
dsh-plugin-discover status [--index-dir <dir>] [--json]
|
|
30
|
+
dsh-plugin-discover search <query> [--alt <text>]... [--limit <n>]
|
|
31
|
+
[--preset strict|balanced|loose|off]
|
|
32
|
+
[--min-verification <0-5>] [--explain <id>] [--json]
|
|
33
|
+
dsh-plugin-discover model status [--index-dir <dir>] [--profile <name>] [--json]
|
|
34
|
+
dsh-plugin-discover model pull [--spec <spec>] [--profile <name>]
|
|
35
|
+
[--index-dir <dir>] [--dry-run] [--no-verify] [--json]
|
|
36
|
+
|
|
37
|
+
Options:
|
|
38
|
+
--index-dir <dir> Override the index directory. Must match the plugin's indexDir.
|
|
39
|
+
--alt <text> A query rewrite; repeat up to three times.
|
|
40
|
+
--json Print machine-readable JSON.
|
|
41
|
+
--quiet Suppress build progress.
|
|
42
|
+
--spec <spec> Model package spec for model pull, such as
|
|
43
|
+
link:D:\\path\\to\\model or file:D:\\path\\model.tgz.
|
|
44
|
+
--profile <name> DSH profile for model pull (default: DSH_PROFILE or web).
|
|
45
|
+
--dry-run Print the model install command without running it.
|
|
46
|
+
--no-verify Skip SHA-256 verification of the model package.
|
|
47
|
+
-h, --help Show this help.
|
|
48
|
+
|
|
49
|
+
Default index directory: %LOCALAPPDATA%\\dsh-plugin-discover (Windows),
|
|
50
|
+
~/Library/Caches/dsh-plugin-discover (macOS),
|
|
51
|
+
\${XDG_CACHE_HOME:-~/.cache}/dsh-plugin-discover (Linux).
|
|
52
|
+
|
|
53
|
+
The optional model package is not downloaded automatically. \`model pull\`
|
|
54
|
+
installs it explicitly; the tool never installs anything.
|
|
55
|
+
`
|
|
56
|
+
|
|
57
|
+
function resolveIndexDir(value) {
|
|
58
|
+
const configured = typeof value === 'string' && value.trim() !== ''
|
|
59
|
+
? value.trim()
|
|
60
|
+
: (process.env.DSH_PLUGIN_DISCOVER_INDEX_DIR || defaultIndexDir())
|
|
61
|
+
return configured
|
|
62
|
+
}
|
|
63
|
+
|
|
64
|
+
function formatBytes(bytes) {
|
|
65
|
+
if (!Number.isFinite(bytes)) return '?'
|
|
66
|
+
if (bytes < 1024) return `${bytes} B`
|
|
67
|
+
if (bytes < 1024 * 1024) return `${(bytes / 1024).toFixed(1)} KiB`
|
|
68
|
+
return `${(bytes / 1024 / 1024).toFixed(1)} MiB`
|
|
69
|
+
}
|
|
70
|
+
|
|
71
|
+
function printStatus(indexDir, json) {
|
|
72
|
+
const current = readManifest(indexDir)
|
|
73
|
+
const lock = readLock(indexDir)
|
|
74
|
+
const activeLock = lock && !isLockStale(lock) ? lock : null
|
|
75
|
+
const recordsSize = (() => {
|
|
76
|
+
try { return statSync(join(indexDir, current?.name ?? '', 'records.json')).size } catch { return null }
|
|
77
|
+
})()
|
|
78
|
+
const value = {
|
|
79
|
+
indexDir,
|
|
80
|
+
hasIndex: current !== null,
|
|
81
|
+
current: current?.name ?? null,
|
|
82
|
+
builtAt: current?.manifest?.builtAt ?? null,
|
|
83
|
+
generatedAt: current?.manifest?.sources?.dshSo?.generatedAt ?? null,
|
|
84
|
+
cards: current?.manifest?.stats?.cards ?? 0,
|
|
85
|
+
sources: current?.manifest?.sources ?? null,
|
|
86
|
+
stats: current?.manifest?.stats ?? null,
|
|
87
|
+
recordsBytes: recordsSize,
|
|
88
|
+
building: activeLock !== null,
|
|
89
|
+
lock: activeLock,
|
|
90
|
+
lastBuildLog: readBuildLog(indexDir).slice(-2000),
|
|
91
|
+
}
|
|
92
|
+
if (json) {
|
|
93
|
+
process.stdout.write(`${JSON.stringify(value, null, 2)}\n`)
|
|
94
|
+
return
|
|
95
|
+
}
|
|
96
|
+
const lines = [
|
|
97
|
+
`索引目录:${indexDir}`,
|
|
98
|
+
`当前索引:${current?.name ?? '(无)'}`,
|
|
99
|
+
`卡片数量:${value.cards}`,
|
|
100
|
+
`数据截至:${value.generatedAt ? String(value.generatedAt).slice(0, 10) : '(未知)'}`,
|
|
101
|
+
`建库时间:${value.builtAt ?? '(无)'}`,
|
|
102
|
+
`记录文件:${recordsSize === null ? '(无)' : formatBytes(recordsSize)}`,
|
|
103
|
+
`来源:${value.sources ? Object.entries(value.sources).filter(([, source]) => source?.ok).map(([name]) => name).join(', ') : '(无)'}`,
|
|
104
|
+
`建库中:${value.building ? `是(PID ${activeLock?.pid ?? '?'})` : '否'}`,
|
|
105
|
+
`DSH 版本:${detectRuntimeVersion() ?? '(未检测到)'}`,
|
|
106
|
+
]
|
|
107
|
+
process.stdout.write(`${lines.join('\n')}\n`)
|
|
108
|
+
}
|
|
109
|
+
|
|
110
|
+
function printReport(value, current) {
|
|
111
|
+
const report = textReport(value, { manifest: current?.manifest ?? null })
|
|
112
|
+
process.stdout.write(`${report}\n`)
|
|
113
|
+
}
|
|
114
|
+
|
|
115
|
+
function profileDirOf(profile) {
|
|
116
|
+
const name = profile || process.env.DSH_PROFILE || 'web'
|
|
117
|
+
if (process.env.DSH_PROFILE_DIR && (!process.env.DSH_PROFILE || process.env.DSH_PROFILE === name)) {
|
|
118
|
+
return { profile: name, dir: process.env.DSH_PROFILE_DIR }
|
|
119
|
+
}
|
|
120
|
+
const home = process.env.DSH_HOME || join(homedir(), '.dsh')
|
|
121
|
+
return { profile: name, dir: join(home, 'profiles', name) }
|
|
122
|
+
}
|
|
123
|
+
|
|
124
|
+
function resolveInstalledModel(profileDir) {
|
|
125
|
+
const manifestFile = join(profileDir, 'package.json')
|
|
126
|
+
if (!existsSync(manifestFile)) return null
|
|
127
|
+
try {
|
|
128
|
+
const require = createRequire(manifestFile)
|
|
129
|
+
const manifestPath = require.resolve('dsh-plugin-discover-model/manifest.json')
|
|
130
|
+
const packageDir = dirname(manifestPath)
|
|
131
|
+
const manifest = JSON.parse(readFileSync(manifestPath, 'utf8'))
|
|
132
|
+
const pkg = JSON.parse(readFileSync(join(packageDir, 'package.json'), 'utf8'))
|
|
133
|
+
return { packageDir, version: pkg.version, manifest }
|
|
134
|
+
} catch {
|
|
135
|
+
return null
|
|
136
|
+
}
|
|
137
|
+
}
|
|
138
|
+
|
|
139
|
+
function readModelPointer(indexDir) {
|
|
140
|
+
try {
|
|
141
|
+
const value = JSON.parse(readFileSync(join(indexDir, 'model.json'), 'utf8'))
|
|
142
|
+
return value && typeof value === 'object' ? value : null
|
|
143
|
+
} catch {
|
|
144
|
+
return null
|
|
145
|
+
}
|
|
146
|
+
}
|
|
147
|
+
|
|
148
|
+
function runtimeDependency(packageDir, name) {
|
|
149
|
+
try {
|
|
150
|
+
const require = createRequire(join(packageDir, 'package.json'))
|
|
151
|
+
const entry = require.resolve(name)
|
|
152
|
+
let version = null
|
|
153
|
+
try {
|
|
154
|
+
const pkg = JSON.parse(readFileSync(join(dirname(entry), '..', 'package.json'), 'utf8'))
|
|
155
|
+
version = pkg.version ?? null
|
|
156
|
+
} catch {}
|
|
157
|
+
return { installed: true, entry, version }
|
|
158
|
+
} catch {
|
|
159
|
+
return { installed: false }
|
|
160
|
+
}
|
|
161
|
+
}
|
|
162
|
+
|
|
163
|
+
function modelStatusValue({ profile, profileDir, indexDir }) {
|
|
164
|
+
const installed = resolveInstalledModel(profileDir)
|
|
165
|
+
const pointer = readModelPointer(indexDir)
|
|
166
|
+
const value = {
|
|
167
|
+
profile,
|
|
168
|
+
profileDir,
|
|
169
|
+
indexDir,
|
|
170
|
+
pointerPath: join(indexDir, 'model.json'),
|
|
171
|
+
installed: installed !== null,
|
|
172
|
+
packageDir: installed?.packageDir ?? null,
|
|
173
|
+
version: installed?.version ?? null,
|
|
174
|
+
manifest: installed?.manifest ?? null,
|
|
175
|
+
runtime: installed
|
|
176
|
+
? {
|
|
177
|
+
tokenizer: runtimeDependency(installed.packageDir, '@huggingface/tokenizers'),
|
|
178
|
+
onnxruntime: runtimeDependency(installed.packageDir, 'onnxruntime-node'),
|
|
179
|
+
}
|
|
180
|
+
: null,
|
|
181
|
+
pointer,
|
|
182
|
+
pointerMatches: Boolean(
|
|
183
|
+
installed && pointer && typeof pointer.packageDir === 'string'
|
|
184
|
+
&& pointer.packageDir.toLowerCase() === installed.packageDir.toLowerCase()),
|
|
185
|
+
}
|
|
186
|
+
if (installed) {
|
|
187
|
+
const modelBytes = installed.manifest?.files?.['model_quantized.onnx']?.bytes ?? null
|
|
188
|
+
const tokenizerBytes = installed.manifest?.files?.['tokenizer.json']?.bytes ?? null
|
|
189
|
+
value.modelBytes = modelBytes
|
|
190
|
+
value.tokenizerBytes = tokenizerBytes
|
|
191
|
+
}
|
|
192
|
+
return value
|
|
193
|
+
}
|
|
194
|
+
|
|
195
|
+
function printModelStatus(value, json) {
|
|
196
|
+
if (json) {
|
|
197
|
+
process.stdout.write(`${JSON.stringify(value, null, 2)}\n`)
|
|
198
|
+
return
|
|
199
|
+
}
|
|
200
|
+
const lines = [
|
|
201
|
+
`模型包:${value.installed ? '已安装' : '未安装'}`,
|
|
202
|
+
`profile:${value.profile}(${value.profileDir})`,
|
|
203
|
+
`索引目录:${value.indexDir}`,
|
|
204
|
+
`指针文件:${value.pointerPath}${value.pointerMatches ? '(已记录)' : '(未记录或与安装目录不一致)'}`,
|
|
205
|
+
]
|
|
206
|
+
if (value.installed) {
|
|
207
|
+
lines.push(`包目录:${value.packageDir}`)
|
|
208
|
+
lines.push(`版本:${value.version}`)
|
|
209
|
+
lines.push(`模型:${value.modelBytes ? formatBytes(value.modelBytes) : '(manifest 无大小)'}`)
|
|
210
|
+
lines.push(`tokenizer:${value.tokenizerBytes ? formatBytes(value.tokenizerBytes) : '(manifest 无大小)'}`)
|
|
211
|
+
lines.push(`tokenizer 运行时:${value.runtime.tokenizer.installed ? '已安装' : '缺失'}`)
|
|
212
|
+
lines.push(`ONNX 运行时:${value.runtime.onnxruntime.installed ? '已安装' : '缺失'}`)
|
|
213
|
+
if (value.pointerMatches) lines.push(`安装时间:${value.pointer.installedAt ?? '(未记录)'}`)
|
|
214
|
+
} else {
|
|
215
|
+
lines.push('拉取:dsh-plugin-discover model pull --spec <模型包 spec>')
|
|
216
|
+
}
|
|
217
|
+
process.stdout.write(`${lines.join('\n')}\n`)
|
|
218
|
+
}
|
|
219
|
+
|
|
220
|
+
async function sha256File(file) {
|
|
221
|
+
const hash = createHash('sha256')
|
|
222
|
+
for await (const chunk of createReadStream(file)) hash.update(chunk)
|
|
223
|
+
return hash.digest('hex')
|
|
224
|
+
}
|
|
225
|
+
|
|
226
|
+
async function verifyModelFiles(packageDir, manifest, verifyHashes) {
|
|
227
|
+
const errors = []
|
|
228
|
+
const files = manifest?.files && typeof manifest.files === 'object' ? manifest.files : {}
|
|
229
|
+
for (const [name, expected] of Object.entries(files)) {
|
|
230
|
+
const file = join(packageDir, name)
|
|
231
|
+
if (!existsSync(file)) {
|
|
232
|
+
errors.push(`${name}: missing`)
|
|
233
|
+
continue
|
|
234
|
+
}
|
|
235
|
+
const size = statSync(file).size
|
|
236
|
+
if (Number.isFinite(expected.bytes) && size !== expected.bytes) {
|
|
237
|
+
errors.push(`${name}: size ${size} != ${expected.bytes}`)
|
|
238
|
+
}
|
|
239
|
+
if (verifyHashes && typeof expected.sha256 === 'string') {
|
|
240
|
+
const hash = await sha256File(file)
|
|
241
|
+
if (hash.toLowerCase() !== expected.sha256.toLowerCase()) {
|
|
242
|
+
errors.push(`${name}: sha256 mismatch`)
|
|
243
|
+
}
|
|
244
|
+
}
|
|
245
|
+
}
|
|
246
|
+
return { ok: errors.length === 0, errors, hashes: verifyHashes, files: Object.keys(files).length }
|
|
247
|
+
}
|
|
248
|
+
|
|
249
|
+
function installModelPackage(spec, profileDir, dryRun) {
|
|
250
|
+
if (!/^[^\s;&|<>^"]+$/.test(spec)) {
|
|
251
|
+
throw new Error('模型包 spec 不能包含空白或 shell 元字符')
|
|
252
|
+
}
|
|
253
|
+
if (dryRun) return { dryRun: true, display: `pnpm add ${spec}` }
|
|
254
|
+
const result = process.platform === 'win32'
|
|
255
|
+
? spawnSync(process.env.ComSpec || 'cmd.exe', ['/d', '/s', '/c', `pnpm add ${spec}`], {
|
|
256
|
+
cwd: profileDir, stdio: 'inherit', windowsHide: true,
|
|
257
|
+
})
|
|
258
|
+
: spawnSync('pnpm', ['add', spec], { cwd: profileDir, stdio: 'inherit' })
|
|
259
|
+
return {
|
|
260
|
+
dryRun: false,
|
|
261
|
+
status: result.status,
|
|
262
|
+
signal: result.signal,
|
|
263
|
+
error: result.error ? String(result.error.message ?? result.error) : null,
|
|
264
|
+
}
|
|
265
|
+
}
|
|
266
|
+
|
|
267
|
+
function buildModelPointer(packageDir, version, manifest) {
|
|
268
|
+
return {
|
|
269
|
+
format: 1,
|
|
270
|
+
package: 'dsh-plugin-discover-model',
|
|
271
|
+
version,
|
|
272
|
+
packageDir,
|
|
273
|
+
server: join(packageDir, 'server.js'),
|
|
274
|
+
manifest: join(packageDir, 'manifest.json'),
|
|
275
|
+
model: join(packageDir, 'model_quantized.onnx'),
|
|
276
|
+
tokenizer: join(packageDir, 'tokenizer.json'),
|
|
277
|
+
dimensions: manifest?.dimensions ?? null,
|
|
278
|
+
maxTokens: manifest?.maxTokens ?? null,
|
|
279
|
+
installedAt: new Date().toISOString(),
|
|
280
|
+
}
|
|
281
|
+
}
|
|
282
|
+
|
|
283
|
+
function writeModelPointer(indexDir, pointer) {
|
|
284
|
+
mkdirSync(indexDir, { recursive: true })
|
|
285
|
+
const file = join(indexDir, 'model.json')
|
|
286
|
+
writeFileSync(file, `${JSON.stringify(pointer, null, 2)}\n`)
|
|
287
|
+
return file
|
|
288
|
+
}
|
|
289
|
+
|
|
290
|
+
|
|
291
|
+
async function main() {
|
|
292
|
+
const argv = process.argv.slice(2)
|
|
293
|
+
const command = argv[0]
|
|
294
|
+
if (!command || command === '-h' || command === '--help' || command === 'help') {
|
|
295
|
+
process.stdout.write(USAGE)
|
|
296
|
+
return 0
|
|
297
|
+
}
|
|
298
|
+
|
|
299
|
+
if (command === 'build') {
|
|
300
|
+
const { values } = parseArgs({
|
|
301
|
+
args: argv.slice(1),
|
|
302
|
+
options: {
|
|
303
|
+
'index-dir': { type: 'string' },
|
|
304
|
+
json: { type: 'boolean', default: false },
|
|
305
|
+
quiet: { type: 'boolean', default: false },
|
|
306
|
+
},
|
|
307
|
+
allowPositionals: true,
|
|
308
|
+
})
|
|
309
|
+
const indexDir = resolveIndexDir(values['index-dir'])
|
|
310
|
+
const result = await buildIndex({
|
|
311
|
+
indexDir,
|
|
312
|
+
onProgress: message => { if (!values.quiet) process.stderr.write(`${message}\n`) },
|
|
313
|
+
})
|
|
314
|
+
if (values.json) process.stdout.write(`${JSON.stringify(result, null, 2)}\n`)
|
|
315
|
+
else if (result.busy) process.stdout.write(`跳过:另一个建库进程正在运行(PID ${result.lock?.pid ?? '?'})\n`)
|
|
316
|
+
else {
|
|
317
|
+
process.stdout.write(`索引已发布:${result.name}\n`)
|
|
318
|
+
process.stdout.write(`卡片:${result.stats.cards} · 数据截至:${String(result.manifest.sources.dshSo.generatedAt).slice(0, 10)}\n`)
|
|
319
|
+
}
|
|
320
|
+
return result.ok || result.busy ? 0 : 1
|
|
321
|
+
}
|
|
322
|
+
|
|
323
|
+
if (command === 'status') {
|
|
324
|
+
const { values } = parseArgs({
|
|
325
|
+
args: argv.slice(1),
|
|
326
|
+
options: {
|
|
327
|
+
'index-dir': { type: 'string' },
|
|
328
|
+
json: { type: 'boolean', default: false },
|
|
329
|
+
},
|
|
330
|
+
allowPositionals: true,
|
|
331
|
+
})
|
|
332
|
+
printStatus(resolveIndexDir(values['index-dir']), values.json)
|
|
333
|
+
return 0
|
|
334
|
+
}
|
|
335
|
+
|
|
336
|
+
if (command === 'model') {
|
|
337
|
+
const sub = argv[1]
|
|
338
|
+
if (sub !== 'status' && sub !== 'pull') {
|
|
339
|
+
process.stderr.write('usage: dsh-plugin-discover model status|pull [options]\n')
|
|
340
|
+
return 2
|
|
341
|
+
}
|
|
342
|
+
const { values } = parseArgs({
|
|
343
|
+
args: argv.slice(2),
|
|
344
|
+
options: {
|
|
345
|
+
'index-dir': { type: 'string' },
|
|
346
|
+
profile: { type: 'string' },
|
|
347
|
+
spec: { type: 'string' },
|
|
348
|
+
'dry-run': { type: 'boolean', default: false },
|
|
349
|
+
'no-verify': { type: 'boolean', default: false },
|
|
350
|
+
json: { type: 'boolean', default: false },
|
|
351
|
+
},
|
|
352
|
+
allowPositionals: true,
|
|
353
|
+
})
|
|
354
|
+
const indexDir = resolveIndexDir(values['index-dir'])
|
|
355
|
+
const { profile, dir } = profileDirOf(values.profile)
|
|
356
|
+
|
|
357
|
+
if (sub === 'status') {
|
|
358
|
+
printModelStatus(modelStatusValue({ profile, profileDir: dir, indexDir }), values.json)
|
|
359
|
+
return 0
|
|
360
|
+
}
|
|
361
|
+
|
|
362
|
+
const spec = values.spec
|
|
363
|
+
|| process.env.DSH_PLUGIN_DISCOVER_MODEL_SPEC
|
|
364
|
+
|| 'dsh-plugin-discover-model@0.1.0'
|
|
365
|
+
if (!existsSync(dir)) {
|
|
366
|
+
process.stderr.write(`profile 目录不存在:${dir}\n`)
|
|
367
|
+
return 1
|
|
368
|
+
}
|
|
369
|
+
if (values['dry-run']) {
|
|
370
|
+
try {
|
|
371
|
+
const dry = installModelPackage(spec, dir, true)
|
|
372
|
+
if (values.json) process.stdout.write(`${JSON.stringify({ spec, profile, profileDir: dir, ...dry }, null, 2)}\n`)
|
|
373
|
+
else process.stdout.write(`cd ${dir}\n${dry.display}\n`)
|
|
374
|
+
return 0
|
|
375
|
+
} catch (error) {
|
|
376
|
+
process.stderr.write(`${error instanceof Error ? error.message : String(error)}\n`)
|
|
377
|
+
return 1
|
|
378
|
+
}
|
|
379
|
+
}
|
|
380
|
+
|
|
381
|
+
let install
|
|
382
|
+
try {
|
|
383
|
+
install = installModelPackage(spec, dir, false)
|
|
384
|
+
} catch (error) {
|
|
385
|
+
process.stderr.write(`${error instanceof Error ? error.message : String(error)}\n`)
|
|
386
|
+
return 1
|
|
387
|
+
}
|
|
388
|
+
if (install.error) {
|
|
389
|
+
process.stderr.write(`模型包安装失败:${install.error}\n`)
|
|
390
|
+
return 1
|
|
391
|
+
}
|
|
392
|
+
const installed = resolveInstalledModel(dir)
|
|
393
|
+
if (!installed) {
|
|
394
|
+
process.stderr.write(`模型包安装后仍无法解析(pnpm exit ${install.status});检查 pnpm 输出与 profile 依赖。\n`)
|
|
395
|
+
return 1
|
|
396
|
+
}
|
|
397
|
+
const verification = await verifyModelFiles(
|
|
398
|
+
installed.packageDir,
|
|
399
|
+
installed.manifest,
|
|
400
|
+
!values['no-verify'],
|
|
401
|
+
)
|
|
402
|
+
if (!verification.ok) {
|
|
403
|
+
process.stderr.write(`模型包校验失败:\n- ${verification.errors.join('\n- ')}\n`)
|
|
404
|
+
return 1
|
|
405
|
+
}
|
|
406
|
+
// pnpm exits non-zero when it ignores a dependency's build script. The
|
|
407
|
+
// design deliberately does not bypass that approval; the prebuilt CPU
|
|
408
|
+
// binary works, so a verified package is still a success with a warning.
|
|
409
|
+
const warning = install.status !== 0
|
|
410
|
+
? `pnpm exit ${install.status};如果这是 ERR_PNPM_IGNORED_BUILDS,onnxruntime-node 的 postinstall 被跳过,预编译二进制仍可用。`
|
|
411
|
+
: null
|
|
412
|
+
const pointer = buildModelPointer(installed.packageDir, installed.version, installed.manifest)
|
|
413
|
+
let pointerPath
|
|
414
|
+
try {
|
|
415
|
+
pointerPath = writeModelPointer(indexDir, pointer)
|
|
416
|
+
} catch (error) {
|
|
417
|
+
process.stderr.write(`写入 model.json 失败:${error instanceof Error ? error.message : String(error)}\n`)
|
|
418
|
+
return 1
|
|
419
|
+
}
|
|
420
|
+
const status = modelStatusValue({ profile, profileDir: dir, indexDir })
|
|
421
|
+
if (values.json) {
|
|
422
|
+
process.stdout.write(`${JSON.stringify({
|
|
423
|
+
spec,
|
|
424
|
+
profile,
|
|
425
|
+
profileDir: dir,
|
|
426
|
+
install,
|
|
427
|
+
warning,
|
|
428
|
+
verification,
|
|
429
|
+
pointerPath,
|
|
430
|
+
pointer,
|
|
431
|
+
status,
|
|
432
|
+
}, null, 2)}\n`)
|
|
433
|
+
} else {
|
|
434
|
+
process.stdout.write(`模型包已安装:${installed.packageDir}\n`)
|
|
435
|
+
process.stdout.write(`校验:${verification.files} 个文件${verification.hashes ? ',SHA-256 通过' : '(仅大小)'}\n`)
|
|
436
|
+
if (warning) process.stdout.write(`注意:${warning}\n`)
|
|
437
|
+
process.stdout.write(`已记录:${pointerPath}\n`)
|
|
438
|
+
printModelStatus(status, false)
|
|
439
|
+
}
|
|
440
|
+
return 0
|
|
441
|
+
}
|
|
442
|
+
|
|
443
|
+
if (command === 'search') {
|
|
444
|
+
const { values, positionals } = parseArgs({
|
|
445
|
+
args: argv.slice(1),
|
|
446
|
+
options: {
|
|
447
|
+
'index-dir': { type: 'string' },
|
|
448
|
+
alt: { type: 'string', multiple: true, default: [] },
|
|
449
|
+
limit: { type: 'string', default: '10' },
|
|
450
|
+
preset: { type: 'string', default: 'balanced' },
|
|
451
|
+
'min-verification': { type: 'string', default: '4' },
|
|
452
|
+
explain: { type: 'string' },
|
|
453
|
+
json: { type: 'boolean', default: false },
|
|
454
|
+
},
|
|
455
|
+
allowPositionals: true,
|
|
456
|
+
})
|
|
457
|
+
const query = positionals.join(' ').trim()
|
|
458
|
+
if (query === '' && !values.explain) {
|
|
459
|
+
process.stderr.write('search requires a query (or --explain <id>)\n')
|
|
460
|
+
return 2
|
|
461
|
+
}
|
|
462
|
+
const indexDir = resolveIndexDir(values['index-dir'])
|
|
463
|
+
const records = readRecords(indexDir)
|
|
464
|
+
if (!records) {
|
|
465
|
+
process.stderr.write(`没有可用索引:${indexDir}\n先执行:dsh-plugin-discover build\n`)
|
|
466
|
+
return 1
|
|
467
|
+
}
|
|
468
|
+
const manifest = readManifest(indexDir)
|
|
469
|
+
const preset = PRESETS.includes(values.preset) ? values.preset : 'balanced'
|
|
470
|
+
const limit = Math.min(20, Math.max(1, Number(values.limit) || 10))
|
|
471
|
+
const searcher = createSearcher(records.records, { runtimeVersion: detectRuntimeVersion() })
|
|
472
|
+
const value = values.explain
|
|
473
|
+
? explain(searcher, values.explain, { query })
|
|
474
|
+
: await search(searcher, {
|
|
475
|
+
query,
|
|
476
|
+
queryAlts: values.alt,
|
|
477
|
+
limit,
|
|
478
|
+
preset,
|
|
479
|
+
minVerificationLevel: Number(values['min-verification']) || 0,
|
|
480
|
+
indexDir,
|
|
481
|
+
})
|
|
482
|
+
if (values.json) process.stdout.write(`${JSON.stringify(value, null, 2)}\n`)
|
|
483
|
+
else printReport(value, manifest)
|
|
484
|
+
return 0
|
|
485
|
+
}
|
|
486
|
+
|
|
487
|
+
process.stderr.write(`unknown command: ${command}\n\n${USAGE}`)
|
|
488
|
+
return 2
|
|
489
|
+
}
|
|
490
|
+
|
|
491
|
+
const code = await main().catch(error => {
|
|
492
|
+
process.stderr.write(`${error instanceof Error ? (error.stack ?? error.message) : String(error)}\n`)
|
|
493
|
+
return 1
|
|
494
|
+
})
|
|
495
|
+
process.exitCode = code
|