dsh-podwave 0.0.0-stage → 0.10.19
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/README.md +183 -2
- package/cordis.patch.yml +4 -0
- package/dist/client.js +1328 -0
- package/package.json +58 -4
- package/scripts/__pycache__/asr_sherpa.cpython-312.pyc +0 -0
- package/scripts/asr_sherpa.py +69 -0
- package/scripts/asr_sherpa_batch.py +40 -0
- package/scripts/diarize.py +79 -0
- package/scripts/release.mjs +79 -0
- package/scripts/translate_local.py +59 -0
- package/src/catalog.js +238 -0
- package/src/config.js +103 -0
- package/src/diarize.js +87 -0
- package/src/downloader.js +364 -0
- package/src/host.js +903 -0
- package/src/jobs.js +86 -0
- package/src/library.js +248 -0
- package/src/llama.js +166 -0
- package/src/models.js +42 -0
- package/src/platform.js +163 -0
- package/src/setup.js +539 -0
- package/src/state.js +111 -0
- package/src/transcribe.js +1123 -0
- package/src/translate.js +256 -0
package/src/config.js
ADDED
|
@@ -0,0 +1,103 @@
|
|
|
1
|
+
// 统一配置:~/.podwave/config.json(schemaVersion 1)。
|
|
2
|
+
// 首次启动时把旧的 ui.json / translate.json / models.json 迁入,并把旧文件改名为 *.bak。
|
|
3
|
+
import fs from 'node:fs'
|
|
4
|
+
import path from 'node:path'
|
|
5
|
+
import { DATA_DIR } from './state.js'
|
|
6
|
+
|
|
7
|
+
export const CONFIG_PATH = path.join(DATA_DIR, 'config.json')
|
|
8
|
+
|
|
9
|
+
export const CONFIG_DEFAULTS = {
|
|
10
|
+
schemaVersion: 1,
|
|
11
|
+
ui: { showSidebarPanel: true, sidebarCollapsed: false },
|
|
12
|
+
translate: {
|
|
13
|
+
provider: 'api',
|
|
14
|
+
target: 'zh',
|
|
15
|
+
apiBase: 'https://api.deepseek.com',
|
|
16
|
+
apiModel: 'deepseek-chat',
|
|
17
|
+
apiKey: '',
|
|
18
|
+
},
|
|
19
|
+
models: { asrModel: 'qwen3-asr-1.7b-8bit', translateModel: 'hy-mt2-1.8b-4bit' },
|
|
20
|
+
transcribe: { hotwords: [], chunkMinutes: 1, diarize: true },
|
|
21
|
+
import: { mode: 'copy' },
|
|
22
|
+
}
|
|
23
|
+
|
|
24
|
+
const LEGACY_FILES = [
|
|
25
|
+
['ui', 'ui.json'],
|
|
26
|
+
['translate', 'translate.json'],
|
|
27
|
+
['models', 'models.json'],
|
|
28
|
+
]
|
|
29
|
+
|
|
30
|
+
function clone(value) {
|
|
31
|
+
return JSON.parse(JSON.stringify(value))
|
|
32
|
+
}
|
|
33
|
+
|
|
34
|
+
function merge(base, patch) {
|
|
35
|
+
if (Array.isArray(base) || Array.isArray(patch) || typeof base !== 'object' || base === null || typeof patch !== 'object' || patch === null) {
|
|
36
|
+
return patch === undefined ? base : patch
|
|
37
|
+
}
|
|
38
|
+
const next = { ...base }
|
|
39
|
+
for (const [key, value] of Object.entries(patch)) next[key] = merge(base[key], value)
|
|
40
|
+
return next
|
|
41
|
+
}
|
|
42
|
+
|
|
43
|
+
function readJson(file) {
|
|
44
|
+
try {
|
|
45
|
+
return JSON.parse(fs.readFileSync(file, 'utf8'))
|
|
46
|
+
} catch {
|
|
47
|
+
return null
|
|
48
|
+
}
|
|
49
|
+
}
|
|
50
|
+
|
|
51
|
+
function writeConfig(config) {
|
|
52
|
+
fs.mkdirSync(DATA_DIR, { recursive: true })
|
|
53
|
+
const tmp = `${CONFIG_PATH}.tmp`
|
|
54
|
+
fs.writeFileSync(tmp, JSON.stringify(config, null, 2), { mode: 0o600 })
|
|
55
|
+
fs.renameSync(tmp, CONFIG_PATH)
|
|
56
|
+
}
|
|
57
|
+
|
|
58
|
+
function migrateLegacy() {
|
|
59
|
+
const config = clone(CONFIG_DEFAULTS)
|
|
60
|
+
for (const [section, name] of LEGACY_FILES) {
|
|
61
|
+
const file = path.join(DATA_DIR, name)
|
|
62
|
+
const data = readJson(file)
|
|
63
|
+
if (data && typeof data === 'object') {
|
|
64
|
+
config[section] = merge(config[section], data)
|
|
65
|
+
try {
|
|
66
|
+
fs.renameSync(file, `${file}.bak`)
|
|
67
|
+
} catch {
|
|
68
|
+
/* ignore */
|
|
69
|
+
}
|
|
70
|
+
}
|
|
71
|
+
}
|
|
72
|
+
writeConfig(config)
|
|
73
|
+
return config
|
|
74
|
+
}
|
|
75
|
+
|
|
76
|
+
let cache = null
|
|
77
|
+
|
|
78
|
+
export function loadConfig() {
|
|
79
|
+
if (!cache) {
|
|
80
|
+
const stored = readJson(CONFIG_PATH)
|
|
81
|
+
cache = stored ? merge(clone(CONFIG_DEFAULTS), stored) : migrateLegacy()
|
|
82
|
+
}
|
|
83
|
+
return clone(cache)
|
|
84
|
+
}
|
|
85
|
+
|
|
86
|
+
export function saveConfig(patch) {
|
|
87
|
+
const next = merge(loadConfig(), patch)
|
|
88
|
+
cache = next
|
|
89
|
+
writeConfig(next)
|
|
90
|
+
return clone(next)
|
|
91
|
+
}
|
|
92
|
+
|
|
93
|
+
export function getSection(name) {
|
|
94
|
+
return loadConfig()[name]
|
|
95
|
+
}
|
|
96
|
+
|
|
97
|
+
export function updateSection(name, patch) {
|
|
98
|
+
return saveConfig({ [name]: patch })[name]
|
|
99
|
+
}
|
|
100
|
+
|
|
101
|
+
export function configFile() {
|
|
102
|
+
return CONFIG_PATH
|
|
103
|
+
}
|
package/src/diarize.js
ADDED
|
@@ -0,0 +1,87 @@
|
|
|
1
|
+
// 说话人分离:sherpa-onnx(pyannote segmentation + 3D-Speaker embedding),模型在 ~/.podwave/models/diarization。
|
|
2
|
+
import fs from 'node:fs'
|
|
3
|
+
import path from 'node:path'
|
|
4
|
+
import { spawn } from 'node:child_process'
|
|
5
|
+
import { fileURLToPath } from 'node:url'
|
|
6
|
+
import { DATA_DIR } from './state.js'
|
|
7
|
+
import { ffmpegPath, venvPython } from './platform.js'
|
|
8
|
+
|
|
9
|
+
const SCRIPT = path.join(path.dirname(fileURLToPath(import.meta.url)), '..', 'scripts', 'diarize.py')
|
|
10
|
+
const PYTHON = venvPython()
|
|
11
|
+
export const DIARIZATION_DIR = path.join(DATA_DIR, 'models', 'diarization')
|
|
12
|
+
export const SEG_MODEL = path.join(DIARIZATION_DIR, 'sherpa-onnx-pyannote-segmentation-3-0', 'model.onnx')
|
|
13
|
+
export const EMB_MODEL = path.join(DIARIZATION_DIR, '3dspeaker_speech_campplus_sv_zh_en_16k-common_advanced.onnx')
|
|
14
|
+
|
|
15
|
+
export function diarizeInfo() {
|
|
16
|
+
return {
|
|
17
|
+
installed: fs.existsSync(PYTHON) && fs.existsSync(SCRIPT),
|
|
18
|
+
modelsReady: fs.existsSync(SEG_MODEL) && fs.existsSync(EMB_MODEL),
|
|
19
|
+
segModel: SEG_MODEL,
|
|
20
|
+
embModel: EMB_MODEL,
|
|
21
|
+
}
|
|
22
|
+
}
|
|
23
|
+
|
|
24
|
+
export function runDiarization(audioPath, { threshold = 0.6, numSpeakers = -1 } = {}) {
|
|
25
|
+
return new Promise((resolve, reject) => {
|
|
26
|
+
const args = [
|
|
27
|
+
SCRIPT,
|
|
28
|
+
'--audio', audioPath,
|
|
29
|
+
'--seg-model', SEG_MODEL,
|
|
30
|
+
'--emb-model', EMB_MODEL,
|
|
31
|
+
'--threshold', String(threshold),
|
|
32
|
+
]
|
|
33
|
+
if (Number(numSpeakers) > 0) args.push('--num-speakers', String(numSpeakers))
|
|
34
|
+
const child = spawn(PYTHON, args, {
|
|
35
|
+
stdio: ['ignore', 'pipe', 'pipe'],
|
|
36
|
+
env: { ...process.env, PYTHONUNBUFFERED: '1', PODWAVE_FFMPEG: process.env.PODWAVE_FFMPEG || ffmpegPath() || 'ffmpeg' },
|
|
37
|
+
})
|
|
38
|
+
let stdout = ''
|
|
39
|
+
let stderr = ''
|
|
40
|
+
child.stdout.on('data', (chunk) => { stdout += chunk })
|
|
41
|
+
child.stderr.on('data', (chunk) => { stderr += chunk })
|
|
42
|
+
child.on('error', reject)
|
|
43
|
+
child.on('close', (code) => {
|
|
44
|
+
if (code !== 0) {
|
|
45
|
+
const detail = stderr.trim().split('\n').slice(-3).join(' ')
|
|
46
|
+
reject(new Error(`说话人分离失败(退出码 ${code})${detail ? `:${detail}` : ''}`))
|
|
47
|
+
return
|
|
48
|
+
}
|
|
49
|
+
try {
|
|
50
|
+
resolve(JSON.parse(stdout.trim().split('\n').pop()))
|
|
51
|
+
} catch {
|
|
52
|
+
reject(new Error('说话人分离输出解析失败'))
|
|
53
|
+
}
|
|
54
|
+
})
|
|
55
|
+
})
|
|
56
|
+
}
|
|
57
|
+
|
|
58
|
+
// 把 diarization 段按最大重叠分配到字幕行;返回 [{id,label}](id 是重排后的 0..n-1)。
|
|
59
|
+
export function assignSpeakers(lines, segments) {
|
|
60
|
+
const mapping = new Map()
|
|
61
|
+
for (const segment of segments) {
|
|
62
|
+
const key = Number(segment.speaker)
|
|
63
|
+
if (!mapping.has(key)) mapping.set(key, mapping.size)
|
|
64
|
+
}
|
|
65
|
+
const speakers = [...mapping.entries()].map(([, index]) => ({ id: index, label: `说话人 ${index + 1}` }))
|
|
66
|
+
for (const line of lines) {
|
|
67
|
+
const overlap = new Map()
|
|
68
|
+
for (const segment of segments) {
|
|
69
|
+
const value = Math.min(line.end, Number(segment.end)) - Math.max(line.start, Number(segment.start))
|
|
70
|
+
if (value > 0) {
|
|
71
|
+
const key = Number(segment.speaker)
|
|
72
|
+
overlap.set(key, (overlap.get(key) || 0) + value)
|
|
73
|
+
}
|
|
74
|
+
}
|
|
75
|
+
let best = null
|
|
76
|
+
let bestValue = 0
|
|
77
|
+
for (const [key, value] of overlap) {
|
|
78
|
+
if (value > bestValue) {
|
|
79
|
+
best = key
|
|
80
|
+
bestValue = value
|
|
81
|
+
}
|
|
82
|
+
}
|
|
83
|
+
const threshold = Math.min(0.25, Math.max(0, (line.end - line.start) * 0.25))
|
|
84
|
+
line.speaker = best !== null && bestValue >= threshold ? mapping.get(best) : null
|
|
85
|
+
}
|
|
86
|
+
return speakers
|
|
87
|
+
}
|
|
@@ -0,0 +1,364 @@
|
|
|
1
|
+
// 多来源下载器:Hugging Face / hf-mirror / ModelScope / GitHub / gh-proxy。
|
|
2
|
+
// 支持断点续传(.part + Range)、跳过大文件已完成的情况、逐文件进度回调。
|
|
3
|
+
import fs from 'node:fs'
|
|
4
|
+
import fsp from 'node:fs/promises'
|
|
5
|
+
import path from 'node:path'
|
|
6
|
+
import { spawn } from 'node:child_process'
|
|
7
|
+
import crypto from 'node:crypto'
|
|
8
|
+
import { Readable } from 'node:stream'
|
|
9
|
+
import { pipeline } from 'node:stream/promises'
|
|
10
|
+
|
|
11
|
+
const UA = 'dsh-podwave-setup/1.0'
|
|
12
|
+
|
|
13
|
+
function timeout(ms) {
|
|
14
|
+
return AbortSignal.timeout(ms)
|
|
15
|
+
}
|
|
16
|
+
|
|
17
|
+
export async function probe(url, ms = 8000) {
|
|
18
|
+
const started = Date.now()
|
|
19
|
+
try {
|
|
20
|
+
const res = await fetch(url, {
|
|
21
|
+
headers: { 'user-agent': UA, range: 'bytes=0-0' },
|
|
22
|
+
redirect: 'follow',
|
|
23
|
+
signal: timeout(ms),
|
|
24
|
+
})
|
|
25
|
+
const ok = res.ok || res.status === 206 || res.status === 416
|
|
26
|
+
await res.body?.cancel().catch(() => {})
|
|
27
|
+
return { ok, status: res.status, ms: Date.now() - started }
|
|
28
|
+
} catch (error) {
|
|
29
|
+
return { ok: false, status: 0, ms: Date.now() - started, error: String(error.message || error) }
|
|
30
|
+
}
|
|
31
|
+
}
|
|
32
|
+
|
|
33
|
+
export async function listHfRepo(repo, base, { include, skip } = {}) {
|
|
34
|
+
const url = `${base}/api/models/${repo}/tree/main?recursive=true`
|
|
35
|
+
const res = await fetch(url, { headers: { 'user-agent': UA }, redirect: 'follow', signal: timeout(20000) })
|
|
36
|
+
if (!res.ok) throw new Error(`读取文件列表失败(${base} ${res.status})`)
|
|
37
|
+
const entries = await res.json()
|
|
38
|
+
return entries
|
|
39
|
+
.filter((entry) => entry.type === 'file')
|
|
40
|
+
.filter((entry) => (include ? include.includes(entry.path) : true))
|
|
41
|
+
.filter((entry) => (skip ? !skip.some((re) => re.test(entry.path)) : true))
|
|
42
|
+
.map((entry) => ({ path: entry.path, size: entry.size || 0, sha256: entry.lfs?.oid || '' }))
|
|
43
|
+
}
|
|
44
|
+
|
|
45
|
+
export async function directHfFiles(repo, base, { include, files, skip } = {}) {
|
|
46
|
+
const wanted = include || files
|
|
47
|
+
if (!wanted || !wanted.length) throw new Error('该资源没有配置固定文件清单,无法在镜像上直接下载')
|
|
48
|
+
const result = []
|
|
49
|
+
for (const file of wanted) {
|
|
50
|
+
if (skip && skip.some((re) => re.test(file))) continue
|
|
51
|
+
const encoded = file.split('/').map(encodeURIComponent).join('/')
|
|
52
|
+
const url = `${base}/${repo}/resolve/main/${encoded}`
|
|
53
|
+
let size = 0
|
|
54
|
+
try {
|
|
55
|
+
const res = await fetch(url, { method: 'HEAD', redirect: 'follow', signal: timeout(20000) })
|
|
56
|
+
size = Number(res.headers.get('content-length') || 0)
|
|
57
|
+
if (!res.ok) continue
|
|
58
|
+
} catch {
|
|
59
|
+
// 允许大小未知,下载时再统计
|
|
60
|
+
}
|
|
61
|
+
result.push({ path: file, size })
|
|
62
|
+
}
|
|
63
|
+
if (!result.length) throw new Error('镜像上没有找到清单里的文件')
|
|
64
|
+
return result
|
|
65
|
+
}
|
|
66
|
+
|
|
67
|
+
export async function listMsRepo(repo) {
|
|
68
|
+
const url = `https://modelscope.cn/api/v1/models/${repo}/repo/files?Revision=master&Recursive=true`
|
|
69
|
+
const res = await fetch(url, { headers: { 'user-agent': UA }, redirect: 'follow', signal: timeout(20000) })
|
|
70
|
+
if (!res.ok) throw new Error(`ModelScope 文件列表失败(${res.status})`)
|
|
71
|
+
const data = await res.json()
|
|
72
|
+
const files = data?.Data?.Files || []
|
|
73
|
+
return files
|
|
74
|
+
.filter((file) => file.Type !== 'tree' && file.Size > 0)
|
|
75
|
+
.map((file) => ({ path: file.Path, size: file.Size || 0 }))
|
|
76
|
+
}
|
|
77
|
+
|
|
78
|
+
export function isSafeArchivePath(name) {
|
|
79
|
+
const value = String(name || '').replace(/\\/g, '/')
|
|
80
|
+
if (!value || value.startsWith('/')) return false
|
|
81
|
+
if (/^[a-zA-Z]:/.test(value)) return false
|
|
82
|
+
return !value.split('/').some((part) => part === '..')
|
|
83
|
+
}
|
|
84
|
+
|
|
85
|
+
function runCapture(cmd, args) {
|
|
86
|
+
return new Promise((resolve) => {
|
|
87
|
+
let output = ''
|
|
88
|
+
const child = spawn(cmd, args, { stdio: ['ignore', 'pipe', 'ignore'] })
|
|
89
|
+
child.stdout.on('data', (chunk) => {
|
|
90
|
+
output += chunk.toString('utf8')
|
|
91
|
+
})
|
|
92
|
+
child.on('error', () => resolve(null))
|
|
93
|
+
child.on('exit', (code) => resolve(code === 0 ? output : null))
|
|
94
|
+
})
|
|
95
|
+
}
|
|
96
|
+
|
|
97
|
+
async function listArchiveEntries(file) {
|
|
98
|
+
const lower = file.toLowerCase()
|
|
99
|
+
if (lower.endsWith('.zip')) {
|
|
100
|
+
const candidates = [
|
|
101
|
+
['unzip', ['-Z1', file]],
|
|
102
|
+
['tar', ['-tf', file]],
|
|
103
|
+
['python3', ['-c', 'import zipfile,sys;print("\n".join(zipfile.ZipFile(sys.argv[1]).namelist()))', file]],
|
|
104
|
+
]
|
|
105
|
+
for (const [cmd, args] of candidates) {
|
|
106
|
+
const output = await runCapture(cmd, args)
|
|
107
|
+
if (output) return output.split(/\r?\n/).filter(Boolean)
|
|
108
|
+
}
|
|
109
|
+
return null
|
|
110
|
+
}
|
|
111
|
+
const output = await runCapture('tar', ['-tf', file])
|
|
112
|
+
return output ? output.split(/\r?\n/).filter(Boolean) : null
|
|
113
|
+
}
|
|
114
|
+
|
|
115
|
+
async function sha256File(file) {
|
|
116
|
+
const hash = crypto.createHash('sha256')
|
|
117
|
+
await pipeline(fs.createReadStream(file), hash)
|
|
118
|
+
return hash.digest('hex')
|
|
119
|
+
}
|
|
120
|
+
|
|
121
|
+
const digestCache = new Map()
|
|
122
|
+
|
|
123
|
+
async function githubDigest(url) {
|
|
124
|
+
try {
|
|
125
|
+
const match = /^https:\/\/github\.com\/([^/]+)\/([^/]+)\/releases\/download\/([^/]+)\/(.+)$/.exec(url)
|
|
126
|
+
if (!match) return ''
|
|
127
|
+
const [, owner, repo, tag, assetName] = match
|
|
128
|
+
const key = `${owner}/${repo}@${tag}#${assetName}`
|
|
129
|
+
if (digestCache.has(key)) return digestCache.get(key)
|
|
130
|
+
const api =
|
|
131
|
+
tag === 'latest'
|
|
132
|
+
? `https://api.github.com/repos/${owner}/${repo}/releases/latest`
|
|
133
|
+
: `https://api.github.com/repos/${owner}/${repo}/releases/tags/${tag}`
|
|
134
|
+
const res = await fetch(api, {
|
|
135
|
+
headers: { 'user-agent': UA, accept: 'application/vnd.github+json' },
|
|
136
|
+
signal: timeout(15000),
|
|
137
|
+
})
|
|
138
|
+
if (!res.ok) {
|
|
139
|
+
digestCache.set(key, '')
|
|
140
|
+
return ''
|
|
141
|
+
}
|
|
142
|
+
const data = await res.json()
|
|
143
|
+
const asset = (data.assets || []).find((item) => item.name === decodeURIComponent(assetName))
|
|
144
|
+
const digest = asset?.digest ? String(asset.digest).replace(/^sha256:/, '') : ''
|
|
145
|
+
digestCache.set(key, digest)
|
|
146
|
+
return digest
|
|
147
|
+
} catch {
|
|
148
|
+
return ''
|
|
149
|
+
}
|
|
150
|
+
}
|
|
151
|
+
|
|
152
|
+
async function downloadOne(url, dest, { expected = 0, sha256 = '', onProgress, signal } = {}) {
|
|
153
|
+
await fsp.mkdir(path.dirname(dest), { recursive: true })
|
|
154
|
+
try {
|
|
155
|
+
const stat = await fsp.stat(dest)
|
|
156
|
+
if (expected && stat.size === expected) {
|
|
157
|
+
if (sha256) {
|
|
158
|
+
const actual = await sha256File(dest)
|
|
159
|
+
if (actual.toLowerCase() === String(sha256).toLowerCase()) return { bytes: stat.size, skipped: true }
|
|
160
|
+
await fsp.rm(dest, { force: true })
|
|
161
|
+
} else {
|
|
162
|
+
return { bytes: stat.size, skipped: true }
|
|
163
|
+
}
|
|
164
|
+
}
|
|
165
|
+
if (expected && stat.size > expected) await fsp.rm(dest, { force: true })
|
|
166
|
+
} catch {
|
|
167
|
+
// 文件不存在
|
|
168
|
+
}
|
|
169
|
+
const part = `${dest}.part`
|
|
170
|
+
let offset = 0
|
|
171
|
+
try {
|
|
172
|
+
offset = (await fsp.stat(part)).size
|
|
173
|
+
} catch {
|
|
174
|
+
offset = 0
|
|
175
|
+
}
|
|
176
|
+
const headers = { 'user-agent': UA }
|
|
177
|
+
if (offset > 0) headers.range = `bytes=${offset}-`
|
|
178
|
+
const res = await fetch(url, { headers, redirect: 'follow', signal: signal || timeout(600000) })
|
|
179
|
+
if (res.status === 416) {
|
|
180
|
+
await fsp.rename(part, dest)
|
|
181
|
+
return { bytes: offset, skipped: false }
|
|
182
|
+
}
|
|
183
|
+
if (!res.ok) throw new Error(`下载失败 ${res.status}:${url}`)
|
|
184
|
+
const write = fs.createWriteStream(part, { flags: offset > 0 ? 'a' : 'w' })
|
|
185
|
+
let received = 0
|
|
186
|
+
const source = Readable.fromWeb(res.body)
|
|
187
|
+
source.on('data', (chunk) => {
|
|
188
|
+
received += chunk.length
|
|
189
|
+
onProgress?.({ delta: chunk.length, bytes: offset + received, expected })
|
|
190
|
+
})
|
|
191
|
+
await pipeline(source, write)
|
|
192
|
+
const stat = await fsp.stat(part)
|
|
193
|
+
if (expected && stat.size !== expected) throw new Error(`文件大小不符(期望 ${expected},实际 ${stat.size})`)
|
|
194
|
+
if (sha256) {
|
|
195
|
+
const actual = await sha256File(part)
|
|
196
|
+
if (actual.toLowerCase() !== String(sha256).toLowerCase()) {
|
|
197
|
+
await fsp.rm(part, { force: true })
|
|
198
|
+
throw new Error('文件校验失败(sha256 不匹配),已丢弃下载内容')
|
|
199
|
+
}
|
|
200
|
+
}
|
|
201
|
+
await fsp.rename(part, dest)
|
|
202
|
+
return { bytes: stat.size, skipped: false }
|
|
203
|
+
}
|
|
204
|
+
|
|
205
|
+
export async function pickSource(sources, probeUrlFor) {
|
|
206
|
+
const checks = sources.map(async (source) => {
|
|
207
|
+
const url = probeUrlFor(source)
|
|
208
|
+
const result = url ? await probe(url) : { ok: false, ms: 99999 }
|
|
209
|
+
return { source, ...result }
|
|
210
|
+
})
|
|
211
|
+
const results = await Promise.all(checks)
|
|
212
|
+
const ok = results.filter((item) => item.ok).sort((a, b) => a.ms - b.ms)
|
|
213
|
+
if (!ok.length) throw new Error('所有下载来源都不可用,请检查网络')
|
|
214
|
+
return ok[0].source
|
|
215
|
+
}
|
|
216
|
+
|
|
217
|
+
function sourceProbeUrl(source, artifact) {
|
|
218
|
+
if (source.id === 'ms') {
|
|
219
|
+
return `https://modelscope.cn/api/v1/models/${source.ms}/repo/files?Revision=master&Recursive=false`
|
|
220
|
+
}
|
|
221
|
+
if (source.id === 'cn' && artifact.source.type === 'github-assets') {
|
|
222
|
+
return `https://gh-proxy.com/${artifact.source.assets[0].url}`
|
|
223
|
+
}
|
|
224
|
+
if (artifact.source.type === 'hf-repo') {
|
|
225
|
+
return `${source.base}/api/models/${artifact.source.repo}`
|
|
226
|
+
}
|
|
227
|
+
if (artifact.source.type === 'hf-file') {
|
|
228
|
+
return `${source.base}/${artifact.source.repo}/resolve/main/${artifact.source.file}`
|
|
229
|
+
}
|
|
230
|
+
if (artifact.source.type === 'github-assets') {
|
|
231
|
+
return artifact.source.assets[0].url
|
|
232
|
+
}
|
|
233
|
+
return null
|
|
234
|
+
}
|
|
235
|
+
|
|
236
|
+
export async function downloadArtifact(artifact, { sourceId = 'auto', onProgress, signal } = {}) {
|
|
237
|
+
const { sourcesFor } = await import('./catalog.js')
|
|
238
|
+
const sources = sourcesFor(artifact.source)
|
|
239
|
+
if (!sources.length) throw new Error('该资源没有配置下载来源')
|
|
240
|
+
let source = sources[0]
|
|
241
|
+
if (sourceId === 'intl' || sourceId === 'cn' || sourceId === 'ms') {
|
|
242
|
+
source = sources.find((item) => item.id === sourceId) || sources[0]
|
|
243
|
+
} else {
|
|
244
|
+
source = await pickSource(sources, (item) => sourceProbeUrl(item, artifact))
|
|
245
|
+
}
|
|
246
|
+
const destDir = path.join(await dataDir(), 'models', artifact.dir)
|
|
247
|
+
const result = { source: source.id, sourceLabel: source.label, files: [], bytes: 0 }
|
|
248
|
+
|
|
249
|
+
if (artifact.source.type === 'github-assets') {
|
|
250
|
+
for (const asset of artifact.source.assets) {
|
|
251
|
+
const url = source.id === 'cn' ? `https://gh-proxy.com/${asset.url}` : asset.url
|
|
252
|
+
const dest = path.join(destDir, asset.file)
|
|
253
|
+
const before = result.bytes
|
|
254
|
+
const sha256 = await githubDigest(asset.url)
|
|
255
|
+
const { bytes } = await downloadOne(url, dest, { sha256, onProgress, signal })
|
|
256
|
+
result.bytes += bytes
|
|
257
|
+
result.files.push({ path: asset.file, bytes })
|
|
258
|
+
if (asset.extract) await extractArchive(dest, destDir)
|
|
259
|
+
onProgress?.({ file: asset.file, delta: 0, bytes: result.bytes, expected: result.bytes })
|
|
260
|
+
void before
|
|
261
|
+
}
|
|
262
|
+
return result
|
|
263
|
+
}
|
|
264
|
+
|
|
265
|
+
let files
|
|
266
|
+
if (source.id === 'ms' && artifact.source.type === 'hf-file') {
|
|
267
|
+
files = [{ path: artifact.source.file, size: artifact.approxBytes || 0 }]
|
|
268
|
+
} else if (source.id === 'ms') {
|
|
269
|
+
const all = await listMsRepo(source.ms)
|
|
270
|
+
const wanted = artifact.source.include
|
|
271
|
+
files = wanted ? all.filter((file) => wanted.includes(file.path)) : all.filter((file) => !isJunk(file.path))
|
|
272
|
+
} else if (artifact.source.type === 'hf-repo') {
|
|
273
|
+
const known = artifact.source.include || artifact.source.files
|
|
274
|
+
if (source.id === 'cn' && known) {
|
|
275
|
+
files = await directHfFiles(artifact.source.repo, source.base, {
|
|
276
|
+
include: artifact.source.include,
|
|
277
|
+
files: artifact.source.files,
|
|
278
|
+
skip: artifact.source.skip,
|
|
279
|
+
})
|
|
280
|
+
} else {
|
|
281
|
+
files = await listHfRepo(artifact.source.repo, source.base, {
|
|
282
|
+
include: artifact.source.include,
|
|
283
|
+
skip: artifact.source.skip,
|
|
284
|
+
}).catch(async (error) => {
|
|
285
|
+
if (!known) throw error
|
|
286
|
+
return directHfFiles(artifact.source.repo, source.base, {
|
|
287
|
+
include: artifact.source.include,
|
|
288
|
+
files: artifact.source.files,
|
|
289
|
+
skip: artifact.source.skip,
|
|
290
|
+
})
|
|
291
|
+
})
|
|
292
|
+
}
|
|
293
|
+
} else {
|
|
294
|
+
files = [{ path: artifact.source.file, size: artifact.approxBytes || 0 }]
|
|
295
|
+
}
|
|
296
|
+
if (!files.length) throw new Error('这个下载源没有找到所需的文件,请换一条线路')
|
|
297
|
+
const total = files.reduce((sum, file) => sum + file.size, 0)
|
|
298
|
+
let done = 0
|
|
299
|
+
for (const file of files) {
|
|
300
|
+
const encoded = file.path.split('/').map(encodeURIComponent).join('/')
|
|
301
|
+
const url =
|
|
302
|
+
source.id === 'ms'
|
|
303
|
+
? `https://modelscope.cn/models/${source.ms}/resolve/master/${encoded}`
|
|
304
|
+
: `${source.base}/${artifact.source.repo}/resolve/main/${encoded}`
|
|
305
|
+
const dest = path.join(destDir, file.path)
|
|
306
|
+
const { bytes } = await downloadOne(url, dest, {
|
|
307
|
+
expected: file.size,
|
|
308
|
+
sha256: file.sha256 || '',
|
|
309
|
+
signal,
|
|
310
|
+
onProgress: (event) => {
|
|
311
|
+
done += event.delta
|
|
312
|
+
onProgress?.({ file: file.path, delta: event.delta, bytes: done, expected: total })
|
|
313
|
+
},
|
|
314
|
+
})
|
|
315
|
+
result.files.push({ path: file.path, bytes })
|
|
316
|
+
result.bytes += bytes
|
|
317
|
+
}
|
|
318
|
+
return result
|
|
319
|
+
}
|
|
320
|
+
|
|
321
|
+
function isJunk(p) {
|
|
322
|
+
return /^\.gitattributes$|^README|^LICENSE|^imgs\/|^train\/|\.md$/i.test(p)
|
|
323
|
+
}
|
|
324
|
+
|
|
325
|
+
function runExtract(cmd, args) {
|
|
326
|
+
return new Promise((resolve, reject) => {
|
|
327
|
+
const child = spawn(cmd, args, { stdio: 'ignore' })
|
|
328
|
+
child.on('error', reject)
|
|
329
|
+
child.on('exit', (code) => (code === 0 ? resolve() : reject(new Error(`${cmd} 解压失败(${code})`))))
|
|
330
|
+
})
|
|
331
|
+
}
|
|
332
|
+
|
|
333
|
+
export async function extractArchive(file, destDir) {
|
|
334
|
+
const entries = await listArchiveEntries(file)
|
|
335
|
+
if (entries && entries.length) {
|
|
336
|
+
const unsafe = entries.find((entry) => !isSafeArchivePath(entry))
|
|
337
|
+
if (unsafe) throw new Error(`压缩包包含不安全路径:${unsafe}`)
|
|
338
|
+
}
|
|
339
|
+
const lower = file.toLowerCase()
|
|
340
|
+
const attempts = []
|
|
341
|
+
if (lower.endsWith('.zip')) {
|
|
342
|
+
attempts.push(['unzip', ['-q', '-o', file, '-d', destDir]])
|
|
343
|
+
attempts.push(['tar', ['-xf', file, '-C', destDir]])
|
|
344
|
+
attempts.push(['powershell.exe', ['-NoProfile', '-Command', `Expand-Archive -LiteralPath '${file}' -DestinationPath '${destDir}' -Force`]])
|
|
345
|
+
attempts.push(['python3', ['-m', 'zipfile', '-e', file, destDir]])
|
|
346
|
+
} else {
|
|
347
|
+
attempts.push(['tar', ['-xf', file, '-C', destDir]])
|
|
348
|
+
}
|
|
349
|
+
let lastError = null
|
|
350
|
+
for (const [cmd, args] of attempts) {
|
|
351
|
+
try {
|
|
352
|
+
await runExtract(cmd, args)
|
|
353
|
+
return
|
|
354
|
+
} catch (error) {
|
|
355
|
+
lastError = error
|
|
356
|
+
}
|
|
357
|
+
}
|
|
358
|
+
throw lastError || new Error('解压失败')
|
|
359
|
+
}
|
|
360
|
+
|
|
361
|
+
async function dataDir() {
|
|
362
|
+
const { DATA_DIR } = await import('./state.js')
|
|
363
|
+
return DATA_DIR
|
|
364
|
+
}
|