dsh-hackertube 0.0.0-stage → 0.1.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/README.en.md +180 -0
- package/README.md +179 -2
- package/client.js +2568 -0
- package/cordis.patch.yml +5 -0
- package/data/zh-taxonomy.json +1 -0
- package/data/zh-titles.json +1 -0
- package/icon.svg +5 -0
- package/index.js +3142 -0
- package/locale/en.json +6 -0
- package/locale/zh.json +6 -0
- package/package.json +59 -4
package/index.js
ADDED
|
@@ -0,0 +1,3142 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* HackerTube Host 半。
|
|
3
|
+
*
|
|
4
|
+
* 浏览器端不能直接取上游数据:hackertube.org 的 /api/* 不带 CORS 头,
|
|
5
|
+
* 而数据主体是 8.2MB 的 videos.jsonl。这里负责拉取、瘦身、落盘缓存,
|
|
6
|
+
* 再通过 /hackertube/api/* 交给面板。
|
|
7
|
+
*
|
|
8
|
+
* webServer 是可选服务:headless 场景没有它,写进 inject 会让插件永远 park,
|
|
9
|
+
* 所以只在运行时用 ctx.get 探测,并给两次延迟重试。
|
|
10
|
+
*/
|
|
11
|
+
|
|
12
|
+
import { spawn } from 'node:child_process'
|
|
13
|
+
import { createReadStream } from 'node:fs'
|
|
14
|
+
import { createServer } from 'node:http'
|
|
15
|
+
import { mkdir, readdir, readFile, rename, rm, stat, unlink, writeFile } from 'node:fs/promises'
|
|
16
|
+
import { homedir } from 'node:os'
|
|
17
|
+
import { dirname, join } from 'node:path'
|
|
18
|
+
import { fileURLToPath } from 'node:url'
|
|
19
|
+
import { inflateRawSync } from 'node:zlib'
|
|
20
|
+
|
|
21
|
+
/* 仓库里预生成的译文,首次安装时直接铺到缓存目录 */
|
|
22
|
+
const HERE = dirname(fileURLToPath(import.meta.url))
|
|
23
|
+
const SEED_DIR = join(HERE, 'data')
|
|
24
|
+
|
|
25
|
+
const SOURCE = 'https://hackertube.org'
|
|
26
|
+
const PREFIX = '/hackertube/api'
|
|
27
|
+
const TTL_MS = 24 * 60 * 60 * 1000
|
|
28
|
+
const FETCH_TIMEOUT_MS = 45_000
|
|
29
|
+
/* 机翻接口是第三方免鉴权端点,超时要短,失败就留原文 */
|
|
30
|
+
const MT_TIMEOUT_MS = 12_000
|
|
31
|
+
/* 导出包 17.8 MB,给足下载预算 */
|
|
32
|
+
const EXPORT_DOWNLOAD_MS = 120_000
|
|
33
|
+
const DETAIL_CACHE_MAX = 60
|
|
34
|
+
/* 单次媒体下载预算;yt-dlp 在本机实测 10 分钟片长约 17 秒 */
|
|
35
|
+
const MEDIA_TIMEOUT_MS = 180_000
|
|
36
|
+
const MEDIA_MAX_HEIGHT = 480
|
|
37
|
+
/* 本地媒体缓存总上限,超了按最近最少使用淘汰 */
|
|
38
|
+
const MEDIA_CAP_BYTES = 512 * 1024 * 1024
|
|
39
|
+
|
|
40
|
+
/* /api/list 用 offset + n 分页;n 的上限是 100,更大的值被静默截到 100 */
|
|
41
|
+
const PAGE_SIZE = 100
|
|
42
|
+
const PAGE_CONCURRENCY = 8
|
|
43
|
+
const PAGE_RETRIES = 3
|
|
44
|
+
const FALLBACK_TOTAL = 6454
|
|
45
|
+
/* 整轮抓取的总预算,网络异常时不至于无限拖住预热 */
|
|
46
|
+
const TOTAL_BUDGET_MS = 120_000
|
|
47
|
+
/* 缓存结构变更时递增,旧缓存直接作废 */
|
|
48
|
+
const CACHE_VERSION = 2
|
|
49
|
+
|
|
50
|
+
const state = {
|
|
51
|
+
ctx: null,
|
|
52
|
+
origin: null,
|
|
53
|
+
authUrl: null,
|
|
54
|
+
mediaOrigin: null,
|
|
55
|
+
list: null,
|
|
56
|
+
listAt: 0,
|
|
57
|
+
listJson: null,
|
|
58
|
+
browse: null,
|
|
59
|
+
playlists: null,
|
|
60
|
+
playlistsAt: 0,
|
|
61
|
+
playlistDetail: new Map(),
|
|
62
|
+
comments: new Map(),
|
|
63
|
+
related: new Map(),
|
|
64
|
+
browseAt: 0,
|
|
65
|
+
browseJson: null,
|
|
66
|
+
inflight: new Map(),
|
|
67
|
+
detail: new Map(),
|
|
68
|
+
}
|
|
69
|
+
|
|
70
|
+
/* ---------------- 缓存位置 ---------------- */
|
|
71
|
+
|
|
72
|
+
function cacheRoot() {
|
|
73
|
+
const base = process.env.DSH_PROFILE_DIR || join(homedir(), '.dsh')
|
|
74
|
+
return join(base, '.dsh-hackertube')
|
|
75
|
+
}
|
|
76
|
+
|
|
77
|
+
async function readCache(name) {
|
|
78
|
+
try {
|
|
79
|
+
return JSON.parse(await readFile(join(cacheRoot(), name), 'utf8'))
|
|
80
|
+
} catch {
|
|
81
|
+
return null
|
|
82
|
+
}
|
|
83
|
+
}
|
|
84
|
+
|
|
85
|
+
async function writeCache(name, value) {
|
|
86
|
+
try {
|
|
87
|
+
await mkdir(cacheRoot(), { recursive: true })
|
|
88
|
+
await writeFile(join(cacheRoot(), name), JSON.stringify(value), 'utf8')
|
|
89
|
+
} catch {
|
|
90
|
+
/* 缓存写失败不影响本次返回 */
|
|
91
|
+
}
|
|
92
|
+
}
|
|
93
|
+
|
|
94
|
+
/* ---------------- 上游取数 ---------------- */
|
|
95
|
+
|
|
96
|
+
async function fetchJson(url) {
|
|
97
|
+
const res = await fetch(url, { signal: AbortSignal.timeout(FETCH_TIMEOUT_MS) })
|
|
98
|
+
if (!res.ok) throw new Error(`上游 ${res.status}: ${url}`)
|
|
99
|
+
return res.json()
|
|
100
|
+
}
|
|
101
|
+
|
|
102
|
+
/** 同一个 key 的并发请求合并成一次。 */
|
|
103
|
+
function once(key, task) {
|
|
104
|
+
const running = state.inflight.get(key)
|
|
105
|
+
if (running) return running
|
|
106
|
+
const p = task().finally(() => state.inflight.delete(key))
|
|
107
|
+
state.inflight.set(key, p)
|
|
108
|
+
return p
|
|
109
|
+
}
|
|
110
|
+
|
|
111
|
+
/* ---------------- 数据整形 ---------------- */
|
|
112
|
+
|
|
113
|
+
/**
|
|
114
|
+
* 列表页只需要这些字段。summary / why_hn / quote / hn 占单条的六成以上,
|
|
115
|
+
* 只在详情页取,所以不进入列表缓存。
|
|
116
|
+
*/
|
|
117
|
+
function toSlim(o) {
|
|
118
|
+
const published = o.published ?? null
|
|
119
|
+
return {
|
|
120
|
+
v: o.vid,
|
|
121
|
+
t: o.title ?? '',
|
|
122
|
+
c: o.channel ?? '',
|
|
123
|
+
d: o.duration_s ?? 0,
|
|
124
|
+
th: o.thumbnail ?? (o.vid ? 'https://i.ytimg.com/vi/' + o.vid + '/hq2.jpg' : ''),
|
|
125
|
+
p: published,
|
|
126
|
+
w: o.views ?? 0,
|
|
127
|
+
ct: o.content_type ?? '',
|
|
128
|
+
lv: o.level ?? '',
|
|
129
|
+
y: published ? Number(String(published).slice(0, 4)) || null : null,
|
|
130
|
+
cats: Array.isArray(o.categories) ? o.categories.map((c) => c.id) : [],
|
|
131
|
+
tags: Array.isArray(o.tags) ? o.tags : [],
|
|
132
|
+
s: typeof o.score === 'number' ? o.score : 0,
|
|
133
|
+
r: o.hn && typeof o.hn.recommenders === 'number' ? o.hn.recommenders : 0,
|
|
134
|
+
}
|
|
135
|
+
}
|
|
136
|
+
|
|
137
|
+
function setList(items, at) {
|
|
138
|
+
state.list = items
|
|
139
|
+
state.listAt = at
|
|
140
|
+
state.listJson = null
|
|
141
|
+
}
|
|
142
|
+
|
|
143
|
+
/* ---------------- 官方数据导出 ---------------- */
|
|
144
|
+
|
|
145
|
+
/**
|
|
146
|
+
* hackertube.org 把整套数据以 zip 形式挂在 /download 上,作者的原话是
|
|
147
|
+
* “Export of the site's data available to give to your agent to easily create
|
|
148
|
+
* your own version”。这里直接吃这份导出:一次 17.8 MB 的下载,换来列表、详情、
|
|
149
|
+
* HN 评论和相关推荐全部本地可得,不必再翻 65 页 /api/list。
|
|
150
|
+
*/
|
|
151
|
+
|
|
152
|
+
const EXPORT_ZIP_URL = `${SOURCE}/download/hackertube-data.zip`
|
|
153
|
+
const EXPORT_FILES = [
|
|
154
|
+
'videos.jsonl',
|
|
155
|
+
'mentions.jsonl',
|
|
156
|
+
'related.jsonl',
|
|
157
|
+
'embeddings.npy',
|
|
158
|
+
'embedding_ids.json',
|
|
159
|
+
]
|
|
160
|
+
const EXPORT_TTL_MS = 7 * 24 * 60 * 60 * 1000
|
|
161
|
+
const EXPORT_VERSION = 1
|
|
162
|
+
const EXPORT_MAX_BYTES = 64 * 1024 * 1024
|
|
163
|
+
|
|
164
|
+
function exportDir() {
|
|
165
|
+
return join(cacheRoot(), 'export')
|
|
166
|
+
}
|
|
167
|
+
|
|
168
|
+
/** 最小 zip 解析:只认 store 与 deflate,够解这一个包。 */
|
|
169
|
+
function unzipEntries(buf, wanted) {
|
|
170
|
+
let eocd = -1
|
|
171
|
+
const floor = Math.max(0, buf.length - 66 * 1024)
|
|
172
|
+
for (let i = buf.length - 22; i >= floor; i -= 1) {
|
|
173
|
+
if (buf.readUInt32LE(i) === 0x06054b50) {
|
|
174
|
+
eocd = i
|
|
175
|
+
break
|
|
176
|
+
}
|
|
177
|
+
}
|
|
178
|
+
if (eocd < 0) throw new Error('zip: 找不到中央目录')
|
|
179
|
+
const count = buf.readUInt16LE(eocd + 10)
|
|
180
|
+
let off = buf.readUInt32LE(eocd + 16)
|
|
181
|
+
const out = {}
|
|
182
|
+
for (let i = 0; i < count; i += 1) {
|
|
183
|
+
if (buf.readUInt32LE(off) !== 0x02014b50) throw new Error('zip: 中央目录损坏')
|
|
184
|
+
const method = buf.readUInt16LE(off + 10)
|
|
185
|
+
const compSize = buf.readUInt32LE(off + 20)
|
|
186
|
+
const nameLen = buf.readUInt16LE(off + 28)
|
|
187
|
+
const extraLen = buf.readUInt16LE(off + 30)
|
|
188
|
+
const commentLen = buf.readUInt16LE(off + 32)
|
|
189
|
+
const localOff = buf.readUInt32LE(off + 42)
|
|
190
|
+
const name = buf.toString('utf8', off + 46, off + 46 + nameLen)
|
|
191
|
+
off += 46 + nameLen + extraLen + commentLen
|
|
192
|
+
const base = name.split('/').pop()
|
|
193
|
+
if (wanted.indexOf(base) < 0) continue
|
|
194
|
+
if (buf.readUInt32LE(localOff) !== 0x04034b50) throw new Error('zip: 本地头损坏')
|
|
195
|
+
const lNameLen = buf.readUInt16LE(localOff + 26)
|
|
196
|
+
const lExtraLen = buf.readUInt16LE(localOff + 28)
|
|
197
|
+
const start = localOff + 30 + lNameLen + lExtraLen
|
|
198
|
+
const raw = buf.subarray(start, start + compSize)
|
|
199
|
+
out[base] = method === 0 ? raw : inflateRawSync(raw)
|
|
200
|
+
}
|
|
201
|
+
return out
|
|
202
|
+
}
|
|
203
|
+
|
|
204
|
+
async function fetchExportZip() {
|
|
205
|
+
const res = await fetch(EXPORT_ZIP_URL, { signal: AbortSignal.timeout(EXPORT_DOWNLOAD_MS) })
|
|
206
|
+
if (!res.ok) throw new Error('export zip ' + res.status)
|
|
207
|
+
const buf = Buffer.from(await res.arrayBuffer())
|
|
208
|
+
if (buf.length > EXPORT_MAX_BYTES) throw new Error('export zip 异常大:' + buf.length)
|
|
209
|
+
if (buf.length < 1024) throw new Error('export zip 异常小:' + buf.length)
|
|
210
|
+
return buf
|
|
211
|
+
}
|
|
212
|
+
|
|
213
|
+
/**
|
|
214
|
+
* 整包一次下载、一次解包,之后各取所需。
|
|
215
|
+
* 分次请求会各触发一次 17.8 MB 下载,所以这里只允许一个在飞的任务。
|
|
216
|
+
*/
|
|
217
|
+
let exportReadyP = null
|
|
218
|
+
|
|
219
|
+
async function ensureExportReady() {
|
|
220
|
+
if (exportReadyP) return exportReadyP
|
|
221
|
+
exportReadyP = (async () => {
|
|
222
|
+
const dir = exportDir()
|
|
223
|
+
const manifest = await readCache('export/manifest.json')
|
|
224
|
+
if (manifest && Number(manifest.v) === EXPORT_VERSION) {
|
|
225
|
+
const fresh = Date.now() - Number(manifest.at || 0) < EXPORT_TTL_MS
|
|
226
|
+
if (fresh) {
|
|
227
|
+
const sizes = await Promise.all(
|
|
228
|
+
EXPORT_FILES.map(async (n) => {
|
|
229
|
+
try {
|
|
230
|
+
return (await stat(join(dir, n))).size > 0
|
|
231
|
+
} catch {
|
|
232
|
+
return false
|
|
233
|
+
}
|
|
234
|
+
}),
|
|
235
|
+
)
|
|
236
|
+
if (sizes.every(Boolean)) return
|
|
237
|
+
}
|
|
238
|
+
}
|
|
239
|
+
await mkdir(dir, { recursive: true })
|
|
240
|
+
const zip = await fetchExportZip()
|
|
241
|
+
const entries = unzipEntries(zip, EXPORT_FILES)
|
|
242
|
+
for (const n of EXPORT_FILES) {
|
|
243
|
+
if (!entries[n]) throw new Error('导出缺少文件:' + n)
|
|
244
|
+
await writeFile(join(dir, n), entries[n])
|
|
245
|
+
}
|
|
246
|
+
await writeCache('export/manifest.json', { v: EXPORT_VERSION, at: Date.now(), bytes: zip.length })
|
|
247
|
+
})().finally(() => {
|
|
248
|
+
exportReadyP = null
|
|
249
|
+
})
|
|
250
|
+
return exportReadyP
|
|
251
|
+
}
|
|
252
|
+
|
|
253
|
+
/** 读若干导出文件,必要时先补齐整包 */
|
|
254
|
+
async function readExport(names) {
|
|
255
|
+
await ensureExportReady()
|
|
256
|
+
const out = {}
|
|
257
|
+
for (const n of names) out[n] = await readFile(join(exportDir(), n))
|
|
258
|
+
return out
|
|
259
|
+
}
|
|
260
|
+
|
|
261
|
+
/**
|
|
262
|
+
* 把导出解析成内存结构。videos 用与列表一致的瘦身形状,detail 保留原始记录
|
|
263
|
+
* 供详情页使用(summary / why_hn / quote / tags / categories 都在里面)。
|
|
264
|
+
*/
|
|
265
|
+
async function loadDataset() {
|
|
266
|
+
if (state.dataset) return state.dataset
|
|
267
|
+
if (state.datasetP) return state.datasetP
|
|
268
|
+
state.datasetP = (async () => {
|
|
269
|
+
const files = await readExport(['videos.jsonl', 'related.jsonl'])
|
|
270
|
+
const videos = []
|
|
271
|
+
const detail = new Map()
|
|
272
|
+
for (const line of files['videos.jsonl'].toString('utf8').split('\n')) {
|
|
273
|
+
if (!line || line.charCodeAt(0) !== 123) continue
|
|
274
|
+
let o
|
|
275
|
+
try {
|
|
276
|
+
o = JSON.parse(line)
|
|
277
|
+
} catch {
|
|
278
|
+
continue
|
|
279
|
+
}
|
|
280
|
+
if (!o || !o.vid) continue
|
|
281
|
+
detail.set(o.vid, o)
|
|
282
|
+
videos.push(toSlim(o))
|
|
283
|
+
}
|
|
284
|
+
if (!videos.length) throw new Error('导出里没有视频')
|
|
285
|
+
|
|
286
|
+
const related = new Map()
|
|
287
|
+
for (const line of files['related.jsonl'].toString('utf8').split('\n')) {
|
|
288
|
+
if (!line || line.charCodeAt(0) !== 123) continue
|
|
289
|
+
let o
|
|
290
|
+
try {
|
|
291
|
+
o = JSON.parse(line)
|
|
292
|
+
} catch {
|
|
293
|
+
continue
|
|
294
|
+
}
|
|
295
|
+
if (!o || !o.vid || !Array.isArray(o.related)) continue
|
|
296
|
+
related.set(
|
|
297
|
+
o.vid,
|
|
298
|
+
o.related.map((r) => r.vid).filter(Boolean),
|
|
299
|
+
)
|
|
300
|
+
}
|
|
301
|
+
|
|
302
|
+
state.dataset = { videos, detail, related, at: Date.now() }
|
|
303
|
+
void writeCache('export/stats.json', { at: Date.now(), videos: videos.length, related: related.size })
|
|
304
|
+
return state.dataset
|
|
305
|
+
})()
|
|
306
|
+
try {
|
|
307
|
+
return await state.datasetP
|
|
308
|
+
} finally {
|
|
309
|
+
state.datasetP = null
|
|
310
|
+
}
|
|
311
|
+
}
|
|
312
|
+
|
|
313
|
+
/** HN 评论按 vid 归组,按推荐力度排序后截取前 12 条 */
|
|
314
|
+
const STANCE_RANK = { strong_rec: 0, rec: 1, reference: 2, joke: 3, critical: 4 }
|
|
315
|
+
|
|
316
|
+
async function loadMentions() {
|
|
317
|
+
if (state.mentions) return state.mentions
|
|
318
|
+
if (state.mentionsP) return state.mentionsP
|
|
319
|
+
state.mentionsP = (async () => {
|
|
320
|
+
const files = await readExport(['mentions.jsonl'])
|
|
321
|
+
const map = new Map()
|
|
322
|
+
for (const line of files['mentions.jsonl'].toString('utf8').split('\n')) {
|
|
323
|
+
if (!line || line.charCodeAt(0) !== 123) continue
|
|
324
|
+
let m
|
|
325
|
+
try {
|
|
326
|
+
m = JSON.parse(line)
|
|
327
|
+
} catch {
|
|
328
|
+
continue
|
|
329
|
+
}
|
|
330
|
+
if (!m || !m.vid) continue
|
|
331
|
+
const bucket = map.get(m.vid)
|
|
332
|
+
if (bucket) bucket.push(m)
|
|
333
|
+
else map.set(m.vid, [m])
|
|
334
|
+
}
|
|
335
|
+
state.mentions = map
|
|
336
|
+
return map
|
|
337
|
+
})()
|
|
338
|
+
try {
|
|
339
|
+
return await state.mentionsP
|
|
340
|
+
} finally {
|
|
341
|
+
state.mentionsP = null
|
|
342
|
+
}
|
|
343
|
+
}
|
|
344
|
+
|
|
345
|
+
function commentsFromMentions(list) {
|
|
346
|
+
return (list || [])
|
|
347
|
+
.slice()
|
|
348
|
+
.sort((a, b) => {
|
|
349
|
+
const ra = STANCE_RANK[a.stance] ?? 9
|
|
350
|
+
const rb = STANCE_RANK[b.stance] ?? 9
|
|
351
|
+
if (ra !== rb) return ra - rb
|
|
352
|
+
return String(b.month || '').localeCompare(String(a.month || ''))
|
|
353
|
+
})
|
|
354
|
+
.slice(0, 12)
|
|
355
|
+
.map((m) => ({
|
|
356
|
+
id: m.hn_comment_id,
|
|
357
|
+
author: m.author || '',
|
|
358
|
+
ym: m.month || '',
|
|
359
|
+
stance: m.stance || '',
|
|
360
|
+
why: m.why || '',
|
|
361
|
+
text: String(m.text || '').replace(/\s+/g, ' ').slice(0, 400),
|
|
362
|
+
quote: m.quote ? String(m.quote).replace(/\s+/g, ' ').slice(0, 240) : '',
|
|
363
|
+
}))
|
|
364
|
+
}
|
|
365
|
+
|
|
366
|
+
/* 语义近邻:导出里的 384 维 float16 向量,用于 related.jsonl 没覆盖的视频 */
|
|
367
|
+
const npyState = { ids: null, index: null, vec: null, rows: 0, dim: 0 }
|
|
368
|
+
|
|
369
|
+
/* float16 → float32 查表,比逐元素 2**exp 快两个数量级 */
|
|
370
|
+
const F16 = (() => {
|
|
371
|
+
const t = new Float32Array(65536)
|
|
372
|
+
for (let h = 0; h < 65536; h += 1) {
|
|
373
|
+
const sign = h & 0x8000 ? -1 : 1
|
|
374
|
+
const exp = (h >> 10) & 0x1f
|
|
375
|
+
const frac = h & 0x3ff
|
|
376
|
+
let v
|
|
377
|
+
if (exp === 0) v = frac * 2 ** -24
|
|
378
|
+
else if (exp === 31) v = 0
|
|
379
|
+
else v = (1 + frac / 1024) * 2 ** (exp - 15)
|
|
380
|
+
t[h] = sign * v
|
|
381
|
+
}
|
|
382
|
+
return t
|
|
383
|
+
})()
|
|
384
|
+
|
|
385
|
+
async function loadVectors() {
|
|
386
|
+
if (npyState.vec) return npyState
|
|
387
|
+
const files = await readExport(['embeddings.npy', 'embedding_ids.json'])
|
|
388
|
+
const npy = files['embeddings.npy']
|
|
389
|
+
if (npy.toString('latin1', 1, 7) !== 'NUMPY') throw new Error('embeddings.npy 头不对')
|
|
390
|
+
const major = npy[6]
|
|
391
|
+
const headerLen = major === 1 ? npy.readUInt16LE(8) : npy.readUInt32LE(8)
|
|
392
|
+
const headerStart = major === 1 ? 10 : 12
|
|
393
|
+
const header = npy.toString('latin1', headerStart, headerStart + headerLen)
|
|
394
|
+
const m = header.match(/shape'\s*:\s*\((\d+)\s*,\s*(\d+)\)/)
|
|
395
|
+
if (!m) throw new Error('embeddings.npy 缺少 shape')
|
|
396
|
+
const rows = Number(m[1])
|
|
397
|
+
const dim = Number(m[2])
|
|
398
|
+
const dataStart = headerStart + headerLen
|
|
399
|
+
const vec = new Float32Array(rows * dim)
|
|
400
|
+
/* Uint16 视图比逐字节 readUInt16LE 快一个数量级;x86 上小端直接可用 */
|
|
401
|
+
const aligned = (npy.byteOffset + dataStart) % 2 === 0
|
|
402
|
+
if (aligned) {
|
|
403
|
+
const u16 = new Uint16Array(npy.buffer, npy.byteOffset + dataStart, rows * dim)
|
|
404
|
+
for (let i = 0; i < rows * dim; i += 1) vec[i] = F16[u16[i]]
|
|
405
|
+
} else {
|
|
406
|
+
for (let i = 0; i < rows * dim; i += 1) {
|
|
407
|
+
vec[i] = F16[npy.readUInt16LE(dataStart + i * 2)]
|
|
408
|
+
}
|
|
409
|
+
}
|
|
410
|
+
const ids = JSON.parse(files['embedding_ids.json'].toString('utf8'))
|
|
411
|
+
/* 向量是 L2 归一化的,模长预先算好,检索时只做点积 */
|
|
412
|
+
const norms = new Float32Array(rows)
|
|
413
|
+
for (let i = 0; i < rows; i += 1) {
|
|
414
|
+
let n = 0
|
|
415
|
+
const base = i * dim
|
|
416
|
+
for (let d = 0; d < dim; d += 1) n += vec[base + d] * vec[base + d]
|
|
417
|
+
norms[i] = Math.sqrt(n) || 1
|
|
418
|
+
}
|
|
419
|
+
npyState.ids = ids
|
|
420
|
+
npyState.norms = norms
|
|
421
|
+
npyState.index = new Map(ids.map((vid, i) => [vid, i]))
|
|
422
|
+
npyState.vec = vec
|
|
423
|
+
npyState.rows = rows
|
|
424
|
+
npyState.dim = dim
|
|
425
|
+
return npyState
|
|
426
|
+
}
|
|
427
|
+
|
|
428
|
+
async function relatedFromVectors(vid, n) {
|
|
429
|
+
const s = await loadVectors()
|
|
430
|
+
const at = s.index.get(vid)
|
|
431
|
+
if (at === undefined) return []
|
|
432
|
+
const { vec, dim, rows, norms } = s
|
|
433
|
+
const off = at * dim
|
|
434
|
+
const an = norms[at] || 1
|
|
435
|
+
const scores = new Float32Array(rows)
|
|
436
|
+
for (let i = 0; i < rows; i += 1) {
|
|
437
|
+
if (i === at) {
|
|
438
|
+
scores[i] = -1
|
|
439
|
+
continue
|
|
440
|
+
}
|
|
441
|
+
const base = i * dim
|
|
442
|
+
let dot = 0
|
|
443
|
+
for (let d = 0; d < dim; d += 1) dot += vec[off + d] * vec[base + d]
|
|
444
|
+
scores[i] = dot / (an * (norms[i] || 1))
|
|
445
|
+
}
|
|
446
|
+
const order = Array.from({ length: rows }, (_, i) => i).sort((a, b) => scores[b] - scores[a])
|
|
447
|
+
const out = []
|
|
448
|
+
for (const i of order.slice(0, n)) {
|
|
449
|
+
const id = s.ids[i]
|
|
450
|
+
if (id && id !== vid) out.push(id)
|
|
451
|
+
}
|
|
452
|
+
return out
|
|
453
|
+
}
|
|
454
|
+
|
|
455
|
+
/* ---------------- 三个数据方法 ---------------- */
|
|
456
|
+
|
|
457
|
+
/** 单页抓取,带重试;offset 定位,n 是每页条数(上限 100)。 */
|
|
458
|
+
async function fetchPage(page) {
|
|
459
|
+
let lastError
|
|
460
|
+
const offset = page * PAGE_SIZE
|
|
461
|
+
for (let attempt = 0; attempt < PAGE_RETRIES; attempt += 1) {
|
|
462
|
+
try {
|
|
463
|
+
return await fetchJson(`${SOURCE}/api/list?offset=${offset}&n=${PAGE_SIZE}`)
|
|
464
|
+
} catch (err) {
|
|
465
|
+
lastError = err
|
|
466
|
+
if (attempt + 1 < PAGE_RETRIES) {
|
|
467
|
+
await new Promise((r) => setTimeout(r, 250 * (attempt + 1)))
|
|
468
|
+
}
|
|
469
|
+
}
|
|
470
|
+
}
|
|
471
|
+
throw lastError
|
|
472
|
+
}
|
|
473
|
+
|
|
474
|
+
/**
|
|
475
|
+
* 全量列表。videos.jsonl 虽然一次给 6454 条,但没有 thumbnail / published /
|
|
476
|
+
* duration_s / views,卡片会缺封面和时长;只有 /api/list 带齐这些字段,
|
|
477
|
+
* 代价是按 offset + n 翻完 65 页。gzip 后每页约 28KB。
|
|
478
|
+
*/
|
|
479
|
+
async function fetchAllSlim() {
|
|
480
|
+
let total = FALLBACK_TOTAL
|
|
481
|
+
try {
|
|
482
|
+
const stats = await fetchJson(`${SOURCE}/api/stats`)
|
|
483
|
+
if (stats && typeof stats.videos === 'number' && stats.videos > 0) total = stats.videos
|
|
484
|
+
} catch {
|
|
485
|
+
/* 拿不到统计就用已知总数 */
|
|
486
|
+
}
|
|
487
|
+
|
|
488
|
+
const pages = Math.ceil(total / PAGE_SIZE)
|
|
489
|
+
const deadline = Date.now() + TOTAL_BUDGET_MS
|
|
490
|
+
const out = []
|
|
491
|
+
const failed = []
|
|
492
|
+
let cursor = 0
|
|
493
|
+
|
|
494
|
+
const absorb = (page, value) => {
|
|
495
|
+
if (!Array.isArray(value)) {
|
|
496
|
+
failed.push(page)
|
|
497
|
+
return
|
|
498
|
+
}
|
|
499
|
+
for (const o of value) out.push(toSlim(o))
|
|
500
|
+
}
|
|
501
|
+
|
|
502
|
+
while (cursor < pages) {
|
|
503
|
+
if (Date.now() > deadline) throw new Error(`抓取超时:${out.length} / ${total}`)
|
|
504
|
+
const batch = []
|
|
505
|
+
for (let i = 0; i < PAGE_CONCURRENCY && cursor + i < pages; i += 1) {
|
|
506
|
+
batch.push(cursor + i)
|
|
507
|
+
}
|
|
508
|
+
const settled = await Promise.allSettled(batch.map((p) => fetchPage(p)))
|
|
509
|
+
settled.forEach((r, i) => {
|
|
510
|
+
if (r.status === 'fulfilled') absorb(batch[i], r.value)
|
|
511
|
+
else failed.push(batch[i])
|
|
512
|
+
})
|
|
513
|
+
cursor += PAGE_CONCURRENCY
|
|
514
|
+
}
|
|
515
|
+
|
|
516
|
+
/* 并发批里偶发失败的页单独补一次,别因为一页丢一百条 */
|
|
517
|
+
if (failed.length) {
|
|
518
|
+
for (const page of failed.slice()) {
|
|
519
|
+
try {
|
|
520
|
+
absorb(page, await fetchPage(page))
|
|
521
|
+
const at = failed.indexOf(page)
|
|
522
|
+
if (at >= 0) failed.splice(at, 1)
|
|
523
|
+
} catch {
|
|
524
|
+
/* 仍失败则记在 failed 里 */
|
|
525
|
+
}
|
|
526
|
+
}
|
|
527
|
+
}
|
|
528
|
+
|
|
529
|
+
const coverage = total ? out.length / total : 0
|
|
530
|
+
if (coverage < 0.95) {
|
|
531
|
+
const err = new Error(
|
|
532
|
+
`列表不完整:拿到 ${out.length} / ${total},${failed.length} 页始终失败`,
|
|
533
|
+
)
|
|
534
|
+
err.partialCount = out.length
|
|
535
|
+
throw err
|
|
536
|
+
}
|
|
537
|
+
return out
|
|
538
|
+
}
|
|
539
|
+
|
|
540
|
+
/** 把 YouTube 元信息(时长/播放量/发布时间/封面)并进导出数据,导出本身没有这几个字段 */
|
|
541
|
+
function mergeMeta(items, metaItems) {
|
|
542
|
+
if (!Array.isArray(metaItems) || !metaItems.length) return items
|
|
543
|
+
const byVid = new Map()
|
|
544
|
+
for (const m of metaItems) if (m && m.v) byVid.set(m.v, m)
|
|
545
|
+
let changed = false
|
|
546
|
+
const out = items.map((it) => {
|
|
547
|
+
const m = byVid.get(it.v)
|
|
548
|
+
if (!m) return it
|
|
549
|
+
if (!(m.d || m.w || m.p || m.th)) return it
|
|
550
|
+
changed = true
|
|
551
|
+
return {
|
|
552
|
+
...it,
|
|
553
|
+
d: m.d || it.d,
|
|
554
|
+
th: it.th || m.th,
|
|
555
|
+
p: m.p || it.p,
|
|
556
|
+
w: m.w || it.w,
|
|
557
|
+
y: it.y || m.y,
|
|
558
|
+
}
|
|
559
|
+
})
|
|
560
|
+
return changed ? out : items
|
|
561
|
+
}
|
|
562
|
+
|
|
563
|
+
async function loadList() {
|
|
564
|
+
const fresh = state.list && Date.now() - state.listAt < TTL_MS
|
|
565
|
+
if (fresh) return state.list
|
|
566
|
+
|
|
567
|
+
if (!state.list) {
|
|
568
|
+
const cached = await readCache('list.json')
|
|
569
|
+
if (
|
|
570
|
+
cached &&
|
|
571
|
+
cached.v === CACHE_VERSION &&
|
|
572
|
+
Array.isArray(cached.items) &&
|
|
573
|
+
cached.items.length
|
|
574
|
+
) {
|
|
575
|
+
setList(cached.items, Number(cached.at) || 0)
|
|
576
|
+
if (Date.now() - state.listAt < TTL_MS) {
|
|
577
|
+
/* 命中缓存时也把导出与评论解析放到后台,打开第一个视频就不用等 */
|
|
578
|
+
void loadDataset()
|
|
579
|
+
.then(() => loadMentions())
|
|
580
|
+
.catch(() => {})
|
|
581
|
+
return state.list
|
|
582
|
+
}
|
|
583
|
+
}
|
|
584
|
+
}
|
|
585
|
+
|
|
586
|
+
return once('list', async () => {
|
|
587
|
+
/* 冷启动并行做两件事:吃官方导出、补 YouTube 元信息。
|
|
588
|
+
导出给内容,/api/list 只给时长、播放量、发布时间和封面。 */
|
|
589
|
+
const metaTask = fetchAllSlim()
|
|
590
|
+
.then((items) => {
|
|
591
|
+
setList(items, Date.now())
|
|
592
|
+
void writeCache('list.json', { v: CACHE_VERSION, at: Date.now(), items })
|
|
593
|
+
return items
|
|
594
|
+
})
|
|
595
|
+
.catch((err) => {
|
|
596
|
+
void writeCache('diag.json', {
|
|
597
|
+
at: Date.now(),
|
|
598
|
+
stage: 'list',
|
|
599
|
+
error: String(err?.message ?? err),
|
|
600
|
+
})
|
|
601
|
+
return null
|
|
602
|
+
})
|
|
603
|
+
|
|
604
|
+
let ds = null
|
|
605
|
+
try {
|
|
606
|
+
ds = await loadDataset()
|
|
607
|
+
} catch (err) {
|
|
608
|
+
void writeCache('diag.json', {
|
|
609
|
+
at: Date.now(),
|
|
610
|
+
stage: 'export',
|
|
611
|
+
error: String(err?.message ?? err),
|
|
612
|
+
})
|
|
613
|
+
}
|
|
614
|
+
|
|
615
|
+
const meta = await metaTask
|
|
616
|
+
|
|
617
|
+
if (ds) {
|
|
618
|
+
const merged = meta ? mergeMeta(ds.videos, meta) : ds.videos
|
|
619
|
+
setList(merged, Date.now())
|
|
620
|
+
/* 评论要等第一次打开视频才用到,先在后台上把 mentions 解析好 */
|
|
621
|
+
void loadMentions().catch(() => {})
|
|
622
|
+
return state.list
|
|
623
|
+
}
|
|
624
|
+
if (meta) return state.list
|
|
625
|
+
if (state.list) return state.list
|
|
626
|
+
throw new Error('列表不可用:导出与上游都失败')
|
|
627
|
+
})
|
|
628
|
+
}
|
|
629
|
+
|
|
630
|
+
/**
|
|
631
|
+
* 27 个播放列表的中文标题与简介。上游只给英文,这里内置一份人工译文,
|
|
632
|
+
* 比让模型临时翻更稳,也不会因为一次调用失败而在界面上露出英文。
|
|
633
|
+
*/
|
|
634
|
+
const PLAYLIST_ZH = {
|
|
635
|
+
"hn-canon": [
|
|
636
|
+
"HN 必看清单",
|
|
637
|
+
"Hacker News 最常被推荐给彼此的十个视频,一个帖子接一个帖子,年复一年。数学、金钱、编程、芯片、生物,还有怎么活。"
|
|
638
|
+
],
|
|
639
|
+
"talks-every-programmer-should-watch": [
|
|
640
|
+
"每个程序员都该看的演讲",
|
|
641
|
+
"HN 反复推荐给开发者看的编程演讲,不论你写什么语言。关于简洁、设计、手艺,以及如何思考软件。"
|
|
642
|
+
],
|
|
643
|
+
"computers-from-the-transistor-up": [
|
|
644
|
+
"从晶体管讲起的计算机",
|
|
645
|
+
"一层一层讲清计算机怎么工作:晶体管与逻辑门、芯片怎么造、CPU 与内存、操作系统,最后是编译器与语言。"
|
|
646
|
+
],
|
|
647
|
+
"learn-deep-learning-from-zero": [
|
|
648
|
+
"从零学深度学习",
|
|
649
|
+
"从「什么是神经元」到亲手用代码搭一个 GPT 的学习路径,每一步都用 HN 最推荐的那个讲解。"
|
|
650
|
+
],
|
|
651
|
+
"programming-history-in-12-talks": [
|
|
652
|
+
"十二场演讲里的编程史",
|
|
653
|
+
"用 HN 推荐的演讲与影像讲编程的故事:从战时的密码破译机器,到 Unix、PARC、家用电脑热潮、自由软件与万维网。"
|
|
654
|
+
],
|
|
655
|
+
"lectures-worth-a-weekend": [
|
|
656
|
+
"值得一个周末的讲座",
|
|
657
|
+
"HN 一直劝人去看的课堂录像:Feynman、SICP、Strang、Sapolsky、Sandel 等好老师,每人一场开篇讲座。"
|
|
658
|
+
],
|
|
659
|
+
"documentaries-hn-loves": [
|
|
660
|
+
"HN 爱看的纪录片",
|
|
661
|
+
"适合慢悠悠一个晚上的长篇纪录片:AlphaGo、Connections、Aaron Swartz、Adam Curtis 等等,横跨技术、科学、历史与人。"
|
|
662
|
+
],
|
|
663
|
+
"how-things-are-made": [
|
|
664
|
+
"东西是怎么造出来的",
|
|
665
|
+
"工厂参观与制造讲解,专治「这玩意儿到底怎么做出来的」:从易拉罐到 EUV 光刻机,再到火箭工厂。"
|
|
666
|
+
],
|
|
667
|
+
"security-war-stories": [
|
|
668
|
+
"安全圈的血泪史",
|
|
669
|
+
"真实的入侵、骗局与数据泄露,由干过的人或追查过的人亲口讲。从 1986 年追捕克格勃间谍,到几乎波及每台 Linux 服务器的 xz 后门。"
|
|
670
|
+
],
|
|
671
|
+
"systems-and-databases": [
|
|
672
|
+
"系统与数据库",
|
|
673
|
+
"HN 推荐来理解真实系统怎么运转的硬核技术演讲:数据库、共识、分布式代码测试、性能,以及怎么让生产环境不倒。"
|
|
674
|
+
],
|
|
675
|
+
"hidden-gems": [
|
|
676
|
+
"被埋没的好东西",
|
|
677
|
+
"HN 很喜欢、YouTube 几乎没注意到的东西,多数远不到五万播放。物理、音乐、航天器、遗传学、能源,还有软件史。"
|
|
678
|
+
],
|
|
679
|
+
"comedy-break-for-programmers": [
|
|
680
|
+
"程序员的喜剧时间",
|
|
681
|
+
"HN 帖子里用来逗乐的短剧、恶搞与荒诞演讲:从需求根本不可能实现的会议,到用 PowerPoint 搭出来的图灵机。"
|
|
682
|
+
],
|
|
683
|
+
"indistinguishable-from-magic": [
|
|
684
|
+
"与魔法无异",
|
|
685
|
+
"从厨房桌上的一只晶体管,到 ASML 机器里的锡滴激光,再讲到为这套东西争抢的人与国家。"
|
|
686
|
+
],
|
|
687
|
+
"mechanical-sympathy": [
|
|
688
|
+
"机械同理心",
|
|
689
|
+
"为什么你的快电脑跑着慢代码,以及写快代码的人对缓存、数据布局和测量知道些什么。"
|
|
690
|
+
],
|
|
691
|
+
"things-you-never-look-at": [
|
|
692
|
+
"你从没细看过东西",
|
|
693
|
+
"门、插头、易拉罐、烤面包机和转向灯,由显然很爱这些东西的人来讲。关于日常物品的短片设计随笔。"
|
|
694
|
+
],
|
|
695
|
+
"debugging-at-a-distance": [
|
|
696
|
+
"远程调试",
|
|
697
|
+
"从阿波罗导航计算机的 1202 警报,到隔着 150 亿英里给旅行者一号打补丁:史上最难的远程调试,由干过的人或重建过那套硬件的人讲述。"
|
|
698
|
+
],
|
|
699
|
+
"brass-brains": [
|
|
700
|
+
"黄铜大脑:用齿轮计算",
|
|
701
|
+
"晶体管之前,人们用齿轮、凸轮和水做微积分。从一枚差速齿轮,一路讲到为战列舰瞄准、预测潮汐、跑经济模型的机器。"
|
|
702
|
+
],
|
|
703
|
+
"objects-on-trial": [
|
|
704
|
+
"面向对象受审记",
|
|
705
|
+
"HN 吵了十五年的 OOP 之争,这次排成一场庭审:辩方、控方、创始人的证词,还有两小时的结案陈词。"
|
|
706
|
+
],
|
|
707
|
+
"before-it-worked": [
|
|
708
|
+
"在它成真之前",
|
|
709
|
+
"用领域起飞之前录下的演讲讲 AI 史:从 1973 年那场促成了第一次 AI 寒冬的辩论,到 2017 年把这场热潮称作炼金术的演讲。"
|
|
710
|
+
],
|
|
711
|
+
"how-to-count-a-vote": [
|
|
712
|
+
"怎么数一张选票",
|
|
713
|
+
"投票是一种算法,HN 就按算法在吵。从为什么简单多数养出两党制,到 Arrow 定理与评分投票,再到杰利蝾螈,以及为什么纸质选票仍然胜过计算机。"
|
|
714
|
+
],
|
|
715
|
+
"speedrunners-are-reverse-engineers": [
|
|
716
|
+
"速通玩家都是逆向工程师",
|
|
717
|
+
"从马里奥赛车里的一条捷径,到只用一只手柄在《超级马里奥世界》里跑《Flappy Bird》:玩家是怎么把自己的游戏拆开的。"
|
|
718
|
+
],
|
|
719
|
+
"cheating-the-hardware": [
|
|
720
|
+
"骗过硬件",
|
|
721
|
+
"游戏开发者怎么绕开可怜的内存、稀少的颜色和慢慢转的磁盘,从《爆破彗星》讲到《古惑狼》,大致按时间顺序。"
|
|
722
|
+
],
|
|
723
|
+
"everything-is-circles": [
|
|
724
|
+
"万物皆圆",
|
|
725
|
+
"一个想法——傅里叶变换——从用本轮画出荷马·辛普森,一路跟到 JPEG、数字音频、无线电、镜头和测不准原理。"
|
|
726
|
+
],
|
|
727
|
+
"biology-for-programmers": [
|
|
728
|
+
"给程序员看的生物学",
|
|
729
|
+
"细胞是纳米级机加工车间,生物电网络存着身体蓝图,遗传病被当成漏洞利用,还有一门用来设计分子的 Lisp 编译器。用 HN 的眼光看生物学:这是世界上最奇怪的代码库。"
|
|
730
|
+
],
|
|
731
|
+
"who-watches-the-silicon": [
|
|
732
|
+
"硅片由谁看守",
|
|
733
|
+
"你的威胁模型大概止步于操作系统。这些演讲往下再挖一层:没有文档的 CPU 指令、隐藏的协处理器、恶意 USB 固件,以及主板上的植入物。"
|
|
734
|
+
],
|
|
735
|
+
"how-things-fall-over": [
|
|
736
|
+
"东西是怎么塌的",
|
|
737
|
+
"被删掉的数据库、一次核事故、因为一个笔误被重启的数据中心,以及安全科学里那些解释「这一切都很正常」的概念。"
|
|
738
|
+
],
|
|
739
|
+
"small-profitable-nobodys-unicorn": [
|
|
740
|
+
"小而赚钱,不当独角兽",
|
|
741
|
+
"HN 发给每个想做生意而不是造火箭的人:DHH 2008 年那篇异端,Jason Cohen 的白手起家手册,Maciej 谈勉强成功,还有一家从 1994 年活到今天的游戏工作室。"
|
|
742
|
+
]
|
|
743
|
+
}
|
|
744
|
+
|
|
745
|
+
function withChinese(p) {
|
|
746
|
+
if (!p || !p.slug) return p
|
|
747
|
+
const z = PLAYLIST_ZH[p.slug]
|
|
748
|
+
if (!z) return p
|
|
749
|
+
return Object.assign({}, p, { titleZh: z[0], pitchZh: z[1] })
|
|
750
|
+
}
|
|
751
|
+
|
|
752
|
+
/**
|
|
753
|
+
* 站点策划的播放列表。上游一次给全部 27 个(标题、简介、封面、时长),
|
|
754
|
+
* 详情按 slug 单独取,缓存 30 分钟。
|
|
755
|
+
*/
|
|
756
|
+
async function loadPlaylists(force = false) {
|
|
757
|
+
const fresh = state.playlists && Date.now() - state.playlistsAt < TTL_MS
|
|
758
|
+
if (fresh && !force) return state.playlists
|
|
759
|
+
return once('playlists', async () => {
|
|
760
|
+
const list = await fetchJson(`${SOURCE}/api/playlists`)
|
|
761
|
+
state.playlists = Array.isArray(list) ? list : []
|
|
762
|
+
state.playlistsAt = Date.now()
|
|
763
|
+
return state.playlists
|
|
764
|
+
})
|
|
765
|
+
}
|
|
766
|
+
|
|
767
|
+
async function loadPlaylist(slug) {
|
|
768
|
+
const hit = state.playlistDetail.get(slug)
|
|
769
|
+
if (hit && Date.now() - hit.at < TTL_MS) return hit.value
|
|
770
|
+
const value = await fetchJson(`${SOURCE}/api/playlist?slug=${encodeURIComponent(slug)}`)
|
|
771
|
+
state.playlistDetail.set(slug, { at: Date.now(), value })
|
|
772
|
+
return value
|
|
773
|
+
}
|
|
774
|
+
|
|
775
|
+
async function loadBrowse() {
|
|
776
|
+
const fresh = state.browse && Date.now() - state.browseAt < TTL_MS
|
|
777
|
+
if (fresh) return state.browse
|
|
778
|
+
|
|
779
|
+
if (!state.browse) {
|
|
780
|
+
const cached = await readCache('browse.json')
|
|
781
|
+
if (cached && cached.taxonomy) {
|
|
782
|
+
state.browse = cached.taxonomy
|
|
783
|
+
state.browseAt = Number(cached.at) || 0
|
|
784
|
+
if (Date.now() - state.browseAt < TTL_MS) return state.browse
|
|
785
|
+
}
|
|
786
|
+
}
|
|
787
|
+
|
|
788
|
+
return once('browse', async () => {
|
|
789
|
+
try {
|
|
790
|
+
const data = await fetchJson(`${SOURCE}/download/taxonomy.json`)
|
|
791
|
+
state.browse = data
|
|
792
|
+
state.browseAt = Date.now()
|
|
793
|
+
state.browseJson = null
|
|
794
|
+
void writeCache('browse.json', { at: state.browseAt, taxonomy: data })
|
|
795
|
+
return data
|
|
796
|
+
} catch (err) {
|
|
797
|
+
if (state.browse) return state.browse
|
|
798
|
+
throw err
|
|
799
|
+
}
|
|
800
|
+
})
|
|
801
|
+
}
|
|
802
|
+
|
|
803
|
+
/**
|
|
804
|
+
* 详情页只展示 12 条:每条带作者、年月、立场、一句为什么。
|
|
805
|
+
* 正文和引用可能很长,截到 400 字,避免一次响应过大。
|
|
806
|
+
*/
|
|
807
|
+
function pickComments(vid) {
|
|
808
|
+
const list = state.comments.get(vid) || []
|
|
809
|
+
return list.slice(0, 12).map((c) => ({
|
|
810
|
+
id: c.id,
|
|
811
|
+
author: c.author || '',
|
|
812
|
+
ym: c.ym || '',
|
|
813
|
+
stance: c.stance || '',
|
|
814
|
+
why: c.why || '',
|
|
815
|
+
text: String(c.text || '').replace(/\s+/g, ' ').slice(0, 400),
|
|
816
|
+
quote: c.quote ? String(c.quote).replace(/\s+/g, ' ').slice(0, 240) : '',
|
|
817
|
+
}))
|
|
818
|
+
}
|
|
819
|
+
|
|
820
|
+
/**
|
|
821
|
+
* 相关视频:上游按 HN 共同推荐度排序。详情页右栏用,最多 12 条。
|
|
822
|
+
*/
|
|
823
|
+
async function loadRelated(vid) {
|
|
824
|
+
const hit = state.related.get(vid)
|
|
825
|
+
if (hit && Date.now() - hit.at < TTL_MS) return hit.value
|
|
826
|
+
/* 先查导出里的共推荐图,没有再退到向量近邻,最后才问上游 */
|
|
827
|
+
try {
|
|
828
|
+
const ds = await loadDataset()
|
|
829
|
+
let ids = ds.related.get(vid) || []
|
|
830
|
+
if (!ids.length) ids = await relatedFromVectors(vid, 14).catch(() => [])
|
|
831
|
+
if (ids.length) {
|
|
832
|
+
const source = state.list && state.list.length ? state.list : ds.videos
|
|
833
|
+
const byId = new Map(source.map((it) => [it.v, it]))
|
|
834
|
+
const value = []
|
|
835
|
+
for (const id of ids) {
|
|
836
|
+
const it = byId.get(id)
|
|
837
|
+
if (it && value.length < 12) value.push(it)
|
|
838
|
+
}
|
|
839
|
+
if (value.length) {
|
|
840
|
+
state.related.set(vid, { at: Date.now(), value })
|
|
841
|
+
return value
|
|
842
|
+
}
|
|
843
|
+
}
|
|
844
|
+
} catch {
|
|
845
|
+
/* 导出不可用时退回上游 */
|
|
846
|
+
}
|
|
847
|
+
const list = await fetchJson(`${SOURCE}/api/related?vid=${encodeURIComponent(vid)}&n=12`)
|
|
848
|
+
const value = (Array.isArray(list) ? list : []).slice(0, 12).map(toSlim)
|
|
849
|
+
state.related.set(vid, { at: Date.now(), value })
|
|
850
|
+
return value
|
|
851
|
+
}
|
|
852
|
+
|
|
853
|
+
async function loadVideo(vid) {
|
|
854
|
+
const hit = state.detail.get(vid)
|
|
855
|
+
if (hit) return hit
|
|
856
|
+
/* 导出里带着 summary / why_hn / quote / tags / categories,详情页不用再联网 */
|
|
857
|
+
try {
|
|
858
|
+
const ds = await loadDataset()
|
|
859
|
+
const rec = ds.detail.get(vid)
|
|
860
|
+
if (rec) {
|
|
861
|
+
/* 导出里的引文用 hn_comment_id,前端要的是 id,这里统一 */
|
|
862
|
+
if (rec.quote && typeof rec.quote === 'object' && rec.quote.hn_comment_id && !rec.quote.id) {
|
|
863
|
+
rec.quote = { ...rec.quote, id: rec.quote.hn_comment_id }
|
|
864
|
+
}
|
|
865
|
+
const mentions = await loadMentions().catch(() => null)
|
|
866
|
+
state.comments.set(vid, commentsFromMentions(mentions ? mentions.get(vid) : null))
|
|
867
|
+
rememberDetail(vid, rec)
|
|
868
|
+
return rec
|
|
869
|
+
}
|
|
870
|
+
} catch {
|
|
871
|
+
/* 导出不可用时退回上游接口 */
|
|
872
|
+
}
|
|
873
|
+
const data = await fetchJson(`${SOURCE}/api/video?vid=${encodeURIComponent(vid)}`)
|
|
874
|
+
const payload = data && data.video ? data.video : data
|
|
875
|
+
const comments = data && Array.isArray(data.comments) ? data.comments : []
|
|
876
|
+
state.comments.set(vid, comments)
|
|
877
|
+
rememberDetail(vid, payload)
|
|
878
|
+
return payload
|
|
879
|
+
}
|
|
880
|
+
|
|
881
|
+
function rememberDetail(vid, payload) {
|
|
882
|
+
if (state.detail.size >= DETAIL_CACHE_MAX) {
|
|
883
|
+
const oldest = state.detail.keys().next().value
|
|
884
|
+
if (oldest !== undefined) state.detail.delete(oldest)
|
|
885
|
+
}
|
|
886
|
+
state.detail.set(vid, payload)
|
|
887
|
+
}
|
|
888
|
+
|
|
889
|
+
/* ---------------- 本地媒体 ---------------- */
|
|
890
|
+
|
|
891
|
+
function mediaDir() {
|
|
892
|
+
return join(cacheRoot(), 'media')
|
|
893
|
+
}
|
|
894
|
+
|
|
895
|
+
/** 调 yt-dlp 落盘:480p 合并 mp4,音视频轨由 yt-dlp 自行调 ffmpeg 合并。 */
|
|
896
|
+
function downloadMedia(vid, dest) {
|
|
897
|
+
return new Promise((resolve, reject) => {
|
|
898
|
+
const args = [
|
|
899
|
+
'-f',
|
|
900
|
+
`bv*[height<=${MEDIA_MAX_HEIGHT}]+ba/b[height<=${MEDIA_MAX_HEIGHT}]`,
|
|
901
|
+
'--merge-output-format',
|
|
902
|
+
'mp4',
|
|
903
|
+
'--no-warnings',
|
|
904
|
+
'--no-playlist',
|
|
905
|
+
'-o',
|
|
906
|
+
dest,
|
|
907
|
+
`https://www.youtube.com/watch?v=${vid}`,
|
|
908
|
+
]
|
|
909
|
+
const child = spawn('yt-dlp', args, { windowsHide: true })
|
|
910
|
+
let stderr = ''
|
|
911
|
+
const timer = setTimeout(() => {
|
|
912
|
+
try {
|
|
913
|
+
child.kill()
|
|
914
|
+
} catch {
|
|
915
|
+
/* 已退出 */
|
|
916
|
+
}
|
|
917
|
+
reject(new Error('媒体下载超时'))
|
|
918
|
+
}, MEDIA_TIMEOUT_MS)
|
|
919
|
+
child.stderr.on('data', (d) => {
|
|
920
|
+
stderr = (stderr + d.toString()).slice(-400)
|
|
921
|
+
})
|
|
922
|
+
child.on('error', (err) => {
|
|
923
|
+
clearTimeout(timer)
|
|
924
|
+
reject(err)
|
|
925
|
+
})
|
|
926
|
+
child.on('close', (code) => {
|
|
927
|
+
clearTimeout(timer)
|
|
928
|
+
if (code === 0) resolve()
|
|
929
|
+
else reject(new Error(`yt-dlp 退出码 ${code}:${stderr}`))
|
|
930
|
+
})
|
|
931
|
+
})
|
|
932
|
+
}
|
|
933
|
+
|
|
934
|
+
/**
|
|
935
|
+
* 中转页:宿主页面不发 Referer,YouTube 因此报 153。
|
|
936
|
+
* 这里返回一个自己声明 Referrer-Policy 的文档,让内层 embed 请求带上来源。
|
|
937
|
+
*/
|
|
938
|
+
function serveEmbed(req, res, vid, lang) {
|
|
939
|
+
/* 播放器界面语言跟随面板:英文界面下 YouTube 不该弹中文的『前往平台观看』 */
|
|
940
|
+
const hl = lang === 'zh' ? 'zh-CN' : 'en'
|
|
941
|
+
const host = String(req.headers?.host || '')
|
|
942
|
+
const relayOrigin = /^[\w.:-]+$/.test(host) ? 'http://' + host : 'http://127.0.0.1:18090'
|
|
943
|
+
/* 用 YouTube IFrame API 托管播放器,才能读到播放位置;每秒把时间戳递回父页面,
|
|
944
|
+
面板据此把「当前在讲什么」作为上下文交给模型 */
|
|
945
|
+
const html =
|
|
946
|
+
'<!doctype html><html><head><meta charset="utf-8">' +
|
|
947
|
+
'<style>html,body{margin:0;height:100%;background:#000;overflow:hidden}#yt{width:100%;height:100%}' +
|
|
948
|
+
'#yt iframe{width:100%;height:100%;border:0;display:block}</style>' +
|
|
949
|
+
'</head><body><div id="yt"></div><script>' +
|
|
950
|
+
'var player=null,lastT=-1,timer=null;' +
|
|
951
|
+
'function tell(m){try{parent.postMessage(m,"*")}catch(e){}}' +
|
|
952
|
+
'function tick(){if(player&&typeof player.getCurrentTime==="function"){try{' +
|
|
953
|
+
'var t=player.getCurrentTime(),d=player.getDuration();' +
|
|
954
|
+
'if(Math.abs(t-lastT)>=1){lastT=t;tell({__ht:"time",t:t,d:d})}' +
|
|
955
|
+
'}catch(e){}}timer=setTimeout(tick,800)}' +
|
|
956
|
+
'function ready(){player=new YT.Player("yt",{videoId:' +
|
|
957
|
+
JSON.stringify(vid) +
|
|
958
|
+
',playerVars:{rel:0,playsinline:1,hl:' + JSON.stringify(hl) + ',origin:' +
|
|
959
|
+
JSON.stringify(relayOrigin) +
|
|
960
|
+
'},events:{' +
|
|
961
|
+
'onReady:function(){tell({__ht:"relay-loaded",secure:!!window.isSecureContext});tick()},' +
|
|
962
|
+
'onStateChange:function(e){tell({__ht:"state",s:e.data})},' +
|
|
963
|
+
'onError:function(e){tell({__ht:"yt-error",code:e.data})}}});}' +
|
|
964
|
+
'window.onYouTubeIframeAPIReady=ready;' +
|
|
965
|
+
'var s=document.createElement("script");s.src="https://www.youtube.com/iframe_api";' +
|
|
966
|
+
's.onerror=function(){tell({__ht:"api-error"})};document.head.appendChild(s);' +
|
|
967
|
+
'</script></body></html>'
|
|
968
|
+
res.writeHead(200, {
|
|
969
|
+
'Content-Type': 'text/html; charset=utf-8',
|
|
970
|
+
'Referrer-Policy': 'strict-origin-when-cross-origin',
|
|
971
|
+
/* 不能设 X-Frame-Options:父页面是 dsh-app://,与本服务器不同源,
|
|
972
|
+
SAMEORIGIN 会让浏览器直接拒绝渲染这个 iframe */
|
|
973
|
+
'Cache-Control': 'no-store',
|
|
974
|
+
...(state.origin ? { 'X-Hackertube-Origin': state.origin } : {}),
|
|
975
|
+
})
|
|
976
|
+
res.end(html)
|
|
977
|
+
}
|
|
978
|
+
|
|
979
|
+
/**
|
|
980
|
+
* 流式播放:yt-dlp 输出 MPEG-TS,ffmpeg 转封装成 fragmented MP4 直接透传。
|
|
981
|
+
* 不落磁盘、不依赖 YouTube 的嵌入策略;代价是流式不可 seek。
|
|
982
|
+
*/
|
|
983
|
+
function serveStream(req, res, vid) {
|
|
984
|
+
res.writeHead(200, {
|
|
985
|
+
'Content-Type': 'video/mp4',
|
|
986
|
+
'Cache-Control': 'no-store',
|
|
987
|
+
'Accept-Ranges': 'none',
|
|
988
|
+
})
|
|
989
|
+
|
|
990
|
+
const ytdlp = spawn(
|
|
991
|
+
'yt-dlp',
|
|
992
|
+
[
|
|
993
|
+
'-f',
|
|
994
|
+
`bv*[height<=${MEDIA_MAX_HEIGHT}]+ba/b[height<=${MEDIA_MAX_HEIGHT}]`,
|
|
995
|
+
'--merge-output-format',
|
|
996
|
+
'mp4',
|
|
997
|
+
'-o',
|
|
998
|
+
'-',
|
|
999
|
+
'--no-warnings',
|
|
1000
|
+
'--no-playlist',
|
|
1001
|
+
`https://www.youtube.com/watch?v=${vid}`,
|
|
1002
|
+
],
|
|
1003
|
+
{ stdio: ['ignore', 'pipe', 'ignore'], windowsHide: true },
|
|
1004
|
+
)
|
|
1005
|
+
|
|
1006
|
+
const ff = spawn(
|
|
1007
|
+
'ffmpeg',
|
|
1008
|
+
[
|
|
1009
|
+
'-hide_banner',
|
|
1010
|
+
'-loglevel',
|
|
1011
|
+
'error',
|
|
1012
|
+
'-i',
|
|
1013
|
+
'pipe:0',
|
|
1014
|
+
'-c',
|
|
1015
|
+
'copy',
|
|
1016
|
+
'-movflags',
|
|
1017
|
+
'frag_keyframe+empty_moov+default_base_moof',
|
|
1018
|
+
'-f',
|
|
1019
|
+
'mp4',
|
|
1020
|
+
'pipe:1',
|
|
1021
|
+
],
|
|
1022
|
+
{ stdio: ['pipe', 'pipe', 'ignore'], windowsHide: true },
|
|
1023
|
+
)
|
|
1024
|
+
|
|
1025
|
+
let closed = false
|
|
1026
|
+
const cleanup = () => {
|
|
1027
|
+
if (closed) return
|
|
1028
|
+
closed = true
|
|
1029
|
+
for (const child of [ytdlp, ff]) {
|
|
1030
|
+
try {
|
|
1031
|
+
child.kill()
|
|
1032
|
+
} catch {
|
|
1033
|
+
/* 已退出 */
|
|
1034
|
+
}
|
|
1035
|
+
}
|
|
1036
|
+
}
|
|
1037
|
+
|
|
1038
|
+
ytdlp.stdout.pipe(ff.stdin)
|
|
1039
|
+
ff.stdout.pipe(res)
|
|
1040
|
+
|
|
1041
|
+
req.on('close', cleanup)
|
|
1042
|
+
res.on('close', cleanup)
|
|
1043
|
+
ytdlp.on('error', cleanup)
|
|
1044
|
+
ff.on('error', cleanup)
|
|
1045
|
+
ff.on('close', () => {
|
|
1046
|
+
try {
|
|
1047
|
+
res.end()
|
|
1048
|
+
} catch {
|
|
1049
|
+
/* 已断开 */
|
|
1050
|
+
}
|
|
1051
|
+
cleanup()
|
|
1052
|
+
})
|
|
1053
|
+
}
|
|
1054
|
+
|
|
1055
|
+
/** 支持 Range 的静态投递,<video> 拖动进度条必须靠它。 */
|
|
1056
|
+
function streamMedia(req, res, file, size) {
|
|
1057
|
+
const range = req.headers.range
|
|
1058
|
+
if (range) {
|
|
1059
|
+
const m = /^bytes=(\d*)-(\d*)$/.exec(range)
|
|
1060
|
+
let start = m && m[1] ? Number(m[1]) : 0
|
|
1061
|
+
let end = m && m[2] ? Number(m[2]) : size - 1
|
|
1062
|
+
if (!Number.isFinite(start) || start < 0) start = 0
|
|
1063
|
+
if (!Number.isFinite(end) || end >= size) end = size - 1
|
|
1064
|
+
if (start > end) {
|
|
1065
|
+
res.writeHead(416, { 'Content-Range': `bytes */${size}` })
|
|
1066
|
+
res.end()
|
|
1067
|
+
return
|
|
1068
|
+
}
|
|
1069
|
+
res.writeHead(206, {
|
|
1070
|
+
'Content-Type': 'video/mp4',
|
|
1071
|
+
'Content-Length': end - start + 1,
|
|
1072
|
+
'Content-Range': `bytes ${start}-${end}/${size}`,
|
|
1073
|
+
'Accept-Ranges': 'bytes',
|
|
1074
|
+
'Cache-Control': 'no-store',
|
|
1075
|
+
})
|
|
1076
|
+
createReadStream(file, { start, end }).pipe(res)
|
|
1077
|
+
return
|
|
1078
|
+
}
|
|
1079
|
+
res.writeHead(200, {
|
|
1080
|
+
'Content-Type': 'video/mp4',
|
|
1081
|
+
'Content-Length': size,
|
|
1082
|
+
'Accept-Ranges': 'bytes',
|
|
1083
|
+
'Cache-Control': 'no-store',
|
|
1084
|
+
})
|
|
1085
|
+
createReadStream(file).pipe(res)
|
|
1086
|
+
}
|
|
1087
|
+
|
|
1088
|
+
/** 超过总配额就按最近访问时间从旧到新删除,避免无限占盘。 */
|
|
1089
|
+
async function trimMedia(keepName) {
|
|
1090
|
+
const dir = mediaDir()
|
|
1091
|
+
let names
|
|
1092
|
+
try {
|
|
1093
|
+
names = await readdir(dir)
|
|
1094
|
+
} catch {
|
|
1095
|
+
return
|
|
1096
|
+
}
|
|
1097
|
+
const files = []
|
|
1098
|
+
let total = 0
|
|
1099
|
+
for (const name of names) {
|
|
1100
|
+
if (!name.endsWith('.mp4')) continue
|
|
1101
|
+
const full = join(dir, name)
|
|
1102
|
+
try {
|
|
1103
|
+
const st = await stat(full)
|
|
1104
|
+
files.push({ full, size: st.size, at: st.atimeMs || st.mtimeMs, name })
|
|
1105
|
+
total += st.size
|
|
1106
|
+
} catch {
|
|
1107
|
+
/* 并发删除,忽略 */
|
|
1108
|
+
}
|
|
1109
|
+
}
|
|
1110
|
+
if (total <= MEDIA_CAP_BYTES) return
|
|
1111
|
+
files.sort((a, b) => a.at - b.at)
|
|
1112
|
+
for (const f of files) {
|
|
1113
|
+
if (total <= MEDIA_CAP_BYTES) break
|
|
1114
|
+
if (f.name === keepName) continue
|
|
1115
|
+
try {
|
|
1116
|
+
await unlink(f.full)
|
|
1117
|
+
total -= f.size
|
|
1118
|
+
} catch {
|
|
1119
|
+
/* 删不掉不影响播放 */
|
|
1120
|
+
}
|
|
1121
|
+
}
|
|
1122
|
+
}
|
|
1123
|
+
|
|
1124
|
+
/** 首次请求触发下载并缓存,之后直接读盘;同一视频的并发请求合并成一次。 */
|
|
1125
|
+
async function serveMedia(req, res, vid) {
|
|
1126
|
+
const dir = mediaDir()
|
|
1127
|
+
const file = join(dir, `${vid}.mp4`)
|
|
1128
|
+
|
|
1129
|
+
try {
|
|
1130
|
+
const st = await stat(file)
|
|
1131
|
+
if (st.size > 1024) return streamMedia(req, res, file, st.size)
|
|
1132
|
+
} catch {
|
|
1133
|
+
/* 尚未缓存 */
|
|
1134
|
+
}
|
|
1135
|
+
|
|
1136
|
+
try {
|
|
1137
|
+
await once(`media:${vid}`, async () => {
|
|
1138
|
+
await mkdir(dir, { recursive: true })
|
|
1139
|
+
const part = `${file}.part`
|
|
1140
|
+
try {
|
|
1141
|
+
await downloadMedia(vid, part)
|
|
1142
|
+
await rename(part, file)
|
|
1143
|
+
void trimMedia(`${vid}.mp4`)
|
|
1144
|
+
} catch (err) {
|
|
1145
|
+
try {
|
|
1146
|
+
await unlink(part)
|
|
1147
|
+
} catch {
|
|
1148
|
+
/* 清理失败忽略 */
|
|
1149
|
+
}
|
|
1150
|
+
throw err
|
|
1151
|
+
}
|
|
1152
|
+
})
|
|
1153
|
+
const st = await stat(file)
|
|
1154
|
+
return streamMedia(req, res, file, st.size)
|
|
1155
|
+
} catch (err) {
|
|
1156
|
+
return json(res, 502, { error: String(err?.message ?? err) })
|
|
1157
|
+
}
|
|
1158
|
+
}
|
|
1159
|
+
|
|
1160
|
+
/* ---------------- 路由 ---------------- */
|
|
1161
|
+
|
|
1162
|
+
function trusted(req) {
|
|
1163
|
+
const host = String(req.headers?.host ?? '')
|
|
1164
|
+
const origin = String(req.headers?.origin ?? '')
|
|
1165
|
+
if (origin && host) {
|
|
1166
|
+
try {
|
|
1167
|
+
return new URL(origin).host === host
|
|
1168
|
+
} catch {
|
|
1169
|
+
return false
|
|
1170
|
+
}
|
|
1171
|
+
}
|
|
1172
|
+
if (!origin) return /^(localhost|127\.0\.0\.1|\[::1\])(:\d+)?$/i.test(host)
|
|
1173
|
+
return false
|
|
1174
|
+
}
|
|
1175
|
+
|
|
1176
|
+
/* ---------------- LLM ---------------- */
|
|
1177
|
+
|
|
1178
|
+
/** 插件自己的模型通道:复用当前 profile 已配置的 provider,不需要额外 key。 */
|
|
1179
|
+
const llmState = { candidates: null, error: null, working: null }
|
|
1180
|
+
|
|
1181
|
+
/** 列出所有可用的文本模型路由,按 provider 顺序排好,供失败时逐个回退。 */
|
|
1182
|
+
async function listCandidates(ctx) {
|
|
1183
|
+
if (llmState.candidates) return llmState.candidates
|
|
1184
|
+
const out = []
|
|
1185
|
+
try {
|
|
1186
|
+
const llm = ctx?.get?.('llm')
|
|
1187
|
+
if (!llm) {
|
|
1188
|
+
llmState.error = 'no llm service'
|
|
1189
|
+
llmState.candidates = []
|
|
1190
|
+
return llmState.candidates
|
|
1191
|
+
}
|
|
1192
|
+
const providers = (await llm.listProviders?.()) || []
|
|
1193
|
+
for (const p of providers) {
|
|
1194
|
+
let models = []
|
|
1195
|
+
try {
|
|
1196
|
+
models = (await llm.listModels(p.id)) || []
|
|
1197
|
+
} catch {
|
|
1198
|
+
continue
|
|
1199
|
+
}
|
|
1200
|
+
for (const m of models) {
|
|
1201
|
+
if (m.inputModalities && !m.inputModalities.includes('text')) continue
|
|
1202
|
+
out.push({ provider: p.id, model: m.id })
|
|
1203
|
+
}
|
|
1204
|
+
}
|
|
1205
|
+
} catch (err) {
|
|
1206
|
+
llmState.error = String(err?.message ?? err).slice(0, 150)
|
|
1207
|
+
}
|
|
1208
|
+
llmState.candidates = out
|
|
1209
|
+
return out
|
|
1210
|
+
}
|
|
1211
|
+
|
|
1212
|
+
async function streamOnce(ctx, route, prompt, opts) {
|
|
1213
|
+
/* 单个路由挂死会拖住整批,加硬超时把它变成可重试的失败 */
|
|
1214
|
+
return Promise.race([
|
|
1215
|
+
streamOnceInner(ctx, route, prompt, opts),
|
|
1216
|
+
new Promise((_, reject) =>
|
|
1217
|
+
setTimeout(() => reject(new Error('llm timeout')), 90_000),
|
|
1218
|
+
),
|
|
1219
|
+
])
|
|
1220
|
+
}
|
|
1221
|
+
|
|
1222
|
+
async function streamOnceInner(ctx, route, prompt, opts) {
|
|
1223
|
+
const llm = ctx.get('llm')
|
|
1224
|
+
const stream = llm.stream({
|
|
1225
|
+
provider: route.provider,
|
|
1226
|
+
model: route.model,
|
|
1227
|
+
messages: [{ role: 'user', content: [{ type: 'text', text: prompt }] }],
|
|
1228
|
+
...(opts.maxTokens ? { maxOutputTokens: opts.maxTokens } : {}),
|
|
1229
|
+
})
|
|
1230
|
+
let out = ''
|
|
1231
|
+
const seen = new Set()
|
|
1232
|
+
/* 流里只有 block-start 带 blockType,后续增量靠当前块类型判断归属 */
|
|
1233
|
+
let currentBlock = null
|
|
1234
|
+
for await (const chunk of stream) {
|
|
1235
|
+
if (!chunk) continue
|
|
1236
|
+
if (typeof chunk === 'string') {
|
|
1237
|
+
if (currentBlock !== 'reasoning') {
|
|
1238
|
+
out += chunk
|
|
1239
|
+
if (opts.onDelta) opts.onDelta(chunk)
|
|
1240
|
+
}
|
|
1241
|
+
continue
|
|
1242
|
+
}
|
|
1243
|
+
if (typeof chunk !== 'object') continue
|
|
1244
|
+
if (chunk.type) seen.add(String(chunk.type))
|
|
1245
|
+
if (chunk.type === 'finish') {
|
|
1246
|
+
if (chunk.reason && chunk.reason.kind === 'error') {
|
|
1247
|
+
throw new Error(chunk.reason.failure?.message || 'llm error')
|
|
1248
|
+
}
|
|
1249
|
+
break
|
|
1250
|
+
}
|
|
1251
|
+
if (chunk.type === 'block-start') {
|
|
1252
|
+
currentBlock = chunk.blockType || null
|
|
1253
|
+
continue
|
|
1254
|
+
}
|
|
1255
|
+
if (chunk.type === 'block-stop' || chunk.type === 'block-end' || chunk.type === 'block-finish') {
|
|
1256
|
+
currentBlock = null
|
|
1257
|
+
continue
|
|
1258
|
+
}
|
|
1259
|
+
if (currentBlock === 'reasoning') continue
|
|
1260
|
+
if (typeof chunk.text === 'string') {
|
|
1261
|
+
out += chunk.text
|
|
1262
|
+
if (opts.onDelta) opts.onDelta(chunk.text)
|
|
1263
|
+
} else if (typeof chunk.delta === 'string') {
|
|
1264
|
+
out += chunk.delta
|
|
1265
|
+
if (opts.onDelta) opts.onDelta(chunk.delta)
|
|
1266
|
+
} else if (chunk.delta && typeof chunk.delta.text === 'string') {
|
|
1267
|
+
out += chunk.delta.text
|
|
1268
|
+
if (opts.onDelta) opts.onDelta(chunk.delta.text)
|
|
1269
|
+
}
|
|
1270
|
+
}
|
|
1271
|
+
if (!out.trim()) {
|
|
1272
|
+
throw new Error('empty response, chunk types: ' + Array.from(seen).join('|'))
|
|
1273
|
+
}
|
|
1274
|
+
return out.trim()
|
|
1275
|
+
}
|
|
1276
|
+
|
|
1277
|
+
/** 一次性问模型,返回完整文本;某个 provider 余额不足或不可用时自动换下一个。 */
|
|
1278
|
+
async function askLlm(ctx, prompt, opts = {}) {
|
|
1279
|
+
if (llmState.working) {
|
|
1280
|
+
try {
|
|
1281
|
+
return await streamOnce(ctx, llmState.working, prompt, opts)
|
|
1282
|
+
} catch (err) {
|
|
1283
|
+
if (!/balance|quota|unavailable|not found|no adapter/i.test(String(err?.message))) throw err
|
|
1284
|
+
llmState.working = null
|
|
1285
|
+
}
|
|
1286
|
+
}
|
|
1287
|
+
const candidates = await listCandidates(ctx)
|
|
1288
|
+
if (!candidates.length) throw new Error(llmState.error || 'no model available')
|
|
1289
|
+
let lastErr = null
|
|
1290
|
+
for (const route of candidates) {
|
|
1291
|
+
try {
|
|
1292
|
+
const text = await streamOnce(ctx, route, prompt, opts)
|
|
1293
|
+
llmState.working = route
|
|
1294
|
+
return text
|
|
1295
|
+
} catch (err) {
|
|
1296
|
+
lastErr = err
|
|
1297
|
+
const msg = String(err?.message ?? err)
|
|
1298
|
+
if (/balance|quota|unavailable|not found|no adapter|unauthorized/i.test(msg)) continue
|
|
1299
|
+
throw err
|
|
1300
|
+
}
|
|
1301
|
+
}
|
|
1302
|
+
throw lastErr || new Error('all providers failed')
|
|
1303
|
+
}
|
|
1304
|
+
|
|
1305
|
+
/* ---------------- 翻译 ---------------- */
|
|
1306
|
+
|
|
1307
|
+
const I18N_VERSION = 1
|
|
1308
|
+
|
|
1309
|
+
function i18nDir() {
|
|
1310
|
+
return join(cacheRoot(), 'i18n')
|
|
1311
|
+
}
|
|
1312
|
+
|
|
1313
|
+
function i18nFile(lang) {
|
|
1314
|
+
return join(i18nDir(), `${lang}.json`)
|
|
1315
|
+
}
|
|
1316
|
+
|
|
1317
|
+
const i18nState = { zh: null, dirty: false, timer: null }
|
|
1318
|
+
|
|
1319
|
+
async function loadI18n(lang) {
|
|
1320
|
+
if (i18nState[lang]) return i18nState[lang]
|
|
1321
|
+
try {
|
|
1322
|
+
const raw = await readFile(i18nFile(lang), 'utf8')
|
|
1323
|
+
const parsed = JSON.parse(raw)
|
|
1324
|
+
i18nState[lang] = parsed.entries || {}
|
|
1325
|
+
} catch {
|
|
1326
|
+
/* 缓存里没有时用仓库自带的译文打底,省掉几小时的模型翻译 */
|
|
1327
|
+
const seeded = await readSeed(lang)
|
|
1328
|
+
i18nState[lang] = seeded
|
|
1329
|
+
if (Object.keys(seeded).length) {
|
|
1330
|
+
void mkdir(i18nDir(), { recursive: true })
|
|
1331
|
+
.then(() => writeFile(i18nFile(lang), JSON.stringify({ v: I18N_VERSION, entries: seeded })))
|
|
1332
|
+
.catch(() => {})
|
|
1333
|
+
}
|
|
1334
|
+
}
|
|
1335
|
+
return i18nState[lang]
|
|
1336
|
+
}
|
|
1337
|
+
|
|
1338
|
+
async function readSeed(name) {
|
|
1339
|
+
try {
|
|
1340
|
+
const parsed = JSON.parse(await readFile(join(SEED_DIR, name + '.json'), 'utf8'))
|
|
1341
|
+
return parsed.entries || {}
|
|
1342
|
+
} catch {
|
|
1343
|
+
return {}
|
|
1344
|
+
}
|
|
1345
|
+
}
|
|
1346
|
+
|
|
1347
|
+
/** 只回写本轮改动过的译文表;共用同一个定时器,避免互相顶掉 */
|
|
1348
|
+
function scheduleI18nSave(lang) {
|
|
1349
|
+
if (!(i18nState.dirty instanceof Set)) i18nState.dirty = new Set()
|
|
1350
|
+
if (lang) i18nState.dirty.add(lang)
|
|
1351
|
+
if (i18nState.timer) return
|
|
1352
|
+
i18nState.timer = setTimeout(() => {
|
|
1353
|
+
i18nState.timer = null
|
|
1354
|
+
const names = i18nState.dirty instanceof Set ? [...i18nState.dirty] : []
|
|
1355
|
+
if (i18nState.dirty instanceof Set) i18nState.dirty.clear()
|
|
1356
|
+
if (!names.length) return
|
|
1357
|
+
void mkdir(i18nDir(), { recursive: true })
|
|
1358
|
+
.then(async () => {
|
|
1359
|
+
for (const name of names) {
|
|
1360
|
+
const entries = i18nState[name]
|
|
1361
|
+
if (!entries || typeof entries !== 'object') continue
|
|
1362
|
+
await writeFile(i18nFile(name), JSON.stringify({ v: I18N_VERSION, entries }))
|
|
1363
|
+
}
|
|
1364
|
+
})
|
|
1365
|
+
.catch(() => {})
|
|
1366
|
+
}, 2000)
|
|
1367
|
+
if (typeof i18nState.timer.unref === 'function') i18nState.timer.unref()
|
|
1368
|
+
}
|
|
1369
|
+
|
|
1370
|
+
const TERMS = [
|
|
1371
|
+
['transformer', 'Transformer'],
|
|
1372
|
+
['kernel', '内核'],
|
|
1373
|
+
['gradient', '梯度'],
|
|
1374
|
+
['backpropagation', '反向传播'],
|
|
1375
|
+
['compiler', '编译器'],
|
|
1376
|
+
['garbage collection', '垃圾回收'],
|
|
1377
|
+
['lock-free', '无锁'],
|
|
1378
|
+
['type inference', '类型推断'],
|
|
1379
|
+
]
|
|
1380
|
+
|
|
1381
|
+
/**
|
|
1382
|
+
* 把一批英文推荐理由翻成中文。
|
|
1383
|
+
* 只翻没有缓存的条目;返回 vid → 译文 的映射。
|
|
1384
|
+
*/
|
|
1385
|
+
async function translateEntries(ctx, items, lang = 'zh') {
|
|
1386
|
+
const cache = await loadI18n(lang)
|
|
1387
|
+
const todo = items.filter((it) => it.r && !cache[it.v])
|
|
1388
|
+
if (!todo.length) return cache
|
|
1389
|
+
|
|
1390
|
+
const glossary = TERMS.map(([en, zh]) => `${en}=${zh}`).join('; ')
|
|
1391
|
+
for (let i = 0; i < todo.length; i += 12) {
|
|
1392
|
+
const batch = todo.slice(i, i + 12)
|
|
1393
|
+
const numbered = batch
|
|
1394
|
+
.map((it, idx) => `${idx + 1}. ${String(it.r).replace(/\s+/g, ' ').slice(0, 600)}`)
|
|
1395
|
+
.join('\n')
|
|
1396
|
+
const prompt =
|
|
1397
|
+
'把这些 Hacker News 上推荐技术视频的理由翻译成简体中文。' +
|
|
1398
|
+
'保持技术准确性,语气自然,不要添加解释。\n' +
|
|
1399
|
+
`术语对照:${glossary}\n` +
|
|
1400
|
+
'严格输出 JSON 数组,只包含译文,顺序与输入一致,例如 ["译文一","译文二"]。\n\n' +
|
|
1401
|
+
numbered
|
|
1402
|
+
let text
|
|
1403
|
+
try {
|
|
1404
|
+
text = await askLlm(ctx, prompt)
|
|
1405
|
+
} catch {
|
|
1406
|
+
break
|
|
1407
|
+
}
|
|
1408
|
+
const m = text.match(/\[[\s\S]*\]/)
|
|
1409
|
+
if (!m) continue
|
|
1410
|
+
let arr
|
|
1411
|
+
try {
|
|
1412
|
+
arr = JSON.parse(m[0])
|
|
1413
|
+
} catch {
|
|
1414
|
+
continue
|
|
1415
|
+
}
|
|
1416
|
+
if (!Array.isArray(arr)) continue
|
|
1417
|
+
batch.forEach((it, idx) => {
|
|
1418
|
+
if (typeof arr[idx] === 'string' && arr[idx].trim()) cache[it.v] = arr[idx].trim()
|
|
1419
|
+
})
|
|
1420
|
+
scheduleI18nSave(lang)
|
|
1421
|
+
}
|
|
1422
|
+
return cache
|
|
1423
|
+
}
|
|
1424
|
+
|
|
1425
|
+
/* 查询扩展用的通用机翻:中文查询先翻成英文,才能命中英文标签 */
|
|
1426
|
+
const mtQueryState = { map: new Map() }
|
|
1427
|
+
const MT_QUERY_MAX = 400
|
|
1428
|
+
|
|
1429
|
+
async function mtQuery(text, to) {
|
|
1430
|
+
const key = to + ':' + text
|
|
1431
|
+
if (mtQueryState.map.has(key)) return mtQueryState.map.get(key)
|
|
1432
|
+
const [from, target] = to === 'en' ? ['zh-CN', 'en'] : ['en', 'zh-CN']
|
|
1433
|
+
let out = ''
|
|
1434
|
+
try {
|
|
1435
|
+
out = (await mtBatch([text], from, target))[0] || ''
|
|
1436
|
+
} catch {
|
|
1437
|
+
try {
|
|
1438
|
+
out = await mtSingle(text, from, target)
|
|
1439
|
+
} catch {
|
|
1440
|
+
out = ''
|
|
1441
|
+
}
|
|
1442
|
+
}
|
|
1443
|
+
if (mtQueryState.map.size > MT_QUERY_MAX) mtQueryState.map.clear()
|
|
1444
|
+
mtQueryState.map.set(key, out)
|
|
1445
|
+
return out
|
|
1446
|
+
}
|
|
1447
|
+
|
|
1448
|
+
|
|
1449
|
+
/* ---------------- HN 评论机翻 ---------------- */
|
|
1450
|
+
|
|
1451
|
+
const commentTrState = { map: null, timer: null }
|
|
1452
|
+
|
|
1453
|
+
/* Chrome 内置翻译的批量端点:一次可交多条,质量明显好于 translate_a 系列 */
|
|
1454
|
+
const MT_ENDPOINT = 'https://translate-pa.googleapis.com/v1/translateHtml'
|
|
1455
|
+
/* 这是 Chrome 浏览器自带翻译随包发布的公开网页 key,不是账号凭证,也不计费 */
|
|
1456
|
+
const MT_KEY = 'AIzaSyATBXajvzQLTDHEQbcpq0Ihe0vWDHmO520'
|
|
1457
|
+
const MT_BATCH = 10
|
|
1458
|
+
/* 缓存键带引擎标记,换引擎后旧条目自然失效,不必手动清文件 */
|
|
1459
|
+
const MT_TAG = 'g'
|
|
1460
|
+
|
|
1461
|
+
function commentTrFile() {
|
|
1462
|
+
return join(i18nDir(), 'zh-comments.json')
|
|
1463
|
+
}
|
|
1464
|
+
|
|
1465
|
+
async function loadCommentTr() {
|
|
1466
|
+
if (commentTrState.map) return commentTrState.map
|
|
1467
|
+
try {
|
|
1468
|
+
const parsed = JSON.parse(await readFile(commentTrFile(), 'utf8'))
|
|
1469
|
+
commentTrState.map = parsed.entries || {}
|
|
1470
|
+
} catch {
|
|
1471
|
+
commentTrState.map = {}
|
|
1472
|
+
}
|
|
1473
|
+
return commentTrState.map
|
|
1474
|
+
}
|
|
1475
|
+
|
|
1476
|
+
function scheduleCommentTrSave() {
|
|
1477
|
+
if (commentTrState.timer) return
|
|
1478
|
+
commentTrState.timer = setTimeout(() => {
|
|
1479
|
+
commentTrState.timer = null
|
|
1480
|
+
const entries = commentTrState.map || {}
|
|
1481
|
+
void mkdir(i18nDir(), { recursive: true })
|
|
1482
|
+
.then(() => writeFile(commentTrFile(), JSON.stringify({ v: I18N_VERSION, entries })))
|
|
1483
|
+
.catch(() => {})
|
|
1484
|
+
}, 2000)
|
|
1485
|
+
if (typeof commentTrState.timer.unref === 'function') commentTrState.timer.unref()
|
|
1486
|
+
}
|
|
1487
|
+
|
|
1488
|
+
function mtFetch(url, init) {
|
|
1489
|
+
const ctrl = typeof AbortController === 'function' ? new AbortController() : null
|
|
1490
|
+
const timer = ctrl ? setTimeout(() => ctrl.abort(), MT_TIMEOUT_MS) : null
|
|
1491
|
+
const opts = Object.assign({}, init, ctrl ? { signal: ctrl.signal } : {})
|
|
1492
|
+
return fetch(url, opts).finally(() => {
|
|
1493
|
+
if (timer) clearTimeout(timer)
|
|
1494
|
+
})
|
|
1495
|
+
}
|
|
1496
|
+
|
|
1497
|
+
/** 主通道:批量提交,返回与输入等长的数组 */
|
|
1498
|
+
async function mtBatch(texts, from = 'en', to = 'zh-CN') {
|
|
1499
|
+
const res = await mtFetch(MT_ENDPOINT, {
|
|
1500
|
+
method: 'POST',
|
|
1501
|
+
headers: {
|
|
1502
|
+
'Content-Type': 'application/json+protobuf',
|
|
1503
|
+
'X-Goog-API-Key': MT_KEY,
|
|
1504
|
+
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 Chrome/124.0 Safari/537.36',
|
|
1505
|
+
},
|
|
1506
|
+
body: JSON.stringify([[texts, from, to], 'wt_lib']),
|
|
1507
|
+
})
|
|
1508
|
+
if (!res.ok) throw new Error('mt http ' + res.status)
|
|
1509
|
+
const data = await res.json()
|
|
1510
|
+
const arr = Array.isArray(data) && Array.isArray(data[0]) ? data[0] : null
|
|
1511
|
+
if (!arr) throw new Error('mt bad payload')
|
|
1512
|
+
return texts.map((_, i) => (typeof arr[i] === 'string' ? arr[i].trim() : ''))
|
|
1513
|
+
}
|
|
1514
|
+
|
|
1515
|
+
/** 备用通道:translate_a 单条接口,主通道被拒时逐条补 */
|
|
1516
|
+
async function mtSingle(text, from = 'en', to = 'zh-CN') {
|
|
1517
|
+
const url =
|
|
1518
|
+
'https://translate.googleapis.com/translate_a/single?client=at&sl=' +
|
|
1519
|
+
from +
|
|
1520
|
+
'&tl=' +
|
|
1521
|
+
to +
|
|
1522
|
+
'&dt=t&q=' +
|
|
1523
|
+
encodeURIComponent(text)
|
|
1524
|
+
const res = await mtFetch(url, {
|
|
1525
|
+
headers: {
|
|
1526
|
+
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 Chrome/124.0 Safari/537.36',
|
|
1527
|
+
Accept: '*/*',
|
|
1528
|
+
},
|
|
1529
|
+
})
|
|
1530
|
+
if (!res.ok) throw new Error('mt http ' + res.status)
|
|
1531
|
+
const data = await res.json()
|
|
1532
|
+
if (!Array.isArray(data) || !Array.isArray(data[0])) return ''
|
|
1533
|
+
return data[0]
|
|
1534
|
+
.map((seg) => (Array.isArray(seg) && typeof seg[0] === 'string' ? seg[0] : ''))
|
|
1535
|
+
.join('')
|
|
1536
|
+
.trim()
|
|
1537
|
+
}
|
|
1538
|
+
|
|
1539
|
+
async function mtTranslateAll(texts) {
|
|
1540
|
+
try {
|
|
1541
|
+
return await mtBatch(texts)
|
|
1542
|
+
} catch {
|
|
1543
|
+
/* 主通道失败时逐条走备用,单条失败留空由调用方保留原文 */
|
|
1544
|
+
const out = []
|
|
1545
|
+
for (const t of texts) {
|
|
1546
|
+
try {
|
|
1547
|
+
out.push(await mtSingle(t))
|
|
1548
|
+
} catch {
|
|
1549
|
+
out.push('')
|
|
1550
|
+
}
|
|
1551
|
+
}
|
|
1552
|
+
return out
|
|
1553
|
+
}
|
|
1554
|
+
}
|
|
1555
|
+
|
|
1556
|
+
/**
|
|
1557
|
+
* 逐条机翻 HN 评论,按评论 id 缓存,同一条只翻一次。
|
|
1558
|
+
* 十条一批提交,失败的条目不写入缓存,下次点击会重试。
|
|
1559
|
+
*/
|
|
1560
|
+
async function translateComments(comments) {
|
|
1561
|
+
const cache = await loadCommentTr()
|
|
1562
|
+
const keyOf = (c) => MT_TAG + ':' + c.id
|
|
1563
|
+
const todo = comments.filter((c) => c.text && c.text.trim().length >= 4 && !cache[keyOf(c)])
|
|
1564
|
+
if (!todo.length) return cache
|
|
1565
|
+
for (let i = 0; i < todo.length; i += MT_BATCH) {
|
|
1566
|
+
const batch = todo.slice(i, i + MT_BATCH)
|
|
1567
|
+
const texts = batch.map((c) => String(c.text).slice(0, 900))
|
|
1568
|
+
const out = await mtTranslateAll(texts)
|
|
1569
|
+
batch.forEach((c, idx) => {
|
|
1570
|
+
if (out[idx]) cache[keyOf(c)] = out[idx]
|
|
1571
|
+
})
|
|
1572
|
+
}
|
|
1573
|
+
scheduleCommentTrSave()
|
|
1574
|
+
return cache
|
|
1575
|
+
}
|
|
1576
|
+
|
|
1577
|
+
/** 从可能夹带推理过程的文本里,抠出第一个完整 JSON 数组。 */
|
|
1578
|
+
function extractJsonArray(text) {
|
|
1579
|
+
const start = text.indexOf('[')
|
|
1580
|
+
if (start < 0) return null
|
|
1581
|
+
let depth = 0
|
|
1582
|
+
let inStr = false
|
|
1583
|
+
let esc = false
|
|
1584
|
+
for (let i = start; i < text.length; i += 1) {
|
|
1585
|
+
const ch = text[i]
|
|
1586
|
+
if (inStr) {
|
|
1587
|
+
if (esc) esc = false
|
|
1588
|
+
else if (ch === '\\') esc = true
|
|
1589
|
+
else if (ch === '"') inStr = false
|
|
1590
|
+
continue
|
|
1591
|
+
}
|
|
1592
|
+
if (ch === '"') inStr = true
|
|
1593
|
+
else if (ch === '[') depth += 1
|
|
1594
|
+
else if (ch === ']') {
|
|
1595
|
+
depth -= 1
|
|
1596
|
+
if (depth === 0) {
|
|
1597
|
+
try {
|
|
1598
|
+
return JSON.parse(text.slice(start, i + 1))
|
|
1599
|
+
} catch {
|
|
1600
|
+
return null
|
|
1601
|
+
}
|
|
1602
|
+
}
|
|
1603
|
+
}
|
|
1604
|
+
}
|
|
1605
|
+
return null
|
|
1606
|
+
}
|
|
1607
|
+
|
|
1608
|
+
/**
|
|
1609
|
+
* 翻译单条视频的详情文本。
|
|
1610
|
+
* 列表里没有推荐理由,只有详情带 quote / summary / why_hn,所以翻译按详情触发。
|
|
1611
|
+
*/
|
|
1612
|
+
async function translateDetail(ctx, detail, lang = 'zh') {
|
|
1613
|
+
const cache = await loadI18n(lang)
|
|
1614
|
+
const jobs = []
|
|
1615
|
+
const push = (field, text) => {
|
|
1616
|
+
if (typeof text !== 'string' || text.trim().length < 8) return
|
|
1617
|
+
const key = `${detail.vid}:${field}`
|
|
1618
|
+
if (cache[key]) return
|
|
1619
|
+
jobs.push({ key, text: text.replace(/\s+/g, ' ').slice(0, 700) })
|
|
1620
|
+
}
|
|
1621
|
+
push('summary', detail.summary)
|
|
1622
|
+
push('why_hn', detail.why_hn)
|
|
1623
|
+
if (detail.quote) push('quote', detail.quote.text)
|
|
1624
|
+
|
|
1625
|
+
if (!jobs.length) return cache
|
|
1626
|
+
|
|
1627
|
+
const glossary = TERMS.map(([en, zh]) => `${en}=${zh}`).join('; ')
|
|
1628
|
+
const numbered = jobs.map((j, i) => `${i + 1}. ${j.text}`).join('\n')
|
|
1629
|
+
const prompt =
|
|
1630
|
+
'把下面这些 Hacker News 上推荐技术视频的英文说明翻成简体中文。' +
|
|
1631
|
+
'保持技术准确、语气自然,不要添加任何解释或前言。\n' +
|
|
1632
|
+
`术语对照:${glossary}\n` +
|
|
1633
|
+
'严格输出 JSON 数组,只含译文,顺序与输入一致,例如 ["译文一","译文二"]。\n\n' +
|
|
1634
|
+
numbered
|
|
1635
|
+
|
|
1636
|
+
try {
|
|
1637
|
+
const text = await askLlm(ctx, prompt)
|
|
1638
|
+
void writeCache('translate-last.json', { ok: true, at: Date.now(), raw: String(text).slice(0, 400) })
|
|
1639
|
+
const arr = extractJsonArray(text)
|
|
1640
|
+
if (Array.isArray(arr)) {
|
|
1641
|
+
jobs.forEach((j, i) => {
|
|
1642
|
+
if (typeof arr[i] === 'string' && arr[i].trim()) cache[j.key] = arr[i].trim()
|
|
1643
|
+
})
|
|
1644
|
+
scheduleI18nSave(lang)
|
|
1645
|
+
}
|
|
1646
|
+
} catch (err) {
|
|
1647
|
+
void writeCache('translate-last.json', {
|
|
1648
|
+
ok: false,
|
|
1649
|
+
at: Date.now(),
|
|
1650
|
+
message: String(err?.message ?? err).slice(0, 300),
|
|
1651
|
+
})
|
|
1652
|
+
}
|
|
1653
|
+
return cache
|
|
1654
|
+
}
|
|
1655
|
+
|
|
1656
|
+
/**
|
|
1657
|
+
* 模型不可用时的详情机翻:摘要、推荐理由、引用一次批量翻完。
|
|
1658
|
+
* 单独存 zh-mt.json,和模型译文分开,日后配了模型还能覆盖升级。
|
|
1659
|
+
*/
|
|
1660
|
+
async function translateDetailMT(detail) {
|
|
1661
|
+
const cache = await loadI18n('zh-mt')
|
|
1662
|
+
const jobs = []
|
|
1663
|
+
const push = (field, text) => {
|
|
1664
|
+
if (typeof text !== 'string' || text.trim().length < 6) return
|
|
1665
|
+
const key = `${detail.vid}:${field}`
|
|
1666
|
+
if (cache[key]) return
|
|
1667
|
+
jobs.push({ key, text: text.replace(/\s+/g, ' ').slice(0, 900) })
|
|
1668
|
+
}
|
|
1669
|
+
push('summary', detail.summary)
|
|
1670
|
+
push('why_hn', detail.why_hn)
|
|
1671
|
+
if (detail.quote) push('quote', detail.quote.text)
|
|
1672
|
+
if (!jobs.length) return cache
|
|
1673
|
+
|
|
1674
|
+
const out = await mtTranslateAll(jobs.map((j) => j.text))
|
|
1675
|
+
jobs.forEach((j, i) => {
|
|
1676
|
+
if (out[i]) cache[j.key] = out[i]
|
|
1677
|
+
})
|
|
1678
|
+
scheduleI18nSave('zh-mt')
|
|
1679
|
+
return cache
|
|
1680
|
+
}
|
|
1681
|
+
|
|
1682
|
+
/** 模型偶尔把批内编号带进译文,剥掉行首的 "N. " / "N、" / "N)" */
|
|
1683
|
+
function stripLeadingIndex(text) { return String(text)
|
|
1684
|
+
.replace(/^\s*\d+\s*[.、).]\s*/, '')
|
|
1685
|
+
.trim()
|
|
1686
|
+
}
|
|
1687
|
+
|
|
1688
|
+
/* ---------------- 译文出口校验 ---------------- */
|
|
1689
|
+
|
|
1690
|
+
/**
|
|
1691
|
+
* 只认真正的指令泄漏。早先的正则把「要求」「prompt」当特征词,
|
|
1692
|
+
* 误伤了「有人要求我对此保密」「[un]prompted 2026」这类正常译文。
|
|
1693
|
+
*/
|
|
1694
|
+
const TITLE_LEAK = /(只含译文|只输出|输出译文|请翻译以下|以下是译文|翻译成中文[::]|不要解释|格式要求|译文如下)/
|
|
1695
|
+
const TITLE_MAX = 120
|
|
1696
|
+
|
|
1697
|
+
/**
|
|
1698
|
+
* 中文标题不用竖线分层。原站标题是 "A | B | C" 的结构,
|
|
1699
|
+
* 译文照搬竖线会读成机器翻的,这里改写成冒号 / 括号 / 逗号。
|
|
1700
|
+
*/
|
|
1701
|
+
function normalizeTitle(input) {
|
|
1702
|
+
let s = String(input ?? '')
|
|
1703
|
+
.replace(/\s+/g, ' ')
|
|
1704
|
+
.trim()
|
|
1705
|
+
.replace(/|/g, '|')
|
|
1706
|
+
const parts = s
|
|
1707
|
+
.split('|')
|
|
1708
|
+
.map((x) => x.trim())
|
|
1709
|
+
.filter(Boolean)
|
|
1710
|
+
if (parts.length < 2) return s
|
|
1711
|
+
let out = parts[0]
|
|
1712
|
+
for (let i = 1; i < parts.length; i += 1) {
|
|
1713
|
+
const p = parts[i]
|
|
1714
|
+
const endsWithQuestion = /[?!?!]$/.test(out)
|
|
1715
|
+
/* 这一段自己已经带冒号("第1章:线性代数的本质"),再用冒号接就是双冒号 */
|
|
1716
|
+
const partHasColon = /[::]/.test(p)
|
|
1717
|
+
if (i === 1 && endsWithQuestion) out += `(${p})`
|
|
1718
|
+
else if (partHasColon) out += `,${p}`
|
|
1719
|
+
else if (!/[::]/.test(out)) out += `:${p}`
|
|
1720
|
+
else out += `,${p}`
|
|
1721
|
+
}
|
|
1722
|
+
return out
|
|
1723
|
+
.replace(/:\s+/g, ':')
|
|
1724
|
+
.replace(/\s+,/g, ',')
|
|
1725
|
+
/* 历史数据里已经写成「A:第1章:B」的,把中间那段降级成逗号 */
|
|
1726
|
+
.replace(/:([^:,。;!?()]{1,10}):/g, ',$1:')
|
|
1727
|
+
}
|
|
1728
|
+
|
|
1729
|
+
/**
|
|
1730
|
+
* 结构可疑度:中文标题里出现堆叠分层、过长、并列逗号过多,都说明模型
|
|
1731
|
+
* 是照着英文语序搬的。0 分不动,>=2 分优先重排。
|
|
1732
|
+
*/
|
|
1733
|
+
function titleStructureScore(text) {
|
|
1734
|
+
const t = String(text || '')
|
|
1735
|
+
let n = 0
|
|
1736
|
+
if ((t.match(/:/g) || []).length >= 2) n += 1
|
|
1737
|
+
if (t.length > 30) n += 1
|
|
1738
|
+
if (/,[^,]{0,8}:/.test(t)) n += 1
|
|
1739
|
+
if ((t.match(/(/g) || []).length >= 2) n += 1
|
|
1740
|
+
if (t.length > 18 && !t.includes(':') && (t.match(/,/g) || []).length >= 2) n += 1
|
|
1741
|
+
return n
|
|
1742
|
+
}
|
|
1743
|
+
|
|
1744
|
+
/** 译文能不能进缓存:ok / empty / leak / separator / too-long */
|
|
1745
|
+
function titleVerdict(text) {
|
|
1746
|
+
const s = String(text ?? '').trim()
|
|
1747
|
+
if (!s) return 'empty'
|
|
1748
|
+
if (TITLE_LEAK.test(s)) return 'leak'
|
|
1749
|
+
if (/[||]/.test(s)) return 'separator'
|
|
1750
|
+
if (s.length > TITLE_MAX) return 'too-long'
|
|
1751
|
+
return 'ok'
|
|
1752
|
+
}
|
|
1753
|
+
|
|
1754
|
+
/**
|
|
1755
|
+
* 批量翻译视频标题。
|
|
1756
|
+
* 标题短、量大,单独一个缓存文件,避免和详情译文混在一起。
|
|
1757
|
+
*/
|
|
1758
|
+
let titleRun = null
|
|
1759
|
+
|
|
1760
|
+
/**
|
|
1761
|
+
* 单飞:标题翻译同一时刻只跑一轮。前端打开面板会带一次 translate=1,
|
|
1762
|
+
* 和后台补翻撞上就会互相抢并发,把两边都拖成超时。
|
|
1763
|
+
*/
|
|
1764
|
+
function translateTitles(ctx, items, lang = 'zh') {
|
|
1765
|
+
if (titleRun) return titleRun
|
|
1766
|
+
titleRun = runTitleTranslation(ctx, items, lang).finally(() => {
|
|
1767
|
+
titleRun = null
|
|
1768
|
+
})
|
|
1769
|
+
return titleRun
|
|
1770
|
+
}
|
|
1771
|
+
|
|
1772
|
+
async function runTitleTranslation(ctx, items, lang = 'zh') {
|
|
1773
|
+
const store = lang + '-titles'
|
|
1774
|
+
const cache = await loadI18n(store)
|
|
1775
|
+
const todo = items.filter((it) => it.t && !cache[it.v])
|
|
1776
|
+
if (!todo.length) return cache
|
|
1777
|
+
|
|
1778
|
+
/* 实测:16 条一批时输出变长、超时增多,自适应把并发压到 4-8,净吞吐反而低于 12 */
|
|
1779
|
+
const BATCH = 12
|
|
1780
|
+
const CONCURRENCY = 6
|
|
1781
|
+
const batches = []
|
|
1782
|
+
for (let i = 0; i < todo.length; i += BATCH) batches.push(todo.slice(i, i + BATCH))
|
|
1783
|
+
|
|
1784
|
+
const buildPrompt = (batch) => {
|
|
1785
|
+
const lines = batch
|
|
1786
|
+
.map((it, idx) => {
|
|
1787
|
+
const hint = []
|
|
1788
|
+
if (Array.isArray(it.tags) && it.tags.length) hint.push('标签:' + it.tags.slice(0, 5).join(', '))
|
|
1789
|
+
if (it.c) hint.push('频道:' + it.c)
|
|
1790
|
+
if (it.y) hint.push('年份:' + it.y)
|
|
1791
|
+
const meta = hint.length ? `(${hint.join(';')})` : ''
|
|
1792
|
+
return `${idx + 1}. ${String(it.t).slice(0, 220)}${meta}`
|
|
1793
|
+
})
|
|
1794
|
+
.join('\n')
|
|
1795
|
+
return (
|
|
1796
|
+
'你是一位极其挑剔的中文作家,痛恨从英文直译而来的不自然表达。\n' +
|
|
1797
|
+
'下面是一批英文技术视频标题。请用中文重写它们——不是逐词翻译,是重写:\n' +
|
|
1798
|
+
'先想清楚这个视频讲什么,再用一个中国人会写出来的标题表达。\n\n' +
|
|
1799
|
+
'【必须遵守】\n' +
|
|
1800
|
+
'1. 拒绝英文语序。英文的连词、冠词、介词在中文里常常不该出现。\n' +
|
|
1801
|
+
' 一句话读不通就是错的,宁可换说法也不要逐词对应。\n' +
|
|
1802
|
+
'2. 拆解英文复合名词,用中文的名词修饰语法重建。\n' +
|
|
1803
|
+
' 例:mom creators 不是「母亲创作者」,是「宝妈博主」\n' +
|
|
1804
|
+
'3. 专有名词、产品名、人名、项目名、缩写保留原文:Transformer、Rust、Karpathy、TCP、micrograd\n' +
|
|
1805
|
+
'4. 副标题用冒号「:」或括号承接,绝对不要用竖线「|」或「|」分层;原标题的编号要保留\n' +
|
|
1806
|
+
'5. 原标题是问句就译成问句;有比喻就保留比喻,不要压平成说明文\n' +
|
|
1807
|
+
'6. 中英文之间加空格:API 密钥 而不是 API密钥\n' +
|
|
1808
|
+
'7. 中文标点用全角:。,:;!?()——……;引用一律用「」,不用弯引号\n' +
|
|
1809
|
+
'8. 括号里的标签和频道只用于理解,不要译进标题\n\n' +
|
|
1810
|
+
'【中文标题的层级:先分层,再落笔】\n' +
|
|
1811
|
+
'英文标题把信息平铺成一串(主题 - 副标题 - 讲者 - 活动),几段是平级的;\n' +
|
|
1812
|
+
'中文标题必须有主从:主标题只说「这是什么」,其余降级或删掉。\n' +
|
|
1813
|
+
'1. 主标题一句话,20 字内,不塞年份、活动名、讲者。\n' +
|
|
1814
|
+
'2. 从属信息最多留一层:系列与集数写「第 N 章 / 第 N 讲」,讲者写「人名:」前置或「(人名)」后置。\n' +
|
|
1815
|
+
'3. 三段以上砍到两段,砍的顺序:活动名 → 年份 → 讲者。\n' +
|
|
1816
|
+
'4. 分层靠语序和冒号,不靠竖线,也不要把四段用逗号串起来。\n' +
|
|
1817
|
+
'5. 四种结构按内容选一种,不混用:\n' +
|
|
1818
|
+
' A 系列在前:线性代数的本质 第 1 章:向量\n' +
|
|
1819
|
+
' B 讲者在前:Derek Sivers:怎样发起一场运动\n' +
|
|
1820
|
+
' C 机构在前:GOTO 2019:Erlang 与 Elixir 的灵魂\n' +
|
|
1821
|
+
' D 主题在前:分布式系统 1.1:导论\n' +
|
|
1822
|
+
'【这些是错的,照着避开】\n' +
|
|
1823
|
+
'- "But what is the Fourier Transform?" → 错:「但傅里叶变换是什么?」(中文不用"但"起句)\n' +
|
|
1824
|
+
' 对:「傅里叶变换究竟是什么?」\n' +
|
|
1825
|
+
'- "Simple Made Easy" → 错:「让简单变容易」(原意是简单不等于容易,译反了)\n' +
|
|
1826
|
+
' 对:「简单并不容易」\n' +
|
|
1827
|
+
'- "Around The Corner - How Differential Steering Works" → 错:「拐角处——差速转向如何工作」\n' +
|
|
1828
|
+
' 对:「过弯时发生了什么——差速转向原理」\n' +
|
|
1829
|
+
'- "How to Speak" → 错:「如何演讲」(干瘪,丢掉这是 MIT 经典讲座的分量)\n' +
|
|
1830
|
+
' 对:「怎样做一场好演讲」\n' +
|
|
1831
|
+
'- "makes sense" 类直译 → 错:「这说得通」,对:「这有道理」\n' +
|
|
1832
|
+
'- "at the end of the day" 类直译 → 错:「在一天结束时」,对:「归根结底」\n\n' +
|
|
1833
|
+
'严格输出 JSON 数组,只含译文,顺序与输入一致,不要任何解释。\n\n' +
|
|
1834
|
+
lines
|
|
1835
|
+
)
|
|
1836
|
+
}
|
|
1837
|
+
|
|
1838
|
+
/* 自适应并发:一次 10 条一批,同时在飞的请求数在 MIN..MAX 之间爬。
|
|
1839
|
+
成功连击就加速,失败或超时立刻降速,避免把 provider 打到 429。 */
|
|
1840
|
+
const MIN_INFLIGHT = 4
|
|
1841
|
+
const MAX_INFLIGHT = 24
|
|
1842
|
+
let inflight = 10
|
|
1843
|
+
let streak = 0
|
|
1844
|
+
let consecutiveFails = 0
|
|
1845
|
+
let shortCount = 0
|
|
1846
|
+
const refilled = new Set()
|
|
1847
|
+
const errors = []
|
|
1848
|
+
let cursor = 0
|
|
1849
|
+
let done = 0
|
|
1850
|
+
let failed = 0
|
|
1851
|
+
let active = 0
|
|
1852
|
+
const startedAt = Date.now()
|
|
1853
|
+
|
|
1854
|
+
const report = () => {
|
|
1855
|
+
const elapsed = Math.max(1, (Date.now() - startedAt) / 1000)
|
|
1856
|
+
const rate = done / elapsed
|
|
1857
|
+
void writeCache('titles-progress.json', {
|
|
1858
|
+
at: Date.now(),
|
|
1859
|
+
total: batches.length,
|
|
1860
|
+
done,
|
|
1861
|
+
failed,
|
|
1862
|
+
inflight,
|
|
1863
|
+
ratePerSec: Number(rate.toFixed(2)),
|
|
1864
|
+
refilled: shortCount,
|
|
1865
|
+
etaSec: rate > 0 ? Math.round((batches.length - done - failed) / rate) : null,
|
|
1866
|
+
})
|
|
1867
|
+
}
|
|
1868
|
+
|
|
1869
|
+
const runBatch = async (batch) => {
|
|
1870
|
+
for (let attempt = 0; attempt < 3; attempt += 1) {
|
|
1871
|
+
try {
|
|
1872
|
+
const text = await askLlm(ctx, buildPrompt(batch))
|
|
1873
|
+
const arr = extractJsonArray(text)
|
|
1874
|
+
if (!Array.isArray(arr)) throw new Error('bad json')
|
|
1875
|
+
const rejected = []
|
|
1876
|
+
batch.forEach((it, i2) => {
|
|
1877
|
+
if (typeof arr[i2] !== 'string' || !arr[i2].trim()) return
|
|
1878
|
+
const clean = normalizeTitle(stripLeadingIndex(arr[i2]))
|
|
1879
|
+
const verdict = titleVerdict(clean)
|
|
1880
|
+
/* 指令泄漏和空串不进缓存,下次还会重翻;其余规范化后收下 */
|
|
1881
|
+
if (verdict === 'leak' || verdict === 'empty') {
|
|
1882
|
+
rejected.push({ v: it.v, verdict, text: clean.slice(0, 90) })
|
|
1883
|
+
return
|
|
1884
|
+
}
|
|
1885
|
+
cache[it.v] = clean
|
|
1886
|
+
})
|
|
1887
|
+
/* 批次变大以后模型偶尔少给几条:把没拿到的重新排队,只回填一次 */
|
|
1888
|
+
if (arr.length < batch.length) {
|
|
1889
|
+
const missing = batch.slice(arr.length).filter((it) => !refilled.has(it.v))
|
|
1890
|
+
if (missing.length) {
|
|
1891
|
+
for (const it of missing) refilled.add(it.v)
|
|
1892
|
+
for (let i = 0; i < missing.length; i += BATCH) batches.push(missing.slice(i, i + BATCH))
|
|
1893
|
+
shortCount += missing.length
|
|
1894
|
+
}
|
|
1895
|
+
}
|
|
1896
|
+
if (rejected.length) {
|
|
1897
|
+
void writeCache('titles-reject.json', { at: Date.now(), count: rejected.length, samples: rejected.slice(0, 20) })
|
|
1898
|
+
}
|
|
1899
|
+
scheduleI18nSave(store)
|
|
1900
|
+
streak += 1
|
|
1901
|
+
consecutiveFails = 0
|
|
1902
|
+
if (streak >= 6 && inflight < MAX_INFLIGHT) {
|
|
1903
|
+
inflight += 1
|
|
1904
|
+
streak = 0
|
|
1905
|
+
}
|
|
1906
|
+
return true
|
|
1907
|
+
} catch (err) {
|
|
1908
|
+
streak = 0
|
|
1909
|
+
consecutiveFails += 1
|
|
1910
|
+
if (errors.length < 12) errors.push(String(err?.message ?? err).slice(0, 160))
|
|
1911
|
+
/* 单批失败很常见(超时、JSON 截断),连续两次才降速,且一次只降一档 */
|
|
1912
|
+
if (consecutiveFails >= 2) {
|
|
1913
|
+
inflight = Math.max(MIN_INFLIGHT, inflight - 2)
|
|
1914
|
+
consecutiveFails = 0
|
|
1915
|
+
}
|
|
1916
|
+
if (attempt < 2) await new Promise((r) => setTimeout(r, 500 * (attempt + 1)))
|
|
1917
|
+
}
|
|
1918
|
+
}
|
|
1919
|
+
return false
|
|
1920
|
+
}
|
|
1921
|
+
|
|
1922
|
+
const lane = async () => {
|
|
1923
|
+
while (cursor < batches.length) {
|
|
1924
|
+
if (active >= inflight) {
|
|
1925
|
+
await new Promise((r) => setTimeout(r, 40))
|
|
1926
|
+
continue
|
|
1927
|
+
}
|
|
1928
|
+
const idx = cursor
|
|
1929
|
+
cursor += 1
|
|
1930
|
+
active += 1
|
|
1931
|
+
const ok = await runBatch(batches[idx])
|
|
1932
|
+
active -= 1
|
|
1933
|
+
if (ok) done += 1
|
|
1934
|
+
else failed += 1
|
|
1935
|
+
if ((done + failed) % 10 === 0) report()
|
|
1936
|
+
}
|
|
1937
|
+
}
|
|
1938
|
+
|
|
1939
|
+
await Promise.all(Array.from({ length: MAX_INFLIGHT }, lane))
|
|
1940
|
+
report()
|
|
1941
|
+
if (errors.length) {
|
|
1942
|
+
void writeCache('titles-llm-errors.json', { at: Date.now(), inflight, errors })
|
|
1943
|
+
}
|
|
1944
|
+
return cache
|
|
1945
|
+
}
|
|
1946
|
+
|
|
1947
|
+
/** 翻译领域名和分类名,一共 166 条,一次跑完 */
|
|
1948
|
+
async function translateTaxonomy(ctx, jobs, lang = 'zh') {
|
|
1949
|
+
const store = lang + '-taxonomy'
|
|
1950
|
+
const cache = await loadI18n(store)
|
|
1951
|
+
const todo = jobs.filter((j) => !cache[j.key])
|
|
1952
|
+
if (!todo.length) return cache
|
|
1953
|
+
|
|
1954
|
+
const BATCH = 30
|
|
1955
|
+
for (let i = 0; i < todo.length; i += BATCH) {
|
|
1956
|
+
const batch = todo.slice(i, i + BATCH)
|
|
1957
|
+
const numbered = batch.map((j, idx) => `${idx + 1}. ${j.text}`).join('\n')
|
|
1958
|
+
const prompt =
|
|
1959
|
+
'把这些技术视频平台的领域名和分类名翻成简体中文。\n' +
|
|
1960
|
+
'要求:\n' +
|
|
1961
|
+
'- 简洁准确,符合中文技术社区习惯\n' +
|
|
1962
|
+
'- 保留专有名词与缩写:AI、GPU、Rust、TCP、VR\n' +
|
|
1963
|
+
'- 并列结构保留并列,例如 "Business, Startups & Economics" → "商业、创业与经济"\n' +
|
|
1964
|
+
'- 分类名用名词短语,不加句号\n\n' +
|
|
1965
|
+
'严格输出 JSON 数组,只含译文,顺序与输入一致。\n\n' +
|
|
1966
|
+
numbered
|
|
1967
|
+
try {
|
|
1968
|
+
const text = await askLlm(ctx, prompt)
|
|
1969
|
+
const arr = extractJsonArray(text)
|
|
1970
|
+
if (Array.isArray(arr)) {
|
|
1971
|
+
batch.forEach((j, idx) => {
|
|
1972
|
+
if (typeof arr[idx] === 'string' && arr[idx].trim()) cache[j.key] = arr[idx].trim()
|
|
1973
|
+
})
|
|
1974
|
+
scheduleI18nSave(store)
|
|
1975
|
+
}
|
|
1976
|
+
} catch {
|
|
1977
|
+
break
|
|
1978
|
+
}
|
|
1979
|
+
}
|
|
1980
|
+
return cache
|
|
1981
|
+
}
|
|
1982
|
+
|
|
1983
|
+
/* ---------------- 依赖探测 ---------------- */
|
|
1984
|
+
|
|
1985
|
+
/** 跑一条命令拿输出;命令不存在、超时、非零退出都收敛成同一个形状。 */
|
|
1986
|
+
function runCmd(cmd, args, timeoutMs = 8000) {
|
|
1987
|
+
return new Promise((resolve) => {
|
|
1988
|
+
let out = ''
|
|
1989
|
+
let err = ''
|
|
1990
|
+
let settled = false
|
|
1991
|
+
const finish = (r) => {
|
|
1992
|
+
if (settled) return
|
|
1993
|
+
settled = true
|
|
1994
|
+
resolve(r)
|
|
1995
|
+
}
|
|
1996
|
+
let child
|
|
1997
|
+
try {
|
|
1998
|
+
child = spawn(cmd, args, { windowsHide: true, stdio: ['ignore', 'pipe', 'pipe'] })
|
|
1999
|
+
} catch (e) {
|
|
2000
|
+
return finish({ ok: false, out: '', err: String(e?.message ?? e) })
|
|
2001
|
+
}
|
|
2002
|
+
const timer = setTimeout(() => {
|
|
2003
|
+
try {
|
|
2004
|
+
child.kill()
|
|
2005
|
+
} catch {
|
|
2006
|
+
/* 已退出 */
|
|
2007
|
+
}
|
|
2008
|
+
finish({ ok: false, out, err: 'timeout' })
|
|
2009
|
+
}, timeoutMs)
|
|
2010
|
+
child.stdout.on('data', (d) => {
|
|
2011
|
+
out = (out + d).slice(0, 8000)
|
|
2012
|
+
})
|
|
2013
|
+
child.stderr.on('data', (d) => {
|
|
2014
|
+
err = (err + d).slice(0, 8000)
|
|
2015
|
+
})
|
|
2016
|
+
child.on('error', (e) => {
|
|
2017
|
+
clearTimeout(timer)
|
|
2018
|
+
finish({ ok: false, out, err: String(e?.message ?? e) })
|
|
2019
|
+
})
|
|
2020
|
+
child.on('close', (code) => {
|
|
2021
|
+
clearTimeout(timer)
|
|
2022
|
+
finish({ ok: code === 0, out, err })
|
|
2023
|
+
})
|
|
2024
|
+
})
|
|
2025
|
+
}
|
|
2026
|
+
|
|
2027
|
+
const depsState = { at: 0, value: null }
|
|
2028
|
+
|
|
2029
|
+
/** 探测字幕管线依赖。缺失时给安装提示,不替用户装。 */
|
|
2030
|
+
async function probeDeps(force) {
|
|
2031
|
+
if (!force && depsState.value && Date.now() - depsState.at < 60_000) return depsState.value
|
|
2032
|
+
const [ytdlp, ffmpeg, py] = await Promise.all([
|
|
2033
|
+
runCmd('yt-dlp', ['--version'], 10_000),
|
|
2034
|
+
runCmd('ffmpeg', ['-version'], 10_000),
|
|
2035
|
+
runCmd('python', ['-c', 'import faster_whisper;print(faster_whisper.__version__)'], 15_000),
|
|
2036
|
+
])
|
|
2037
|
+
const value = {
|
|
2038
|
+
ytdlp: ytdlp.ok
|
|
2039
|
+
? { ok: true, version: ytdlp.out.trim().split('\n')[0] }
|
|
2040
|
+
: { ok: false, hint: 'pip install -U yt-dlp' },
|
|
2041
|
+
ffmpeg: ffmpeg.ok
|
|
2042
|
+
? {
|
|
2043
|
+
ok: true,
|
|
2044
|
+
version: (/ffmpeg version (\S+)/.exec(ffmpeg.out) || [])[1] || 'ok',
|
|
2045
|
+
}
|
|
2046
|
+
: { ok: false, hint: '安装 ffmpeg 并加入 PATH' },
|
|
2047
|
+
whisper: py.ok
|
|
2048
|
+
? { ok: true, version: py.out.trim() }
|
|
2049
|
+
: { ok: false, hint: 'pip install faster-whisper' },
|
|
2050
|
+
}
|
|
2051
|
+
depsState.at = Date.now()
|
|
2052
|
+
depsState.value = value
|
|
2053
|
+
return value
|
|
2054
|
+
}
|
|
2055
|
+
|
|
2056
|
+
/* ---------------- 字幕 ---------------- */
|
|
2057
|
+
|
|
2058
|
+
function subsDir() {
|
|
2059
|
+
return join(cacheRoot(), 'subs')
|
|
2060
|
+
}
|
|
2061
|
+
|
|
2062
|
+
function vttSeconds(s) {
|
|
2063
|
+
const m = /(\d+):(\d+):(\d+)\.(\d+)/.exec(s)
|
|
2064
|
+
if (!m) return 0
|
|
2065
|
+
return Number(m[1]) * 3600 + Number(m[2]) * 60 + Number(m[3]) + Number(m[4]) / 1000
|
|
2066
|
+
}
|
|
2067
|
+
|
|
2068
|
+
function parseVtt(text) {
|
|
2069
|
+
const cues = []
|
|
2070
|
+
let cur = null
|
|
2071
|
+
for (const line of String(text).replace(/\r/g, '').split('\n')) {
|
|
2072
|
+
if (/^\d{2}:\d{2}:\d{2}\.\d{3}\s*-->/.test(line)) {
|
|
2073
|
+
const [a, b] = line.split('-->')
|
|
2074
|
+
cur = {
|
|
2075
|
+
start: vttSeconds(a.trim()),
|
|
2076
|
+
end: vttSeconds((b || '').trim().split(/\s+/)[0] || ''),
|
|
2077
|
+
text: '',
|
|
2078
|
+
}
|
|
2079
|
+
cues.push(cur)
|
|
2080
|
+
} else if (cur && line.trim() && !/^(WEBVTT|NOTE|Kind:|Language:|STYLE)/.test(line)) {
|
|
2081
|
+
const clean = line.replace(/<[^>]+>/g, '').trim()
|
|
2082
|
+
if (clean) cur.text = cur.text ? cur.text + ' ' + clean : clean
|
|
2083
|
+
}
|
|
2084
|
+
}
|
|
2085
|
+
return cues.filter((c) => c.text)
|
|
2086
|
+
}
|
|
2087
|
+
|
|
2088
|
+
/* 中文优先,回退到英文 */
|
|
2089
|
+
const SUBS_ZH = ['zh-Hans', 'zh-CN', 'zh', 'zh-Hant', 'zh-TW']
|
|
2090
|
+
const SUBS_EN = ['en', 'en-US', 'en-GB']
|
|
2091
|
+
|
|
2092
|
+
/**
|
|
2093
|
+
* 取一条视频的字幕。
|
|
2094
|
+
* 只做三件事:查缓存、调 yt-dlp、把失败翻译成可执行的提示。
|
|
2095
|
+
* 依赖缺失或上游风控时如实上报,不尝试自行修复环境。
|
|
2096
|
+
*/
|
|
2097
|
+
async function loadSubs(vid, wantZh) {
|
|
2098
|
+
const dir = subsDir()
|
|
2099
|
+
const cacheFile = join(dir, `${vid}.${wantZh ? 'zh' : 'en'}.json`)
|
|
2100
|
+
try {
|
|
2101
|
+
const raw = await readFile(cacheFile, 'utf8')
|
|
2102
|
+
return { status: 200, body: JSON.parse(raw) }
|
|
2103
|
+
} catch {
|
|
2104
|
+
/* 未缓存 */
|
|
2105
|
+
}
|
|
2106
|
+
|
|
2107
|
+
const deps = await probeDeps()
|
|
2108
|
+
if (!deps.ytdlp.ok) {
|
|
2109
|
+
return {
|
|
2110
|
+
status: 424,
|
|
2111
|
+
body: { reason: 'missing-dependency', need: 'yt-dlp', hint: deps.ytdlp.hint },
|
|
2112
|
+
}
|
|
2113
|
+
}
|
|
2114
|
+
|
|
2115
|
+
await mkdir(dir, { recursive: true })
|
|
2116
|
+
const tmp = join(dir, `.tmp-${vid}-${Date.now()}`)
|
|
2117
|
+
await mkdir(tmp, { recursive: true })
|
|
2118
|
+
|
|
2119
|
+
/* 约定路径:用户把导出的 cookie 放到插件缓存目录下,管线自动使用 */
|
|
2120
|
+
const cookieFile = join(cacheRoot(), 'cookies.txt')
|
|
2121
|
+
const hasCookies = await stat(cookieFile)
|
|
2122
|
+
.then((s) => s.size > 0)
|
|
2123
|
+
.catch(() => false)
|
|
2124
|
+
|
|
2125
|
+
const langs = (wantZh ? SUBS_ZH.concat(SUBS_EN) : SUBS_EN).join(',')
|
|
2126
|
+
const args = [
|
|
2127
|
+
'--skip-download',
|
|
2128
|
+
'--write-subs',
|
|
2129
|
+
'--write-auto-subs',
|
|
2130
|
+
'--sub-langs',
|
|
2131
|
+
langs,
|
|
2132
|
+
'--sub-format',
|
|
2133
|
+
'vtt',
|
|
2134
|
+
'--no-warnings',
|
|
2135
|
+
'-o',
|
|
2136
|
+
join(tmp, '%(id)s.%(ext)s'),
|
|
2137
|
+
]
|
|
2138
|
+
if (hasCookies) args.push('--cookies', cookieFile)
|
|
2139
|
+
args.push(`https://www.youtube.com/watch?v=${vid}`)
|
|
2140
|
+
|
|
2141
|
+
const res = await runCmd('yt-dlp', args, 120_000)
|
|
2142
|
+
|
|
2143
|
+
let files = []
|
|
2144
|
+
try {
|
|
2145
|
+
files = (await readdir(tmp)).filter((f) => f.endsWith('.vtt'))
|
|
2146
|
+
} catch {
|
|
2147
|
+
/* 目录都没建出来 */
|
|
2148
|
+
}
|
|
2149
|
+
|
|
2150
|
+
if (!files.length) {
|
|
2151
|
+
await rm(tmp, { recursive: true, force: true }).catch(() => {})
|
|
2152
|
+
if (/sign in to confirm|not a bot|cookies/i.test(res.err)) {
|
|
2153
|
+
return {
|
|
2154
|
+
status: 409,
|
|
2155
|
+
body: {
|
|
2156
|
+
reason: 'needs-cookies',
|
|
2157
|
+
hint:
|
|
2158
|
+
'YouTube 要求登录校验。用浏览器扩展导出 Netscape 格式 cookies.txt,' +
|
|
2159
|
+
`放到 ${cookieFile},然后重试。`,
|
|
2160
|
+
cookiePath: cookieFile,
|
|
2161
|
+
},
|
|
2162
|
+
}
|
|
2163
|
+
}
|
|
2164
|
+
return {
|
|
2165
|
+
status: 409,
|
|
2166
|
+
body: { reason: 'no-captions', hint: '这条视频没有可用字幕,需要本地转写' },
|
|
2167
|
+
}
|
|
2168
|
+
}
|
|
2169
|
+
|
|
2170
|
+
/* 文件名形如 <vid>.<lang>.vtt,按优先级挑一条 */
|
|
2171
|
+
const order = wantZh ? SUBS_ZH.concat(SUBS_EN) : SUBS_EN
|
|
2172
|
+
let picked = null
|
|
2173
|
+
let pickedLang = null
|
|
2174
|
+
for (const lang of order) {
|
|
2175
|
+
const hit = files.find((f) => f.startsWith(`${vid}.${lang}.`))
|
|
2176
|
+
if (hit) {
|
|
2177
|
+
picked = hit
|
|
2178
|
+
pickedLang = lang
|
|
2179
|
+
break
|
|
2180
|
+
}
|
|
2181
|
+
}
|
|
2182
|
+
if (!picked) {
|
|
2183
|
+
picked = files[0]
|
|
2184
|
+
pickedLang = picked.slice(vid.length + 1, picked.length - 4)
|
|
2185
|
+
}
|
|
2186
|
+
|
|
2187
|
+
let cues = []
|
|
2188
|
+
try {
|
|
2189
|
+
cues = parseVtt(await readFile(join(tmp, picked), 'utf8'))
|
|
2190
|
+
} catch {
|
|
2191
|
+
cues = []
|
|
2192
|
+
}
|
|
2193
|
+
await rm(tmp, { recursive: true, force: true }).catch(() => {})
|
|
2194
|
+
|
|
2195
|
+
if (!cues.length) {
|
|
2196
|
+
return { status: 409, body: { reason: 'no-captions', hint: '字幕文件为空' } }
|
|
2197
|
+
}
|
|
2198
|
+
|
|
2199
|
+
const payload = {
|
|
2200
|
+
v: 1,
|
|
2201
|
+
vid,
|
|
2202
|
+
lang: pickedLang,
|
|
2203
|
+
auto: /^(zh|en)/.test(pickedLang) ? undefined : undefined,
|
|
2204
|
+
cues,
|
|
2205
|
+
at: Date.now(),
|
|
2206
|
+
}
|
|
2207
|
+
void writeFile(cacheFile, JSON.stringify(payload)).catch(() => {})
|
|
2208
|
+
return { status: 200, body: payload }
|
|
2209
|
+
}
|
|
2210
|
+
|
|
2211
|
+
function send(res, code, body, type = 'application/json; charset=utf-8') {
|
|
2212
|
+
const headers = {
|
|
2213
|
+
'Content-Type': type,
|
|
2214
|
+
'Cache-Control': 'no-store',
|
|
2215
|
+
'X-Content-Type-Options': 'nosniff',
|
|
2216
|
+
}
|
|
2217
|
+
/* 桌面端页面在 dsh-app:// 下拿不到 HTTP 地址,借每次响应把它带过去,
|
|
2218
|
+
前端才能把播放器中转页放回安全上下文 */
|
|
2219
|
+
if (state.origin) headers['X-Hackertube-Origin'] = state.origin
|
|
2220
|
+
res.writeHead(code, headers)
|
|
2221
|
+
res.end(typeof body === 'string' ? body : JSON.stringify(body))
|
|
2222
|
+
}
|
|
2223
|
+
|
|
2224
|
+
function json(res, code, body) {
|
|
2225
|
+
send(res, code, body)
|
|
2226
|
+
}
|
|
2227
|
+
|
|
2228
|
+
async function handler(req, res) {
|
|
2229
|
+
const url = new URL(req.url ?? '/', 'http://local')
|
|
2230
|
+
const route = url.pathname.slice(PREFIX.length)
|
|
2231
|
+
|
|
2232
|
+
if (!trusted(req)) return json(res, 403, { error: 'forbidden' })
|
|
2233
|
+
|
|
2234
|
+
/* 浏览器半的执行回执:client.js 在各阶段 POST 上来,落盘供事后核查 */
|
|
2235
|
+
if (route === '/chat' && req.method === 'POST') {
|
|
2236
|
+
let size = 0
|
|
2237
|
+
const chunks = []
|
|
2238
|
+
req.on('data', (c) => {
|
|
2239
|
+
size += c.length
|
|
2240
|
+
if (size > 262144) {
|
|
2241
|
+
req.destroy()
|
|
2242
|
+
return
|
|
2243
|
+
}
|
|
2244
|
+
chunks.push(c)
|
|
2245
|
+
})
|
|
2246
|
+
req.on('end', async () => {
|
|
2247
|
+
let body = {}
|
|
2248
|
+
try {
|
|
2249
|
+
body = JSON.parse(Buffer.concat(chunks).toString('utf8'))
|
|
2250
|
+
} catch {
|
|
2251
|
+
return json(res, 400, { error: 'bad json' })
|
|
2252
|
+
}
|
|
2253
|
+
const vid = String(body.vid || '')
|
|
2254
|
+
const question = String(body.question || '').slice(0, 2000)
|
|
2255
|
+
if (!/^[A-Za-z0-9_-]{6,20}$/.test(vid) || !question) {
|
|
2256
|
+
return json(res, 400, { error: 'vid and question required' })
|
|
2257
|
+
}
|
|
2258
|
+
if (!state.ctx) return json(res, 503, { error: 'llm unavailable' })
|
|
2259
|
+
try {
|
|
2260
|
+
const detail = await loadVideo(vid).catch(() => null)
|
|
2261
|
+
const title = (detail && (detail.title || detail.t)) || vid
|
|
2262
|
+
/* 有字幕就带上播放位置附近的片段,没有就只靠元数据回答 */
|
|
2263
|
+
let subtitleContext = ''
|
|
2264
|
+
try {
|
|
2265
|
+
const subs = await loadSubs(vid, false)
|
|
2266
|
+
if (subs.status === 200 && subs.body.cues && subs.body.cues.length) {
|
|
2267
|
+
const at = Number(body.at)
|
|
2268
|
+
const cues = subs.body.cues
|
|
2269
|
+
let slice = cues
|
|
2270
|
+
if (Number.isFinite(at) && at > 0) {
|
|
2271
|
+
const from = Math.max(0, at - 180)
|
|
2272
|
+
const to = at + 180
|
|
2273
|
+
slice = cues.filter((c) => c.end >= from && c.start <= to)
|
|
2274
|
+
if (!slice.length) slice = cues.slice(0, 60)
|
|
2275
|
+
} else {
|
|
2276
|
+
slice = cues.slice(0, 120)
|
|
2277
|
+
}
|
|
2278
|
+
subtitleContext =
|
|
2279
|
+
'\n\n以下是该视频的字幕片段(带时间戳):\n' +
|
|
2280
|
+
slice
|
|
2281
|
+
.slice(0, 200)
|
|
2282
|
+
.map((c) => `[${Math.floor(c.start / 60)}:${String(Math.floor(c.start % 60)).padStart(2, '0')}] ${c.text}`)
|
|
2283
|
+
.join('\n')
|
|
2284
|
+
}
|
|
2285
|
+
} catch {
|
|
2286
|
+
/* 字幕不可用时退化为只读元数据 */
|
|
2287
|
+
}
|
|
2288
|
+
const answerLang = body.lang === 'en' ? 'en' : 'zh'
|
|
2289
|
+
const prompt =
|
|
2290
|
+
`你在帮用户理解一个 Hacker News 推荐过的技术视频。\n` +
|
|
2291
|
+
`视频标题:${title}\n` +
|
|
2292
|
+
(detail && detail.channel ? `频道:${detail.channel}\n` : '') +
|
|
2293
|
+
(detail && detail.summary ? `简介:${detail.summary}\n` : '') +
|
|
2294
|
+
(detail && detail.why_hn ? `HN 推荐理由:${detail.why_hn}\n` : '') +
|
|
2295
|
+
subtitleContext +
|
|
2296
|
+
`\n\n用户的问题:${question}\n\n` +
|
|
2297
|
+
(answerLang === 'en'
|
|
2298
|
+
? '要求:用英文回答,紧扣视频内容;如果上下文里没有相关信息,直接说明看不到这部分内容,不要编造。'
|
|
2299
|
+
: '要求:用简体中文回答,紧扣视频内容;如果上下文里没有相关信息,直接说明看不到这部分内容,不要编造。')
|
|
2300
|
+
if (body.stream === true) {
|
|
2301
|
+
res.writeHead(200, {
|
|
2302
|
+
'Content-Type': 'text/event-stream; charset=utf-8',
|
|
2303
|
+
'Cache-Control': 'no-store',
|
|
2304
|
+
Connection: 'keep-alive',
|
|
2305
|
+
'X-Accel-Buffering': 'no',
|
|
2306
|
+
})
|
|
2307
|
+
if (typeof res.flushHeaders === 'function') res.flushHeaders()
|
|
2308
|
+
let closed = false
|
|
2309
|
+
const finish = (payload) => {
|
|
2310
|
+
if (closed) return
|
|
2311
|
+
closed = true
|
|
2312
|
+
try {
|
|
2313
|
+
res.write('data: ' + JSON.stringify(payload) + '\n\n')
|
|
2314
|
+
res.write('data: [DONE]\n\n')
|
|
2315
|
+
res.end()
|
|
2316
|
+
} catch {
|
|
2317
|
+
/* 客户端提前断开 */
|
|
2318
|
+
}
|
|
2319
|
+
}
|
|
2320
|
+
res.on('close', () => {
|
|
2321
|
+
closed = true
|
|
2322
|
+
})
|
|
2323
|
+
try {
|
|
2324
|
+
const answer = await askLlm(state.ctx, prompt, {
|
|
2325
|
+
onDelta: (piece) => {
|
|
2326
|
+
if (closed) return
|
|
2327
|
+
try {
|
|
2328
|
+
res.write('data: ' + JSON.stringify({ delta: piece }) + '\n\n')
|
|
2329
|
+
} catch {
|
|
2330
|
+
closed = true
|
|
2331
|
+
}
|
|
2332
|
+
},
|
|
2333
|
+
})
|
|
2334
|
+
finish({ done: true, answer, hasSubtitles: !!subtitleContext })
|
|
2335
|
+
} catch (err) {
|
|
2336
|
+
finish({ error: String(err?.message ?? err).slice(0, 200) })
|
|
2337
|
+
}
|
|
2338
|
+
return
|
|
2339
|
+
}
|
|
2340
|
+
const answer = await askLlm(state.ctx, prompt)
|
|
2341
|
+
return json(res, 200, { answer, hasSubtitles: !!subtitleContext })
|
|
2342
|
+
} catch (err) {
|
|
2343
|
+
return json(res, 502, { error: String(err?.message ?? err).slice(0, 200) })
|
|
2344
|
+
}
|
|
2345
|
+
})
|
|
2346
|
+
return
|
|
2347
|
+
}
|
|
2348
|
+
|
|
2349
|
+
if (route === '/beacon' && req.method === 'POST') {
|
|
2350
|
+
let size = 0
|
|
2351
|
+
const chunks = []
|
|
2352
|
+
req.on('data', (c) => {
|
|
2353
|
+
size += c.length
|
|
2354
|
+
if (size > 65536) {
|
|
2355
|
+
req.destroy()
|
|
2356
|
+
return
|
|
2357
|
+
}
|
|
2358
|
+
chunks.push(c)
|
|
2359
|
+
})
|
|
2360
|
+
req.on('end', () => {
|
|
2361
|
+
try {
|
|
2362
|
+
const payload = JSON.parse(Buffer.concat(chunks).toString('utf8'))
|
|
2363
|
+
/* 记请求头时避开 payload 自己的 origin 字段,免得覆盖客户端上报的值 */
|
|
2364
|
+
payload.reqReferer = req.headers.referer ?? req.headers.referrer ?? null
|
|
2365
|
+
payload.reqOrigin = req.headers.origin ?? null
|
|
2366
|
+
payload.reqSite = req.headers['sec-fetch-site'] ?? null
|
|
2367
|
+
const prev = state.beaconLog || []
|
|
2368
|
+
prev.push(payload)
|
|
2369
|
+
state.beaconLog = prev.slice(-20)
|
|
2370
|
+
void writeCache('beacon.json', state.beaconLog)
|
|
2371
|
+
} catch {
|
|
2372
|
+
/* 回执坏了不影响主流程 */
|
|
2373
|
+
}
|
|
2374
|
+
json(res, 200, { ok: true })
|
|
2375
|
+
})
|
|
2376
|
+
return
|
|
2377
|
+
}
|
|
2378
|
+
|
|
2379
|
+
if (route === '/prefs') {
|
|
2380
|
+
if (req.method === 'POST') {
|
|
2381
|
+
let size = 0
|
|
2382
|
+
const chunks = []
|
|
2383
|
+
req.on('data', (c) => {
|
|
2384
|
+
size += c.length
|
|
2385
|
+
if (size > 262144) {
|
|
2386
|
+
req.destroy()
|
|
2387
|
+
return
|
|
2388
|
+
}
|
|
2389
|
+
chunks.push(c)
|
|
2390
|
+
})
|
|
2391
|
+
req.on('end', async () => {
|
|
2392
|
+
let body = null
|
|
2393
|
+
try {
|
|
2394
|
+
body = JSON.parse(Buffer.concat(chunks).toString('utf8'))
|
|
2395
|
+
} catch {
|
|
2396
|
+
return json(res, 400, { error: 'bad json' })
|
|
2397
|
+
}
|
|
2398
|
+
const clean = {}
|
|
2399
|
+
for (const key of ['likes', 'later', 'seen']) {
|
|
2400
|
+
const list = body && Array.isArray(body[key]) ? body[key] : []
|
|
2401
|
+
clean[key] = list.map((x) => String(x)).filter((x) => /^[A-Za-z0-9_-]{6,20}$/.test(x)).slice(-400)
|
|
2402
|
+
}
|
|
2403
|
+
clean.at = Date.now()
|
|
2404
|
+
await writeCache('prefs.json', clean)
|
|
2405
|
+
return json(res, 200, { ok: true, counts: { likes: clean.likes.length, later: clean.later.length, seen: clean.seen.length } })
|
|
2406
|
+
})
|
|
2407
|
+
return
|
|
2408
|
+
}
|
|
2409
|
+
const saved = await readCache('prefs.json')
|
|
2410
|
+
return json(res, 200, {
|
|
2411
|
+
likes: Array.isArray(saved && saved.likes) ? saved.likes : [],
|
|
2412
|
+
later: Array.isArray(saved && saved.later) ? saved.later : [],
|
|
2413
|
+
seen: Array.isArray(saved && saved.seen) ? saved.seen : [],
|
|
2414
|
+
})
|
|
2415
|
+
}
|
|
2416
|
+
/* 搜索框的中文查询先机翻成英文,再去命中英文标签与频道 */
|
|
2417
|
+
if (route === '/mt' && req.method === 'POST') {
|
|
2418
|
+
let size = 0
|
|
2419
|
+
const chunks = []
|
|
2420
|
+
req.on('data', (c) => {
|
|
2421
|
+
size += c.length
|
|
2422
|
+
if (size > 8192) {
|
|
2423
|
+
req.destroy()
|
|
2424
|
+
return
|
|
2425
|
+
}
|
|
2426
|
+
chunks.push(c)
|
|
2427
|
+
})
|
|
2428
|
+
req.on('end', async () => {
|
|
2429
|
+
let body = {}
|
|
2430
|
+
try {
|
|
2431
|
+
body = JSON.parse(Buffer.concat(chunks).toString('utf8'))
|
|
2432
|
+
} catch {
|
|
2433
|
+
return json(res, 400, { error: 'bad json' })
|
|
2434
|
+
}
|
|
2435
|
+
const text = String(body.text || '').trim().slice(0, 200)
|
|
2436
|
+
const to = body.to === 'zh' ? 'zh' : 'en'
|
|
2437
|
+
if (!text) return json(res, 400, { error: 'text required' })
|
|
2438
|
+
const out = await mtQuery(text, to)
|
|
2439
|
+
return json(res, 200, { text: out })
|
|
2440
|
+
})
|
|
2441
|
+
return
|
|
2442
|
+
}
|
|
2443
|
+
|
|
2444
|
+
if (route === '/comments/translate' && req.method === 'POST') {
|
|
2445
|
+
let size = 0
|
|
2446
|
+
const chunks = []
|
|
2447
|
+
req.on('data', (c) => {
|
|
2448
|
+
size += c.length
|
|
2449
|
+
if (size > 262144) {
|
|
2450
|
+
req.destroy()
|
|
2451
|
+
return
|
|
2452
|
+
}
|
|
2453
|
+
chunks.push(c)
|
|
2454
|
+
})
|
|
2455
|
+
req.on('end', async () => {
|
|
2456
|
+
let body = {}
|
|
2457
|
+
try {
|
|
2458
|
+
body = JSON.parse(Buffer.concat(chunks).toString('utf8'))
|
|
2459
|
+
} catch {
|
|
2460
|
+
return json(res, 400, { error: 'bad json' })
|
|
2461
|
+
}
|
|
2462
|
+
const vid = String(body.vid || '')
|
|
2463
|
+
if (!/^[A-Za-z0-9_-]{6,20}$/.test(vid)) return json(res, 400, { error: 'bad vid' })
|
|
2464
|
+
const wanted = Array.isArray(body.ids) ? body.ids.map(String) : null
|
|
2465
|
+
const list = pickComments(vid).filter((c) => !wanted || wanted.indexOf(String(c.id)) >= 0)
|
|
2466
|
+
if (!list.length) return json(res, 200, { entries: {} })
|
|
2467
|
+
try {
|
|
2468
|
+
const cache = await translateComments(list)
|
|
2469
|
+
const entries = {}
|
|
2470
|
+
for (const c of list) {
|
|
2471
|
+
const zh = cache[MT_TAG + ':' + c.id]
|
|
2472
|
+
if (zh) entries[c.id] = zh
|
|
2473
|
+
}
|
|
2474
|
+
return json(res, 200, { entries })
|
|
2475
|
+
} catch (err) {
|
|
2476
|
+
return json(res, 500, { error: String(err?.message ?? err).slice(0, 200) })
|
|
2477
|
+
}
|
|
2478
|
+
})
|
|
2479
|
+
return
|
|
2480
|
+
}
|
|
2481
|
+
|
|
2482
|
+
if (req.method !== 'GET') return json(res, 405, { error: 'method not allowed' })
|
|
2483
|
+
|
|
2484
|
+
try {
|
|
2485
|
+
if (route === '/list') {
|
|
2486
|
+
const items = await loadList()
|
|
2487
|
+
if (!state.listJson) state.listJson = JSON.stringify(items)
|
|
2488
|
+
return send(res, 200, state.listJson)
|
|
2489
|
+
}
|
|
2490
|
+
if (route === '/playlists') {
|
|
2491
|
+
const list = await loadPlaylists(url.searchParams.get('force') === '1')
|
|
2492
|
+
return json(res, 200, list.map(withChinese))
|
|
2493
|
+
}
|
|
2494
|
+
if (route === '/playlist') {
|
|
2495
|
+
const slug = url.searchParams.get('slug')
|
|
2496
|
+
if (!slug || !/^[a-z0-9-]{2,64}$/.test(slug)) return json(res, 400, { error: 'bad-slug' })
|
|
2497
|
+
return json(res, 200, withChinese(await loadPlaylist(slug)))
|
|
2498
|
+
}
|
|
2499
|
+
if (route === '/browse') {
|
|
2500
|
+
if (!state.browseJson) state.browseJson = JSON.stringify(await loadBrowse())
|
|
2501
|
+
return send(res, 200, state.browseJson)
|
|
2502
|
+
}
|
|
2503
|
+
if (route === '/related') {
|
|
2504
|
+
const vid = url.searchParams.get('vid')
|
|
2505
|
+
if (!vid || !/^[A-Za-z0-9_-]{6,20}$/.test(vid)) return json(res, 400, { error: 'bad vid' })
|
|
2506
|
+
return json(res, 200, await loadRelated(vid))
|
|
2507
|
+
}
|
|
2508
|
+
if (route === '/video') {
|
|
2509
|
+
const vid = url.searchParams.get('vid')
|
|
2510
|
+
if (!vid) return json(res, 400, { error: 'vid required' })
|
|
2511
|
+
const detail = await loadVideo(vid)
|
|
2512
|
+
if (url.searchParams.get('lang') === 'zh') {
|
|
2513
|
+
/* 模型在就用模型翻;没有模型或没翻出来,退到机翻,同样落盘缓存 */
|
|
2514
|
+
let llmCache = {}
|
|
2515
|
+
if (state.ctx) {
|
|
2516
|
+
try {
|
|
2517
|
+
llmCache = await translateDetail(state.ctx, detail, 'zh')
|
|
2518
|
+
} catch {
|
|
2519
|
+
llmCache = {}
|
|
2520
|
+
}
|
|
2521
|
+
}
|
|
2522
|
+
const pick = (field) => llmCache[`${vid}:${field}`] || null
|
|
2523
|
+
const needMt = !pick('summary') || !pick('why_hn') || !!(detail.quote && !pick('quote'))
|
|
2524
|
+
let mtCache = {}
|
|
2525
|
+
if (needMt) {
|
|
2526
|
+
try {
|
|
2527
|
+
mtCache = await translateDetailMT(detail)
|
|
2528
|
+
} catch {
|
|
2529
|
+
mtCache = {}
|
|
2530
|
+
}
|
|
2531
|
+
}
|
|
2532
|
+
const merge = (field) => llmCache[`${vid}:${field}`] || mtCache[`${vid}:${field}`] || null
|
|
2533
|
+
return json(res, 200, {
|
|
2534
|
+
...detail,
|
|
2535
|
+
comments: pickComments(vid),
|
|
2536
|
+
i18n: {
|
|
2537
|
+
summary: merge('summary'),
|
|
2538
|
+
why_hn: merge('why_hn'),
|
|
2539
|
+
quote: merge('quote'),
|
|
2540
|
+
},
|
|
2541
|
+
})
|
|
2542
|
+
}
|
|
2543
|
+
return json(res, 200, Object.assign({}, detail, { comments: pickComments(vid) }))
|
|
2544
|
+
}
|
|
2545
|
+
if (route === '/media') {
|
|
2546
|
+
const vid = url.searchParams.get('vid')
|
|
2547
|
+
if (!vid || !/^[A-Za-z0-9_-]{6,20}$/.test(vid)) {
|
|
2548
|
+
return json(res, 400, { error: 'bad vid' })
|
|
2549
|
+
}
|
|
2550
|
+
return serveMedia(req, res, vid)
|
|
2551
|
+
}
|
|
2552
|
+
if (route === '/stream') {
|
|
2553
|
+
const vid = url.searchParams.get('vid')
|
|
2554
|
+
if (!vid || !/^[A-Za-z0-9_-]{6,20}$/.test(vid)) {
|
|
2555
|
+
return json(res, 400, { error: 'bad vid' })
|
|
2556
|
+
}
|
|
2557
|
+
return serveStream(req, res, vid)
|
|
2558
|
+
}
|
|
2559
|
+
if (route === '/embed') {
|
|
2560
|
+
const vid = url.searchParams.get('vid')
|
|
2561
|
+
if (!vid || !/^[A-Za-z0-9_-]{6,20}$/.test(vid)) {
|
|
2562
|
+
return json(res, 400, { error: 'bad vid' })
|
|
2563
|
+
}
|
|
2564
|
+
return serveEmbed(req, res, vid, url.searchParams.get('lang') === 'zh' ? 'zh' : 'en')
|
|
2565
|
+
}
|
|
2566
|
+
if (route === '/config') {
|
|
2567
|
+
return json(res, 200, {
|
|
2568
|
+
origin: state.origin,
|
|
2569
|
+
authUrl: state.authUrl,
|
|
2570
|
+
mediaOrigin: state.mediaOrigin,
|
|
2571
|
+
})
|
|
2572
|
+
}
|
|
2573
|
+
if (route === '/taxonomy') {
|
|
2574
|
+
const lang = url.searchParams.get('lang') || 'en'
|
|
2575
|
+
const tax = await loadBrowse()
|
|
2576
|
+
if (lang !== 'zh') return json(res, 200, tax)
|
|
2577
|
+
const cache = await loadI18n('zh-taxonomy')
|
|
2578
|
+
if (url.searchParams.get('translate') === '1' && state.ctx) {
|
|
2579
|
+
const jobs = []
|
|
2580
|
+
for (const a of tax.areas || []) {
|
|
2581
|
+
if (a.name && !cache['a:' + a.id]) jobs.push({ key: 'a:' + a.id, text: a.name })
|
|
2582
|
+
for (const c of a.categories || []) {
|
|
2583
|
+
if (c.name && !cache['c:' + c.id]) jobs.push({ key: 'c:' + c.id, text: c.name })
|
|
2584
|
+
}
|
|
2585
|
+
}
|
|
2586
|
+
if (jobs.length) void translateTaxonomy(state.ctx, jobs).catch(() => {})
|
|
2587
|
+
}
|
|
2588
|
+
const out = {
|
|
2589
|
+
version: tax.version,
|
|
2590
|
+
areas: (tax.areas || []).map((a) => ({
|
|
2591
|
+
...a,
|
|
2592
|
+
nameZh: cache['a:' + a.id] || null,
|
|
2593
|
+
categories: (a.categories || []).map((c) => ({
|
|
2594
|
+
...c,
|
|
2595
|
+
nameZh: cache['c:' + c.id] || null,
|
|
2596
|
+
})),
|
|
2597
|
+
})),
|
|
2598
|
+
}
|
|
2599
|
+
return json(res, 200, out)
|
|
2600
|
+
}
|
|
2601
|
+
if (route === '/titles') {
|
|
2602
|
+
const lang = url.searchParams.get('lang') || 'zh'
|
|
2603
|
+
const store = lang + '-titles'
|
|
2604
|
+
const cache = await loadI18n(store)
|
|
2605
|
+
if (url.searchParams.get('translate') === '1') {
|
|
2606
|
+
const items = await loadList()
|
|
2607
|
+
const missing = items.filter((it) => it.t && !cache[it.v])
|
|
2608
|
+
void writeCache('titles-trigger.json', {
|
|
2609
|
+
at: Date.now(),
|
|
2610
|
+
hasCtx: !!state.ctx,
|
|
2611
|
+
items: items.length,
|
|
2612
|
+
missing: missing.length,
|
|
2613
|
+
})
|
|
2614
|
+
if (missing.length && state.ctx) {
|
|
2615
|
+
const limit = Number(url.searchParams.get('n') || 400)
|
|
2616
|
+
void translateTitles(state.ctx, missing.slice(0, limit), lang).catch(() => {})
|
|
2617
|
+
}
|
|
2618
|
+
}
|
|
2619
|
+
return json(res, 200, { lang, total: Object.keys(cache).length, entries: cache })
|
|
2620
|
+
}
|
|
2621
|
+
/**
|
|
2622
|
+
* 标题译文体检。没有 apply 只统计,apply=1 时把能修的修掉、
|
|
2623
|
+
* 把指令泄漏的条目删掉让下一轮重翻。可反复调用,幂等。
|
|
2624
|
+
*/
|
|
2625
|
+
if (route === '/titles/lint') {
|
|
2626
|
+
const lang = url.searchParams.get('lang') || 'zh'
|
|
2627
|
+
const store = lang + '-titles'
|
|
2628
|
+
const cache = await loadI18n(store)
|
|
2629
|
+
const apply = url.searchParams.get('apply') === '1'
|
|
2630
|
+
const out = {
|
|
2631
|
+
lang,
|
|
2632
|
+
total: 0,
|
|
2633
|
+
changed: 0,
|
|
2634
|
+
removed: 0,
|
|
2635
|
+
leaks: 0,
|
|
2636
|
+
separators: 0,
|
|
2637
|
+
rawSeparators: 0,
|
|
2638
|
+
empty: 0,
|
|
2639
|
+
applied: apply,
|
|
2640
|
+
samples: [],
|
|
2641
|
+
}
|
|
2642
|
+
for (const [vid, raw] of Object.entries(cache)) {
|
|
2643
|
+
out.total += 1
|
|
2644
|
+
const clean = normalizeTitle(raw)
|
|
2645
|
+
const verdict = titleVerdict(clean)
|
|
2646
|
+
if (verdict === 'leak') {
|
|
2647
|
+
out.leaks += 1
|
|
2648
|
+
if (out.samples.length < 8) {
|
|
2649
|
+
out.samples.push({ vid, verdict, text: String(raw).slice(0, 90) })
|
|
2650
|
+
}
|
|
2651
|
+
}
|
|
2652
|
+
if (verdict === 'separator') out.separators += 1
|
|
2653
|
+
if (/[||]/.test(String(raw))) out.rawSeparators += 1
|
|
2654
|
+
if (verdict === 'empty') out.empty += 1
|
|
2655
|
+
const changed = clean !== raw
|
|
2656
|
+
if (changed) out.changed += 1
|
|
2657
|
+
if (!apply) continue
|
|
2658
|
+
if (verdict === 'leak' || verdict === 'empty') {
|
|
2659
|
+
delete cache[vid]
|
|
2660
|
+
out.removed += 1
|
|
2661
|
+
} else if (changed) {
|
|
2662
|
+
cache[vid] = clean
|
|
2663
|
+
}
|
|
2664
|
+
}
|
|
2665
|
+
if (apply && (out.changed || out.removed)) scheduleI18nSave(store)
|
|
2666
|
+
return json(res, 200, out)
|
|
2667
|
+
}
|
|
2668
|
+
/** 指定 id 强制重译:先删缓存再排队,用来验证新的提示词。 */
|
|
2669
|
+
if (route === '/titles/retranslate') {
|
|
2670
|
+
const store = 'zh-titles'
|
|
2671
|
+
let ids = String(url.searchParams.get('ids') || '')
|
|
2672
|
+
.split(',')
|
|
2673
|
+
.map((x) => x.trim())
|
|
2674
|
+
.filter(Boolean)
|
|
2675
|
+
if (!ids.length && url.searchParams.get('filter') === 'title-missing') {
|
|
2676
|
+
const limit = Number(url.searchParams.get('n') || 200)
|
|
2677
|
+
const list = await loadList()
|
|
2678
|
+
const cacheNow = await loadI18n(store)
|
|
2679
|
+
ids = list
|
|
2680
|
+
.filter((it) => it.t && !cacheNow[it.v])
|
|
2681
|
+
.slice(0, limit)
|
|
2682
|
+
.map((it) => it.v)
|
|
2683
|
+
}
|
|
2684
|
+
if (!ids.length && url.searchParams.get('filter') === 'title-structure') {
|
|
2685
|
+
const limit = Number(url.searchParams.get('n') || 200)
|
|
2686
|
+
const minScore = Number(url.searchParams.get('min') || 1)
|
|
2687
|
+
const list = await loadList()
|
|
2688
|
+
const cacheNow = await loadI18n(store)
|
|
2689
|
+
ids = list
|
|
2690
|
+
.filter((it) => {
|
|
2691
|
+
const z = cacheNow[it.v]
|
|
2692
|
+
return z && titleStructureScore(z) >= minScore
|
|
2693
|
+
})
|
|
2694
|
+
.slice(0, limit)
|
|
2695
|
+
.map((it) => it.v)
|
|
2696
|
+
}
|
|
2697
|
+
if (!ids.length) return json(res, 400, { error: 'ids required' })
|
|
2698
|
+
const cache = await loadI18n(store)
|
|
2699
|
+
let removed = 0
|
|
2700
|
+
for (const id of ids) {
|
|
2701
|
+
if (cache[id] !== undefined) {
|
|
2702
|
+
delete cache[id]
|
|
2703
|
+
removed += 1
|
|
2704
|
+
}
|
|
2705
|
+
}
|
|
2706
|
+
scheduleI18nSave(store)
|
|
2707
|
+
const list = await loadList()
|
|
2708
|
+
const items = list.filter((it) => ids.includes(it.v))
|
|
2709
|
+
if (state.ctx && items.length) void translateTitles(state.ctx, items, 'zh').catch(() => {})
|
|
2710
|
+
return json(res, 200, { requested: ids.length, removed, queued: items.length })
|
|
2711
|
+
}
|
|
2712
|
+
if (route === '/titles/progress') {
|
|
2713
|
+
const p = await readCache('titles-progress.json')
|
|
2714
|
+
return json(res, 200, p || { done: 0, total: 0 })
|
|
2715
|
+
}
|
|
2716
|
+
if (route === '/tools') {
|
|
2717
|
+
const out = { registered: null, names: [] }
|
|
2718
|
+
try {
|
|
2719
|
+
const tools = state.ctx?.get?.('tools')
|
|
2720
|
+
if (tools) {
|
|
2721
|
+
out.hasService = true
|
|
2722
|
+
const api = []
|
|
2723
|
+
let p = tools
|
|
2724
|
+
while (p && p !== Object.prototype) {
|
|
2725
|
+
api.push(...Object.getOwnPropertyNames(p))
|
|
2726
|
+
p = Object.getPrototypeOf(p)
|
|
2727
|
+
}
|
|
2728
|
+
out.api = Array.from(new Set(api)).slice(0, 30)
|
|
2729
|
+
for (const key of ['get', 'schemas', 'view', 'wireSchemas']) {
|
|
2730
|
+
if (typeof tools[key] !== 'function') continue
|
|
2731
|
+
try {
|
|
2732
|
+
const v = await tools[key]()
|
|
2733
|
+
if (Array.isArray(v) && v.length) {
|
|
2734
|
+
out.names = v
|
|
2735
|
+
.map((x) => (typeof x === 'string' ? x : x && (x.name || x.id || x.tool)))
|
|
2736
|
+
.filter(Boolean)
|
|
2737
|
+
out.via = key
|
|
2738
|
+
if (out.names.length) break
|
|
2739
|
+
} else if (v && typeof v === 'object') {
|
|
2740
|
+
out.names = Object.keys(v).slice(0, 80)
|
|
2741
|
+
out.via = key
|
|
2742
|
+
if (out.names.length) break
|
|
2743
|
+
}
|
|
2744
|
+
} catch {
|
|
2745
|
+
/* 换下一个 */
|
|
2746
|
+
}
|
|
2747
|
+
}
|
|
2748
|
+
}
|
|
2749
|
+
} catch (err) {
|
|
2750
|
+
out.error = String(err?.message ?? err).slice(0, 150)
|
|
2751
|
+
}
|
|
2752
|
+
return json(res, 200, out)
|
|
2753
|
+
}
|
|
2754
|
+
if (route === '/thumb') {
|
|
2755
|
+
const vid = url.searchParams.get('vid')
|
|
2756
|
+
if (!vid || !/^[A-Za-z0-9_-]{6,20}$/.test(vid)) {
|
|
2757
|
+
return json(res, 400, { error: 'bad vid' })
|
|
2758
|
+
}
|
|
2759
|
+
try {
|
|
2760
|
+
const upstream = await fetch(`https://i.ytimg.com/vi/${vid}/hqdefault.jpg`, {
|
|
2761
|
+
signal: AbortSignal.timeout(15_000),
|
|
2762
|
+
})
|
|
2763
|
+
if (!upstream.ok) {
|
|
2764
|
+
res.writeHead(502, { 'Content-Type': 'text/plain' })
|
|
2765
|
+
return res.end('upstream ' + upstream.status)
|
|
2766
|
+
}
|
|
2767
|
+
const buf = Buffer.from(await upstream.arrayBuffer())
|
|
2768
|
+
/* 从本域返回,画布才不会被跨域图片污染 */
|
|
2769
|
+
res.writeHead(200, {
|
|
2770
|
+
'Content-Type': 'image/jpeg',
|
|
2771
|
+
'Cache-Control': 'public, max-age=86400',
|
|
2772
|
+
'Content-Length': buf.length,
|
|
2773
|
+
})
|
|
2774
|
+
return res.end(buf)
|
|
2775
|
+
} catch (err) {
|
|
2776
|
+
res.writeHead(502, { 'Content-Type': 'text/plain' })
|
|
2777
|
+
return res.end(String(err?.message ?? err).slice(0, 120))
|
|
2778
|
+
}
|
|
2779
|
+
}
|
|
2780
|
+
if (route === '/i18n') {
|
|
2781
|
+
const lang = url.searchParams.get('lang') || 'zh'
|
|
2782
|
+
const entries = await loadI18n(lang)
|
|
2783
|
+
/* 有 pending=1 时顺手把未翻译的条目排进后台,返回当前已有的 */
|
|
2784
|
+
if (url.searchParams.get('translate') === '1' && state.ctx) {
|
|
2785
|
+
const items = await loadList()
|
|
2786
|
+
const missing = items.filter((it) => it.r && !entries[it.v])
|
|
2787
|
+
if (missing.length) {
|
|
2788
|
+
const limit = Number(url.searchParams.get('n') || 240)
|
|
2789
|
+
void translateEntries(state.ctx, missing.slice(0, limit), lang).catch(() => {})
|
|
2790
|
+
}
|
|
2791
|
+
}
|
|
2792
|
+
return json(res, 200, { lang, total: Object.keys(entries).length, entries })
|
|
2793
|
+
}
|
|
2794
|
+
if (route === '/deps') {
|
|
2795
|
+
const force = url.searchParams.get('force') === '1'
|
|
2796
|
+
return json(res, 200, await probeDeps(force))
|
|
2797
|
+
}
|
|
2798
|
+
if (route === '/subs') {
|
|
2799
|
+
const vid = url.searchParams.get('vid')
|
|
2800
|
+
if (!vid || !/^[A-Za-z0-9_-]{6,20}$/.test(vid)) {
|
|
2801
|
+
return json(res, 400, { reason: 'bad-vid' })
|
|
2802
|
+
}
|
|
2803
|
+
const lang = url.searchParams.get('lang') || 'zh'
|
|
2804
|
+
const result = await loadSubs(vid, lang !== 'en')
|
|
2805
|
+
return json(res, result.status, result.body)
|
|
2806
|
+
}
|
|
2807
|
+
return json(res, 404, { error: 'not found' })
|
|
2808
|
+
} catch (err) {
|
|
2809
|
+
return json(res, 502, { error: String(err?.message ?? err) })
|
|
2810
|
+
}
|
|
2811
|
+
}
|
|
2812
|
+
|
|
2813
|
+
export function apply(ctx) {
|
|
2814
|
+
state.ctx = ctx
|
|
2815
|
+
/* 落一个启动标记:能不能写出这个文件,本身就说明 apply 有没有被调用 */
|
|
2816
|
+
void writeCache('boot.json', {
|
|
2817
|
+
at: Date.now(),
|
|
2818
|
+
pid: process.pid,
|
|
2819
|
+
profileDir: process.env.DSH_PROFILE_DIR || null,
|
|
2820
|
+
})
|
|
2821
|
+
|
|
2822
|
+
let attempt = 0
|
|
2823
|
+
let done = false
|
|
2824
|
+
|
|
2825
|
+
const tryRegister = () => {
|
|
2826
|
+
if (done) return
|
|
2827
|
+
let webServer = null
|
|
2828
|
+
try {
|
|
2829
|
+
webServer = ctx?.get?.('webServer')
|
|
2830
|
+
} catch {
|
|
2831
|
+
webServer = null
|
|
2832
|
+
}
|
|
2833
|
+
if (!webServer || typeof webServer.register !== 'function') {
|
|
2834
|
+
if (attempt < 2) {
|
|
2835
|
+
attempt += 1
|
|
2836
|
+
const t = setTimeout(tryRegister, 1500 * attempt)
|
|
2837
|
+
if (t && typeof t.unref === 'function') t.unref()
|
|
2838
|
+
}
|
|
2839
|
+
return
|
|
2840
|
+
}
|
|
2841
|
+
done = true
|
|
2842
|
+
/* 桌面端页面跑在 dsh-app:// 自定义协议下,不发 Referer,YouTube 因此报 153;
|
|
2843
|
+
记下 HTTP origin,让前端用绝对地址把中转页放进安全上下文 */
|
|
2844
|
+
try {
|
|
2845
|
+
const origin = typeof webServer.getOrigin === 'function' ? webServer.getOrigin() : null
|
|
2846
|
+
if (origin) state.origin = String(origin)
|
|
2847
|
+
const names = []
|
|
2848
|
+
let proto = webServer
|
|
2849
|
+
while (proto && proto !== Object.prototype) {
|
|
2850
|
+
for (const n of Object.getOwnPropertyNames(proto)) names.push(n)
|
|
2851
|
+
proto = Object.getPrototypeOf(proto)
|
|
2852
|
+
}
|
|
2853
|
+
void writeCache('webserver.json', {
|
|
2854
|
+
origin: state.origin,
|
|
2855
|
+
methods: Array.from(new Set(names)).slice(0, 40),
|
|
2856
|
+
})
|
|
2857
|
+
} catch (err) {
|
|
2858
|
+
void writeCache('webserver.json', { error: String(err?.message ?? err) })
|
|
2859
|
+
}
|
|
2860
|
+
|
|
2861
|
+
/* 浏览器打开本应用的入口:dsh-app:// 不发 Referer,导致内嵌播放器全部失效;
|
|
2862
|
+
拿到带 launch token 的 HTTP 地址后,整个页面就能落到安全上下文里 */
|
|
2863
|
+
void (async () => {
|
|
2864
|
+
try {
|
|
2865
|
+
const conn = ctx?.get?.('connection')
|
|
2866
|
+
let info = { hasConn: !!conn }
|
|
2867
|
+
if (conn) {
|
|
2868
|
+
let proto = conn
|
|
2869
|
+
const keys = []
|
|
2870
|
+
while (proto && proto !== Object.prototype) {
|
|
2871
|
+
keys.push(...Object.getOwnPropertyNames(proto))
|
|
2872
|
+
proto = Object.getPrototypeOf(proto)
|
|
2873
|
+
}
|
|
2874
|
+
info.methods = Array.from(new Set(keys)).slice(0, 40)
|
|
2875
|
+
if (typeof conn.authenticatedUrl === 'function' && state.origin) {
|
|
2876
|
+
try {
|
|
2877
|
+
info.url = conn.authenticatedUrl(state.origin + '/')
|
|
2878
|
+
state.authUrl = info.url
|
|
2879
|
+
} catch (e) {
|
|
2880
|
+
info.urlError = String(e?.message ?? e)
|
|
2881
|
+
}
|
|
2882
|
+
}
|
|
2883
|
+
}
|
|
2884
|
+
await writeCache('auth-url.json', info)
|
|
2885
|
+
|
|
2886
|
+
/* connection 可能比 apply 晚就绪,重试补齐 authUrl */
|
|
2887
|
+
if (!state.authUrl) {
|
|
2888
|
+
for (let i = 0; i < 15 && !state.authUrl; i += 1) {
|
|
2889
|
+
await new Promise((r) => setTimeout(r, 1000))
|
|
2890
|
+
try {
|
|
2891
|
+
const c2 = ctx?.get?.('connection')
|
|
2892
|
+
if (c2 && typeof c2.authenticatedUrl === 'function' && state.origin) {
|
|
2893
|
+
state.authUrl = c2.authenticatedUrl(state.origin + '/')
|
|
2894
|
+
await writeCache('auth-url.json', { hasConn: true, url: state.authUrl })
|
|
2895
|
+
break
|
|
2896
|
+
}
|
|
2897
|
+
} catch {
|
|
2898
|
+
/* 继续重试 */
|
|
2899
|
+
}
|
|
2900
|
+
}
|
|
2901
|
+
}
|
|
2902
|
+
} catch (err) {
|
|
2903
|
+
await writeCache('auth-url.json', { error: String(err?.message ?? err) })
|
|
2904
|
+
}
|
|
2905
|
+
})()
|
|
2906
|
+
|
|
2907
|
+
/* 探针:试一次真实 LLM 调用,确认插件能用当前 profile 的模型 */
|
|
2908
|
+
void (async () => {
|
|
2909
|
+
const out = { steps: [] }
|
|
2910
|
+
try {
|
|
2911
|
+
const llm = ctx.get?.('llm')
|
|
2912
|
+
if (!llm) {
|
|
2913
|
+
await writeCache('llm-call.json', { steps: ['no llm service'] })
|
|
2914
|
+
return
|
|
2915
|
+
}
|
|
2916
|
+
const provider = 'magpie'
|
|
2917
|
+
const model = 'opencode-go/deepseek-v4.1-flash'
|
|
2918
|
+
const stream = llm.stream({
|
|
2919
|
+
provider,
|
|
2920
|
+
model,
|
|
2921
|
+
messages: [
|
|
2922
|
+
{ role: 'user', content: [{ type: 'text', text: 'Reply with exactly: OK' }] },
|
|
2923
|
+
],
|
|
2924
|
+
})
|
|
2925
|
+
let text = ''
|
|
2926
|
+
let raw = null
|
|
2927
|
+
for await (const chunk of stream) {
|
|
2928
|
+
if (raw === null) raw = JSON.stringify(chunk).slice(0, 400)
|
|
2929
|
+
if (typeof chunk === 'string') text += chunk
|
|
2930
|
+
else if (chunk && typeof chunk.text === 'string') text += chunk.text
|
|
2931
|
+
else if (chunk && typeof chunk.delta === 'string') text += chunk.delta
|
|
2932
|
+
else if (chunk && chunk.delta && typeof chunk.delta.text === 'string') text += chunk.delta.text
|
|
2933
|
+
if (text.length > 300) break
|
|
2934
|
+
}
|
|
2935
|
+
out.steps.push('chunk shape: ' + raw)
|
|
2936
|
+
out.steps.push('text: ' + JSON.stringify(text.slice(0, 150)))
|
|
2937
|
+
} catch (err) {
|
|
2938
|
+
out.steps.push('ERR: ' + String(err?.message ?? err).slice(0, 250))
|
|
2939
|
+
}
|
|
2940
|
+
await writeCache('llm-call.json', out)
|
|
2941
|
+
})()
|
|
2942
|
+
ctx.effect?.(
|
|
2943
|
+
() => webServer.register({ kind: 'prefix', path: PREFIX, handler }),
|
|
2944
|
+
'dsh-hackertube: routes',
|
|
2945
|
+
)
|
|
2946
|
+
}
|
|
2947
|
+
|
|
2948
|
+
tryRegister()
|
|
2949
|
+
|
|
2950
|
+
/* 让主对话能查这个视频库:Agent 在回答技术问题时可以主动检索 */
|
|
2951
|
+
try {
|
|
2952
|
+
ctx.inject(['tools'], (scope) => {
|
|
2953
|
+
scope.tools.register({
|
|
2954
|
+
name: 'hackertube_search',
|
|
2955
|
+
description:
|
|
2956
|
+
'在 Hacker News 二十年推荐的 6454 个技术视频里检索。' +
|
|
2957
|
+
'当用户在讨论某个技术概念、想找深入讲解,或问有没有相关演讲、课程、论文讲解时使用。' +
|
|
2958
|
+
'返回标题、频道、时长、播放量、HN 推荐理由和链接。',
|
|
2959
|
+
parameters: {
|
|
2960
|
+
query: {
|
|
2961
|
+
type: 'string',
|
|
2962
|
+
required: true,
|
|
2963
|
+
description: '检索词,可以是技术概念、语言、框架名或自然语言描述',
|
|
2964
|
+
},
|
|
2965
|
+
limit: {
|
|
2966
|
+
type: 'number',
|
|
2967
|
+
description: '返回条数,默认 5,最多 15',
|
|
2968
|
+
},
|
|
2969
|
+
},
|
|
2970
|
+
output: {
|
|
2971
|
+
schema: { type: 'string' },
|
|
2972
|
+
render: (_args, value) => [{ type: 'text', text: String(value) }],
|
|
2973
|
+
},
|
|
2974
|
+
async execute(args) {
|
|
2975
|
+
const items = await loadList()
|
|
2976
|
+
const terms = String(args.query || '')
|
|
2977
|
+
.toLowerCase()
|
|
2978
|
+
.split(/\s+/)
|
|
2979
|
+
.filter(Boolean)
|
|
2980
|
+
if (!terms.length) return '请提供检索词。'
|
|
2981
|
+
const scored = items
|
|
2982
|
+
.map((it) => {
|
|
2983
|
+
const hay = [it.t, it.c, (it.tags || []).join(' '), (it.cats || []).join(' ')]
|
|
2984
|
+
.join(' ')
|
|
2985
|
+
.toLowerCase()
|
|
2986
|
+
let score = 0
|
|
2987
|
+
for (const t of terms) if (hay.includes(t)) score += 1
|
|
2988
|
+
return { it, score }
|
|
2989
|
+
})
|
|
2990
|
+
.filter((x) => x.score > 0)
|
|
2991
|
+
.sort((a, b) => b.score - a.score || (b.it.w || 0) - (a.it.w || 0))
|
|
2992
|
+
.slice(0, Math.min(Number(args.limit) || 5, 15))
|
|
2993
|
+
if (!scored.length) return '没有找到匹配的视频。'
|
|
2994
|
+
return scored
|
|
2995
|
+
.map((x, i) => {
|
|
2996
|
+
const it = x.it
|
|
2997
|
+
const mins = Math.round((it.d || 0) / 60)
|
|
2998
|
+
return (
|
|
2999
|
+
`${i + 1}. ${it.t} — ${it.c} · ${mins} 分钟 · ${it.w || 0} 次观看\n` +
|
|
3000
|
+
` https://www.youtube.com/watch?v=${it.v}`
|
|
3001
|
+
)
|
|
3002
|
+
})
|
|
3003
|
+
.join('\n')
|
|
3004
|
+
},
|
|
3005
|
+
})
|
|
3006
|
+
void writeCache('tool-registered.json', { ok: true, at: Date.now() })
|
|
3007
|
+
})
|
|
3008
|
+
} catch (err) {
|
|
3009
|
+
void writeCache('tool-registered.json', {
|
|
3010
|
+
ok: false,
|
|
3011
|
+
error: String(err?.message ?? err).slice(0, 200),
|
|
3012
|
+
})
|
|
3013
|
+
}
|
|
3014
|
+
|
|
3015
|
+
/* 探针:列出 ctx 上可用的服务,决定推荐流能否直接读会话数据 */
|
|
3016
|
+
try {
|
|
3017
|
+
const own = Object.getOwnPropertyNames(ctx).filter((n) => n.startsWith('$'))
|
|
3018
|
+
const protoNames = []
|
|
3019
|
+
let p = Object.getPrototypeOf(ctx)
|
|
3020
|
+
while (p && p !== Object.prototype) {
|
|
3021
|
+
protoNames.push(...Object.getOwnPropertyNames(p))
|
|
3022
|
+
p = Object.getPrototypeOf(p)
|
|
3023
|
+
}
|
|
3024
|
+
const candidates = [
|
|
3025
|
+
'sessions',
|
|
3026
|
+
'agent',
|
|
3027
|
+
'agents',
|
|
3028
|
+
'llm',
|
|
3029
|
+
'model',
|
|
3030
|
+
'models',
|
|
3031
|
+
'completion',
|
|
3032
|
+
'inference',
|
|
3033
|
+
'provider',
|
|
3034
|
+
'providers',
|
|
3035
|
+
'ai',
|
|
3036
|
+
'chat',
|
|
3037
|
+
'prompt',
|
|
3038
|
+
]
|
|
3039
|
+
const resolved = {}
|
|
3040
|
+
for (const name of candidates) {
|
|
3041
|
+
try {
|
|
3042
|
+
const svc = ctx.get?.(name)
|
|
3043
|
+
resolved[name] = svc ? 'yes' : 'no'
|
|
3044
|
+
} catch (err) {
|
|
3045
|
+
resolved[name] = 'throw: ' + String(err?.message ?? err).slice(0, 60)
|
|
3046
|
+
}
|
|
3047
|
+
}
|
|
3048
|
+
void writeCache('ctx-services.json', {
|
|
3049
|
+
dollarProps: own.slice(0, 60),
|
|
3050
|
+
proto: Array.from(new Set(protoNames)).slice(0, 60),
|
|
3051
|
+
resolved,
|
|
3052
|
+
apiOf: Object.fromEntries(
|
|
3053
|
+
['sessions', 'llm', 'agents'].map((name) => {
|
|
3054
|
+
try {
|
|
3055
|
+
const svc = ctx.get?.(name)
|
|
3056
|
+
if (!svc) return [name, null]
|
|
3057
|
+
const keys = []
|
|
3058
|
+
let q = svc
|
|
3059
|
+
while (q && q !== Object.prototype) {
|
|
3060
|
+
keys.push(...Object.getOwnPropertyNames(q))
|
|
3061
|
+
q = Object.getPrototypeOf(q)
|
|
3062
|
+
}
|
|
3063
|
+
return [name, Array.from(new Set(keys)).slice(0, 40)]
|
|
3064
|
+
} catch (err) {
|
|
3065
|
+
return [name, 'throw: ' + String(err?.message ?? err).slice(0, 60)]
|
|
3066
|
+
}
|
|
3067
|
+
}),
|
|
3068
|
+
),
|
|
3069
|
+
})
|
|
3070
|
+
} catch (err) {
|
|
3071
|
+
void writeCache('ctx-services.json', { error: String(err?.message ?? err) })
|
|
3072
|
+
}
|
|
3073
|
+
|
|
3074
|
+
/**
|
|
3075
|
+
* 独立的中转服务器:DSH 的 webServer 在认证层后面,dsh-app:// 页面发起的
|
|
3076
|
+
* 跨源 iframe 请求带不上 cookie,会被 401 挡掉。这里另起一个只服务中转页、
|
|
3077
|
+
* 免认证、仅监听回环地址的小服务器,让播放器真正落在 HTTP 安全上下文。
|
|
3078
|
+
*/
|
|
3079
|
+
try {
|
|
3080
|
+
/* 重载时旧实例的监听不会自动消失,先显式关掉,避免端口漂移后
|
|
3081
|
+
前端还指向旧版中转页(旧版带 X-Frame-Options,会被浏览器拒绝渲染) */
|
|
3082
|
+
const stale = globalThis.__htMediaServer
|
|
3083
|
+
if (stale && typeof stale.close === 'function') {
|
|
3084
|
+
try {
|
|
3085
|
+
stale.close()
|
|
3086
|
+
} catch {
|
|
3087
|
+
/* 已关闭 */
|
|
3088
|
+
}
|
|
3089
|
+
}
|
|
3090
|
+
|
|
3091
|
+
const mediaServer = createServer((req, res) => {
|
|
3092
|
+
const u = new URL(req.url ?? '/', 'http://local')
|
|
3093
|
+
if (u.pathname === '/embed') {
|
|
3094
|
+
const vid = u.searchParams.get('vid')
|
|
3095
|
+
if (!vid || !/^[A-Za-z0-9_-]{6,20}$/.test(vid)) {
|
|
3096
|
+
res.writeHead(400, { 'Content-Type': 'text/plain; charset=utf-8' })
|
|
3097
|
+
res.end('bad vid')
|
|
3098
|
+
return
|
|
3099
|
+
}
|
|
3100
|
+
const lang = u.searchParams.get('lang') === 'zh' ? 'zh' : 'en'
|
|
3101
|
+
return serveEmbed(req, res, vid, lang)
|
|
3102
|
+
}
|
|
3103
|
+
res.writeHead(404, { 'Content-Type': 'text/plain; charset=utf-8' })
|
|
3104
|
+
res.end('not found')
|
|
3105
|
+
})
|
|
3106
|
+
mediaServer.on('error', () => {
|
|
3107
|
+
/* 端口占用等异常不影响主面板 */
|
|
3108
|
+
})
|
|
3109
|
+
globalThis.__htMediaServer = mediaServer
|
|
3110
|
+
|
|
3111
|
+
/* 优先用固定端口:前端可能缓存了地址,端口稳定才不会被旧值误导 */
|
|
3112
|
+
const tryListen = (ports) => {
|
|
3113
|
+
const port = ports[0]
|
|
3114
|
+
mediaServer.listen(port, '127.0.0.1', () => {
|
|
3115
|
+
state.mediaOrigin = `http://127.0.0.1:${port}`
|
|
3116
|
+
void writeCache('media-origin.json', { origin: state.mediaOrigin, port })
|
|
3117
|
+
})
|
|
3118
|
+
mediaServer.once('error', () => {
|
|
3119
|
+
if (ports.length > 1) tryListen(ports.slice(1))
|
|
3120
|
+
})
|
|
3121
|
+
}
|
|
3122
|
+
tryListen([18090, 18091, 18092, 18093, 0])
|
|
3123
|
+
|
|
3124
|
+
ctx.effect(
|
|
3125
|
+
() => () => {
|
|
3126
|
+
try {
|
|
3127
|
+
mediaServer.close()
|
|
3128
|
+
} catch {
|
|
3129
|
+
/* 已关闭 */
|
|
3130
|
+
}
|
|
3131
|
+
if (globalThis.__htMediaServer === mediaServer) globalThis.__htMediaServer = null
|
|
3132
|
+
},
|
|
3133
|
+
'dsh-hackertube: media server',
|
|
3134
|
+
)
|
|
3135
|
+
} catch (err) {
|
|
3136
|
+
void writeCache('media-origin.json', { error: String(err?.message ?? err) })
|
|
3137
|
+
}
|
|
3138
|
+
|
|
3139
|
+
/* 预热:把首次十几秒的抓取挪到 Host 启动阶段,面板打开时多半已就绪 */
|
|
3140
|
+
void loadList().catch(() => {})
|
|
3141
|
+
void loadBrowse().catch(() => {})
|
|
3142
|
+
}
|