dsh-baleen 0.1.1 → 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/AGENTS.md +4 -0
- package/README.md +7 -3
- package/client.js +509 -31
- package/index.js +153 -3
- package/lib/adapters/index.js +5 -4
- package/lib/adapters/xhs.js +173 -0
- package/lib/dedup.js +16 -1
- package/lib/fulltext.js +222 -0
- package/lib/jobs.js +4 -0
- package/lib/mcp/xhs-client.js +206 -0
- package/lib/normalize.js +4 -2
- package/lib/preference.js +23 -2
- package/lib/runner.js +53 -2
- package/lib/store.js +11 -0
- package/package.json +1 -1
- package/src/client/_footer.txt +173 -4
- package/src/client/css.js +48 -10
- package/src/client/format.js +15 -0
- package/src/client/sidebar-tab.js +56 -0
- package/src/client/view-api.js +10 -0
- package/src/client/views/article.js +37 -3
- package/src/client/views/drawer.js +100 -7
- package/src/client/views/player-tab.js +33 -0
- package/src/client/views/settings.js +27 -3
- package/src/client/views/today.js +5 -2
- package/src/client/views/video.js +9 -5
- package/tools/build-client.mjs +2 -1
- package/tools/client-loader.mjs +4 -3
- package/tools/preview.mjs +19 -2
package/lib/jobs.js
CHANGED
|
@@ -58,7 +58,11 @@ export function planJobs({ db, now = Date.now, force = false }) {
|
|
|
58
58
|
// yt 是 flat 列表 + 每条两次 yt-dlp(dump-single-json + 字幕轨落盘,一条 ≈2-6s)。
|
|
59
59
|
// 通用 20s 在真机 run 17 上跑不完(run_end 之后适配器还在打 detail——孤儿调用)。
|
|
60
60
|
// 更宽的预算配 pool 的"超时即 abort":真超了也断得干净,不白烧配额不顶风控。
|
|
61
|
+
// xhs 同理(冒烟实证:search_feeds 冷启动会顶到工具内部 60s 限时);再叠
|
|
62
|
+
// spawn 等待 + 登录探测 + 详情批(并发 2),90s/180s 真机两轮实证都不够
|
|
63
|
+
// (12 条详情装不下),预算放宽到 300s 并把每轮详情上限降到 6(分批续传)。
|
|
61
64
|
if (s.type === 'bili' || s.type === 'yt') job.timeout_ms = 90000
|
|
65
|
+
if (s.type === 'xhs') job.timeout_ms = 300000
|
|
62
66
|
return job
|
|
63
67
|
})
|
|
64
68
|
}
|
|
@@ -0,0 +1,206 @@
|
|
|
1
|
+
import { spawn } from 'node:child_process'
|
|
2
|
+
import { randomBytes } from 'node:crypto'
|
|
3
|
+
import net from 'node:net'
|
|
4
|
+
import fs from 'node:fs'
|
|
5
|
+
import path from 'node:path'
|
|
6
|
+
import { homeDir } from '../paths.js'
|
|
7
|
+
|
|
8
|
+
/**
|
|
9
|
+
* xhs-client —— 小红书 MCP(xiaohongshu-mcp 官方 exe)的 HTTP 客户端 + 子进程生命周期。
|
|
10
|
+
*
|
|
11
|
+
* 选型与边界见 docs/v3-xhs-mcp-audit.md(V3)与 docs/superpowers/plans/2026-11-01-baleen-m6-xhs.md:
|
|
12
|
+
* - 只读白名单:9 个只读工具之外的名字在这里就被拒掉,零 HTTP 往返——写平台动作
|
|
13
|
+
* (发布/点赞/收藏/评论,共 8 个)永远不从 baleen 调用,也永远不进 agent 工具面(红线);
|
|
14
|
+
* - 子进程由 baleen 拉起:回环绑定 + 每 boot 随机 token(spawn 侧与 HTTP 侧同源,
|
|
15
|
+
* 不落盘不进日志);cookie 收敛在 <home>/credentials/xhs-cookies.json(凭据只落本机);
|
|
16
|
+
* - 惰性 spawn:配了 xhs 源才会拉子进程;首次运行官方 exe 会先下载内置浏览器
|
|
17
|
+
* (约 140-190MB,带 SHA256 校验),那一轮的源请求会按 mcp_unready 失败,下一轮自然恢复。
|
|
18
|
+
*/
|
|
19
|
+
|
|
20
|
+
/** 只读工具白名单(V3 审查清单逐字)。工具面对 agent 仍恰好六个,这里是摄取内部面。 */
|
|
21
|
+
export const XHS_READ_TOOLS = new Set([
|
|
22
|
+
'check_login_status', 'get_login_qrcode', 'get_feed_detail', 'get_unread_count',
|
|
23
|
+
'list_feeds', 'list_notifications', 'search_feeds', 'user_profile', 'get_my_profile'
|
|
24
|
+
])
|
|
25
|
+
|
|
26
|
+
/** 端口就绪探测预算:浏览器已缓存时秒级;首次下载内置浏览器的那一轮等不到,如实报 mcp_unready。 */
|
|
27
|
+
const READY_TIMEOUT_MS = 20000
|
|
28
|
+
const READY_POLL_MS = 300
|
|
29
|
+
|
|
30
|
+
function fail(message, props = {}) {
|
|
31
|
+
return Object.assign(new Error(message), props)
|
|
32
|
+
}
|
|
33
|
+
|
|
34
|
+
let active = null // { key, baseUrl, token, port, child, client }
|
|
35
|
+
|
|
36
|
+
/** 供测试与 baleen 的 ctx.effect 清理用。 */
|
|
37
|
+
export function stopXhsMcp() {
|
|
38
|
+
if (active?.child) {
|
|
39
|
+
try { active.child.kill() } catch { /* 已死即成功 */ }
|
|
40
|
+
}
|
|
41
|
+
active = null
|
|
42
|
+
}
|
|
43
|
+
|
|
44
|
+
/**
|
|
45
|
+
* 取(必要时拉起)小红书 MCP 的 HTTP 客户端。幂等:同一份 (exePath, port) 配置复用
|
|
46
|
+
* 已在跑的实例;配置变了才重启。mcp_path 空 / exe 不存在 → mcp_missing。
|
|
47
|
+
*/
|
|
48
|
+
export async function getXhsMcp(pref, log = () => {}) {
|
|
49
|
+
const cfg = pref?.xhs
|
|
50
|
+
if (!cfg?.mcp_path || !String(cfg.mcp_path).trim()) {
|
|
51
|
+
throw fail('xhs.mcp_path 未配置——小红书通道不启用(要启用就填官方 exe 路径)', { error_class: 'mcp_missing' })
|
|
52
|
+
}
|
|
53
|
+
const exePath = String(cfg.mcp_path).trim()
|
|
54
|
+
if (!fs.existsSync(exePath)) {
|
|
55
|
+
throw fail(`xhs MCP exe 不存在: ${exePath}`, { error_class: 'mcp_missing' })
|
|
56
|
+
}
|
|
57
|
+
const port = Number.isInteger(cfg.port) && cfg.port > 0 ? cfg.port : 18060
|
|
58
|
+
const key = `${exePath}|${port}`
|
|
59
|
+
if (active?.key === key && active.listening) return active.client
|
|
60
|
+
|
|
61
|
+
stopXhsMcp()
|
|
62
|
+
const token = randomBytes(24).toString('hex')
|
|
63
|
+
const cookiesPath = path.join(homeDir(), 'credentials', 'xhs-cookies.json')
|
|
64
|
+
fs.mkdirSync(path.dirname(cookiesPath), { recursive: true })
|
|
65
|
+
|
|
66
|
+
const child = spawn(exePath, ['-port', `127.0.0.1:${port}`, '-token', token], {
|
|
67
|
+
cwd: path.dirname(cookiesPath),
|
|
68
|
+
env: { ...process.env, AUTH_TOKEN: token, COOKIES_PATH: cookiesPath },
|
|
69
|
+
stdio: 'ignore'
|
|
70
|
+
})
|
|
71
|
+
const record = { key, baseUrl: `http://127.0.0.1:${port}`, token, port, child, listening: false, client: null }
|
|
72
|
+
active = record
|
|
73
|
+
|
|
74
|
+
await waitListening(port, READY_TIMEOUT_MS)
|
|
75
|
+
record.listening = true
|
|
76
|
+
record.client = makeClient(record, log)
|
|
77
|
+
log('xhs_mcp_up', `小红书 MCP 已就绪(127.0.0.1:${port})`)
|
|
78
|
+
return record.client
|
|
79
|
+
}
|
|
80
|
+
|
|
81
|
+
/** 供 ctx.effect 注册清理;没拉起过就是空操作。 */
|
|
82
|
+
export function xhsMcpRunning() {
|
|
83
|
+
return Boolean(active?.child)
|
|
84
|
+
}
|
|
85
|
+
|
|
86
|
+
function waitListening(port, timeoutMs) {
|
|
87
|
+
return new Promise((resolve, reject) => {
|
|
88
|
+
const deadline = Date.now() + timeoutMs
|
|
89
|
+
const tryOnce = () => {
|
|
90
|
+
const socket = net.connect(port, '127.0.0.1')
|
|
91
|
+
socket.once('connect', () => { socket.destroy(); resolve() })
|
|
92
|
+
socket.once('error', () => {
|
|
93
|
+
socket.destroy()
|
|
94
|
+
if (Date.now() > deadline) {
|
|
95
|
+
reject(fail(`xhs MCP 在 ${timeoutMs}ms 内没监听上 127.0.0.1:${port}——若是首次运行,多半还在下载内置浏览器(约 140-190MB),下一轮自然恢复`, { error_class: 'mcp_unready' }))
|
|
96
|
+
} else {
|
|
97
|
+
setTimeout(tryOnce, READY_POLL_MS)
|
|
98
|
+
}
|
|
99
|
+
})
|
|
100
|
+
}
|
|
101
|
+
tryOnce()
|
|
102
|
+
})
|
|
103
|
+
}
|
|
104
|
+
|
|
105
|
+
/** 单个 MCP 客户端实例:会话惰性建立、404 过期自动重初始化一次。 */
|
|
106
|
+
function makeClient(record, log = () => {}) {
|
|
107
|
+
let sessionId = null
|
|
108
|
+
let rpcId = 0
|
|
109
|
+
|
|
110
|
+
const headers = () => ({
|
|
111
|
+
'Content-Type': 'application/json',
|
|
112
|
+
Accept: 'application/json, text/event-stream',
|
|
113
|
+
Authorization: 'Bearer ' + record.token,
|
|
114
|
+
...(sessionId ? { 'Mcp-Session-Id': sessionId } : {})
|
|
115
|
+
})
|
|
116
|
+
|
|
117
|
+
async function rpc(method, params, isNotification = false) {
|
|
118
|
+
const body = { jsonrpc: '2.0', method, ...(params ? { params } : {}), ...(isNotification ? {} : { id: ++rpcId }) }
|
|
119
|
+
const res = await fetch(record.baseUrl + '/mcp', { method: 'POST', headers: headers(), body: JSON.stringify(body), signal: params?.signal })
|
|
120
|
+
const sid = res.headers.get('mcp-session-id')
|
|
121
|
+
if (sid && !sessionId) sessionId = sid
|
|
122
|
+
const ct = res.headers.get('content-type') || ''
|
|
123
|
+
let payload = null
|
|
124
|
+
if (ct.includes('event-stream')) {
|
|
125
|
+
const text = await res.text()
|
|
126
|
+
const dataLines = text.split('\n').filter((l) => l.startsWith('data:')).map((l) => l.slice(5).trim())
|
|
127
|
+
payload = dataLines.length ? JSON.parse(dataLines[dataLines.length - 1]) : null
|
|
128
|
+
} else {
|
|
129
|
+
const text = await res.text()
|
|
130
|
+
payload = text ? JSON.parse(text) : null
|
|
131
|
+
}
|
|
132
|
+
return { res, payload }
|
|
133
|
+
}
|
|
134
|
+
|
|
135
|
+
async function ensureSession() {
|
|
136
|
+
const { res } = await rpc('initialize', {
|
|
137
|
+
protocolVersion: '2025-03-26', capabilities: {},
|
|
138
|
+
clientInfo: { name: 'baleen-xhs', version: '0.1.0' }
|
|
139
|
+
})
|
|
140
|
+
if (res.status !== 200) throw fail(`xhs MCP initialize HTTP ${res.status}`, { error_class: 'mcp_unready' })
|
|
141
|
+
await rpc('notifications/initialized', undefined, true)
|
|
142
|
+
}
|
|
143
|
+
|
|
144
|
+
/**
|
|
145
|
+
* 调一个只读工具,返回 { texts: string[], images: [] }。白名单外的名字在这里
|
|
146
|
+
* 就地拒绝(零 HTTP 往返)——写平台动作从结构上进不了调用链。
|
|
147
|
+
*/
|
|
148
|
+
async function callTool(name, args = {}) {
|
|
149
|
+
if (!XHS_READ_TOOLS.has(name)) {
|
|
150
|
+
throw fail(`工具 "${name}" 不在小红书只读白名单里——baleen 对写平台动作零调用`, { error_class: 'xhs_tool_forbidden' })
|
|
151
|
+
}
|
|
152
|
+
for (let attempt = 0; attempt < 2; attempt++) {
|
|
153
|
+
if (!sessionId) await ensureSession()
|
|
154
|
+
const { res, payload } = await rpc('tools/call', { name, arguments: args })
|
|
155
|
+
if (res.status === 404 || (payload?.error && /session/i.test(payload.error.message || ''))) {
|
|
156
|
+
sessionId = null // 会话过期:重初始化一次再试
|
|
157
|
+
continue
|
|
158
|
+
}
|
|
159
|
+
if (res.status === 401) throw fail('xhs MCP 鉴权被拒——token 与子进程不同源,属内部错误', { error_class: 'mcp_unready' })
|
|
160
|
+
if (payload?.error) throw fail(`xhs MCP ${name}: ${payload.error.message ?? '未知错误'}`, { error_class: 'xhs_tool_error' })
|
|
161
|
+
const content = payload?.result?.content ?? []
|
|
162
|
+
return {
|
|
163
|
+
texts: content.filter((x) => x.type === 'text').map((x) => x.text),
|
|
164
|
+
images: content.filter((x) => x.type === 'image')
|
|
165
|
+
}
|
|
166
|
+
}
|
|
167
|
+
throw fail('xhs MCP 会话两次初始化都失败', { error_class: 'mcp_unready' })
|
|
168
|
+
}
|
|
169
|
+
|
|
170
|
+
/** check_login_status 的结构化视图:{ loggedIn, username?, raw }。 */
|
|
171
|
+
async function checkLoginStatus() {
|
|
172
|
+
const { texts } = await callTool('check_login_status')
|
|
173
|
+
const text = texts.join('\n')
|
|
174
|
+
const loggedIn = /✅\s*已登录/.test(text)
|
|
175
|
+
const name = loggedIn ? (/用户名:\s*(.+)/.exec(text)?.[1]?.trim() ?? '') : ''
|
|
176
|
+
return { loggedIn, username: name, raw: text }
|
|
177
|
+
}
|
|
178
|
+
|
|
179
|
+
/** search_feeds:返回解析后的 { feeds: [...] }(工具以 JSON 文本回内容)。 */
|
|
180
|
+
async function searchFeeds(keyword) {
|
|
181
|
+
const { texts } = await callTool('search_feeds', { keyword })
|
|
182
|
+
return parseJsonTool(texts, 'search_feeds')
|
|
183
|
+
}
|
|
184
|
+
|
|
185
|
+
/** get_feed_detail:返回解析后的笔记详情 JSON。 */
|
|
186
|
+
async function getFeedDetail(noteId, xsecToken) {
|
|
187
|
+
const { texts } = await callTool('get_feed_detail', { feed_id: noteId, xsec_token: xsecToken })
|
|
188
|
+
return parseJsonTool(texts, 'get_feed_detail')
|
|
189
|
+
}
|
|
190
|
+
|
|
191
|
+
return { callTool, checkLoginStatus, searchFeeds, getFeedDetail }
|
|
192
|
+
}
|
|
193
|
+
|
|
194
|
+
/** 供测试打桩:对固定 baseUrl/token 的裸 HTTP 客户端(不经 spawn、不碰磁盘)。 */
|
|
195
|
+
export function createXhsHttpClient(baseUrl, token, log = () => {}) {
|
|
196
|
+
return makeClient({ baseUrl, token }, log)
|
|
197
|
+
}
|
|
198
|
+
|
|
199
|
+
/** 工具把 JSON 以 text 形态回(搜索/详情都这样),这里统一解析;坏 JSON 如实抛。 */
|
|
200
|
+
function parseJsonTool(texts, tool) {
|
|
201
|
+
const text = texts.join('\n').trim()
|
|
202
|
+
if (!text) throw fail(`xhs MCP ${tool} 返回空内容`, { error_class: 'xhs_tool_error' })
|
|
203
|
+
try { return JSON.parse(text) } catch (e) {
|
|
204
|
+
throw fail(`xhs MCP ${tool} 返回的不是 JSON: ${text.slice(0, 80)}`, { error_class: 'xhs_tool_error', parse: true })
|
|
205
|
+
}
|
|
206
|
+
}
|
package/lib/normalize.js
CHANGED
|
@@ -33,6 +33,8 @@ function kwHit(hayLower, hayRaw, kw) {
|
|
|
33
33
|
* missing(无语音)与 burned_in(字烧在画面上)绝不能合成一个值。
|
|
34
34
|
*/
|
|
35
35
|
export function inferBodyState(raw) {
|
|
36
|
+
// 现取到的正文优先:它就是"全文",不必再去比摘要长度
|
|
37
|
+
if (raw.fulltext_text) return 'full'
|
|
36
38
|
if (raw.subtitle_text) return 'subtitle'
|
|
37
39
|
if (raw.transcribed_text) return 'transcribed'
|
|
38
40
|
const full = stripTags(raw.body_html)
|
|
@@ -86,7 +88,7 @@ export function matchThemes(text, pref) {
|
|
|
86
88
|
* 输出可直接插库:判断与预筛需要的额外信息都塞在 match_json 里,不污染列。
|
|
87
89
|
*/
|
|
88
90
|
export function normalize(raw, source, pref, now = Date.now) {
|
|
89
|
-
const haystack = [raw.title, (raw.tags ?? []).join(' '), stripTags(raw.body_html ?? ''), stripTags(raw.summary_html ?? '')]
|
|
91
|
+
const haystack = [raw.title, (raw.tags ?? []).join(' '), stripTags(raw.fulltext_text ?? ''), stripTags(raw.body_html ?? ''), stripTags(raw.summary_html ?? '')]
|
|
90
92
|
.filter(Boolean).join('\n')
|
|
91
93
|
const m = matchThemes(haystack, pref)
|
|
92
94
|
const srcWeight = Number.isFinite(source?.weight) ? source.weight : 1
|
|
@@ -94,7 +96,7 @@ export function normalize(raw, source, pref, now = Date.now) {
|
|
|
94
96
|
const srcUrl = cleanUrl(source?.url)
|
|
95
97
|
const followed = !!srcUrl && (pref?.follow?.feeds ?? []).some(f => cleanUrl(f) === srcUrl)
|
|
96
98
|
const score_follow = followed ? 0.6 : 0
|
|
97
|
-
const bodyText = stripTags(raw.body_html ?? raw.summary_html ?? '')
|
|
99
|
+
const bodyText = stripTags(raw.fulltext_text ?? raw.body_html ?? raw.summary_html ?? '')
|
|
98
100
|
|
|
99
101
|
return {
|
|
100
102
|
source_id: raw.source_id,
|
package/lib/preference.js
CHANGED
|
@@ -7,7 +7,7 @@ const FILE = 'preference.yaml'
|
|
|
7
7
|
/** 模板与 writePreference 共用的文件头注释:写回时没有用户注释就带这句 */
|
|
8
8
|
const FILE_HEADER = '# 改这里:themes / follow / sources 是你的,tuning 只由面板那两个按钮写。\n'
|
|
9
9
|
const CLAMP = 0.5
|
|
10
|
-
const KNOWN_TYPES = ['rss', 'bili', 'yt'] // xhs
|
|
10
|
+
const KNOWN_TYPES = ['rss', 'bili', 'yt', 'xhs'] // xhs:M6 起接入,源主键是搜索关键词(query)
|
|
11
11
|
const WINDOW_RE = /^([01]\d|2[0-3]):[0-5]\d-([01]\d|2[0-3]):[0-5]\d$/
|
|
12
12
|
|
|
13
13
|
export const DEFAULTS = {
|
|
@@ -19,11 +19,15 @@ export const DEFAULTS = {
|
|
|
19
19
|
limits: { max_age_days: 14, min_duration_sec: 90, skip_kinds: [] },
|
|
20
20
|
require_theme_hit: true,
|
|
21
21
|
inject_prompt: true,
|
|
22
|
+
sidebar_takeover: true, // 会话里点 B 站/YouTube 链接是否落进侧栏播放 tab(M7)
|
|
23
|
+
sidebar_crop: true, // 侧栏播 B 站时裁掉完整页自带的相关推荐(M7)
|
|
24
|
+
fulltext: true, // RSS 只给摘要时去抓原文正文(抽不到就退回摘要,不覆盖已有内容)
|
|
22
25
|
tuning: { source_delta: {} },
|
|
23
26
|
redlines: ['广告代理', '引流私聊', '付费课程推销'],
|
|
24
27
|
charter: '',
|
|
25
28
|
yt_dlp_path: 'yt-dlp', // yt 源(M4b)的 yt-dlp 可执行名/路径,缺省走 PATH
|
|
26
|
-
network: { proxy: '' }
|
|
29
|
+
network: { proxy: '' }, // 只传给 yt-dlp 的 --proxy,不动全局 fetch;空 = 不走代理
|
|
30
|
+
xhs: { mcp_path: '', port: 18060 } // 小红书 MCP(M6):官方 exe 路径,空 = 通道不启用;token 每 boot 随机生成不落盘
|
|
27
31
|
}
|
|
28
32
|
|
|
29
33
|
export function prefFile(dir = homeDir()) {
|
|
@@ -205,6 +209,7 @@ export function validate(obj) {
|
|
|
205
209
|
if (s.type === 'rss' && (!str(s.url) || !/^https?:\/\//i.test(s.url))) errs.push(`sources[${i}].url: rss 源必须是 http(s) URL`)
|
|
206
210
|
if (s.type === 'bili' && (!str(s.url) || !s.url.trim())) errs.push(`sources[${i}].url: bili 源要填 UP 主 space URL(space.bilibili.com/<uid>)`)
|
|
207
211
|
if (s.type === 'yt' && (!str(s.url) || !/^https?:\/\//i.test(s.url))) errs.push(`sources[${i}].url: yt 源必须是 http(s) URL(频道 /videos、playlist 或单视频)`)
|
|
212
|
+
if (s.type === 'xhs' && (!str(s.url) || !/^https?:\/\//i.test(s.url))) errs.push(`sources[${i}].url: xhs 源要填搜索 URL(xiaohongshu.com/search?query=…)——sources 表只回传 url,关键词从它的 query 参数派生`)
|
|
208
213
|
if (s.enabled !== undefined && typeof s.enabled !== 'boolean') errs.push(`sources[${i}].enabled: 必须是布尔`)
|
|
209
214
|
if (s.interval_min !== undefined && !(Number.isInteger(s.interval_min) && s.interval_min >= 15)) errs.push(`sources[${i}].interval_min: ≥15 的整数`)
|
|
210
215
|
if (s.weight !== undefined && !(finite(s.weight) && s.weight >= 0 && s.weight <= 2)) errs.push(`sources[${i}].weight: 0–2`)
|
|
@@ -237,6 +242,13 @@ export function validate(obj) {
|
|
|
237
242
|
}
|
|
238
243
|
|
|
239
244
|
if (obj.yt_dlp_path !== undefined && !str(obj.yt_dlp_path)) errs.push('yt_dlp_path: 必须是字符串(yt-dlp 可执行名或绝对路径,缺省 yt-dlp 走 PATH)')
|
|
245
|
+
if (obj.xhs !== undefined) {
|
|
246
|
+
if (!obj.xhs || typeof obj.xhs !== 'object' || isArr(obj.xhs)) errs.push('xhs: 必须是映射')
|
|
247
|
+
else {
|
|
248
|
+
if (obj.xhs.mcp_path !== undefined && !str(obj.xhs.mcp_path)) errs.push('xhs.mcp_path: 必须是字符串(空 = 通道不启用)')
|
|
249
|
+
if (obj.xhs.port !== undefined && !(Number.isInteger(obj.xhs.port) && obj.xhs.port > 0 && obj.xhs.port < 65536)) errs.push('xhs.port: 1-65535 的整数')
|
|
250
|
+
}
|
|
251
|
+
}
|
|
240
252
|
if (obj.network !== undefined) {
|
|
241
253
|
if (!obj.network || typeof obj.network !== 'object' || isArr(obj.network)) errs.push('network: 必须是映射')
|
|
242
254
|
else if (obj.network.proxy !== undefined && !str(obj.network.proxy)) errs.push('network.proxy: 必须是字符串(空 = 不走代理;只传给 yt-dlp 的 --proxy,不动全局 fetch)')
|
|
@@ -248,6 +260,15 @@ export function validate(obj) {
|
|
|
248
260
|
if (obj.inject_prompt !== undefined && typeof obj.inject_prompt !== 'boolean') {
|
|
249
261
|
errs.push('inject_prompt: 必须是布尔')
|
|
250
262
|
}
|
|
263
|
+
if (obj.sidebar_takeover !== undefined && typeof obj.sidebar_takeover !== 'boolean') {
|
|
264
|
+
errs.push('sidebar_takeover: 必须是布尔')
|
|
265
|
+
}
|
|
266
|
+
if (obj.sidebar_crop !== undefined && typeof obj.sidebar_crop !== 'boolean') {
|
|
267
|
+
errs.push('sidebar_crop: 必须是布尔')
|
|
268
|
+
}
|
|
269
|
+
if (obj.fulltext !== undefined && typeof obj.fulltext !== 'boolean') {
|
|
270
|
+
errs.push('fulltext: 必须是布尔')
|
|
271
|
+
}
|
|
251
272
|
if (obj.redlines !== undefined && (!isArr(obj.redlines) || obj.redlines.some(s => !str(s)))) errs.push('redlines: 必须是字符串数组')
|
|
252
273
|
|
|
253
274
|
if (obj.tuning?.source_delta !== undefined) {
|
package/lib/runner.js
CHANGED
|
@@ -4,10 +4,18 @@ import { normalize } from './normalize.js'
|
|
|
4
4
|
import { putDraft } from './dedup.js'
|
|
5
5
|
import { gate, summarizeGaps } from './prefilter.js'
|
|
6
6
|
import { planJobs, syncSources } from './jobs.js'
|
|
7
|
-
import {
|
|
7
|
+
import {
|
|
8
|
+
createRun, finishRun, setCursor, setSourceResult, setStatus, isCollected, knownExternalIds,
|
|
9
|
+
listSources, fulltextCandidates, refreshItem, getMeta, setMeta
|
|
10
|
+
} from './store.js'
|
|
8
11
|
|
|
9
12
|
export { planJobs, syncSources }
|
|
10
13
|
|
|
14
|
+
/** 每轮最多补抓几条正文。故意小:这是"顺手把摘要补全",不是另一个抓取系统。 */
|
|
15
|
+
const FULLTEXT_PER_RUN = 8
|
|
16
|
+
/** 同一条失败这么多次就不再试了——付费墙/需要登录的站重试一万次也不会开门。 */
|
|
17
|
+
const FULLTEXT_MAX_TRIES = 2
|
|
18
|
+
|
|
11
19
|
/**
|
|
12
20
|
* 一轮摄取:计划 → 抓取 → 归一 → 去重 → 预筛 → 记账。
|
|
13
21
|
* 不做判断(那是 M3 的 headless 会话),不碰网络细节(那是适配器),不改偏好。
|
|
@@ -16,7 +24,13 @@ export { planJobs, syncSources }
|
|
|
16
24
|
* 1. **失败不推进游标**:只有成功的源才 setCursor,失败源下一轮自动补上缺口。
|
|
17
25
|
* 2. **丢弃必须留痕**:预筛丢弃与"带标记保留"都进 stats.gaps,汇报区靠它写诚实的那一行。
|
|
18
26
|
*/
|
|
19
|
-
export async function runOnce({
|
|
27
|
+
export async function runOnce({
|
|
28
|
+
db, pref, now = Date.now, log = () => {}, fetchImpl, adapters, trigger = 'schedule', force = false,
|
|
29
|
+
// 正文补抓的实现(lib/fulltext.js 的 fetchFulltext)。刻意做成注入口而不是直接 import:
|
|
30
|
+
// ① 测试要能断言"没有实现就一条网络请求都不发";② 宿主那边才有权决定用哪个 fetch。
|
|
31
|
+
// 不传 = 这一轮不补抓(29 处既有测试用例都不传,天然不受影响)。
|
|
32
|
+
fetchFulltextImpl = null
|
|
33
|
+
} = {}) {
|
|
20
34
|
const at = now()
|
|
21
35
|
|
|
22
36
|
syncSources(db, pref)
|
|
@@ -129,6 +143,43 @@ export async function runOnce({ db, pref, now = Date.now, log = () => {}, fetchI
|
|
|
129
143
|
emit('info', 'source_ok', `抓 ${local.fetched} → 候选 ${local.candidates}(丢弃 ${local.archived})`, id)
|
|
130
144
|
}
|
|
131
145
|
|
|
146
|
+
// 正文补抓(M8):RSS 只搬 feed 的摘要,正文得回原文去抓。放在源循环之后、记账之前——
|
|
147
|
+
// 这样同一轮里"抓完正文的条目"直接进候选,面板上点开就是全文而不是一句话。
|
|
148
|
+
// 三条硬规矩:① 不注入实现就不抓(测试与默认路径零网络副作用);② 只补 status='new'
|
|
149
|
+
// 且 kind='article' 的(判过的不必改正文,视频的正文是字幕、不该去爬播放页);
|
|
150
|
+
// ③ 同一条失败满 2 次就记进 meta 不再试——付费墙不会因为我们坚持而打开。
|
|
151
|
+
if (pref?.fulltext !== false && typeof fetchFulltextImpl === 'function') {
|
|
152
|
+
const tried = fulltextCandidates(db, FULLTEXT_PER_RUN * 5)
|
|
153
|
+
.filter((it) => Number(getMeta(db, `fulltext:fail:${it.id}`) ?? 0) < FULLTEXT_MAX_TRIES)
|
|
154
|
+
.slice(0, FULLTEXT_PER_RUN)
|
|
155
|
+
stats.fulltext = { tried: tried.length, ok: 0, failed: 0 }
|
|
156
|
+
if (tried.length) {
|
|
157
|
+
// 复用 runJobs:超时即 abort、错误分类与源抓取同一套口径,不用另写一遍。
|
|
158
|
+
const out = await runJobs(tried.map((it) => ({ item: it, timeout_ms: 15000 })), {
|
|
159
|
+
limit: 2, // 回源是逐个打外部站点,并发给太大等于同时压好几个站
|
|
160
|
+
timeoutMs: 15000,
|
|
161
|
+
retries: 0, // 重试上限 1 次的红线:这里一次都不重试,失败留给下一轮
|
|
162
|
+
run: async (job, { signal } = {}) => fetchFulltextImpl(job.item.url, { signal })
|
|
163
|
+
})
|
|
164
|
+
for (const o of out) {
|
|
165
|
+
const id = o.job.item.id
|
|
166
|
+
if (o.ok && typeof o.value?.text === 'string' && o.value.text.length) {
|
|
167
|
+
// 只改正文两列 + fetched_at:不动 status(它还是候选),也不动 prefilter_score
|
|
168
|
+
// (分数是入库那轮的判断依据,抓到全文不该回头把已经做出的排序重写一遍)。
|
|
169
|
+
refreshItem(db, id, { body_text: o.value.text, body_state: 'full', fetched_at: now() })
|
|
170
|
+
setMeta(db, `fulltext:fail:${id}`, 0)
|
|
171
|
+
stats.fulltext.ok++
|
|
172
|
+
emit('info', 'fulltext_ok', `条目#${id} 补到 ${o.value.chars ?? o.value.text.length} 字`, null)
|
|
173
|
+
} else {
|
|
174
|
+
const n = Number(getMeta(db, `fulltext:fail:${id}`) ?? 0) + 1
|
|
175
|
+
setMeta(db, `fulltext:fail:${id}`, n)
|
|
176
|
+
stats.fulltext.failed++
|
|
177
|
+
emit('warn', 'fulltext_failed', `条目#${id} ${o.error_class ?? 'empty'}(第 ${n} 次)`, null)
|
|
178
|
+
}
|
|
179
|
+
}
|
|
180
|
+
}
|
|
181
|
+
}
|
|
182
|
+
|
|
132
183
|
stats.gaps = summarizeGaps(gapLog)
|
|
133
184
|
// held 挂进 gaps 才能活到面板:finishRun 只持久化 {status, candidates, gaps, per_source}。
|
|
134
185
|
stats.gaps.held = stats.held
|
package/lib/store.js
CHANGED
|
@@ -181,6 +181,17 @@ export function candidates(db, limit) {
|
|
|
181
181
|
ORDER BY prefilter_score DESC, published_at DESC, id ASC LIMIT ?`).all(limit)
|
|
182
182
|
}
|
|
183
183
|
|
|
184
|
+
/**
|
|
185
|
+
* 补抓正文的候选(M8):还没判过、正文只有摘要、有链接的那些条目。
|
|
186
|
+
* 只挑 status='new'——判过的条目再改正文没有意义(判断结论已经落库),
|
|
187
|
+
* 归档的更不该因为"抓到全文"而复活。排序照 candidates:先高分的。
|
|
188
|
+
*/
|
|
189
|
+
export function fulltextCandidates(db, limit = 8) {
|
|
190
|
+
return db.prepare(`SELECT ${ITEM_COLS} FROM items
|
|
191
|
+
WHERE status = 'new' AND body_state = 'summary_only' AND kind = 'article'
|
|
192
|
+
ORDER BY prefilter_score DESC, published_at DESC, id ASC LIMIT ?`).all(limit)
|
|
193
|
+
}
|
|
194
|
+
|
|
184
195
|
export function counts(db) {
|
|
185
196
|
const rows = db.prepare('SELECT status, COUNT(*) AS n FROM items GROUP BY status').all()
|
|
186
197
|
const out = { new: 0, judged: 0, digest: 0, archived: 0, total: 0 }
|