dsh-social-trace 0.1.13 → 0.1.18
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +12 -5
- package/assets/NOTICE.md +2 -0
- package/assets/douyin.ico +0 -0
- package/lib/agent.js +4 -4
- package/lib/business.js +74 -1
- package/lib/chat-context.js +9 -6
- package/lib/chat-model.js +5 -4
- package/lib/chat-runtime.js +104 -0
- package/lib/chat.js +129 -82
- package/lib/client.js +130 -40
- package/lib/documents.js +13 -1
- package/lib/index.js +69 -39
- package/lib/plain-text.js +10 -0
- package/lib/platforms/douyin/client.js +44 -0
- package/lib/platforms/douyin/config.js +33 -0
- package/lib/platforms/douyin/data.js +49 -0
- package/lib/platforms/douyin/index.js +163 -0
- package/lib/platforms/douyin/library.js +154 -0
- package/lib/platforms/douyin/media.js +12 -0
- package/lib/platforms/douyin/module.js +18 -0
- package/lib/platforms/douyin/service.js +34 -0
- package/lib/platforms/douyin/storage.js +39 -0
- package/lib/platforms/douyin/transcribe.py +27 -0
- package/lib/platforms/registry.js +7 -0
- package/lib/platforms/weibo/index.js +1 -1
- package/lib/platforms/weibo/media.js +19 -0
- package/lib/platforms/weibo/module.js +5 -0
- package/lib/platforms/weibo/recovery.js +17 -0
- package/lib/platforms/xueqiu/module.js +5 -0
- package/lib/store.js +42 -11
- package/lib/text.js +10 -35
- package/lib/worker.js +71 -4
- package/package.json +9 -4
package/lib/index.js
CHANGED
|
@@ -4,11 +4,17 @@ import { parseHistoryRange } from './history-range.js';
|
|
|
4
4
|
import { Store } from './store.js';
|
|
5
5
|
import { Worker } from './worker.js';
|
|
6
6
|
import { WeiboBrowser } from './platforms/weibo/index.js';
|
|
7
|
+
import { DouyinBrowser } from './platforms/douyin/index.js';
|
|
8
|
+
import { VideoLibrary } from './platforms/douyin/library.js';
|
|
9
|
+
import { saveConfig as saveDouyinConfig, validatePublicConfig as validateDouyinPublicConfig, status as douyinStatus } from './platforms/douyin/service.js';
|
|
10
|
+
import { platformModule } from './platforms/registry.js';
|
|
11
|
+
import { createReadStream, statSync } from 'node:fs';
|
|
7
12
|
import { XueqiuBrowser } from './platforms/xueqiu/index.js';
|
|
8
13
|
import { dataDirectory, workspaceDirectory, saveWorkspaceDirectory, chooseDirectory } from './paths.js';
|
|
9
|
-
import { Documents } from './documents.js';
|
|
14
|
+
import { Documents, videoMarkdown } from './documents.js';
|
|
10
15
|
import { Avatars } from './avatars.js';
|
|
11
16
|
import { SocialAgent } from './agent.js';
|
|
17
|
+
import { hostAgentRuntime } from './chat-runtime.js';
|
|
12
18
|
import { hostModel } from './chat-model.js';
|
|
13
19
|
import { BusinessChat } from './chat.js';
|
|
14
20
|
import { createBusiness } from './business.js';
|
|
@@ -47,7 +53,7 @@ function localRequest(req) {
|
|
|
47
53
|
return local && ['127.0.0.1', '::1', '::ffff:127.0.0.1'].includes(address);
|
|
48
54
|
}
|
|
49
55
|
|
|
50
|
-
export function createRuntime(directory = dataDirectory(), { adapterFactories = { weibo: WeiboBrowser, xueqiu: XueqiuBrowser }, chooseFolder = chooseDirectory, avatarFetch = fetch, getIM = () => null, model = hostModel(() => null), updates = createUpdates() } = {}) {
|
|
56
|
+
export function createRuntime(directory = dataDirectory(), { adapterFactories = { weibo: WeiboBrowser, xueqiu: XueqiuBrowser, douyin: DouyinBrowser }, chooseFolder = chooseDirectory, avatarFetch = fetch, getIM = () => null, model = hostModel(() => null), updates = createUpdates(), videoOptions = {} } = {}) {
|
|
51
57
|
let workspace = realpathSync(workspaceDirectory(directory));
|
|
52
58
|
let store = new Store(workspace);
|
|
53
59
|
let documents = new Documents(store, workspace);
|
|
@@ -66,9 +72,13 @@ export function createRuntime(directory = dataDirectory(), { adapterFactories =
|
|
|
66
72
|
requestGapMs: Math.max(browser.defaults?.requestGapMs || 15000, store.getSetting(id === 'weibo' ? 'requestGapMs' : `requestGapMs:${id}`, browser.defaults?.requestGapMs || 15000)) });
|
|
67
73
|
return [id, { browser, worker, loginBusy: false, historyBusy: false }];
|
|
68
74
|
}));
|
|
75
|
+
const videos = new VideoLibrary(() => store, () => documents, videoOptions);
|
|
69
76
|
const { browser, worker } = sources.weibo;
|
|
70
77
|
let closing = false;
|
|
71
|
-
const business = createBusiness(() => store, sources, () => documents.drainImages()
|
|
78
|
+
const business = createBusiness(() => store, sources, () => documents.drainImages(), {
|
|
79
|
+
validate(target, options) { if (target.platform !== 'douyin') throw new CaptureError('input', '此平台不支持视频保存或每日时间点设置'); return validateDouyinPublicConfig(store, target.uid, options); },
|
|
80
|
+
save(target, options) { saveDouyinConfig(store, { platform: target.platform, uid: target.uid, inherit: false, config: options }, { media: videos, worker: sources[target.platform].worker }); },
|
|
81
|
+
});
|
|
72
82
|
const chat = new BusinessChat(() => store, business, model, { paused: () => changingWorkspace || applyingUpdate || closing || !!serviceError });
|
|
73
83
|
|
|
74
84
|
function enableAuthorJob(uid, kind) {
|
|
@@ -95,7 +105,7 @@ export function createRuntime(directory = dataDirectory(), { adapterFactories =
|
|
|
95
105
|
if (destination === workspace) { saveWorkspaceDirectory(directory, destination); await documents.sync({ backgroundImages: true }); return; }
|
|
96
106
|
const database = join(destination, 'archive.sqlite');
|
|
97
107
|
if (existsSync(database)) throw new CaptureError('storage', '所选文件夹已有资料库,为避免覆盖,请选择一个新的文件夹');
|
|
98
|
-
if (agent.busy || chat.busy) throw new CaptureError('storage', '请等当前消息投递或 Chat 执行结束后再切换信息保存位置');
|
|
108
|
+
if (agent.busy || chat.busy || chat.confirmWork || videos.busy) throw new CaptureError('storage', '请等当前消息投递或 Chat 执行结束后再切换信息保存位置');
|
|
99
109
|
if (exporting) throw new CaptureError('storage', '正在导出文档,请等导出完成后再切换信息保存位置');
|
|
100
110
|
if (Object.values(sources).some(source => source.loginBusy || source.historyBusy)) throw new CaptureError('storage', '请等登录窗口打开完成后再切换信息保存位置');
|
|
101
111
|
changingWorkspace = true;
|
|
@@ -157,6 +167,10 @@ export function createRuntime(directory = dataDirectory(), { adapterFactories =
|
|
|
157
167
|
res.writeHead(200, { 'Content-Type': 'image/png', 'Cache-Control': 'public, max-age=86400', 'X-Content-Type-Options': 'nosniff' });
|
|
158
168
|
return res.end(readFileSync(new URL('../assets/xueqiu.png', import.meta.url)));
|
|
159
169
|
}
|
|
170
|
+
if (path === '/assets/douyin.ico') {
|
|
171
|
+
res.writeHead(200, { 'Content-Type': 'image/x-icon', 'Cache-Control': 'public, max-age=86400' });
|
|
172
|
+
return res.end(readFileSync(new URL('../assets/douyin.ico', import.meta.url)));
|
|
173
|
+
}
|
|
160
174
|
if (path === '/assets/weibo.ico') {
|
|
161
175
|
res.writeHead(200, { 'Content-Type': 'image/x-icon', 'Cache-Control': 'public, max-age=86400', 'X-Content-Type-Options': 'nosniff' });
|
|
162
176
|
return res.end(readFileSync(new URL('../assets/weibo.ico', import.meta.url)));
|
|
@@ -168,11 +182,11 @@ export function createRuntime(directory = dataDirectory(), { adapterFactories =
|
|
|
168
182
|
update: updates.view(), agent: agent.status(), enabled: store.getSetting('monitorEnabled', true),
|
|
169
183
|
account: store.account(), targets: store.targets().map(target => ({ ...target, historyOriginalOnly: store.getSetting('historyOriginalOnly:' + target.uid, false), policy: store.getSetting('targetPolicy:' + target.uid, null), enabled: store.getSetting('targetEnabled:' + target.uid, true) })), jobs: store.jobs().map(({ checkpoint, ...job }) => ({
|
|
170
184
|
...job, page: checkpoint.page || 1, pending: checkpoint.pending?.length || 0,
|
|
171
|
-
reportedTotal: checkpoint.reportedTotal, oldestAt: checkpoint.oldestAt,
|
|
185
|
+
reportedTotal: checkpoint.reportedTotal, oldestAt: checkpoint.oldestAt, contentIssue: checkpoint.contentIssue || null,
|
|
172
186
|
platform: store.target(job.uid).platform,
|
|
173
187
|
})), notices: store.notices(), unreadNotices: store.unreadNotices(), count: store.count(), todayNewCount: store.todayNewCount(),
|
|
174
188
|
requestGapMs: worker.requestGapMs, loginBusy: sources.weibo.loginBusy, stopped: Object.values(sources).every(source => source.worker.stopping),
|
|
175
|
-
serviceError,
|
|
189
|
+
serviceError, douyin: douyinStatus(store, videos),
|
|
176
190
|
unreadContent: store.unreadContent(), archiveSummary: store.archiveSummary(), progress: store.targetProgress(), activeJobId: worker.currentJobId,
|
|
177
191
|
storage: { workspaceDirectory: workspace, workspaceConfigured: existsSync(join(directory, 'workspace.json')), documentsDirectory: documents.directory,
|
|
178
192
|
changing: changingWorkspace, syncing: documents.syncing, mediaSyncing: documents.imageTasks.size > 0, mediaQueued: documents.imageQueue.size, error: documents.error, mediaError: documents.mediaError },
|
|
@@ -213,6 +227,27 @@ export function createRuntime(directory = dataDirectory(), { adapterFactories =
|
|
|
213
227
|
return res.end(bytes);
|
|
214
228
|
} finally { exporting--; }
|
|
215
229
|
}
|
|
230
|
+
if (path === '/video') {
|
|
231
|
+
const row = store.db.prepare('SELECT * FROM video_records WHERE post_id=?').get(url.searchParams.get('id') || '');
|
|
232
|
+
const type = url.searchParams.get('type') === 'audio' ? 'audio' : 'video';
|
|
233
|
+
if (!row || row[type + '_status'] !== 'done') return json(res, 404, { error: '该媒体尚未保存在本地' });
|
|
234
|
+
const file = videos.paths(row)[type];
|
|
235
|
+
if (!existsSync(file)) return json(res, 404, { error: '本地媒体文件已不存在,可重试下载' });
|
|
236
|
+
const length = statSync(file).size, range = req.headers.range;
|
|
237
|
+
let start = 0, end = length - 1;
|
|
238
|
+
if (range) {
|
|
239
|
+
const match = /^bytes=(\d+)-(\d*)$/.exec(range);
|
|
240
|
+
if (!match) { res.writeHead(416, { 'Content-Range': 'bytes */' + length }); return res.end(); }
|
|
241
|
+
start = Number(match[1]); end = match[2] ? Math.min(Number(match[2]), end) : end;
|
|
242
|
+
if (!Number.isSafeInteger(start) || start > end || start >= length) { res.writeHead(416, { 'Content-Range': 'bytes */' + length }); return res.end(); }
|
|
243
|
+
}
|
|
244
|
+
res.writeHead(range ? 206 : 200, { 'Content-Type': type === 'audio' ? 'audio/wav' : 'video/mp4', 'Content-Length': end - start + 1,
|
|
245
|
+
'Accept-Ranges': 'bytes', ...(range ? { 'Content-Range': `bytes ${start}-${end}/${length}` } : {}),
|
|
246
|
+
'Cache-Control': 'private, no-store', 'X-Content-Type-Options': 'nosniff',
|
|
247
|
+
...(url.searchParams.has('download') ? { 'Content-Disposition': `attachment; filename="${row.post_id.replace('douyin:', '')}.${type === 'audio' ? 'wav' : 'mp4'}"` } : {}) });
|
|
248
|
+
const stream = createReadStream(file, { start, end }); exporting++;
|
|
249
|
+
res.once('close', () => { stream.destroy(); exporting--; }); stream.on('error', () => res.destroy()); stream.pipe(res); return;
|
|
250
|
+
}
|
|
216
251
|
if (path === '/posts') return json(res, 200, { items: store.posts({
|
|
217
252
|
ids: url.searchParams.get('postId') ? [url.searchParams.get('postId')] : null,
|
|
218
253
|
uid: url.searchParams.get('uid') || '', query: (url.searchParams.get('q') || '').slice(0, 200), kind: url.searchParams.get('kind') || 'all',
|
|
@@ -235,9 +270,10 @@ export function createRuntime(directory = dataDirectory(), { adapterFactories =
|
|
|
235
270
|
${uid ? 'WHERE uid=?' : ''} ORDER BY published_at DESC,id DESC`);
|
|
236
271
|
for (const post of statement.iterate(...(uid ? [uid] : []))) {
|
|
237
272
|
const quote = post.quote ? JSON.parse(post.quote) : null;
|
|
273
|
+
const video = post.id.startsWith('douyin:') ? store.posts({ ids: [post.id], limit: 1 })[0]?.video : null;
|
|
238
274
|
const text = markdown
|
|
239
|
-
? `## ${post.author} · ${new Date(post.published_at).toISOString()}\n\n${post.url}\n\n${post.text}\n\n${quote ? '转发原文 · ' + quote.author + '\n\n' + quote.text + '\n\n' : ''}---\n\n`
|
|
240
|
-
: JSON.stringify({ ...post, quote }) + '\n';
|
|
275
|
+
? `## ${post.author} · ${new Date(post.published_at).toISOString()}\n\n${post.url}\n\n${post.text}\n\n${quote ? '转发原文 · ' + quote.author + '\n\n' + quote.text + '\n\n' : ''}${videoMarkdown(video)}---\n\n`
|
|
276
|
+
: JSON.stringify({ ...post, quote, ...(video ? { video } : {}) }) + '\n';
|
|
241
277
|
if (!res.write(text)) await once(res, 'drain');
|
|
242
278
|
}
|
|
243
279
|
return res.end();
|
|
@@ -258,25 +294,25 @@ export function createRuntime(directory = dataDirectory(), { adapterFactories =
|
|
|
258
294
|
if (path === '/chat/session/delete') return json(res, 200, await chat.deleteSession(body.id));
|
|
259
295
|
if (path === '/chat/session/restore') return json(res, 200, chat.restoreSession(body.id));
|
|
260
296
|
if (path === '/chat/message') return json(res, 202, chat.start(body.message, body.sessionId || ''));
|
|
261
|
-
if (path === '/chat/confirm') return json(res, 200, chat.confirm(body.id));
|
|
297
|
+
if (path === '/chat/confirm') return json(res, 200, await chat.confirm(body.id));
|
|
262
298
|
chat.cancel(body.id); return json(res, 200, { ok: true });
|
|
263
299
|
} catch (error) { return json(res, 400, { error: error.message }); }
|
|
264
300
|
}
|
|
265
301
|
if (path === '/update') {
|
|
266
|
-
if (chat.busy) return json(res, 409, { error: '请先停止或等候当前 Chat 任务结束后再更新' });
|
|
302
|
+
if (chat.busy || chat.confirmWork) return json(res, 409, { error: '请先停止或等候当前 Chat 任务结束后再更新' });
|
|
267
303
|
if (exporting || Object.values(sources).some(source => source.loginBusy || source.historyBusy)) return json(res, 409, { error: '请等当前登录、缓存设置或导出完成后再更新' });
|
|
268
304
|
try {
|
|
269
305
|
const update = updates.start(body.version, async apply => {
|
|
270
|
-
applyingUpdate = true;
|
|
306
|
+
applyingUpdate = true; videos.suspended = true; videos.controller?.abort();
|
|
271
307
|
Object.values(sources).forEach(source => { source.worker.suspended = true; });
|
|
272
308
|
documents.stopping = true;
|
|
273
309
|
try {
|
|
274
|
-
await Promise.all([agent.busy, documents.task, documents.queueTask, ...documents.imageTasks.values(), ...Object.values(sources).map(source => source.worker.busy)]);
|
|
310
|
+
await Promise.all([videos.busy, agent.busy, documents.task, documents.queueTask, ...documents.imageTasks.values(), ...Object.values(sources).map(source => source.worker.busy)]);
|
|
275
311
|
return await apply();
|
|
276
312
|
} finally {
|
|
277
313
|
applyingUpdate = false;
|
|
278
314
|
if (!closing) {
|
|
279
|
-
documents.stopping = false;
|
|
315
|
+
documents.stopping = false; videos.suspended = false;
|
|
280
316
|
Object.values(sources).forEach(source => { source.worker.suspended = false; source.worker.wake(); });
|
|
281
317
|
documents.drainImages();
|
|
282
318
|
}
|
|
@@ -315,6 +351,7 @@ export function createRuntime(directory = dataDirectory(), { adapterFactories =
|
|
|
315
351
|
store.setSetting(key, body.enabled);
|
|
316
352
|
Object.values(sources).forEach(source => source.worker.wake());
|
|
317
353
|
if (body.enabled) documents.drainImages();
|
|
354
|
+
videos.cancelInactive();
|
|
318
355
|
return json(res, 200, { ok: true });
|
|
319
356
|
}
|
|
320
357
|
const platform = body.platform || 'weibo';
|
|
@@ -323,7 +360,16 @@ export function createRuntime(directory = dataDirectory(), { adapterFactories =
|
|
|
323
360
|
const { browser: platformBrowser, worker: platformWorker } = source;
|
|
324
361
|
const platformName = platformBrowser.name;
|
|
325
362
|
if (source.historyBusy) return json(res, 409, { error: '正在调整缓存范围,请稍候' });
|
|
326
|
-
if (path === '/
|
|
363
|
+
if (path === '/jobs/skip-content') {
|
|
364
|
+
return json(res, 200, business.skipContent(body.id, body.postId));
|
|
365
|
+
}
|
|
366
|
+
if (path === '/douyin/config') {
|
|
367
|
+
saveDouyinConfig(store, { ...body, platform }, { media: videos, worker: platformWorker });
|
|
368
|
+
} else if (path === '/video/retry') {
|
|
369
|
+
if (platform !== 'douyin' || !store.db.prepare('SELECT post_id FROM video_records WHERE post_id=?').get(body.id)) throw new CaptureError('input', '视频不存在');
|
|
370
|
+
videos.retry(body.id);
|
|
371
|
+
} else if (path === '/targets/policy') {
|
|
372
|
+
if (platformModule(platform).intervalScheduling === false) throw new CaptureError('input', platformModule(platform).scheduleHelp);
|
|
327
373
|
business.policy({ ...body, platform });
|
|
328
374
|
} else if (path === '/history/pause') {
|
|
329
375
|
const targets = body.uid ? [store.target(body.uid)] : body.scope === 'platform' ? store.targets().filter(target => target.platform === platform) : [];
|
|
@@ -349,30 +395,12 @@ export function createRuntime(directory = dataDirectory(), { adapterFactories =
|
|
|
349
395
|
store.notice('task', target.name + (body.kind === 'history' ? ':已开启历史缓存' : body.mode === 'history' ? ':已开启历史缓存与新动态监控' : ':从现在开始记录新动态'), Date.now(), target.uid);
|
|
350
396
|
})();
|
|
351
397
|
} else if (path === '/history/range') {
|
|
352
|
-
if (body.enableTarget !== undefined && typeof body.enableTarget !== 'boolean') return json(res, 400, { error: '无效的博主开关值' });
|
|
353
398
|
const targets = body.uid ? [store.target(body.uid)] : body.scope === 'platform' ? store.targets().filter(target => target.platform === platform && target.capture_mode !== 'pending') : [];
|
|
354
|
-
if (!targets.length || targets.some(target => !target || target.platform !== platform))
|
|
399
|
+
if (!targets.length || targets.some(target => !target || target.platform !== platform)) throw new CaptureError('input', '请先添加并设置要缓存的博主');
|
|
400
|
+
if (body.enableTarget !== undefined && typeof body.enableTarget !== 'boolean') throw new CaptureError('input', '无效的博主开关值');
|
|
355
401
|
if (targets.some(target => target.capture_mode === 'pending')) return json(res, 409, { error: '请先确认从何时开始记录' });
|
|
356
402
|
if (source.loginBusy) return json(res, 409, { error: '请完成当前账号操作后重试' });
|
|
357
|
-
|
|
358
|
-
if (body.originalOnly !== undefined && typeof body.originalOnly !== 'boolean') return json(res, 400, { error: '请选择有效的缓存内容类型' });
|
|
359
|
-
source.historyBusy = true; platformWorker.suspended = true;
|
|
360
|
-
try {
|
|
361
|
-
if (platformWorker.busy) await platformWorker.busy;
|
|
362
|
-
store.transaction(() => {
|
|
363
|
-
for (const target of targets) {
|
|
364
|
-
if (body.enableTarget === true && body.uid) enableAuthorJob(target.uid, 'history');
|
|
365
|
-
const job = store.jobs().find(job => job.uid === target.uid && job.kind === 'history');
|
|
366
|
-
const originalOnly = body.originalOnly ?? store.getSetting('historyOriginalOnly:' + target.uid, false);
|
|
367
|
-
const changed = target.history_from !== range.from || target.history_to !== range.to || originalOnly !== store.getSetting('historyOriginalOnly:' + target.uid, false);
|
|
368
|
-
store.db.prepare('UPDATE targets SET history_from=?,history_to=? WHERE uid=?').run(range.from, range.to, target.uid);
|
|
369
|
-
store.setSetting('historyOriginalOnly:' + target.uid, originalOnly);
|
|
370
|
-
store.setSetting('historyRequested:' + target.uid, true);
|
|
371
|
-
if (changed) store.updateJob(job.id, { checkpoint: {}, pages: 0, saved: 0, last_success_at: null, status: 'queued', next_run_at: 0, message: '按选定日期缓存以前的消息,已有内容保留' });
|
|
372
|
-
else if (job.status !== 'complete') store.updateJob(job.id, { status: 'queued', message: '等待继续缓存以前的消息' });
|
|
373
|
-
}
|
|
374
|
-
})();
|
|
375
|
-
} finally { source.historyBusy = false; platformWorker.suspended = false; platformWorker.wake(); }
|
|
403
|
+
await business.scope({ uids: targets.map(target => target.uid), from: body.from || '', to: body.to || '', ...(body.originalOnly === undefined ? {} : { originalOnly: body.originalOnly }), historyEnabled: true });
|
|
376
404
|
} else if (path === '/targets/profile') {
|
|
377
405
|
const target = store.target(body.uid);
|
|
378
406
|
if (!target || target.platform !== platform || !platformBrowser.profile) return json(res, 400, { error: '请选择有效的' + platformName + '博主' });
|
|
@@ -434,6 +462,7 @@ export function createRuntime(directory = dataDirectory(), { adapterFactories =
|
|
|
434
462
|
store.db.prepare('UPDATE targets SET history_gap_ms=? WHERE platform=?').run(seconds * 1000, platform);
|
|
435
463
|
})();
|
|
436
464
|
} else if (path === '/settings') {
|
|
465
|
+
if (platformModule(platform).intervalScheduling === false) throw new CaptureError('input', platformModule(platform).scheduleHelp);
|
|
437
466
|
const gap = body.requestGapSeconds === undefined ? platformWorker.requestGapMs / 1000 : Number(body.requestGapSeconds);
|
|
438
467
|
const history = body.historyGapSeconds === undefined ? store.getSetting(platform === 'weibo' ? 'targetDefaults' : `targetDefaults:${platform}`, { historyGapMs: 60000 }).historyGapMs / 1000 : Number(body.historyGapSeconds);
|
|
439
468
|
const interval = Number(body.intervalMinutes);
|
|
@@ -464,7 +493,7 @@ export function createRuntime(directory = dataDirectory(), { adapterFactories =
|
|
|
464
493
|
}
|
|
465
494
|
const clockChanges = ['/targets/start', '/monitoring', '/verify'].includes(path) || path.startsWith('/jobs/') && path.endsWith('/resume') && store.job(path.split('/')[2])?.kind === 'updates';
|
|
466
495
|
Object.values(sources).forEach(source => { if (clockChanges) source.worker.rescheduleUpdates(); source.worker.wake(); });
|
|
467
|
-
documents.drainImages();
|
|
496
|
+
documents.drainImages(); videos.cancelInactive();
|
|
468
497
|
return json(res, 200, { ok: true });
|
|
469
498
|
} catch (error) {
|
|
470
499
|
if (res.headersSent) return res.destroy();
|
|
@@ -483,6 +512,7 @@ export function createRuntime(directory = dataDirectory(), { adapterFactories =
|
|
|
483
512
|
Object.values(sources).forEach(source => { source.worker.stopping = true; source.worker.running = false; clearTimeout(source.worker.timer); });
|
|
484
513
|
return;
|
|
485
514
|
}
|
|
515
|
+
videos.start();
|
|
486
516
|
chat.recover();
|
|
487
517
|
agent.start();
|
|
488
518
|
chat.startScheduler();
|
|
@@ -490,13 +520,13 @@ export function createRuntime(directory = dataDirectory(), { adapterFactories =
|
|
|
490
520
|
void documents.drainImages();
|
|
491
521
|
}
|
|
492
522
|
return {
|
|
493
|
-
get store() { return store; }, get documents() { return documents; }, updates, agent, chat, worker, browser, sources, handle, start,
|
|
523
|
+
get store() { return store; }, get documents() { return documents; }, updates, agent, chat, videos, worker, browser, sources, handle, start,
|
|
494
524
|
async close() {
|
|
495
525
|
if (closing) return;
|
|
496
526
|
closing = true;
|
|
497
527
|
if (workspaceTask) await workspaceTask.catch(() => {});
|
|
498
528
|
documents.stopping = true;
|
|
499
|
-
await Promise.all([chat.close(), agent.close(), avatars.close(), ...Object.values(sources).map(source => source.worker.stop())]);
|
|
529
|
+
await Promise.all([videos.close(), chat.close(), agent.close(), avatars.close(), ...Object.values(sources).map(source => source.worker.stop())]);
|
|
500
530
|
if (documents.task) await documents.task;
|
|
501
531
|
if (documents.queueTask) await documents.queueTask;
|
|
502
532
|
await Promise.allSettled([...documents.imageTasks.values()]);
|
|
@@ -517,7 +547,7 @@ export function apply(ctx) {
|
|
|
517
547
|
ctx.effect(() => {
|
|
518
548
|
let runtime;
|
|
519
549
|
let startupError;
|
|
520
|
-
try { runtime = createRuntime(undefined, { getIM: () => im, model: hostModel(getService), updates }); runtime.start(); }
|
|
550
|
+
try { runtime = createRuntime(undefined, { getIM: () => im, model: hostAgentRuntime(getService, hostModel(getService)), updates }); runtime.start(); }
|
|
521
551
|
catch { startupError = '无法打开本地资料库,请检查信息保存位置和文件权限后重新启用插件'; }
|
|
522
552
|
const unregister = ctx.webServer.register({ kind: 'prefix', path: PREFIX,
|
|
523
553
|
handler: (req, res) => startupError ? json(res, 503, { error: startupError }) : runtime.handle(req, res) });
|
|
@@ -0,0 +1,10 @@
|
|
|
1
|
+
import he from 'he';
|
|
2
|
+
|
|
3
|
+
export function plainText(html = '') {
|
|
4
|
+
return he.decode(String(html)
|
|
5
|
+
.replace(/<script\b[^>]*>[\s\S]*?<\/script>/gi, '')
|
|
6
|
+
.replace(/<style\b[^>]*>[\s\S]*?<\/style>/gi, '')
|
|
7
|
+
.replace(/<img\b[^>]*alt=["']([^"']*)["'][^>]*>/gi, '$1')
|
|
8
|
+
.replace(/<br\s*\/?>|<\/p>|<\/div>/gi, '\n')
|
|
9
|
+
.replace(/<[^>]+>/g, '')).trim();
|
|
10
|
+
}
|
|
@@ -0,0 +1,44 @@
|
|
|
1
|
+
export function createDouyinUI({ React, h, SettingRow, TaskControl, LegacyNodeSettings }) {
|
|
2
|
+
function DouyinConfigForm({ state, uid = '', act, busy, part = 'all' }) {
|
|
3
|
+
const base = { times: ['20:00'], repeat: 'daily', platformAI: true, saveVideo: false, saveAudio: false, transcribe: false, maxFileMB: 256, python: 'python3', ffmpeg: 'ffmpeg', modelPath: '', threads: 2, ...(state.douyin?.config || {}) };
|
|
4
|
+
const [value, setValue] = React.useState(() => ({ ...base, ...(state.douyin?.overrides?.[uid] || {}) }));
|
|
5
|
+
const [inherit, setInherit] = React.useState(!!uid && !state.douyin?.overrides?.[uid]);
|
|
6
|
+
const [message, setMessage] = React.useState('');
|
|
7
|
+
const change = (key, val) => { if (uid && inherit) { setInherit(false); setMessage('已切换为这位博主的独立配置,保存后生效'); } else setMessage(''); setValue(old => ({ ...old, [key]: val })); };
|
|
8
|
+
const disabled = busy;
|
|
9
|
+
return h('form', { className: 'st-douyin-config', 'aria-label': uid ? '抖音博主配置' : '抖音平台默认配置', onSubmit: async event => {
|
|
10
|
+
event.preventDefault();
|
|
11
|
+
if (await act('/douyin/config', { platform: 'douyin', uid: uid || undefined, inherit, section: part, config: value })) setMessage('抖音配置已保存');
|
|
12
|
+
} },
|
|
13
|
+
uid ? h(SettingRow, { label: '跟随平台', description: '修改下方选项会自动切换为独立配置,其他博主不受影响' }, h('input', { type: 'checkbox', role: 'switch', 'aria-label': '跟随抖音平台配置', checked: inherit, disabled: busy, onChange: event => { setInherit(event.target.checked); if (event.target.checked) setValue(base); } })) : h('p', { className: 'st-caption' }, '只更新跟随平台的博主;独立配置保持不变。'),
|
|
14
|
+
part !== 'media' ? h('section', { 'aria-label': '抖音定时检查' }, h('h3', null, '定时检查'),
|
|
15
|
+
h(SettingRow, { label: '执行日期' }, h('select', { 'aria-label': '抖音检查日期', value: value.repeat, disabled, onChange: event => change('repeat', event.target.value) }, h('option', { value: 'daily' }, '每天'), h('option', { value: 'weekdays' }, '工作日(周一至周五)'))),
|
|
16
|
+
value.times.map((time, index) => h(SettingRow, { key: index, label: '检查时间 ' + (index + 1), description: '北京时间' }, h('div', { className: 'st-actions' }, h('input', { type: 'time', 'aria-label': '抖音检查时间 ' + (index + 1), value: time, required: true, disabled, onChange: event => change('times', value.times.map((item, at) => at === index ? event.target.value : item)) }),
|
|
17
|
+
value.times.length > 1 ? h('button', { type: 'button', disabled, onClick: () => change('times', value.times.filter((item, at) => at !== index)) }, '删除') : null))),
|
|
18
|
+
h('button', { type: 'button', disabled: disabled || value.times.length >= 8, onClick: () => change('times', [...value.times, '']) }, '+ 添加检查时间'),
|
|
19
|
+
h('p', { className: 'st-caption' }, '只在指定时间检查;重启后会补查离线更新。下载与转写单独排队,不阻塞检查。')) : null,
|
|
20
|
+
part !== 'frequency' ? h('section', { 'aria-label': '抖音内容保存' }, h('h3', null, '内容保存'),
|
|
21
|
+
[['platformAI', '平台 AI 解析', '保存平台返回的摘要与章节,不是作者逐字原话'], ['saveVideo', '保存视频', '把新发现的视频下载到本地'], ['saveAudio', '保存音频', '从视频音轨提取,不把背景音乐当人声'], ['transcribe', '本地转写', '用本地 SenseVoice 生成识别稿;一次处理一个作品']].map(([key, label, description]) => h(SettingRow, { key, label, description }, h('input', { type: 'checkbox', role: 'switch', 'aria-label': label, checked: value[key], disabled, onChange: event => change(key, event.target.checked) }))),
|
|
22
|
+
h(SettingRow, { label: '单文件上限', description: 'MB,超过上限会停止该文件下载' }, h('input', { type: 'number', 'aria-label': '抖音单文件上限', min: 16, max: 2048, value: value.maxFileMB, disabled, onChange: event => change('maxFileMB', Number(event.target.value)) })),
|
|
23
|
+
value.saveAudio || value.transcribe ? h(SettingRow, { label: 'ffmpeg', description: '程序名或完整路径;音频和转写需要' }, h('input', { 'aria-label': 'ffmpeg 程序', value: value.ffmpeg, disabled, onChange: event => change('ffmpeg', event.target.value) })) : null,
|
|
24
|
+
value.transcribe ? h(React.Fragment, null,
|
|
25
|
+
h(SettingRow, { label: 'Python', description: '需安装 FunASR、PyTorch 依赖' }, h('input', { 'aria-label': '转写 Python', value: value.python, disabled, onChange: event => change('python', event.target.value) })),
|
|
26
|
+
h(SettingRow, { label: '模型目录', description: '已下载的 SenseVoice 本地目录;不会自动下载模型' }, h('input', { 'aria-label': 'SenseVoice 模型目录', value: value.modelPath, disabled, onChange: event => change('modelPath', event.target.value) })),
|
|
27
|
+
h(SettingRow, { label: 'CPU 线程', description: '1–4;空闲时不常驻模型' }, h('input', { type: 'number', 'aria-label': '转写 CPU 线程', min: 1, max: 4, value: value.threads, disabled, onChange: event => change('threads', Number(event.target.value)) }))) : null,
|
|
28
|
+
h('p', { className: 'st-caption' }, '关闭选项会停止后续处理,已有文件保留;音频和转写支持一小时以内的作品。图集保留配图,不生成音轨。')) : null,
|
|
29
|
+
message ? h('p', { role: 'status', className: 'st-caption' }, message) : null,
|
|
30
|
+
h('button', { className: 'st-primary', disabled: busy, type: 'submit' }, '保存抖音配置'));
|
|
31
|
+
}
|
|
32
|
+
function DouyinNodeSettings(props) {
|
|
33
|
+
const { context, state, act, busy, close, embedded } = props;
|
|
34
|
+
const dialog = React.useRef(null), [tab, setTab] = React.useState(context.tab || 'frequency');
|
|
35
|
+
const target = state.targets.find(item => item.uid === context.uid);
|
|
36
|
+
React.useEffect(() => { if (!embedded) dialog.current?.showModal(); }, []);
|
|
37
|
+
return h(embedded ? 'section' : 'dialog', { ref: embedded ? undefined : dialog, className: embedded ? 'st-inline-settings' : 'st-node-dialog', 'aria-label': '抖音详细设置', onCancel: close },
|
|
38
|
+
!embedded ? h('div', { className: 'st-dialog-head' }, h('h2', null, (target?.name || '抖音平台') + '的详细设置'), h('button', { onClick: close, 'aria-label': '关闭监控设置' }, '×')) : null,
|
|
39
|
+
h('div', { className: 'st-platform-tabs', role: 'tablist', 'aria-label': '抖音配置分组' }, [['frequency', '定时检查'], ['media', '内容保存'], ['history', '历史缓存']].map(([id, title]) => h('button', { key: id, type: 'button', role: 'tab', 'aria-selected': tab === id, onClick: () => setTab(id) }, title))),
|
|
40
|
+
props.error ? h('p', { role: 'alert', className: 'st-inline-error' }, props.error) : null,
|
|
41
|
+
h('div', { hidden: tab === 'history' }, target && tab === 'frequency' ? h(TaskControl, { state, target, kind: 'updates', act, busy, openSettings: props.openSettings }) : null, h(DouyinConfigForm, { state, uid: target?.uid, act, busy, part: tab === 'media' ? 'media' : 'frequency' })), tab === 'history' ? h(LegacyNodeSettings, { ...props, embedded: true, context: { ...context, tab: 'history' } }) : null);
|
|
42
|
+
}
|
|
43
|
+
return { DouyinConfigForm, DouyinNodeSettings };
|
|
44
|
+
}
|
|
@@ -0,0 +1,33 @@
|
|
|
1
|
+
import { isAbsolute } from 'node:path';
|
|
2
|
+
|
|
3
|
+
export const defaults = { requestGapMs: 30000, historyGapMs: 120000, intervalMs: 86400000 };
|
|
4
|
+
export const defaultConfig = Object.freeze({ times: ['20:00'], repeat: 'daily', platformAI: true, saveVideo: false,
|
|
5
|
+
saveAudio: false, transcribe: false, maxFileMB: 256, python: 'python3', ffmpeg: 'ffmpeg', modelPath: '', threads: 2 });
|
|
6
|
+
|
|
7
|
+
export function validateConfig(value) {
|
|
8
|
+
if (!value || typeof value !== 'object' || Array.isArray(value) || Object.keys(value).some(key => !Object.hasOwn(defaultConfig, key))) throw new Error('抖音配置包含无效字段');
|
|
9
|
+
const config = { ...defaultConfig, ...value };
|
|
10
|
+
if (!Array.isArray(config.times) || !config.times.length || config.times.length > 8 || config.times.some(time => typeof time !== 'string' || !/^([01]\d|2[0-3]):[0-5]\d$/.test(time)) || !['daily', 'weekdays'].includes(config.repeat)) throw new Error('请选择每天或工作日,并填写1–8个有效检查时间');
|
|
11
|
+
for (const key of ['platformAI', 'saveVideo', 'saveAudio', 'transcribe']) if (typeof config[key] !== 'boolean') throw new Error('请选择有效的保存选项');
|
|
12
|
+
if (!Number.isInteger(config.maxFileMB) || config.maxFileMB < 16 || config.maxFileMB > 2048 || !Number.isInteger(config.threads) || config.threads < 1 || config.threads > 4) throw new Error('单文件上限16–2048MB,转写线程1–4个');
|
|
13
|
+
for (const key of ['python', 'ffmpeg']) if (typeof config[key] !== 'string' || !config[key] || config[key].length > 500 || /[\0\r\n]/.test(config[key]) || !(isAbsolute(config[key]) || /^[\w.-]+$/.test(config[key])) || config[key].startsWith('-')) throw new Error('请填写有效的程序名或完整路径');
|
|
14
|
+
if (typeof config.modelPath !== 'string' || config.modelPath.length > 1000 || config.modelPath.includes('\0') || config.modelPath && !isAbsolute(config.modelPath)) throw new Error('模型目录需要填写本机完整路径');
|
|
15
|
+
return { ...config, times: [...new Set(config.times)].sort() };
|
|
16
|
+
}
|
|
17
|
+
|
|
18
|
+
export function configFor(store, uid = '') {
|
|
19
|
+
return validateConfig({ ...store.getSetting('douyinConfig', defaultConfig), ...(uid ? store.getSetting('douyinConfig:' + uid, null) || {} : {}) });
|
|
20
|
+
}
|
|
21
|
+
|
|
22
|
+
export function nextCheckAt(config, after, inclusive = false) {
|
|
23
|
+
const start = Math.floor((after + 28800000) / 86400000) * 86400000 - 28800000;
|
|
24
|
+
for (let day = 0; day < 8; day++) {
|
|
25
|
+
const at = start + day * 86400000, weekday = new Date(at + 28800000).getUTCDay();
|
|
26
|
+
if (config.repeat === 'weekdays' && [0, 6].includes(weekday)) continue;
|
|
27
|
+
for (const time of [...config.times].sort()) {
|
|
28
|
+
const candidate = at + (Number(time.slice(0, 2)) * 60 + Number(time.slice(3))) * 60000;
|
|
29
|
+
if (inclusive ? candidate >= after : candidate > after) return candidate;
|
|
30
|
+
}
|
|
31
|
+
}
|
|
32
|
+
throw new Error('抖音检查时间无效');
|
|
33
|
+
}
|
|
@@ -0,0 +1,49 @@
|
|
|
1
|
+
import { CaptureError } from '../../capture-error.js';
|
|
2
|
+
import { plainText } from '../../plain-text.js';
|
|
3
|
+
import { avatar, images, firstURL } from './media.js';
|
|
4
|
+
export { mediaURL, avatar, images } from './media.js';
|
|
5
|
+
|
|
6
|
+
export function parseAuthor(input) {
|
|
7
|
+
const value = String(input || '').trim();
|
|
8
|
+
if (/^MS4w[\w-]{16,180}$/.test(value)) return value;
|
|
9
|
+
try {
|
|
10
|
+
const url = new URL(value);
|
|
11
|
+
if (['www.douyin.com', 'douyin.com'].includes(url.hostname) && ['http:', 'https:'].includes(url.protocol) && !url.username && !url.password && !url.port) {
|
|
12
|
+
const match = /^\/user\/(MS4w[\w-]{16,180})\/?$/.exec(url.pathname);
|
|
13
|
+
if (match) return match[1];
|
|
14
|
+
}
|
|
15
|
+
} catch {}
|
|
16
|
+
throw new Error('请粘贴抖音博主完整主页链接(/user/…),或 sec_uid;抖音号不是内部用户ID');
|
|
17
|
+
}
|
|
18
|
+
|
|
19
|
+
export function normalizePost(raw, uid) {
|
|
20
|
+
if (raw?.author?.sec_uid !== uid) return null;
|
|
21
|
+
if (typeof raw.aweme_id !== 'string' || !/^\d{5,30}$/.test(raw.aweme_id)) throw new CaptureError('schema', '抖音作品ID无效,已停止以防记录错位');
|
|
22
|
+
const publishedAt = Number(raw.create_time) * 1000;
|
|
23
|
+
if (!Number.isSafeInteger(publishedAt) || publishedAt <= 0) throw new CaptureError('schema', '抖音发布时间无效');
|
|
24
|
+
const description = plainText(raw.desc || raw.caption || '') || (raw.images?.length ? '分享图片' : '视频作品');
|
|
25
|
+
const chapters = (Array.isArray(raw.chapter_list) ? raw.chapter_list : []).slice(0, 200).map(chapter => ({
|
|
26
|
+
atMs: Number.isSafeInteger(Number(chapter.timestamp)) && Number(chapter.timestamp) >= 0 ? Number(chapter.timestamp) : 0,
|
|
27
|
+
title: plainText(chapter.desc || '').slice(0, 1000), text: plainText(chapter.detail || '').slice(0, 10000),
|
|
28
|
+
}));
|
|
29
|
+
const reviewed = Number(raw.chapter_review_status) === 2;
|
|
30
|
+
const abstract = plainText(raw.chapter_abstract || '').slice(0, 20000);
|
|
31
|
+
const video = { durationMs: Math.max(0, Number(raw.video?.duration) || 0),
|
|
32
|
+
summary: { status: abstract || chapters.length ? reviewed ? 'available' : 'unconfirmed' : 'not_returned', abstract, chapters,
|
|
33
|
+
reviewStatus: raw.chapter_review_status ?? null, origin: 'platform_ai' },
|
|
34
|
+
videoURL: firstURL(raw.video?.play_addr) || firstURL(raw.video?.play_addr_h264),
|
|
35
|
+
imagePost: !!raw.images?.length };
|
|
36
|
+
return { id: raw.aweme_id, uid, author: plainText(raw.author.nickname || uid), publishedAt, text: description,
|
|
37
|
+
url: `https://www.douyin.com/${video.imagePost ? 'note' : 'video'}/${raw.aweme_id}`, complete: true, quote: null,
|
|
38
|
+
avatar: avatar(raw.author), images: images(raw), video,
|
|
39
|
+
raw: { aweme_id: raw.aweme_id, create_time: raw.create_time, desc: description, author: { sec_uid: uid, nickname: raw.author.nickname }, images: raw.images || [], video } };
|
|
40
|
+
}
|
|
41
|
+
|
|
42
|
+
export function decodeList(body) {
|
|
43
|
+
if (!body || typeof body !== 'object' || Array.isArray(body)) throw new CaptureError('verification', '抖音未返回作品数据,请检查独立登录窗口');
|
|
44
|
+
if (Number(body.status_code) !== 0 || !Array.isArray(body.aweme_list) || ![0, 1, false, true].includes(body.has_more)) throw new CaptureError('verification', '抖音未确认作品列表,可能需要登录或验证;不会当成没有更新');
|
|
45
|
+
const cursor = String(body.max_cursor ?? '');
|
|
46
|
+
if (body.has_more && !/^\d+$/.test(cursor)) throw new CaptureError('schema', '抖音分页游标缺失,断点已保留');
|
|
47
|
+
return { items: body.aweme_list, end: !body.has_more, nextCursor: cursor,
|
|
48
|
+
endMessage: body.has_more ? '' : '已到抖音当前可见作品末尾,不代表完整历史' };
|
|
49
|
+
}
|
|
@@ -0,0 +1,163 @@
|
|
|
1
|
+
import { chromium } from 'playwright';
|
|
2
|
+
import { createHash } from 'node:crypto';
|
|
3
|
+
import { join } from 'node:path';
|
|
4
|
+
import { existsSync, readFileSync, writeFileSync, renameSync, mkdirSync } from 'node:fs';
|
|
5
|
+
import { CaptureError } from '../../capture-error.js';
|
|
6
|
+
import { parseAuthor, normalizePost, decodeList, avatar } from './data.js';
|
|
7
|
+
import { defaults } from './config.js';
|
|
8
|
+
|
|
9
|
+
export { parseAuthor, normalizePost, defaults };
|
|
10
|
+
export function parseResponse(text) {
|
|
11
|
+
return JSON.parse(text, (key, value, context) => {
|
|
12
|
+
if (['aweme_id', 'uid', 'max_cursor'].includes(key) && typeof value === 'number') {
|
|
13
|
+
if (context?.source && /^\d+$/.test(context.source)) return context.source;
|
|
14
|
+
if (!Number.isSafeInteger(value)) throw new CaptureError('schema', '抖音ID无法无损解析');
|
|
15
|
+
return String(value);
|
|
16
|
+
}
|
|
17
|
+
return value;
|
|
18
|
+
});
|
|
19
|
+
}
|
|
20
|
+
|
|
21
|
+
export class DouyinBrowser {
|
|
22
|
+
constructor(directory, beforeRequest, expectedUid = () => null) {
|
|
23
|
+
this.id = 'douyin'; this.name = '抖音'; this.defaults = defaults;
|
|
24
|
+
this.directory = directory; this.beforeRequest = beforeRequest; this.expectedUid = expectedUid;
|
|
25
|
+
this.sessionFile = join(directory, 'douyin-session.json');
|
|
26
|
+
this.context = null; this.browser = null; this.backgroundContext = null; this.page = null; this.idle = null; this.closed = false;
|
|
27
|
+
this.sessionData = null; this.activeUid = ''; this.responses = []; this.profileData = null;
|
|
28
|
+
}
|
|
29
|
+
parseAuthor(input) { return parseAuthor(input); }
|
|
30
|
+
normalizePost(raw, uid) { return normalizePost(raw, uid); }
|
|
31
|
+
isPinned(raw) { return raw.is_top === 1 || raw.is_top === true || raw.is_top === '1'; }
|
|
32
|
+
needsRequest() { return false; }
|
|
33
|
+
async complete(raw) { return raw; }
|
|
34
|
+
|
|
35
|
+
loadSession() {
|
|
36
|
+
if (!this.sessionData && existsSync(this.sessionFile)) {
|
|
37
|
+
try { this.sessionData = JSON.parse(readFileSync(this.sessionFile, 'utf8')); }
|
|
38
|
+
catch { throw new CaptureError('login', '抖音授权文件不可读,请重新登录'); }
|
|
39
|
+
}
|
|
40
|
+
const cookies = this.sessionData?.storageState?.cookies || [];
|
|
41
|
+
if (!cookies.some(cookie => /(^|\.)douyin\.com$/.test(cookie.domain) && ['sessionid', 'sessionid_ss'].includes(cookie.name) && cookie.value && (cookie.expires <= 0 || cookie.expires > Date.now() / 1000))) throw new CaptureError('login', '抖音本地授权缺失或过期,请主动登录;不会自动弹出窗口');
|
|
42
|
+
if (this.expectedUid() && this.expectedUid() !== this.sessionData.uid) throw new CaptureError('login', '抖音授权与资料库绑定账号不一致');
|
|
43
|
+
return this.sessionData;
|
|
44
|
+
}
|
|
45
|
+
async login() {
|
|
46
|
+
await this.release(); this.closed = false;
|
|
47
|
+
this.context = await chromium.launchPersistentContext(join(this.directory, 'douyin-browser'), { channel: 'chrome', headless: false, viewport: null });
|
|
48
|
+
this.context.on('close', () => { this.context = null; });
|
|
49
|
+
const page = this.context.pages()[0] || await this.context.newPage();
|
|
50
|
+
await page.goto('https://www.douyin.com/', { waitUntil: 'domcontentloaded', timeout: 45000 });
|
|
51
|
+
}
|
|
52
|
+
async checkLogin() {
|
|
53
|
+
if (this.context) {
|
|
54
|
+
const storageState = await this.context.storageState();
|
|
55
|
+
const identity = storageState.cookies.find(cookie => /(^|\.)douyin\.com$/.test(cookie.domain) && cookie.name === 'uid_tt');
|
|
56
|
+
if (!identity?.value) throw new CaptureError('login', '尚未确认抖音登录,请在授权窗口完成登录后再继续');
|
|
57
|
+
const uid = createHash('sha256').update(identity.value).digest('hex');
|
|
58
|
+
if (this.expectedUid() && this.expectedUid() !== uid) throw new CaptureError('login', '抖音账号与资料库绑定账号不同');
|
|
59
|
+
const previous = this.sessionData;
|
|
60
|
+
this.sessionData = { uid, storageState };
|
|
61
|
+
try { this.loadSession(); } catch (error) { this.sessionData = previous; throw error; }
|
|
62
|
+
mkdirSync(this.directory, { recursive: true, mode: 0o700 });
|
|
63
|
+
writeFileSync(this.sessionFile + '.tmp', JSON.stringify(this.sessionData), { mode: 0o600 });
|
|
64
|
+
renameSync(this.sessionFile + '.tmp', this.sessionFile);
|
|
65
|
+
await this.context.close(); this.context = null;
|
|
66
|
+
}
|
|
67
|
+
// Only local session validity is checked here. Actual access is confirmed by the next list response.
|
|
68
|
+
return this.loadSession().uid;
|
|
69
|
+
}
|
|
70
|
+
async release() {
|
|
71
|
+
clearTimeout(this.idle); this.idle = null;
|
|
72
|
+
const browser = this.browser; this.browser = null; this.backgroundContext = null; this.page = null; this.activeUid = ''; this.responses = [];
|
|
73
|
+
if (browser) await browser.close();
|
|
74
|
+
}
|
|
75
|
+
armIdle() {
|
|
76
|
+
clearTimeout(this.idle);
|
|
77
|
+
this.idle = setTimeout(() => { void this.release().catch(() => {}); }, 180000); this.idle.unref();
|
|
78
|
+
}
|
|
79
|
+
async finishScan() { this.armIdle(); }
|
|
80
|
+
async openAuthor(uid) {
|
|
81
|
+
clearTimeout(this.idle);
|
|
82
|
+
const session = this.loadSession();
|
|
83
|
+
if (!this.browser) {
|
|
84
|
+
this.browser = await chromium.launch({ headless: true });
|
|
85
|
+
this.backgroundContext = await this.browser.newContext({ storageState: session.storageState, viewport: { width: 1100, height: 800 } });
|
|
86
|
+
}
|
|
87
|
+
if (this.page) await this.page.close();
|
|
88
|
+
this.page = await this.backgroundContext.newPage(); this.activeUid = uid; this.profileData = null; this.responses = [];
|
|
89
|
+
const owner = this.page;
|
|
90
|
+
this.page.on('response', async response => {
|
|
91
|
+
try {
|
|
92
|
+
const url = new URL(response.url());
|
|
93
|
+
if (!['www.douyin.com', 'www-hj.douyin.com', 'douyin.com'].includes(url.hostname)) return;
|
|
94
|
+
if (url.pathname.includes('/aweme/post/') && url.searchParams.get('sec_user_id') === uid) {
|
|
95
|
+
const status = response.status();
|
|
96
|
+
const entry = { cursor: url.searchParams.get('max_cursor') || '0' };
|
|
97
|
+
if ([401, 403, 429].includes(status)) entry.error = new CaptureError(status === 429 ? 'limited' : 'verification', '抖音限制了访问,请在独立授权窗口检查');
|
|
98
|
+
else if (status !== 200) entry.error = new CaptureError('network', '抖音作品请求未成功');
|
|
99
|
+
else { try { entry.data = parseResponse(await response.text()); } catch { entry.error = new CaptureError('schema', '抖音未返回可解析的作品数据'); } }
|
|
100
|
+
if (this.activeUid === uid && this.page === owner) this.responses.push(entry);
|
|
101
|
+
} else if (url.pathname.includes('/user/profile/other/') && url.searchParams.get('sec_user_id') === uid) {
|
|
102
|
+
const body = parseResponse(await response.text());
|
|
103
|
+
if (body.user?.sec_uid === uid && this.page === owner) this.profileData = body.user;
|
|
104
|
+
}
|
|
105
|
+
} catch { /* Ignore unrelated resource responses. */ }
|
|
106
|
+
});
|
|
107
|
+
await this.page.goto('https://www.douyin.com/user/' + uid, { waitUntil: 'domcontentloaded', timeout: 45000 });
|
|
108
|
+
}
|
|
109
|
+
async pageResult(uid, cursor, options) {
|
|
110
|
+
this.closed = false;
|
|
111
|
+
await this.beforeRequest(options);
|
|
112
|
+
const requested = String(cursor ?? '0');
|
|
113
|
+
try {
|
|
114
|
+
if (!this.page || this.activeUid !== uid || requested === '0') await this.openAuthor(uid);
|
|
115
|
+
const deadline = Date.now() + 30000;
|
|
116
|
+
let scrollAt = 0, progressed = false;
|
|
117
|
+
while (Date.now() < deadline && !this.closed) {
|
|
118
|
+
const index = this.responses.findIndex(item => item.cursor === requested);
|
|
119
|
+
if (index >= 0) {
|
|
120
|
+
const [entry] = this.responses.splice(index, 1);
|
|
121
|
+
if (entry.error) throw entry.error;
|
|
122
|
+
const result = decodeList(entry.data);
|
|
123
|
+
if (!result.items.length && !result.end) throw new CaptureError('schema', '抖音返回不明空页,保留断点,不当成没有更新');
|
|
124
|
+
if (!result.end && String(result.nextCursor) === requested) throw new CaptureError('schema', '抖音分页未前进,已停止并保留断点');
|
|
125
|
+
const refreshed = await this.backgroundContext.storageState?.();
|
|
126
|
+
if (refreshed) {
|
|
127
|
+
this.sessionData = { ...this.sessionData, storageState: refreshed };
|
|
128
|
+
writeFileSync(this.sessionFile + '.tmp', JSON.stringify(this.sessionData), { mode: 0o600 });
|
|
129
|
+
renameSync(this.sessionFile + '.tmp', this.sessionFile);
|
|
130
|
+
}
|
|
131
|
+
this.armIdle(); return result;
|
|
132
|
+
}
|
|
133
|
+
if (requested !== '0' && this.responses.length) {
|
|
134
|
+
for (const entry of this.responses.splice(0)) {
|
|
135
|
+
if (entry.error) throw entry.error;
|
|
136
|
+
if (decodeList(entry.data).end) throw new CaptureError('schema', '历史分页位置已失效,请重新选择历史范围');
|
|
137
|
+
progressed = true;
|
|
138
|
+
}
|
|
139
|
+
}
|
|
140
|
+
if (requested !== '0' && Date.now() >= scrollAt) {
|
|
141
|
+
await this.page.evaluate(() => { window.scrollTo(0, document.body.scrollHeight); window.dispatchEvent(new Event('scroll')); });
|
|
142
|
+
await this.page.mouse.wheel(0, 2000); scrollAt = Date.now() + defaults.requestGapMs;
|
|
143
|
+
}
|
|
144
|
+
await new Promise(resolve => setTimeout(resolve, 200));
|
|
145
|
+
}
|
|
146
|
+
if (progressed && requested !== '0') { this.armIdle(); throw new CaptureError('wait', '正在恢复抖音历史分页位置,已保留作品断点', defaults.requestGapMs); }
|
|
147
|
+
throw new CaptureError('network', '抖音作品列表加载超时,已保留进度,将自动重试,无需重新登录');
|
|
148
|
+
} catch (error) {
|
|
149
|
+
if (error.kind !== 'wait') await this.release();
|
|
150
|
+
if (error.kind) throw error;
|
|
151
|
+
throw new CaptureError('network', '抖音读取中断,进度已保留');
|
|
152
|
+
}
|
|
153
|
+
}
|
|
154
|
+
async profile(uid, options) {
|
|
155
|
+
const result = await this.pageResult(uid, '0', options);
|
|
156
|
+
const user = this.profileData || result.items.find(item => item.author?.sec_uid === uid)?.author;
|
|
157
|
+
if (!user?.nickname) throw new CaptureError('schema', '抖音未返回可确认的博主昵称,请稍后重试');
|
|
158
|
+
const total = Number(user.aweme_count);
|
|
159
|
+
return { name: user.nickname, avatar: avatar(user), total: Number.isSafeInteger(total) && total >= 0 ? total : null };
|
|
160
|
+
}
|
|
161
|
+
timeline(uid, page, cursor, options) { return this.pageResult(uid, page === 1 ? '0' : cursor, options); }
|
|
162
|
+
async close() { this.closed = true; if (this.context) await this.context.close(); this.context = null; await this.release(); }
|
|
163
|
+
}
|