dsh-social-trace 0.1.13 → 0.1.18

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,154 @@
1
+ import { existsSync, mkdirSync, renameSync, unlinkSync, statSync, createWriteStream } from 'node:fs';
2
+ import { join } from 'node:path';
3
+ import { fileURLToPath } from 'node:url';
4
+ import { pipeline } from 'node:stream/promises';
5
+ import { Readable, Transform } from 'node:stream';
6
+ import { spawn } from 'node:child_process';
7
+ import { randomUUID } from 'node:crypto';
8
+ import { configFor } from './config.js';
9
+ import { mediaURL } from './data.js';
10
+
11
+ export function runProcess(program, args, { signal, threads = 2, timeout = 900000 } = {}) {
12
+ return new Promise((resolve, reject) => {
13
+ const child = spawn(program, args, { shell: false, signal, stdio: ['ignore', 'pipe', 'pipe'], env: { ...process.env, OMP_NUM_THREADS: String(threads), MKL_NUM_THREADS: String(threads), HF_HUB_OFFLINE: '1', MODELSCOPE_OFFLINE: '1' } });
14
+ let output = '', exceeded = false;
15
+ const timer = setTimeout(() => { exceeded = true; child.kill('SIGKILL'); }, timeout); timer.unref();
16
+ const abort = () => child.kill('SIGKILL'); signal?.addEventListener('abort', abort, { once: true });
17
+ child.stdout.on('data', chunk => { output += chunk; if (output.length > 1000000) { exceeded = true; child.kill('SIGKILL'); } });
18
+ child.stderr.resume(); // Do not store paths, media URLs, or model diagnostics in public errors.
19
+ child.on('error', error => { clearTimeout(timer); signal?.removeEventListener('abort', abort); reject(Object.assign(new Error('本地处理程序不可用,请检查 Python、ffmpeg 与模型配置'), { setup: error.code === 'ENOENT' })); });
20
+ child.on('close', code => { clearTimeout(timer); signal?.removeEventListener('abort', abort); code === 0 && !exceeded ? resolve(output) : reject(Object.assign(new Error(exceeded ? '本地处理超时或输出过大,可稍后重试' : code === 78 ? '转写环境缺少 PyTorch 或 FunASR,请配置已安装依赖的 Python' : '本地处理未成功,请检查依赖、模型和媒体文件'), { setup: code === 78 })); });
21
+ });
22
+ }
23
+
24
+ export function videoView(row) {
25
+ if (!row) return null;
26
+ const metadata = JSON.parse(row.metadata), summary = JSON.parse(row.summary);
27
+ return { durationMs: metadata.durationMs, imagePost: metadata.imagePost, summary, transcript: row.transcript,
28
+ stages: { video: row.video_status, audio: row.audio_status, transcription: row.transcript_status },
29
+ videoAvailable: row.video_status === 'done', audioAvailable: row.audio_status === 'done', error: row.error, updatedAt: row.updated_at };
30
+ }
31
+
32
+ export class VideoLibrary {
33
+ constructor(getStore, getDocuments, { fetchImpl = fetch, processImpl = runProcess } = {}) {
34
+ this.getStore = getStore; this.getDocuments = getDocuments; this.fetchImpl = fetchImpl; this.processImpl = processImpl;
35
+ this.busy = null; this.controller = null; this.timer = null; this.closed = false;
36
+ }
37
+ get store() { return this.getStore(); }
38
+ recover() { this.store.db.exec("UPDATE video_records SET video_status=CASE WHEN video_status='running' THEN 'queued' ELSE video_status END,audio_status=CASE WHEN audio_status='running' THEN 'queued' ELSE audio_status END,transcript_status=CASE WHEN transcript_status='running' THEN 'queued' ELSE transcript_status END"); }
39
+ configure(uid = '') {
40
+ const rows = this.store.db.prepare('SELECT * FROM video_records WHERE (?=\'\' OR uid=?)').all(uid, uid);
41
+ for (const row of rows) {
42
+ const config = configFor(this.store, row.uid), metadata = JSON.parse(row.metadata);
43
+ for (const [column, enabled] of [['video_status', config.saveVideo], ['audio_status', config.saveAudio], ['transcript_status', config.transcribe]]) {
44
+ if (enabled && !metadata.imagePost && row[column] === 'not_requested') this.patch(row.post_id, { [column]: 'queued' });
45
+ if (!enabled && ['queued', 'running'].includes(row[column])) this.patch(row.post_id, { [column]: 'not_requested' });
46
+ }
47
+ }
48
+ this.cancelInactive();
49
+ }
50
+ canRun(row) {
51
+ const target = this.store.target(row.uid), account = this.store.account('douyin');
52
+ return !this.closed && target && this.store.monitoringEnabled('douyin', row.uid) && account?.status === 'ready' && account.cooldown_until <= Date.now()
53
+ && this.store.jobs().some(job => job.uid === row.uid && job.kind === (row.capture_kind === 'history' ? 'history' : 'updates') && ['queued', 'complete'].includes(job.status));
54
+ }
55
+ cancelInactive() {
56
+ if (this.active && (!this.canRun(this.active) || !configFor(this.store, this.active.uid)[this.activeOption])) this.controller?.abort();
57
+ }
58
+ patch(id, patch) {
59
+ const allowed = ['video_status', 'audio_status', 'transcript_status', 'transcript', 'error', 'search_text'];
60
+ const entries = Object.entries(patch);
61
+ if (entries.some(([key]) => !allowed.includes(key))) throw new Error('无效的视频状态');
62
+ this.store.db.prepare('UPDATE video_records SET ' + entries.map(([key]) => key + '=?').join(',') + ',updated_at=? WHERE post_id=?').run(...entries.map(([, value]) => value), Date.now(), id);
63
+ }
64
+ paths(row) {
65
+ const id = row.post_id.replace(/^douyin:/, '');
66
+ if (!/^\d{5,30}$/.test(id)) throw new Error('作品标识无效');
67
+ const directory = join(this.getDocuments().authorDirectory(row.uid), 'media', id);
68
+ mkdirSync(directory, { recursive: true, mode: 0o700 });
69
+ return { directory, video: join(directory, 'video.mp4'), audio: join(directory, 'audio.wav') };
70
+ }
71
+ retry(id) {
72
+ const row = this.store.db.prepare('SELECT * FROM video_records WHERE post_id=?').get(id);
73
+ if (!row) throw new Error('视频记录不存在');
74
+ const config = configFor(this.store, row.uid), paths = this.paths(row);
75
+ for (const type of ['video', 'audio']) if (config[type === 'video' ? 'saveVideo' : 'saveAudio'] && row[type + '_status'] === 'done' && !existsSync(paths[type])) this.patch(id, { [type + '_status']: 'queued' });
76
+ for (const column of ['video_status', 'audio_status', 'transcript_status']) if (['failed', 'needs_setup'].includes(row[column]) && config[{ video_status: 'saveVideo', audio_status: 'saveAudio', transcript_status: 'transcribe' }[column]]) this.patch(id, { [column]: 'queued', error: '' });
77
+ }
78
+ async download(url, destination, maxBytes, signal) {
79
+ for (let redirects = 0; redirects <= 3; redirects++) {
80
+ if (!mediaURL(url)) throw new Error('媒体地址不属于允许的平台域名');
81
+ const response = await this.fetchImpl(url, { redirect: 'manual', signal: AbortSignal.any([signal, AbortSignal.timeout(120000)]), headers: { Referer: 'https://www.douyin.com/' } });
82
+ if ([301, 302, 303, 307, 308].includes(response.status)) {
83
+ await response.body?.cancel(); url = new URL(response.headers.get('location'), url).href; continue;
84
+ }
85
+ if (!response.ok || !response.body || Number(response.headers.get('content-length')) > maxBytes) { await response.body?.cancel(); throw new Error('媒体下载未成功或超过大小上限;地址可能已过期'); }
86
+ const temporary = destination + '.' + randomUUID() + '.part'; let length = 0, head = Buffer.alloc(0);
87
+ try {
88
+ const cap = new Transform({ transform(chunk, encoding, callback) { length += chunk.length; if (head.length < 64) head = Buffer.concat([head, chunk.subarray(0, 64 - head.length)]); callback(length > maxBytes ? new Error('媒体超过大小上限') : null, chunk); } });
89
+ await pipeline(Readable.fromWeb(response.body), cap, createWriteStream(temporary, { mode: 0o600 }), { signal });
90
+ signal.throwIfAborted();
91
+ if (head.subarray(4, 8).toString() !== 'ftyp') throw new Error('未获取到可确认的 MP4 文件,不当成下载成功');
92
+ if (length < 1024) throw new Error('媒体内容过小,未当成下载成功');
93
+ renameSync(temporary, destination); return;
94
+ } finally { if (existsSync(temporary)) unlinkSync(temporary); }
95
+ }
96
+ throw new Error('媒体重定向次数过多');
97
+ }
98
+ tick() {
99
+ if (this.closed || this.suspended || this.busy) return this.busy;
100
+ const rows = this.store.db.prepare("SELECT v.* FROM video_records v JOIN jobs j ON j.uid=v.uid AND j.kind=CASE WHEN v.capture_kind='history' THEN 'history' ELSE 'updates' END WHERE (v.video_status='queued' OR v.audio_status='queued' OR v.transcript_status='queued') AND j.status IN ('queued','complete') AND NOT EXISTS (SELECT 1 FROM settings WHERE key='targetEnabled:'||v.uid AND value='false') ORDER BY v.updated_at,v.post_id LIMIT 100").all();
101
+ const row = rows.find(row => this.canRun(row) && (configFor(this.store, row.uid).saveVideo && row.video_status === 'queued' || configFor(this.store, row.uid).saveAudio && row.audio_status === 'queued' || configFor(this.store, row.uid).transcribe && row.transcript_status === 'queued'));
102
+ if (!row) return;
103
+ this.active = row; this.controller = new AbortController();
104
+ this.busy = this.process(row, this.controller.signal).finally(() => { this.busy = null; this.active = null; this.controller = null; });
105
+ return this.busy;
106
+ }
107
+ async process(row, signal) {
108
+ const config = configFor(this.store, row.uid), metadata = JSON.parse(row.metadata);
109
+ let paths;
110
+ const column = config.saveVideo && row.video_status === 'queued' ? 'video_status' : config.saveAudio && row.audio_status === 'queued' ? 'audio_status' : config.transcribe && row.transcript_status === 'queued' ? 'transcript_status' : null;
111
+ if (!column) return;
112
+ this.activeOption = { video_status: 'saveVideo', audio_status: 'saveAudio', transcript_status: 'transcribe' }[column];
113
+ const current = () => { signal.throwIfAborted(); if (!this.canRun(row) || !configFor(this.store, row.uid)[this.activeOption]) throw new Error('paused'); };
114
+ this.patch(row.post_id, { [column]: 'running', error: '' });
115
+ let temporaryVideo = null, temporaryAudio = null;
116
+ try {
117
+ current(); paths = this.paths(row);
118
+ if (column === 'transcript_status' && (!config.modelPath || !existsSync(join(config.modelPath, 'model.pt')))) throw Object.assign(new Error('请先配置已下载的 SenseVoice 模型目录;不会自动下载大型模型'), { setup: true });
119
+ if (column !== 'video_status' && (!metadata.durationMs || metadata.durationMs > 3600000)) throw new Error('音频或转写仅支持时长明确且不超过一小时的作品');
120
+ let video = paths.video;
121
+ if (!(column === 'transcript_status' && row.audio_status === 'done' && existsSync(paths.audio)) && !(row.video_status === 'done' && existsSync(paths.video))) {
122
+ if (!metadata.videoURL) throw new Error('本次未获取到视频下载地址,等待下次正常检查后重试');
123
+ if (column !== 'video_status') video = temporaryVideo = join(paths.directory, randomUUID() + '.mp4');
124
+ await this.download(metadata.videoURL, video, config.maxFileMB * 1024 * 1024, signal); current();
125
+ }
126
+ if (column === 'video_status') this.patch(row.post_id, { video_status: 'done' });
127
+ else {
128
+ let audio = paths.audio;
129
+ if (!(row.audio_status === 'done' && existsSync(audio))) {
130
+ audio = temporaryAudio = join(paths.directory, randomUUID() + '.wav');
131
+ await this.processImpl(config.ffmpeg, ['-nostdin', '-v', 'error', '-threads', String(config.threads), '-protocol_whitelist', 'file,pipe', '-i', video, '-vn', '-ac', '1', '-ar', '16000', audio], { signal, threads: config.threads });
132
+ current();
133
+ if (!existsSync(audio) || statSync(audio).size < 44) throw new Error('音轨提取未生成有效文件');
134
+ }
135
+ if (column === 'audio_status') { if (audio !== paths.audio) { renameSync(audio, paths.audio); temporaryAudio = null; } this.patch(row.post_id, { audio_status: 'done' }); }
136
+ else {
137
+ if (metadata.durationMs > 3600000) throw new Error('单个视频超过一小时,请分段处理后再转写');
138
+ const output = await this.processImpl(config.python, [fileURLToPath(new URL('./transcribe.py', import.meta.url)), audio, config.modelPath, String(config.threads)], { signal, threads: config.threads });
139
+ current(); const text = JSON.parse(output).text;
140
+ if (typeof text !== 'string' || !text.trim() || text.length > 200000) throw new Error('转写结果为空或超过长度上限');
141
+ this.patch(row.post_id, { transcript: text, transcript_status: 'done', search_text: row.search_text + '\n' + text });
142
+ }
143
+ }
144
+ } catch (error) {
145
+ if (signal.aborted || !this.canRun(row)) this.patch(row.post_id, { [column]: configFor(this.store, row.uid)[this.activeOption] ? 'queued' : 'not_requested' });
146
+ else this.patch(row.post_id, { [column]: error.setup ? 'needs_setup' : 'failed', error: error.setup ? error.message : '媒体处理未完成,请检查文件大小、下载地址或本地依赖后重试' });
147
+ } finally {
148
+ for (const file of [temporaryVideo, temporaryAudio].filter(Boolean)) if (existsSync(file)) unlinkSync(file);
149
+ const post = this.store.posts({ ids: [row.post_id], limit: 1 })[0]; if (post) this.getDocuments().save({ ...post, publishedAt: post.published_at });
150
+ }
151
+ }
152
+ start() { if (this.timer) return; this.closed = false; this.recover(); this.timer = setInterval(() => { void this.tick()?.catch(() => {}); }, 5000); this.timer.unref(); }
153
+ async close() { this.closed = true; clearInterval(this.timer); this.controller?.abort(); if (this.busy) await this.busy; }
154
+ }
@@ -0,0 +1,12 @@
1
+ export function mediaURL(value) {
2
+ try {
3
+ const url = new URL(value);
4
+ if (url.protocol === 'https:' && !url.username && !url.password && !url.port
5
+ && /(^|\.)(douyin\.com|douyinvod\.com|douyinpic\.com|douyinstatic\.com|byteimg\.com|ibytedtos\.com|bytecdn\.cn|pstatp\.com|amemv\.com|snssdk\.com)$/.test(url.hostname)) return url.href;
6
+ } catch {}
7
+ return null;
8
+ }
9
+
10
+ export const firstURL = value => (value?.url_list || []).map(mediaURL).find(Boolean) || null;
11
+ export const avatar = user => firstURL(user?.avatar_larger) || firstURL(user?.avatar_medium) || firstURL(user?.avatar_thumb);
12
+ export const images = raw => (raw?.images || []).slice(0, 18).map(firstURL).filter(Boolean);
@@ -0,0 +1,18 @@
1
+ import { mediaURL, images } from './media.js';
2
+ import { configFor, nextCheckAt } from './config.js';
3
+ import { initializeStorage, saveMedia, preparePost, decoratePost, filterPosts, searchPosts } from './storage.js';
4
+
5
+ export const platformModule = {
6
+ id: 'douyin', name: '抖音', intervalScheduling: false,
7
+ image: mediaURL, images, referer: 'https://www.douyin.com/',
8
+ transientConnectionFailure: error => /抖音作品列表加载超时|未收到抖音作品列表/.test(error.message || ''),
9
+ recoverLegacyConnection: account => account.status === 'needs_verification' && /未收到抖音作品列表/.test(account.reason || ''),
10
+ scheduleHelp: '抖音按每天时间点检查,请在详细设置调整',
11
+ initializeStorage, saveMedia, preparePost, decoratePost, filterPosts, searchPosts,
12
+ nextCheckAt: (store, target, after, inclusive = false) => nextCheckAt(configFor(store, target.uid), after, inclusive),
13
+ publicConfig(store, target) {
14
+ const config = configFor(store, target.uid);
15
+ return { dailySchedule: { times: config.times, repeat: config.repeat },
16
+ contentOptions: Object.fromEntries(['saveVideo', 'saveAudio', 'platformAI', 'transcribe'].map(key => [key, config[key]])) };
17
+ },
18
+ };
@@ -0,0 +1,34 @@
1
+ import { configFor, validateConfig, defaultConfig } from './config.js';
2
+ import { CaptureError } from '../../capture-error.js';
3
+
4
+ const frequencyKeys = ['times', 'repeat'];
5
+ const mediaKeys = Object.keys(defaultConfig).filter(key => !frequencyKeys.includes(key));
6
+
7
+ export function saveConfig(store, body, { media, worker }) {
8
+ if (body.platform !== 'douyin' || body.uid && store.target(body.uid)?.platform !== 'douyin'
9
+ || body.inherit !== undefined && typeof body.inherit !== 'boolean'
10
+ || !['all', 'frequency', 'media'].includes(body.section || 'all')) throw new CaptureError('input', '请选择有效的抖音配置对象');
11
+ let config;
12
+ try {
13
+ if (body.uid && body.inherit === true) config = null;
14
+ else {
15
+ // Validate the whole input, then merge only the section being edited with current saved values.
16
+ const input = validateConfig(body.config), allowed = body.section === 'frequency' ? frequencyKeys : body.section === 'media' ? mediaKeys : Object.keys(defaultConfig);
17
+ const keys = Object.keys(body.config).filter(key => allowed.includes(key));
18
+ config = validateConfig({ ...configFor(store, body.uid || ''), ...Object.fromEntries(keys.map(key => [key, input[key]])) });
19
+ }
20
+ } catch (error) { throw new CaptureError('input', error.message); }
21
+ store.setSetting(body.uid ? 'douyinConfig:' + body.uid : 'douyinConfig', config);
22
+ media.configure(body.uid || ''); worker.rescheduleUpdates(); worker.wake();
23
+ }
24
+
25
+ export function status(store, media) {
26
+ return { config: configFor(store), overrides: Object.fromEntries(store.targets().filter(target => target.platform === 'douyin').map(target => [target.uid, store.getSetting('douyinConfig:' + target.uid, null)])),
27
+ counts: store.db.prepare("SELECT uid,SUM(json_extract(metadata,'$.imagePost')=0) AS videos,SUM(json_extract(metadata,'$.imagePost')=1) AS images FROM video_records GROUP BY uid").all(),
28
+ processing: !!media.busy, revision: store.db.prepare('SELECT MAX(updated_at) AS at FROM video_records').get().at || 0 };
29
+ }
30
+
31
+ export function validatePublicConfig(store, uid, options) {
32
+ if (Object.keys(options).some(key => !['times', 'repeat', 'saveVideo', 'saveAudio', 'platformAI', 'transcribe'].includes(key))) throw new CaptureError('input', '无效的公开抖音选项');
33
+ return validateConfig({ ...configFor(store, uid), ...options });
34
+ }
@@ -0,0 +1,39 @@
1
+ import { configFor } from './config.js';
2
+ import { videoView } from './library.js';
3
+
4
+ export function initializeStorage(store) {
5
+ store.db.exec(`CREATE TABLE IF NOT EXISTS video_records (
6
+ post_id TEXT PRIMARY KEY REFERENCES posts(id),uid TEXT NOT NULL REFERENCES targets(uid),metadata TEXT NOT NULL,summary TEXT NOT NULL,
7
+ transcript TEXT NOT NULL DEFAULT '',search_text TEXT NOT NULL DEFAULT '',capture_kind TEXT NOT NULL DEFAULT 'updates',
8
+ video_status TEXT NOT NULL DEFAULT 'not_requested',audio_status TEXT NOT NULL DEFAULT 'not_requested',transcript_status TEXT NOT NULL DEFAULT 'not_requested',
9
+ error TEXT NOT NULL DEFAULT '',updated_at INTEGER NOT NULL);
10
+ CREATE INDEX IF NOT EXISTS video_owner ON video_records(uid);`);
11
+ }
12
+
13
+ export function saveMedia(store, post, now = Date.now(), captureKind = 'updates') {
14
+ if (!post.video || store.target(post.uid)?.platform !== 'douyin') return;
15
+ const options = configFor(store, post.uid), previous = store.db.prepare('SELECT * FROM video_records WHERE post_id=?').get(post.id);
16
+ const summary = options.platformAI ? post.video.summary : previous ? JSON.parse(previous.summary) : { status: 'disabled', abstract: '', chapters: [], origin: 'platform_ai' };
17
+ const metadata = { durationMs: post.video.durationMs, imagePost: post.video.imagePost, videoURL: post.video.videoURL };
18
+ const search = [summary.abstract, ...(summary.chapters || []).map(chapter => chapter.title + ' ' + chapter.text), previous?.transcript || ''].join('\n');
19
+ store.db.prepare(`INSERT INTO video_records(post_id,uid,metadata,summary,search_text,capture_kind,video_status,audio_status,transcript_status,updated_at)
20
+ VALUES(?,?,?,?,?,?,?,?,?,?) ON CONFLICT(post_id) DO UPDATE SET metadata=excluded.metadata,summary=excluded.summary,search_text=excluded.search_text,updated_at=excluded.updated_at`)
21
+ .run(post.id, post.uid, JSON.stringify(metadata), JSON.stringify(summary), search, captureKind || 'updates',
22
+ !metadata.imagePost && options.saveVideo ? 'queued' : 'not_requested', !metadata.imagePost && options.saveAudio ? 'queued' : 'not_requested', !metadata.imagePost && options.transcribe ? 'queued' : 'not_requested', now);
23
+ }
24
+
25
+ export function preparePost(store, post) {
26
+ return post.video && !configFor(store, post.uid).platformAI ? { ...post, raw: { ...post.raw, video: { ...post.video, summary: undefined } } } : post;
27
+ }
28
+
29
+ export function decoratePost(store, post) {
30
+ return { video: videoView(store.db.prepare('SELECT * FROM video_records WHERE post_id=?').get(post.id)) };
31
+ }
32
+
33
+ export function filterPosts(kind) {
34
+ return ['video', 'images'].includes(kind) ? { sql: "EXISTS (SELECT 1 FROM video_records v WHERE v.post_id=posts.id AND json_extract(v.metadata,'$.imagePost')=?)", values: [kind === 'images' ? 1 : 0] } : null;
35
+ }
36
+
37
+ export function searchPosts(value, literal) {
38
+ return { sql: 'EXISTS (SELECT 1 FROM video_records v WHERE v.post_id=posts.id AND ' + (literal ? 'instr(v.search_text,?)>0' : "v.search_text LIKE ? ESCAPE '\\'") + ')', values: [value] };
39
+ }
@@ -0,0 +1,27 @@
1
+ """Local-only SenseVoice worker. Called with fixed arguments by Social Trace."""
2
+ import contextlib
3
+ import json
4
+ import os
5
+ import re
6
+ import sys
7
+ from pathlib import Path
8
+
9
+ audio, model_path, threads = sys.argv[1:]
10
+ if not Path(audio).is_file() or not Path(model_path, 'model.pt').is_file():
11
+ raise SystemExit('Local audio or model is missing')
12
+ os.environ['MODELSCOPE_OFFLINE'] = '1'
13
+ os.environ['HF_HUB_OFFLINE'] = '1'
14
+ with contextlib.redirect_stdout(sys.stderr):
15
+ try:
16
+ import torch
17
+ from funasr import AutoModel
18
+ except ImportError:
19
+ raise SystemExit(78)
20
+ torch.set_num_threads(int(threads))
21
+ model = AutoModel(model=model_path, device='cpu', disable_update=True, trust_remote_code=False)
22
+ result = model.generate(input=audio, language='zh', use_itn=True, batch_size_s=30)
23
+ text = ' '.join(item.get('text', '') for item in result or [])
24
+ text = re.sub(r'<\|[^|]+\|>', '', text).strip()
25
+ if not text:
26
+ raise SystemExit('Empty transcription')
27
+ print(json.dumps({'text': text}, ensure_ascii=False))
@@ -0,0 +1,7 @@
1
+ import { platformModule as weibo } from './weibo/module.js';
2
+ import { platformModule as xueqiu } from './xueqiu/module.js';
3
+ import { platformModule as douyin } from './douyin/module.js';
4
+
5
+ // The shared scheduler, store and Agent depend on this contract, not on platform implementations.
6
+ export const platformModules = { weibo, xueqiu, douyin };
7
+ export const platformModule = id => platformModules[id] || {};
@@ -216,7 +216,7 @@ export class WeiboBrowser {
216
216
  if (item.isLongText && !item.longTextContent) {
217
217
  const data = await this.request('/ajax/statuses/longtext?id=' + encodeURIComponent(item.idstr || item.id));
218
218
  if (typeof data.longTextContent !== 'string' || !data.longTextContent.trim()) {
219
- throw new CaptureError('schema', '长文接口未返回全文,保留断点并暂停');
219
+ throw Object.assign(new CaptureError('content', '微博暂时没有返回这条内容的全文'), { postId: String(item.idstr || item.id) });
220
220
  }
221
221
  item.longTextContent = data.longTextContent;
222
222
  break;
@@ -0,0 +1,19 @@
1
+ export function weiboAvatar(user) {
2
+ for (const value of [user?.avatar_hd, user?.avatar_large, user?.profile_image_url]) {
3
+ try {
4
+ const url = new URL(value);
5
+ if (['http:', 'https:'].includes(url.protocol) && /(^|\.)sinaimg\.(cn|com)$/.test(url.hostname) && !url.username && !url.password && !url.port) {
6
+ url.protocol = 'https:';
7
+ return url.href;
8
+ }
9
+ } catch { /* Missing or invalid image fields fall back to the author's initial. */ }
10
+ }
11
+ return null;
12
+ }
13
+
14
+ export function weiboImages(raw) {
15
+ const pictures = raw?.pic_infos
16
+ ? (raw.pic_ids?.length ? raw.pic_ids.map(id => raw.pic_infos[id]) : Object.values(raw.pic_infos))
17
+ : Array.isArray(raw?.pics) ? raw.pics : [];
18
+ return [...new Set(pictures.slice(0, 18).map(pic => weiboAvatar({ avatar_hd: pic?.large?.url || pic?.largest?.url || pic?.original?.url || pic?.bmiddle?.url || pic?.url })).filter(Boolean))];
19
+ }
@@ -0,0 +1,5 @@
1
+ import { contentProblem, contentRetryDelays } from './recovery.js';
2
+ import { weiboAvatar, weiboImages } from './media.js';
3
+
4
+ export const platformModule = { id: 'weibo', name: '微博', intervalScheduling: true, contentProblem, contentRetryDelays,
5
+ image: value => weiboAvatar({ avatar_hd: value }), images: weiboImages, referer: 'https://weibo.com/' };
@@ -0,0 +1,17 @@
1
+ import { plainText } from '../../plain-text.js';
2
+
3
+ export const contentRetryDelays = [60000, 300000, 1800000];
4
+
5
+ export function contentProblem(error, job) {
6
+ if (error.kind !== 'content' && !(error.kind === 'schema' && /长文接口未返回全文/.test(error.message || ''))) return null;
7
+ const root = job.checkpoint.pending?.[0];
8
+ if (!root) return null;
9
+ const post = [root, root.retweeted_status].filter(Boolean).find(post => String(post.idstr || post.id) === String(error.postId))
10
+ || [root, root.retweeted_status].filter(Boolean).find(post => post.isLongText && !post.longTextContent) || root;
11
+ const postId = String(post.idstr || post.id || ''), rootId = String(root.idstr || root.id || '');
12
+ if (!/^\d{5,30}$/.test(postId) || !/^\d{5,30}$/.test(rootId)) return null;
13
+ const publishedAt = new Date(post.created_at).getTime();
14
+ return { type: 'full_text', postId, rootId, url: 'https://weibo.com/detail/' + postId,
15
+ author: plainText(post.user?.screen_name || ''), excerpt: plainText(post.text_raw || post.text || '').slice(0, 140),
16
+ publishedAt: Number.isFinite(publishedAt) ? publishedAt : null, stage: post === root ? '微博正文' : '转发引用的原文' };
17
+ }
@@ -0,0 +1,5 @@
1
+ import { xueqiuImage, xueqiuImages } from './media.js';
2
+
3
+ export const platformModule = { id: 'xueqiu', name: '雪球', intervalScheduling: true,
4
+ image: xueqiuImage, images: xueqiuImages, referer: 'https://xueqiu.com/',
5
+ isArticle: raw => String(raw.type) === '3' && !!raw.title };
package/lib/store.js CHANGED
@@ -4,6 +4,7 @@ import { DatabaseSync } from 'node:sqlite';
4
4
  import { randomUUID } from 'node:crypto';
5
5
  import { chmodSync } from 'node:fs';
6
6
  import { join } from 'node:path';
7
+ import { platformModule, platformModules } from './platforms/registry.js';
7
8
  import { weiboAvatar, platformImages } from './text.js';
8
9
 
9
10
  export class Store {
@@ -27,6 +28,7 @@ export class Store {
27
28
  errors INTEGER NOT NULL DEFAULT 0
28
29
  );
29
30
  INSERT OR IGNORE INTO platform_accounts(platform,reason) VALUES('xueqiu','请先登录雪球');
31
+ INSERT OR IGNORE INTO platform_accounts(platform,reason) VALUES('douyin','请先登录抖音');
30
32
  CREATE TABLE IF NOT EXISTS targets (
31
33
  uid TEXT PRIMARY KEY, name TEXT NOT NULL, created_at INTEGER NOT NULL,
32
34
  interval_ms INTEGER NOT NULL, history_gap_ms INTEGER NOT NULL
@@ -53,6 +55,7 @@ export class Store {
53
55
  );
54
56
  CREATE TABLE IF NOT EXISTS settings (key TEXT PRIMARY KEY,value TEXT NOT NULL);
55
57
  `);
58
+ for (const module of Object.values(platformModules)) module.initializeStorage?.(this);
56
59
  this.db.exec(`CREATE TABLE IF NOT EXISTS social_deliveries (
57
60
  id TEXT PRIMARY KEY,post_id TEXT NOT NULL,uid TEXT NOT NULL,author TEXT NOT NULL,
58
61
  bot_id TEXT NOT NULL,target_id TEXT NOT NULL,status TEXT NOT NULL DEFAULT 'queued',
@@ -143,13 +146,13 @@ export class Store {
143
146
  this.db.prepare(`UPDATE ${table} SET ${entries.map(([key]) => `${key}=?`).join(',')} WHERE ${platform === 'weibo' ? 'id=1' : 'platform=?'}`)
144
147
  .run(...entries.map(([, value]) => value), ...(platform === 'weibo' ? [] : [platform]));
145
148
  }
146
- if (['weibo', 'xueqiu'].includes(platform) && patch.status === 'ready') {
149
+ if (['weibo', 'xueqiu', 'douyin'].includes(platform) && patch.status === 'ready') {
147
150
  if (this.getSetting('accountAlerted:' + platform, false)) this.setSetting('accountAlerted:' + platform, false);
148
151
  for (const row of this.db.prepare("SELECT key FROM settings WHERE key LIKE ?").all('accountAlerted:' + platform + ':%')) this.setSetting(row.key, false);
149
152
  this.db.prepare("UPDATE social_deliveries SET status='cancelled',error='平台连接已恢复' WHERE uid=? AND status='queued' AND message<>''").run('platform:' + platform);
150
153
  }
151
- if (notify && ['weibo', 'xueqiu'].includes(platform) && ['needs_login', 'needs_verification'].includes(patch.status)) {
152
- const platformName = platform === 'weibo' ? '微博' : '雪球';
154
+ if (notify && ['weibo', 'xueqiu', 'douyin'].includes(platform) && ['needs_login', 'needs_verification'].includes(patch.status)) {
155
+ const platformName = { weibo: '微博', xueqiu: '雪球', douyin: '抖音' }[platform];
153
156
  const delivery = this.getSetting('agentDelivery', null);
154
157
  for (const destination of activeDestinations(delivery)) {
155
158
  const alertKey = 'accountAlerted:' + platform + ':' + destination.botId + ':' + destination.targetId;
@@ -220,6 +223,7 @@ export class Store {
220
223
  const id = randomUUID();
221
224
  this.db.prepare('INSERT INTO notices(id,kind,message,created_at,read_at,post_id) VALUES(?,?,?,?,NULL,?)').run(id, kind, message, now, postId);
222
225
  if (uid) this.db.prepare('INSERT INTO notice_authors VALUES(?,?)').run(id, uid);
226
+ return id;
223
227
  }
224
228
  notices() { return this.db.prepare('SELECT notices.*,notice_authors.uid,posts.published_at,posts.detected_at,posts.first_seen_at AS saved_at FROM notices LEFT JOIN notice_authors USING(id) LEFT JOIN posts ON posts.id=notices.post_id ORDER BY notices.created_at DESC LIMIT 100').all(); }
225
229
  unreadNotices() {
@@ -243,7 +247,11 @@ export class Store {
243
247
  hasPost(id) { return !!this.db.prepare('SELECT id FROM posts WHERE id=?').get(id); }
244
248
  hasCompletePost(id) { return this.db.prepare('SELECT complete FROM posts WHERE id=?').get(id)?.complete === 1; }
245
249
 
250
+ saveVideo(post, now = Date.now(), captureKind = 'updates') {
251
+ platformModule(this.target(post.uid)?.platform).saveMedia?.(this, post, now, captureKind);
252
+ }
246
253
  savePost(post, now = Date.now(), captureKind = null, detectedAt = null) {
254
+ post = platformModule(this.target(post.uid)?.platform).preparePost?.(this, post) || post;
247
255
  this.summaryCache = null;
248
256
  this.todayCountCache = null;
249
257
  const previous = this.db.prepare('SELECT complete FROM posts WHERE id=?').get(post.id);
@@ -254,6 +262,7 @@ export class Store {
254
262
  raw=excluded.raw,last_seen_at=excluded.last_seen_at`)
255
263
  .run(post.id, post.uid, post.author, post.publishedAt, post.text, post.url, quote,
256
264
  Number(post.complete), JSON.stringify(post.raw), now, now, captureKind, detectedAt);
265
+ this.saveVideo(post, now, captureKind);
257
266
  if (['updates', 'catchup'].includes(captureKind)) this.observeUpdate(post.id, now);
258
267
  return !previous;
259
268
  }
@@ -279,6 +288,28 @@ export class Store {
279
288
  }
280
289
  }
281
290
 
291
+ postSearch(query, literal = false) {
292
+ const value = literal ? query : '%' + query.replace(/[\\%_]/g, '\\$&') + '%';
293
+ const fields = literal ? ['instr(text,?)>0', 'instr(quote,?)>0'] : ["text LIKE ? ESCAPE '\\'", "quote LIKE ? ESCAPE '\\'"];
294
+ const values = [value, value];
295
+ for (const module of Object.values(platformModules)) {
296
+ const search = module.searchPosts?.(value, literal);
297
+ if (search) { fields.push(search.sql); values.push(...search.values); }
298
+ }
299
+ return { sql: '(' + fields.join(' OR ') + ')', values };
300
+ }
301
+
302
+ searchPosts({ uids, from = 0, to = Number.MAX_SAFE_INTEGER, query = '', kind = 'all', offset = 0 }) {
303
+ const where = ['complete=1', 'uid IN (' + uids.map(() => '?').join(',') + ')', 'published_at>=?', 'published_at<?'];
304
+ const values = [...uids, from, to];
305
+ if (query) { const search = this.postSearch(query, true); where.push(search.sql); values.push(...search.values); }
306
+ if (kind === 'original') where.push('quote IS NULL');
307
+ if (kind === 'repost') where.push('quote IS NOT NULL');
308
+ const sql = where.join(' AND '), total = this.db.prepare('SELECT COUNT(*) AS n FROM posts WHERE ' + sql).get(...values).n;
309
+ const ids = this.db.prepare('SELECT id FROM posts WHERE ' + sql + ' ORDER BY published_at DESC,id DESC LIMIT 20 OFFSET ?').all(...values, offset).map(row => row.id);
310
+ return { total, offset, items: this.posts({ ids, limit: 20 }) };
311
+ }
312
+
282
313
  posts({ uid = '', query = '', kind = 'all', offset = 0, limit = 50, modifiedAfter = null, ids = null } = {}) {
283
314
  const where = [], args = [];
284
315
  if (modifiedAfter) { where.push('(last_seen_at,id)>(?,?)'); args.push(modifiedAfter.at, modifiedAfter.id); }
@@ -286,17 +317,17 @@ export class Store {
286
317
  if (ids) { if (!ids.length) return []; where.push('id IN (' + ids.map(() => '?').join(',') + ')'); args.push(...ids); }
287
318
  if (kind === 'original') where.push('quote IS NULL');
288
319
  if (kind === 'repost') where.push('quote IS NOT NULL');
289
- if (query) {
290
- where.push("(text LIKE ? ESCAPE '\\' OR quote LIKE ? ESCAPE '\\')");
291
- const escaped = '%' + query.replace(/[\\%_]/g, '\\$&') + '%';
292
- args.push(escaped, escaped);
320
+ for (const module of Object.values(platformModules)) {
321
+ const filter = module.filterPosts?.(kind);
322
+ if (filter) { where.push(filter.sql); args.push(...filter.values); }
293
323
  }
324
+ if (query) { const search = this.postSearch(query); where.push(search.sql); args.push(...search.values); }
294
325
  return this.db.prepare(`SELECT id,uid,author,published_at,text,url,quote,complete,first_seen_at,last_seen_at,capture_kind,detected_at,raw FROM posts
295
326
  ${where.length ? 'WHERE ' + where.join(' AND ') : ''} ORDER BY ${modifiedAfter ? 'last_seen_at,id' : 'published_at DESC,id DESC'} LIMIT ? OFFSET ?`)
296
327
  .all(...args, limit, offset).map(({ raw, ...row }) => {
297
328
  const source = JSON.parse(raw), quote = row.quote ? JSON.parse(row.quote) : null;
298
- const platform = row.uid.startsWith('xueqiu:') ? 'xueqiu' : 'weibo';
299
- return { ...row, article: platform === 'xueqiu' && String(source.type) === '3' && !!source.title, images: platformImages(source, platform), quote: quote ? { ...quote, images: platformImages(source.retweeted_status, platform) } : null };
329
+ const platform = this.target(row.uid)?.platform || 'weibo';
330
+ return { ...row, ...platformModule(platform).decoratePost?.(this, row, source), article: platformModule(platform).isArticle?.(source) === true, images: platformImages(source, platform), quote: quote ? { ...quote, images: platformImages(source.retweeted_status, platform) } : null };
300
331
  });
301
332
  }
302
333
  postImages(id, quoted = false) {
@@ -329,7 +360,7 @@ export class Store {
329
360
  const policy = this.monitoringPolicy(targets.get(job.uid));
330
361
  const intervalMs = intervalAt(policy.schedule, policy.intervalMs, now);
331
362
  const lastCheckedAt = job.checkpoint.lastScanAt || job.last_success_at || null;
332
- const dueAt = lastCheckedAt ? lastCheckedAt + intervalMs : job.next_run_at;
363
+ const dueAt = lastCheckedAt && platformModule(platform).nextCheckAt ? platformModule(platform).nextCheckAt(this, targets.get(job.uid), lastCheckedAt) : lastCheckedAt ? lastCheckedAt + intervalMs : job.next_run_at;
333
364
  return { uid: job.uid, intervalMs, lastCheckedAt, lateMs: Math.max(0, now - dueAt) };
334
365
  });
335
366
  return { activeAuthors: checks.length, minimumCycleMs: checks.length * requestGapMs,
@@ -353,7 +384,7 @@ export class Store {
353
384
  lastCheckedAt: updates?.last_success_at || null,
354
385
  recentChecks: this.getSetting('checkHistory:' + target.uid, []),
355
386
  nextCheckAt: canCheck ? Math.max(now, updates.next_run_at, account.next_request_at, account.cooldown_until) : null,
356
- history: { from: target.history_from, to: target.history_to, originalOnly, rangeBounded: target.history_from !== null || target.history_to !== null, rangeComplete: !!history?.checkpoint.rangeComplete,
387
+ history: { incompleteCount: this.db.prepare('SELECT COUNT(*) AS n FROM posts WHERE uid=? AND complete=0').get(target.uid).n, from: target.history_from, to: target.history_to, originalOnly, rangeBounded: target.history_from !== null || target.history_to !== null, rangeComplete: !!history?.checkpoint.rangeComplete,
357
388
  rangeSaved: target.history_from === null && target.history_to === null && !originalOnly ? saved : this.db.prepare('SELECT COUNT(*) AS n FROM posts WHERE uid=? AND published_at>=? AND published_at<=? AND (?=0 OR quote IS NULL)').get(target.uid, target.history_from ?? 0, target.history_to ?? Number.MAX_SAFE_INTEGER, Number(originalOnly)).n, id: history?.id, status: history?.status || 'not_started',
358
389
  started: this.getSetting('historyRequested:' + target.uid, false) || !!(history?.pages || history?.checkpoint.pageLoaded || history?.checkpoint.pending?.length),
359
390
  pages: history?.pages || 0, oldestAt: history?.checkpoint.oldestAt || null, reportedTotal: total,
package/lib/text.js CHANGED
@@ -1,35 +1,10 @@
1
- import he from 'he';
2
- import { xueqiuImage, xueqiuImages } from './platforms/xueqiu/media.js';
3
-
4
- export const platformImage = (value, platform) => platform === 'xueqiu' ? xueqiuImage(value) : weiboAvatar({ avatar_hd: value });
5
- export const platformImages = (raw, platform) => platform === 'xueqiu' ? xueqiuImages(raw) : weiboImages(raw);
6
- export const imageReferer = value => /(^|\.)imedao\.com$/.test(new URL(value).hostname) ? 'https://xueqiu.com/' : 'https://weibo.com/';
7
-
8
- export function plainText(html = '') {
9
- return he.decode(String(html)
10
- .replace(/<script\b[^>]*>[\s\S]*?<\/script>/gi, '')
11
- .replace(/<style\b[^>]*>[\s\S]*?<\/style>/gi, '')
12
- .replace(/<img\b[^>]*alt=["']([^"']*)["'][^>]*>/gi, '$1')
13
- .replace(/<br\s*\/?>|<\/p>|<\/div>/gi, '\n')
14
- .replace(/<[^>]+>/g, '')).trim();
15
- }
16
-
17
- export function weiboAvatar(user) {
18
- for (const value of [user?.avatar_hd, user?.avatar_large, user?.profile_image_url]) {
19
- try {
20
- const url = new URL(value);
21
- if (['http:', 'https:'].includes(url.protocol) && /(^|\.)sinaimg\.(cn|com)$/.test(url.hostname) && !url.username && !url.password && !url.port) {
22
- url.protocol = 'https:';
23
- return url.href;
24
- }
25
- } catch { /* Missing or invalid image fields fall back to the author's initial. */ }
26
- }
27
- return null;
28
- }
29
-
30
- export function weiboImages(raw) {
31
- const pictures = raw?.pic_infos
32
- ? (raw.pic_ids?.length ? raw.pic_ids.map(id => raw.pic_infos[id]) : Object.values(raw.pic_infos))
33
- : Array.isArray(raw?.pics) ? raw.pics : [];
34
- return [...new Set(pictures.slice(0, 18).map(pic => weiboAvatar({ avatar_hd: pic?.large?.url || pic?.largest?.url || pic?.original?.url || pic?.bmiddle?.url || pic?.url })).filter(Boolean))];
35
- }
1
+ import { platformModule, platformModules } from './platforms/registry.js';
2
+ export { plainText } from './plain-text.js';
3
+ export { weiboAvatar, weiboImages } from './platforms/weibo/media.js';
4
+
5
+ export const platformImage = (value, platform = 'weibo') => platformModule(platform).image?.(value) || null;
6
+ export const platformImages = (raw, platform = 'weibo') => platformModule(platform).images?.(raw) || [];
7
+ export const imageReferer = value => {
8
+ const url = new URL(value).href;
9
+ return Object.values(platformModules).find(module => module.image?.(url))?.referer || 'https://weibo.com/';
10
+ };