dsh-social-trace 0.1.1 → 0.1.2

package/README.md

@@ -100,7 +100,7 @@ dsh plugin --profile desktop add .
 
 ```sh
 npm ci
-npm test        # 244 项离线测试
+npm test        # 254 项离线测试
 npm run test:ui # 5 项界面测试（需要 Chrome）
 ```
 

package/lib/client.js

@@ -101,6 +101,15 @@
   const taskProblem = job => job.kind === 'history' && /长文接口未返回全文/.test(job.message || '')
     ? '有一条长内容暂时无法完整读取，历史缓存已暂停；已保存内容保留，新动态监控不受此问题影响。'
     : (job.kind === 'history' ? '历史缓存已暂停：' : '新动态检查已暂停：') + (job.message || '读取未完成，请重试');
+  function checkHealthReason(source, uid = '') {
+    const health = source?.health;
+    if (!health) return '';
+    const delayed = health.delayed.filter(check => !uid || check.uid === uid);
+    const reasons = [];
+    if (delayed.length) reasons.push((uid ? '这位博主' : delayed.length + ' 位博主') + '未按设定频率及时检查');
+    if (health.capacityLimited) reasons.push(health.activeAuthors + ' 位博主共用账号，仅检查一轮最新页就至少需要 ' + Math.ceil(health.minimumCycleMs / 1000) + ' 秒，超出了当前频率的处理能力');
+    return reasons.join('；');
+  }
   function monitorStatus(state, platform = 'weibo', uid = '') {
     const target = state?.targets.find(item => item.uid === uid);
     const source = state?.platforms?.find(item => item.id === platform);
@@ -126,6 +135,8 @@
       return result(historyOnly ? names.length + ' 位博主的历史缓存已暂停' : names.length + ' 位博主的新动态检查需处理', jobs.some(job => job.status === 'queued') ? 'partial' : 'blocked', names.join('、') + '：' + taskProblem(problems[0]), 'tasks');
     }
     if (jobs.length && jobs.every(job => ['paused', 'complete'].includes(job.status))) return result('任务已暂停', 'paused', '在大屏开启需要执行的任务', 'tasks');
+    const healthReason = jobs.some(job => job.kind === 'updates' && job.status === 'queued') && checkHealthReason(source, uid);
+    if (healthReason) return result('检查延迟', 'partial', healthReason, 'frequency');
     const active = state.activeJobIds || [state.activeJobId];
     if (jobs.some(job => active.includes(job.id))) return result('正在读取', 'active', '新内容保存后会显示在本地档案中');
     return result(jobs.length ? '监控中' + (uid ? ' · ' + intervalText(checkInterval(state, platform, uid)) : '') : '监控就绪', 'ready', '更新按设置的检查间隔执行，历史内容分批缓存');
@@ -157,6 +168,7 @@
     if (state.enabled === false || state.platforms?.find(item => item.id === target.platform)?.enabled === false || target.enabled === false || job?.status === 'paused') return '已暂停';
     if (job.status === 'needs_review') return '需处理';
     if (!platformReady(state, target.platform || 'weibo')) return '等待连接';
+    if (kind === 'updates' && state.platforms?.find(item => item.id === (target.platform || 'weibo'))?.health?.delayed.some(check => check.uid === target.uid)) return '检查延迟';
     if (kind === 'updates' && progressFor(state, target.uid).catchup) return '补查中';
     return kind === 'history' ? '缓存中' : '监控中';
   }
@@ -494,6 +506,12 @@
           tab === 'frequency' ? h('form', { id: 'st-node-panel-frequency', role: 'tabpanel', 'aria-labelledby': embedded ? 'st-reader-tab-frequency' : 'st-node-tab-frequency', onSubmit: event => { event.preventDefault(); void run(target ? '/targets/policy' : '/settings', target ? { uid: target.uid, inherit: false, intervalMinutes: value.intervalMinutes, schedule: value.schedule } : { intervalMinutes: value.intervalMinutes, schedule: value.schedule }, '监控时间与频率已保存'); } },
             target ? h('p', { className: 'st-caption' }, target.policy ? '当前为这位博主单独设置。' : '已填入' + platformNames[context.platform] + '平台的时间；保存后只对这位博主生效。') : null,
             target ? h(TaskControl, { state, target, kind: 'updates', act, busy, openSettings: context => { setTab(context.tab || 'frequency'); } }) : null,
+            checkHealthReason(source, target?.uid) ? h('p', { className: 'st-inline-error', role: 'status' }, checkHealthReason(source, target?.uid), '。登录正常不代表每次检查都及时完成。') : null,
+            progress?.recentChecks?.length ? h('details', null, h('summary', null, '最近检查记录'), h('table', null,
+              h('thead', null, h('tr', null, ['检查时间', '接口结果', '返回的最新发布时间'].map(label => h('th', { key: label }, label)))),
+              h('tbody', null, progress.recentChecks.slice(-6).reverse().map((check, index) => h('tr', { key: index },
+                h('td', null, date(check.at)), h('td', null, check.status === 'success' ? '第 ' + check.page + ' 页 · 返回 ' + check.count + ' 条' : (check.stage === 'complete' ? '正文补全失败' : '动态列表读取失败')),
+                h('td', null, check.newestPublishedAt ? date(check.newestPublishedAt) : '—')))))) : null,
             h(FrequencyFields, { value, change: setValue, disabled: busy }),
 
             !target ? h('section', { className: 'st-independent', 'aria-label': '独立时间配置的博主' }, h('h3', null, '独立时间配置', independent.length ? ' · ' + independent.length + ' 位' : ''),

package/lib/index.js

@@ -59,7 +59,8 @@ export function createRuntime(directory = dataDirectory(), { adapterFactories =
     let worker;
     const browser = new Adapter(directory, options => worker.beforeRequest(options), () => store.account(id).uid);
     if (browser.defaults && !store.getSetting('targetDefaults:' + id, null)) store.setSetting('targetDefaults:' + id, { intervalMs: browser.defaults.intervalMs, historyGapMs: browser.defaults.historyGapMs });
-    worker = new Worker(store, browser, { documents, requestGapMs: Math.max(browser.defaults?.requestGapMs || 15000, store.getSetting(id === 'weibo' ? 'requestGapMs' : `requestGapMs:${id}`, browser.defaults?.requestGapMs || 15000)) });
+    worker = new Worker(store, browser, { documents, monitoringGapMs: id === 'weibo' ? 3000 : browser.defaults?.requestGapMs || 15000,
+      requestGapMs: Math.max(browser.defaults?.requestGapMs || 15000, store.getSetting(id === 'weibo' ? 'requestGapMs' : `requestGapMs:${id}`, browser.defaults?.requestGapMs || 15000)) });
     return [id, { browser, worker, loginBusy: false, historyBusy: false }];
   }));
   const { browser, worker } = sources.weibo;
@@ -174,6 +175,7 @@ export function createRuntime(directory = dataDirectory(), { adapterFactories =
             return { id, name: source.browser.name, enabled: store.getSetting('platformEnabled:' + id, true), schedule, targetDefaults,
               frequency: { intervalMs: intervalAt(schedule, targetDefaults.intervalMs, now), nextChangeAt: Number.isFinite(change) ? change : null,
                 nextIntervalMs: Number.isFinite(change) ? intervalAt(schedule, targetDefaults.intervalMs, change) : null },
+              health: store.monitoringHealth(id, Math.min(source.worker.requestGapMs, source.worker.monitoringGapMs)),
               requestGapMs: source.worker.requestGapMs, profileRequestAt: store.getSetting('profileRequestAt:' + id, 0), loginBusy: source.loginBusy, stopped: source.worker.stopping, lastFailure: store.getSetting('lastFailure:' + id, null) };
           }),
           targetDefaults: store.getSetting('targetDefaults', { intervalMs: 900000, historyGapMs: 60000 }),

package/lib/platforms/weibo/index.js

@@ -61,7 +61,7 @@ function browserError(error) {
   return new CaptureError('network', /closed|destroyed|disconnected/i.test(message) ? '微博采集页面已关闭或正在跳转，请重新打开登录窗口并确认连接'
     : /timeout|timed out|net::/i.test(message) ? '微博网页或接口连接超时，稍后自动重试，已保存内容不受影响'
     : /executable|browser.*not found/i.test(message) ? '找不到采集所需的 Chrome 浏览器，请检查 Chrome 是否已安装'
-    : '微博采集浏览器暂不可用，请重新打开登录窗口并确认连接');
+    : '微博接口连接失败，稍后自动重试；若持续失败，请检查网络或微博连接');
 }
 
 export class WeiboBrowser {
@@ -158,7 +158,8 @@ export class WeiboBrowser {
     let response;
     try {
       const context = await this.httpSession();
-      response = await context.get('https://weibo.com' + path, { maxRedirects: 0 });
+      response = await context.get('https://weibo.com' + path, { maxRedirects: 0,
+        ...(options?.monitoringHead ? { timeout: 10000, headers: { 'Cache-Control': 'no-cache' } } : {}) });
       let body;
       try { body = await response.json(); } catch { body = null; }
       const retry = response.headers()['retry-after'];
@@ -197,9 +198,9 @@ export class WeiboBrowser {
     return { name: user.screen_name.trim(), avatar: weiboAvatar(user), total: user.statuses_count != null && user.statuses_count !== '' && Number.isSafeInteger(Number(user.statuses_count)) && Number(user.statuses_count) >= 0 ? Number(user.statuses_count) : null };
   }
 
-  async timeline(uid, page, cursor) {
+  async timeline(uid, page, cursor, options) {
     const data = await this.request(`/ajax/statuses/mymblog?uid=${uid}&page=${page}&feature=0`
-      + (cursor ? '&since_id=' + encodeURIComponent(cursor) : ''));
+      + (cursor ? '&since_id=' + encodeURIComponent(cursor) : ''), options);
     if (!Array.isArray(data.list)) throw new CaptureError('schema', '时间线字段变化，已暂停');
     // An explicit end marker is needed: an empty page alone cannot prove completion.
     return { items: data.list, end: data.bottom_tips_visible === true || data.bottom_tips_visible === 1,

package/lib/store.js

@@ -195,13 +195,15 @@ export class Store {
     return this.decodeJob(this.db.prepare(`SELECT jobs.* FROM jobs JOIN targets ON targets.uid=jobs.uid
       WHERE jobs.status='queued' AND next_run_at<=? AND targets.platform=? AND targets.capture_mode<>'pending'
       AND NOT EXISTS (SELECT 1 FROM settings WHERE key='targetEnabled:' || targets.uid AND value='false')
-      ORDER BY CASE kind WHEN 'updates' THEN 0 ELSE 1 END,next_run_at LIMIT 1`).get(now, platform));
+      ORDER BY CASE WHEN kind='updates' AND COALESCE(json_extract(checkpoint,'$.page'),1)=1
+        AND NOT COALESCE(json_extract(checkpoint,'$.pageLoaded'),0) THEN 0 WHEN kind='updates' THEN 1 ELSE 2 END,next_run_at LIMIT 1`).get(now, platform));
   }
 
-  nextJobAt(platform, kind = '') {
+  nextJobAt(platform, kind = '', headOnly = false) {
     return this.db.prepare(`SELECT MIN(next_run_at) AS due FROM jobs JOIN targets ON targets.uid=jobs.uid
       WHERE jobs.status='queued' AND targets.platform=? AND targets.capture_mode<>'pending' AND (?='' OR jobs.kind=?)
-      AND NOT EXISTS (SELECT 1 FROM settings WHERE key='targetEnabled:' || targets.uid AND value='false')`).get(platform, kind, kind).due;
+      AND (?=0 OR COALESCE(json_extract(checkpoint,'$.page'),1)=1 AND NOT COALESCE(json_extract(checkpoint,'$.pageLoaded'),0))
+      AND NOT EXISTS (SELECT 1 FROM settings WHERE key='targetEnabled:' || targets.uid AND value='false')`).get(platform, kind, kind, Number(headOnly)).due;
   }
 
   updateJob(id, patch, now = Date.now()) {
@@ -320,6 +322,21 @@ export class Store {
     return this.getSetting('targetPolicy:' + target.uid, null) || { intervalMs: target.interval_ms, schedule: this.getSetting('schedule:' + target.platform, null) };
   }
 
+  monitoringHealth(platform, requestGapMs, now = Date.now()) {
+    const targets = new Map(this.targets().filter(target => target.platform === platform && target.capture_mode !== 'pending'
+      && this.monitoringEnabled(platform, target.uid)).map(target => [target.uid, target]));
+    const checks = this.jobs().filter(job => job.kind === 'updates' && job.status === 'queued' && targets.has(job.uid)).map(job => {
+      const policy = this.monitoringPolicy(targets.get(job.uid));
+      const intervalMs = intervalAt(policy.schedule, policy.intervalMs, now);
+      const lastCheckedAt = job.checkpoint.lastScanAt || job.last_success_at || null;
+      const dueAt = lastCheckedAt ? lastCheckedAt + intervalMs : job.next_run_at;
+      return { uid: job.uid, intervalMs, lastCheckedAt, lateMs: Math.max(0, now - dueAt) };
+    });
+    return { activeAuthors: checks.length, minimumCycleMs: checks.length * requestGapMs,
+      capacityLimited: checks.reduce((load, check) => load + requestGapMs / check.intervalMs, 0) > 1.001,
+      delayed: checks.filter(check => check.lateMs > Math.max(2 * requestGapMs, 5000)) };
+  }
+
   targetProgress(now = Date.now()) {
     const jobs = this.jobs(), summaries = this.archiveSummary();
     return this.targets().map(target => {
@@ -334,6 +351,7 @@ export class Store {
       const canCheck = updates?.status === 'queued' && this.monitoringEnabled(target.platform, target.uid) && account?.status === 'ready';
       return { uid: target.uid, catchup: updates?.checkpoint.catchupFrom != null ? { from: updates.checkpoint.catchupFrom, page: updates.checkpoint.page || 1, saved: Math.max(0, updates.saved - (updates.checkpoint.catchupSaved || 0)) } : null, intervalMs: intervalAt(policy.schedule, policy.intervalMs, now), nextChangeAt: Number.isFinite(policyChange) ? policyChange : null, nextIntervalMs: Number.isFinite(policyChange) ? intervalAt(policy.schedule, policy.intervalMs, policyChange) : null, saved, originals: summary?.original_count || 0, reposts: summary?.repost_count || 0, lastSavedAt: summary?.last_saved_at || null,
         lastCheckedAt: updates?.last_success_at || null,
+        recentChecks: this.getSetting('checkHistory:' + target.uid, []),
         nextCheckAt: canCheck ? Math.max(now, updates.next_run_at, account.next_request_at, account.cooldown_until) : null,
         history: { from: target.history_from, to: target.history_to, originalOnly, rangeBounded: target.history_from !== null || target.history_to !== null, rangeComplete: !!history?.checkpoint.rangeComplete,
           rangeSaved: target.history_from === null && target.history_to === null && !originalOnly ? saved : this.db.prepare('SELECT COUNT(*) AS n FROM posts WHERE uid=? AND published_at>=? AND published_at<=? AND (?=0 OR quote IS NULL)').get(target.uid, target.history_from ?? 0, target.history_to ?? Number.MAX_SAFE_INTEGER, Number(originalOnly)).n, id: history?.id, status: history?.status || 'not_started',

package/lib/worker.js

@@ -3,7 +3,7 @@ import { alignedCheckAt, intervalAt, nextBoundary } from './schedule.js';
 import { CaptureError } from './capture-error.js';
 
 export class Worker {
-  constructor(store, adapter, { now = Date.now, requestGapMs = 15000, documents = null } = {}) {
+  constructor(store, adapter, { now = Date.now, requestGapMs = 15000, monitoringGapMs = requestGapMs, documents = null } = {}) {
     this.store = store;
     this.adapter = adapter;
     this.documents = documents;
@@ -12,6 +12,7 @@ export class Worker {
     this.leaseKey = this.platform === 'weibo' ? 'worker' : 'worker:' + this.platform;
     this.now = now;
     this.requestGapMs = requestGapMs;
+    this.monitoringGapMs = monitoringGapMs;
     this.busy = null;
     this.currentJobId = null;
     this.stopping = false;
@@ -20,6 +21,8 @@ export class Worker {
     this.running = false;
     this.claimAfter = 0;
     this.boundaryAt = 0;
+    this.lastTickAt = null;
+    this.resumeCheckPending = false;
   }
 
   claim() {
@@ -62,9 +65,9 @@ export class Worker {
 
   pickJob() {
     const job = this.store.dueJob(this.now(), this.platform);
-    if (job?.kind === 'history' && !this.localStep(job)) {
-      const updateAt = this.store.nextJobAt(this.platform, 'updates');
-      if (updateAt > this.now() && updateAt <= this.now() + this.requestGapMs) return { wait: updateAt };
+    if (job && (job.kind === 'history' || job.checkpoint.pageLoaded || job.checkpoint.page > 1) && !this.localStep(job)) {
+      const updateAt = this.store.nextJobAt(this.platform, 'updates', true);
+      if (updateAt > this.now() && updateAt <= this.now() + Math.min(this.requestGapMs, this.monitoringGapMs)) return { wait: updateAt };
     }
     return { job };
   }
@@ -110,25 +113,36 @@ export class Worker {
     if (this.running && !this.stopping) return;
     this.claim();
     this.prepareRestartChecks();
+    this.lastTickAt = this.now();
+    this.resumeCheckPending = false;
     this.running = true;
     this.rescheduleUpdates();
     this.wake();
   }
 
-  async beforeRequest({ profileOnly = false, interactiveProfile = false, verifying = false } = {}) {
+  async beforeRequest({ profileOnly = false, interactiveProfile = false, verifying = false, monitoringHead = false } = {}) {
     const account = this.store.account(this.platform);
     const job = this.currentJobId ? this.store.job(this.currentJobId) : null;
     if (this.stopping || (verifying ? false : profileOnly ? !['ready', 'paused'].includes(account.status) : !this.store.monitoringEnabled(this.platform, job?.uid) || (job && job.status !== 'queued') || account.status !== 'ready')) throw new CaptureError('stopped', '账号已暂停');
     const interactive = profileOnly && interactiveProfile;
     const profileKey = 'profileRequestAt:' + this.platform;
-    const remaining = Math.max(interactive ? this.store.getSetting(profileKey, 0) : account.next_request_at, account.cooldown_until) - this.now();
+    const auxiliary = !monitoringHead && !interactive && !verifying;
+    const auxiliaryKey = 'auxiliaryRequestAt:' + this.platform;
+    const remaining = Math.max(interactive ? this.store.getSetting(profileKey, 0) : account.next_request_at,
+      auxiliary ? this.store.getSetting(auxiliaryKey, 0) : 0, account.cooldown_until) - this.now();
     if (remaining > 0) throw new CaptureError('wait', '等待下一次请求时间', remaining);
     if (interactive) this.store.setSetting(profileKey, this.now() + this.requestGapMs);
-    this.store.setAccount({ next_request_at: Math.max(account.next_request_at, this.now() + this.requestGapMs) }, this.platform);
+    if (auxiliary) this.store.setSetting(auxiliaryKey, this.now() + this.requestGapMs);
+    const gap = Math.min(this.requestGapMs, this.monitoringGapMs);
+    this.store.setAccount({ next_request_at: Math.max(account.next_request_at, this.now() + gap) }, this.platform);
   }
 
   async tick() {
     if (this.stopping || this.suspended || this.busy) return;
+    if (this.running) {
+      if (this.lastTickAt !== null && this.now() - this.lastTickAt > 120000) this.resumeCheckPending = true;
+      this.lastTickAt = this.now();
+    }
     const account = this.store.account(this.platform);
     if (!this.store.monitoringEnabled(this.platform) || account.status !== 'ready' || account.cooldown_until > this.now()) return;
     try { if (this.now() >= this.claimAfter) this.claim(); } catch (error) {
@@ -137,6 +151,10 @@ export class Worker {
       clearTimeout(this.timer);
       throw error;
     }
+    if (this.resumeCheckPending) {
+      this.prepareRestartChecks();
+      this.resumeCheckPending = false;
+    }
     if (this.now() >= this.boundaryAt) this.rescheduleUpdates();
     const { job } = this.pickJob();
     if (!job) return;
@@ -210,7 +228,7 @@ export class Worker {
       return;
     }
     const page = checkpoint.page || 1;
-    const result = await this.adapter.timeline(target.external_uid || job.uid, page, checkpoint.cursor);
+    const result = await this.adapter.timeline(target.external_uid || job.uid, page, checkpoint.cursor, { monitoringHead: job.kind === 'updates' && page === 1 });
     this.requestSucceeded();
     if (!this.canRun(job)) return;
     const detectedAt = this.now();
@@ -218,6 +236,7 @@ export class Worker {
     const pending = [];
     const unpinnedDates = [];
     let foundKnown = false;
+    let reachedBoundary = false;
     const firstPageIds = page === 1 ? [] : checkpoint.firstPageIds || [];
     for (const raw of result.items) {
       const post = this.normalizePost(raw, target);
@@ -229,20 +248,24 @@ export class Worker {
       if (!this.adapter.isPinned(raw)) {
         if (page === 1) firstPageIds.push(post.id);
         if (job.kind === 'history' && known) foundKnown = true;
+        if (checkpoint.boundaryIds?.includes(post.id) && post.publishedAt < scanFrom) reachedBoundary = true;
       }
       if (inRange(post)) {
         if (!this.store.hasCompletePost(post.id)) pending.push(raw);
         else if (job.kind === 'updates') this.store.observeUpdate(post.id, this.now());
       }
     }
-    // A known ID may come from history; only an older non-pinned page closes the time gap.
-    if (job.kind === 'updates') foundKnown = unpinnedDates.length > 0 && unpinnedDates.every(at => at < scanFrom);
+    // Only the previous update head is a baseline; arbitrary history IDs and pinned posts are not.
+    if (job.kind === 'updates') foundKnown = unpinnedDates.length > 0 && (unpinnedDates.every(at => at < scanFrom)
+      || checkpoint.catchupFrom == null && reachedBoundary);
     if (ids.length && JSON.stringify(ids) === JSON.stringify(checkpoint.lastPageIds)) {
       throw new CaptureError('schema', '平台重复返回上一页，已暂停，避免循环请求');
     }
     if (!ids.length && !result.end) {
       throw new CaptureError('schema', '平台返回空页且没有明确末尾说明，已暂停待检查');
     }
+    if (job.kind === 'updates') this.recordCheck(job.uid, { at: detectedAt, page, stage: 'timeline', status: 'success', count: ids.length,
+      newestPublishedAt: unpinnedDates.length ? Math.max(...unpinnedDates) : null });
     this.store.updateJob(job.id, {
       checkpoint: { ...checkpoint, scanFrom, pending, page, pageLoaded: true, lastPageIds: ids, detectedAt,
         firstPageIds, lastScanAt: page === 1 ? this.now() : checkpoint.lastScanAt, retryAt: 0,
@@ -294,6 +317,11 @@ export class Worker {
     if (account.status === 'ready' && (account.errors || account.reason)) this.store.setAccount({ errors: 0, reason: '' }, this.platform);
   }
 
+  recordCheck(uid, check) {
+    const key = 'checkHistory:' + uid;
+    this.store.setSetting(key, [...this.store.getSetting(key, []), check].slice(-32));
+  }
+
   failAccount(error) {
     const account = this.store.account(this.platform);
     let message = error.message;
@@ -318,6 +346,8 @@ export class Worker {
     }
     let message = error.kind && typeof error.message === 'string' ? error.message : '处理采集数据时发生内部错误，已暂停此任务并保留断点';
     this.store.setSetting('lastFailure:' + this.platform, { uid: job.uid, kind, at: this.now(), message, code: error.kind ? null : String(error.name || 'Error') });
+    if (job.kind === 'updates') this.recordCheck(job.uid, { at: this.now(), page: job.checkpoint.page || 1,
+      stage: job.checkpoint.pending?.length ? 'complete' : 'timeline', status: kind });
     if (kind === 'limited' || kind === 'network') {
       message = this.failAccount(error);
       const retryAt = this.store.account(this.platform).cooldown_until;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "dsh-social-trace",
-  "version": "0.1.1",
+  "version": "0.1.2",
   "description": "Social Trace：微博与雪球增量监控、可视化拓扑、历史缓存与本地检索",
   "type": "module",
   "license": "MIT",