@sovovs/bycli 2.1.23 → 2.1.25

package/clis/weixin/download-publish-data.js

@@ -1,4 +1,7 @@
-import { ArgumentError } from '@sovovs/bycli/errors';
+import { constants } from 'node:fs';
+import { access, stat } from 'node:fs/promises';
+import { extname, resolve } from 'node:path';
+import { ArgumentError, CommandExecutionError } from '@sovovs/bycli/errors';
 import { cli, Strategy } from '@sovovs/bycli/registry';
 import { resolveBrowserCredentials } from './_wechat/auth-session.js';
 import { buildSecretSet, redactText } from './_wechat/redact.js';
@@ -10,9 +13,13 @@ import {
   matchPublishedRecord,
   positiveSafeInteger,
   validatePublishDate,
+  validatePublishedQuery,
 } from './_wechat/publish-records.js';
 
-const COLUMNS = ['title', 'publishedAt', 'url', 'status', 'markdownPath', 'dataPath', 'size', 'error'];
+const COLUMNS = [
+  'title', 'publishedAt', 'url', 'status',
+  'markdownPath', 'markdownSize', 'dataPath', 'dataSize', 'error',
+];
 
 function sanitizedError(error, secrets, fallback) {
   const message = error instanceof Error ? error.message : fallback;
@@ -20,19 +27,46 @@ function sanitizedError(error, secrets, fallback) {
     .replace(/https?:\/\/mp\.weixin\.qq\.com\/\S*/giu, '[REDACTED]');
 }
 
+async function validateArtifact(result, { label, expectedStatus, expectedExtension }) {
+  if (!result || result.status !== expectedStatus) {
+    throw new CommandExecutionError(`${label} returned an invalid status`);
+  }
+  if (typeof result.path !== 'string' || !result.path.trim()) {
+    throw new CommandExecutionError(`${label} returned no output path`);
+  }
+  if (!Number.isSafeInteger(result.size) || result.size <= 0) {
+    throw new CommandExecutionError(`${label} returned an invalid size`);
+  }
+  const path = resolve(result.path);
+  if (extname(path).toLowerCase() !== expectedExtension) {
+    throw new CommandExecutionError(`${label} returned an unexpected file type`);
+  }
+  let info;
+  try {
+    await access(path, constants.R_OK);
+    info = await stat(path);
+  } catch {
+    throw new CommandExecutionError(`${label} returned an unreadable file`);
+  }
+  if (!info.isFile() || info.size <= 0 || info.size !== result.size) {
+    throw new CommandExecutionError(`${label} returned an unreadable or mismatched file`);
+  }
+  return { ...result, path, size: info.size };
+}
+
 export const downloadPublishDataCommand = cli({
   site: 'weixin',
   name: 'download-publish-data',
   access: 'write',
   domain: 'mp.weixin.qq.com',
-  description: 'Match a Weixin published article and save its content analysis as Markdown',
+  description: 'Match a Weixin published article and save its Excel data and Markdown analysis',
   strategy: Strategy.INTERCEPT,
   browser: true,
   navigateBefore: false,
   args: [
     { name: 'query', positional: true, required: true, help: 'Exact article URL or title text' },
     { name: 'date', help: 'Optional publication date in YYYY-MM-DD' },
-    { name: 'output', default: './weixin-publish-data', help: 'Directory for generated Markdown reports' },
+    { name: 'output', default: './weixin-publish-data', help: 'Directory for generated Excel data and Markdown reports' },
     { name: 'max-pages', type: 'int', default: 5, help: 'Maximum published-record pages to scan' },
     { name: 'timeout', type: 'int', default: 60, help: 'Maximum seconds for page capture' },
   ],
@@ -40,6 +74,7 @@ export const downloadPublishDataCommand = cli({
   func: async (page, args) => {
     const query = String(args.query ?? '').trim();
     if (!query) throw new ArgumentError('query required');
+    const validatedQuery = validatePublishedQuery(query);
 
     const timeoutSeconds = positiveSafeInteger(args.timeout, 'timeout', 60);
     const maxPages = positiveSafeInteger(args['max-pages'], 'max-pages', 5);
@@ -53,7 +88,7 @@ export const downloadPublishDataCommand = cli({
       maxPages,
       timeout: timeoutSeconds,
     });
-    const record = matchPublishedRecord(rows, query, validatedDate);
+    const record = matchPublishedRecord(rows, validatedQuery, validatedDate);
     const detailUrl = buildDetailUrl(record, token);
     const outputDir = args.output ?? './weixin-publish-data';
     const commonOptions = {
@@ -64,31 +99,45 @@ export const downloadPublishDataCommand = cli({
     };
     const secrets = buildSecretSet({ token, cookie });
 
+    let dataResult = null;
+    let markdownResult = null;
+    const errors = [];
     try {
       const result = await downloadPublishData(page, commonOptions);
-      return [{ title: record.title, publishedAt: record.publishedAt, url: record.url,
-        status: 'saved', markdownPath: null, dataPath: result.path,
-        size: result.size, error: null }];
-    } catch (downloadError) {
-      const downloadMessage = sanitizedError(downloadError, secrets, 'Excel download failed');
-      try {
-        const result = await collectPublishAnalysis(page, {
-          ...commonOptions,
-          publishedAt: record.publishedAt,
-        });
-        return [{ title: record.title, publishedAt: record.publishedAt, url: record.url,
-          status: 'saved', markdownPath: result.path, dataPath: null,
-          size: result.size, error: downloadMessage }];
-      } catch (analysisError) {
-        const analysisMessage = sanitizedError(
-          analysisError,
-          secrets,
-          'Markdown fallback failed',
-        );
-        return [{ title: record.title, publishedAt: record.publishedAt, url: record.url,
-          status: 'failed', markdownPath: null, dataPath: null, size: null,
-          error: `Excel download failed: ${downloadMessage}; Markdown fallback failed: ${analysisMessage}` }];
-      }
+      dataResult = await validateArtifact(result, {
+        label: 'Excel artifact',
+        expectedStatus: 'downloaded',
+        expectedExtension: '.xls',
+      });
+    } catch (error) {
+      errors.push(`Excel download failed: ${sanitizedError(error, secrets, 'Excel download failed')}`);
+    }
+    try {
+      const result = await collectPublishAnalysis(page, {
+        ...commonOptions,
+        publishedAt: record.publishedAt,
+      });
+      markdownResult = await validateArtifact(result, {
+        label: 'Markdown artifact',
+        expectedStatus: 'saved',
+        expectedExtension: '.md',
+      });
+    } catch (error) {
+      errors.push(`Markdown analysis failed: ${sanitizedError(error, secrets, 'Markdown analysis failed')}`);
     }
+
+    const status = dataResult && markdownResult ? 'downloaded'
+      : dataResult || markdownResult ? 'partial' : 'failed';
+    return [{
+      title: record.title,
+      publishedAt: record.publishedAt,
+      url: record.url,
+      status,
+      markdownPath: markdownResult?.path ?? null,
+      markdownSize: markdownResult?.size ?? null,
+      dataPath: dataResult?.path ?? null,
+      dataSize: dataResult?.size ?? null,
+      error: errors.length > 0 ? errors.join('; ') : null,
+    }];
   },
 });

package/clis/weixin/download.js

@@ -10,45 +10,19 @@ import { cli, Strategy } from '@sovovs/bycli/registry';
 import { downloadArticle } from '@sovovs/bycli/download/article-download';
 import { AuthRequiredError } from '@sovovs/bycli/errors';
 import { buildExtractWechatArticleContentJs } from './_wechat/article-content.js';
+import {
+    isTrustedSogouRedirectUrl,
+    isTrustedWechatArticleUrl,
+    normalizeWechatUrl,
+    resolveWechatArticleUrl,
+} from './_wechat/article-link.js';
 export { extractWechatArticleContent } from './_wechat/article-content.js';
-// ============================================================
-// URL Normalization
-// ============================================================
-/**
- * Normalize a pasted WeChat article URL.
- */
-export function normalizeWechatUrl(raw) {
-    let s = (raw || '').trim();
-    if (!s)
-        return s;
-    // Strip wrapping quotes / angle brackets
-    if ((s.startsWith('"') && s.endsWith('"')) || (s.startsWith("'") && s.endsWith("'"))) {
-        s = s.slice(1, -1).trim();
-    }
-    if (s.startsWith('<') && s.endsWith('>')) {
-        s = s.slice(1, -1).trim();
-    }
-    // Remove backslash escapes before URL-significant characters
-    s = s.replace(/\\+([:/&?=#%])/g, '$1');
-    // Decode HTML entities
-    s = s.replace(/&amp;/g, '&').replace(/&lt;/g, '<').replace(/&gt;/g, '>').replace(/&quot;/g, '"');
-    // Allow bare hostnames
-    if (s.startsWith('mp.weixin.qq.com/') || s.startsWith('//mp.weixin.qq.com/')) {
-        s = 'https://' + s.replace(/^\/+/, '');
-    }
-    // Force https for mp.weixin.qq.com
-    try {
-        const parsed = new URL(s);
-        if (['http:', 'https:'].includes(parsed.protocol) && parsed.hostname.toLowerCase() === 'mp.weixin.qq.com') {
-            parsed.protocol = 'https:';
-            s = parsed.toString();
-        }
-    }
-    catch {
-        // Ignore parse errors
-    }
-    return s;
-}
+export {
+    isTrustedSogouRedirectUrl,
+    isTrustedWechatArticleUrl,
+    normalizeWechatUrl,
+    resolveWechatArticleUrl as resolveWechatDownloadUrl,
+};
 /**
  * Format a WeChat article timestamp as a UTC+8 datetime string.
  * Accepts either Unix seconds or milliseconds.
@@ -183,15 +157,12 @@ cli({
         { name: 'output', default: './weixin-articles', help: 'Output directory' },
         { name: 'download-images', type: 'boolean', default: true, help: 'Download images locally' },
     ],
-    columns: ['title', 'author', 'publish_time', 'status', 'size', 'saved'],
+    columns: ['title', 'author', 'publish_time', 'status', 'size', 'saved', 'source_url', 'resolved_url'],
     func: async (page, kwargs) => {
-        const rawUrl = kwargs.url;
-        const url = normalizeWechatUrl(rawUrl);
-        if (!url.startsWith('https://mp.weixin.qq.com/')) {
-            return [{ title: 'Error', author: '-', publish_time: '-', status: 'invalid URL', size: '-', saved: '-' }];
-        }
-        // Navigate and wait for content to load
-        await page.goto(url);
+        const { sourceUrl, resolvedUrl, alreadyNavigated } = await resolveWechatArticleUrl(page, kwargs.url);
+        // Navigate and wait for content to load. Sogou resolution already lands on the article.
+        if (!alreadyNavigated)
+            await page.goto(resolvedUrl);
         await page.wait(5);
         // Extract article data in browser context
         const data = await page.evaluate(`
@@ -255,11 +226,11 @@ cli({
                 'WeChat article page requires environment verification. Complete it in the open browser tab and run the command again.',
             );
         }
-        return downloadArticle({
+        const rows = await downloadArticle({
             title: data?.title || '',
             author: data?.author,
             publishTime: data?.publishTime,
-            sourceUrl: url,
+            sourceUrl: resolvedUrl,
             contentHtml: data?.contentHtml || '',
             codeBlocks: data?.codeBlocks,
             imageUrls: data?.imageUrls,
@@ -274,5 +245,6 @@ cli({
             },
             secureMarkdown: true,
         });
+        return rows.map(row => ({ ...row, source_url: sourceUrl, resolved_url: resolvedUrl }));
     },
 });

package/clis/weixin/save-articles.js

@@ -1,12 +1,21 @@
-import { ArgumentError, AuthRequiredError, CommandExecutionError } from '@sovovs/bycli/errors';
+import {
+  ArgumentError, AuthRequiredError, CommandExecutionError, EmptyResultError,
+} from '@sovovs/bycli/errors';
 import { MAX_WECHAT_HTML_BYTES } from '@sovovs/bycli/download/wechat-article';
 import { cli, Strategy } from '@sovovs/bycli/registry';
 import { readEnvironmentCredentials, resolveBrowserCredentials } from './_wechat/auth-session.js';
 import {
-  callCrawler, collectArticles, createWechatApi, isTrustedWechatArticleUrl, saveArticles,
+  combineArticleFallbackErrors,
+  isEligibleArticleFallbackError,
+  withMissingFallbackName,
+} from './_wechat/article-fallback-policy.js';
+import { createArticleIndexFetcher } from './_wechat/article-index.js';
+import {
+  callCrawler, collectArticles, isTrustedWechatArticleUrl, saveArticles,
 } from './_wechat/crawler-runtime.js';
 import { readAuthSource } from './_wechat/args.js';
 import { wechatArticleToMarkdown } from './_wechat/markdown.js';
+import { collectSogouAccountArticles } from './_wechat/sogou-fallback.js';
 
 const DOMAIN = 'mp.weixin.qq.com';
 const browserRequired = args => readAuthSource(args) === 'browser';
@@ -155,11 +164,11 @@ export const saveArticlesCommand = cli({
   description: 'Download WeChat official-account articles as Markdown files',
   strategy: Strategy.COOKIE, browser: browserRequired,
   args: [
-    { name: 'fakeid', positional: true, required: true, help: 'Official-account fakeid returned by weixin accounts' }, { name: 'name', help: 'Official-account name used in Markdown metadata' },
+    { name: 'fakeid', positional: true, required: true, help: 'Official-account fakeid returned by weixin accounts' }, { name: 'name', help: 'Official-account name; exact case-insensitive match required for browser Sogou fallback' },
     { name: 'output', default: './weixin-articles', help: 'Directory for saved Markdown files' }, { name: 'limit', type: 'int', help: 'Maximum number of articles to save' },
     { name: 'max-pages', type: 'int', help: 'Maximum number of history pages to scan' }, { name: 'auth-source', default: 'browser', choices: ['browser', 'env'], help: 'Credential source: browser session or environment variables' },
   ],
-  columns: ['title', 'status', 'stage', 'path', 'error', 'url'],
+  columns: ['title', 'status', 'stage', 'path', 'error', 'url', 'source', 'coverage'],
   func: async (page, args) => {
     const fakeid = String(args.fakeid ?? '').trim();
     if (!fakeid) throw new ArgumentError('fakeid is required');
@@ -167,21 +176,56 @@ export const saveArticlesCommand = cli({
     const credentials = authSource === 'env'
       ? readEnvironmentCredentials(false) : await resolveBrowserCredentials(page);
     const articleHtmlDownloader = createArticleHtmlDownloader({ authSource, page });
-    const rows = await callCrawler(async () => {
-      const { fetchPage } = createWechatApi(credentials);
-      const { articles } = await collectArticles({ fakeid, fetchPage, limit: args.limit, maxPages: args['max-pages'] });
-      return saveArticles({
+    const fetchPage = createArticleIndexFetcher({ page, source: authSource, credentials });
+    let articles;
+    let resolutionFailures = [];
+    let source = 'wechat';
+    let coverage = null;
+    try {
+      const result = await callCrawler(() => collectArticles({
+        fakeid, fetchPage, limit: args.limit, maxPages: args['max-pages'],
+      }));
+      articles = result.articles;
+      if (articles.length === 0) {
+        throw new EmptyResultError('weixin save-articles', `No published articles were found for ${fakeid}.`);
+      }
+    } catch (primaryError) {
+      if (authSource !== 'browser' || !isEligibleArticleFallbackError(primaryError)) throw primaryError;
+      const accountName = String(args.name ?? '').trim();
+      if (!accountName) throw withMissingFallbackName('weixin save-articles', primaryError);
+      try {
+        const fallback = await collectSogouAccountArticles({
+          page, accountName, limit: args.limit, maxPages: args['max-pages'],
+          resolutionPolicy: 'rows',
+        });
+        articles = fallback.articles;
+        resolutionFailures = fallback.resolutionFailures;
+        source = fallback.source;
+        coverage = fallback.coverage;
+      } catch (fallbackError) {
+        throw combineArticleFallbackErrors({
+          operation: 'weixin save-articles', primaryError, fallbackError, credentials,
+        });
+      }
+    }
+
+    const savedRows = articles.length === 0 ? [] : await callCrawler(() => saveArticles({
         articles, accountName: String(args.name ?? '').trim(),
         outputDir: args.output ?? './weixin-articles', fetchArticleHtml: articleHtmlDownloader,
         buildMarkdown: (article, html) => wechatArticleToMarkdown({
           html, title: article.title, accountName: String(args.name ?? '').trim(), author: article.author,
           publishedAt: article.publishedAt, digest: article.digest, url: article.url,
         }), existingFilePolicy: 'suffix',
-      });
-    });
-    return rows.map(row => ({
+      }));
+    const orderedRows = source === 'sogou'
+      ? [
+        ...savedRows.map((row, index) => ({ ...row, order: articles[index]?.order ?? index })),
+        ...resolutionFailures,
+      ].sort((left, right) => left.order - right.order)
+      : savedRows;
+    return orderedRows.map(row => ({
       title: row.title, status: row.status, stage: row.stage || null, path: row.saved || null,
-      error: row.error || null, url: row.url,
+      error: row.error || null, url: row.url, source, coverage,
     }));
   },
 });

package/clis/weixin/search.js

@@ -1,30 +1,15 @@
-import { ArgumentError, CommandExecutionError, EmptyResultError } from '@sovovs/bycli/errors';
+import { EmptyResultError } from '@sovovs/bycli/errors';
 import { cli, Strategy } from '@sovovs/bycli/registry';
+import {
+  normalizePositiveInteger,
+  searchSogouArticlePage,
+} from './_wechat/sogou-search.js';
 
 const SOGOU_WEIXIN_DOMAIN = 'weixin.sogou.com';
 const DEFAULT_PAGE = 1;
 const DEFAULT_LIMIT = 10;
 const MAX_LIMIT = 10;
 
-function normalizePositiveInteger(value, name, defaultValue, maxValue) {
-  if (value === undefined || value === null) return defaultValue;
-  const text = String(value).trim();
-  if (!/^\d+$/.test(text)) {
-    throw new ArgumentError(
-      `weixin sougousearch --${name} must be a positive integer`,
-      `Pass --${name} as a whole number${maxValue ? ` from 1 to ${maxValue}` : ' greater than 0'}.`,
-    );
-  }
-  const parsed = Number(text);
-  if (!Number.isSafeInteger(parsed) || parsed < 1 || (maxValue && parsed > maxValue)) {
-    throw new ArgumentError(
-      `weixin sougousearch --${name} is out of range`,
-      `Pass --${name} as a whole number${maxValue ? ` from 1 to ${maxValue}` : ' greater than 0'}.`,
-    );
-  }
-  return parsed;
-}
-
 function normalizePage(page) {
   return normalizePositiveInteger(page, 'page', DEFAULT_PAGE);
 }
@@ -33,63 +18,6 @@ function normalizeLimit(limit) {
   return normalizePositiveInteger(limit, 'limit', DEFAULT_LIMIT, MAX_LIMIT);
 }
 
-function buildSearchUrl(query, pageNo) {
-  const searchUrl = new URL('https://weixin.sogou.com/weixin');
-  searchUrl.searchParams.set('query', query);
-  searchUrl.searchParams.set('type', '2');
-  searchUrl.searchParams.set('page', String(pageNo));
-  searchUrl.searchParams.set('ie', 'utf8');
-  return searchUrl.toString();
-}
-
-function buildExtractSearchResultsEvaluate() {
-  return String.raw`(() => {
-    const clean = (value) => {
-      return (value || '')
-        .replace(/\s+/g, ' ')
-        .replace(/<!--red_beg-->|<!--red_end-->/g, '')
-        .replace(/document\.write\(timeConvert\('\d+'\)\)/g, '')
-        .trim();
-    };
-
-    const absolutize = (href) => {
-      if (!href) return '';
-      try {
-        return new URL(href, window.location.origin).toString();
-      } catch {
-        return href;
-      }
-    };
-
-    const bodyText = clean(document.body && document.body.innerText);
-    const blocked = /验证码|安全验证|异常访问|访问过于频繁|请输入验证码/.test(bodyText);
-    const empty = /没有找到相关的微信文章|未找到相关|暂无相关|没有找到/.test(bodyText)
-      || Boolean(document.querySelector('.no-result, .no_result, .s-noresult'));
-    const cards = Array.from(document.querySelectorAll('.news-list li'));
-    const extracted = cards.map((item) => {
-      const linkEl = item.querySelector('h3 a[href]');
-      const summaryEl = item.querySelector('p.txt-info');
-      const accountEl = item.querySelector('.s-p .all-time-y2');
-      const timeEl = item.querySelector('.s-p .s2');
-      return {
-        title: clean(linkEl && linkEl.textContent),
-        account: clean(accountEl && accountEl.textContent),
-        url: absolutize(linkEl && linkEl.getAttribute('href')),
-        summary: clean(summaryEl && summaryEl.textContent),
-        publish_time: clean(timeEl && timeEl.textContent),
-      };
-    });
-    const rows = extracted.filter((row) => row.title && row.url);
-
-    return {
-      blocked,
-      empty,
-      invalidCount: extracted.length - rows.length,
-      rows,
-    };
-  })()`;
-}
-
 export const weixinSearchCommand = cli({
   site: 'weixin',
   name: 'sougousearch',
@@ -106,50 +34,20 @@ export const weixinSearchCommand = cli({
   columns: ['rank', 'page', 'title', 'account', 'url', 'summary', 'publish_time'],
   func: async (page, kwargs) => {
     const query = String(kwargs.query ?? '').trim();
-    if (!query) {
-      throw new ArgumentError('A search query is required.', 'Pass a non-empty keyword to search Weixin articles via Sogou.');
-    }
-
     const pageNo = normalizePage(kwargs.page);
     const limit = normalizeLimit(kwargs.limit);
-    const searchUrl = buildSearchUrl(query, pageNo);
-
-    let payload;
-    try {
-      await page.goto(searchUrl);
-      await page.wait(2);
-      payload = await page.evaluate(buildExtractSearchResultsEvaluate());
-    } catch (error) {
-      const detail = error instanceof Error ? error.message : String(error);
-      throw new CommandExecutionError('weixin sougousearch failed while loading Sogou results', detail);
-    }
-
-    if (!payload || typeof payload !== 'object' || !Array.isArray(payload.rows)) {
-      throw new CommandExecutionError('weixin sougousearch returned an unreadable browser payload', 'Sogou Weixin may have changed its result page structure.');
-    }
-    if (payload.blocked) {
-      throw new CommandExecutionError('Sogou Weixin blocked this search request', 'Open weixin.sogou.com in Chrome and complete any verification before retrying.');
-    }
-    if (payload.invalidCount > 0) {
-      throw new CommandExecutionError('Sogou Weixin returned article cards without required title or URL', 'The result page structure may have changed; refusing to return a partial result set.');
-    }
-
-    const rows = payload.rows;
-    if (rows.length === 0 && payload.empty) {
+    const result = await searchSogouArticlePage(page, { query, pageNo });
+    if (result.state === 'empty') {
       throw new EmptyResultError('weixin sougousearch', 'Try a different keyword or a different page number.');
     }
-    if (rows.length === 0) {
-      throw new CommandExecutionError('weixin sougousearch did not expose article result cards', 'Sogou Weixin may have changed its selectors or returned a transient shell page.');
-    }
-
-    return rows.slice(0, limit).map((row, index) => ({
+    return result.rows.slice(0, limit).map((row, index) => ({
       rank: (pageNo - 1) * 10 + index + 1,
       page: pageNo,
       title: row.title,
       account: row.account,
       url: row.url,
       summary: row.summary,
-      publish_time: row.publish_time,
+      publish_time: row.publishTime,
     }));
   },
 });
@@ -158,6 +56,4 @@ export const __test__ = {
   MAX_LIMIT,
   normalizePage,
   normalizeLimit,
-  buildSearchUrl,
-  buildExtractSearchResultsEvaluate,
 };

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@sovovs/bycli",
-  "version": "2.1.23",
+  "version": "2.1.25",
   "publishConfig": {
     "access": "public"
   },