@sovovs/bycli 2.1.23 → 2.1.24

package/cli-manifest.json

@@ -28084,7 +28084,9 @@
       "publish_time",
       "status",
       "size",
-      "saved"
+      "saved",
+      "source_url",
+      "resolved_url"
     ],
     "type": "js",
     "modulePath": "weixin/download.js",
@@ -28094,7 +28096,7 @@
   {
     "site": "weixin",
     "name": "download-publish-data",
-    "description": "Match a Weixin published article and save its content analysis as Markdown",
+    "description": "Match a Weixin published article and save its Excel data and Markdown analysis",
     "access": "write",
     "domain": "mp.weixin.qq.com",
     "strategy": "intercept",
@@ -28118,7 +28120,7 @@
         "type": "str",
         "default": "./weixin-publish-data",
         "required": false,
-        "help": "Directory for generated Markdown reports"
+        "help": "Directory for generated Excel data and Markdown reports"
       },
       {
         "name": "max-pages",
@@ -28141,8 +28143,9 @@
       "url",
       "status",
       "markdownPath",
+      "markdownSize",
       "dataPath",
-      "size",
+      "dataSize",
       "error"
     ],
     "type": "js",

package/clis/weixin/_wechat/article-index.js

@@ -0,0 +1,163 @@
+import { AuthRequiredError, CommandExecutionError } from '@sovovs/bycli/errors';
+import { buildSecretSet, redactText } from './redact.js';
+
+const DOMAIN = 'mp.weixin.qq.com';
+const ENDPOINT = `https://${DOMAIN}/cgi-bin/appmsgpublish`;
+
+function commandError(message, hint) {
+  return new CommandExecutionError(`WeChat appmsgpublish ${message}`, hint);
+}
+
+function parseNestedJson(value, field) {
+  if (typeof value !== 'string') return value;
+  try {
+    return JSON.parse(value);
+  } catch {
+    throw commandError(`returned invalid ${field} JSON`);
+  }
+}
+
+function requireRecord(value, field) {
+  if (!value || typeof value !== 'object' || Array.isArray(value)) {
+    throw commandError(`returned invalid ${field}`);
+  }
+  return value;
+}
+
+export function mapArticleIndexPayload(payload) {
+  const response = requireRecord(payload, 'response');
+  const baseResponse = requireRecord(response.base_resp, 'base_resp');
+  const ret = baseResponse.ret;
+  const normalizedMessage = typeof baseResponse.err_msg === 'string'
+    ? baseResponse.err_msg.trim().toLowerCase().replace(/\s+/g, ' ') : '';
+  if (ret === 200013 && normalizedMessage === 'invalid credential') {
+    throw new AuthRequiredError(DOMAIN, 'WeChat article-index credentials have expired');
+  }
+  if (ret === 200013 && normalizedMessage === 'freq control') {
+    throw commandError(
+      'was rate limited (ret=200013)',
+      'Wait before retrying the WeChat article-index request; repeated retries may extend frequency control.',
+    );
+  }
+  if (!Number.isInteger(ret) || ret !== 0) {
+    throw commandError(`failed (ret=${String(ret ?? 'unknown')})`);
+  }
+  if (response.publish_page === undefined || response.publish_page === null
+    || response.publish_page === '') {
+    return { total: 0, publishItemCount: 0, articles: [] };
+  }
+
+  const page = requireRecord(parseNestedJson(response.publish_page, 'publish_page'), 'publish_page');
+  if (!Number.isInteger(page.total_count) || page.total_count < 0) {
+    throw commandError('returned invalid total_count');
+  }
+  if (!Array.isArray(page.publish_list)) {
+    throw commandError('returned invalid publish_list');
+  }
+
+  const articles = [];
+  for (const [publishIndex, rawItem] of page.publish_list.entries()) {
+    const item = requireRecord(rawItem, `publish_list[${publishIndex}]`);
+    if (!Object.prototype.hasOwnProperty.call(item, 'publish_info')) {
+      throw commandError(`returned missing publish_info at index ${publishIndex}`);
+    }
+    const info = requireRecord(
+      parseNestedJson(item.publish_info, `publish_info at index ${publishIndex}`),
+      `publish_info at index ${publishIndex}`,
+    );
+    if (!Array.isArray(info.appmsg_info)) {
+      throw commandError(`returned invalid appmsg_info at index ${publishIndex}`);
+    }
+    if (info.sent_info !== undefined) requireRecord(info.sent_info, `sent_info at index ${publishIndex}`);
+    if (info.publish_info !== undefined) requireRecord(info.publish_info, `publish metadata at index ${publishIndex}`);
+    const timestamp = info.sent_info?.time ?? info.publish_info?.create_time ?? 0;
+    if (!Number.isInteger(timestamp) || timestamp < 0) {
+      throw commandError(`returned invalid timestamp at index ${publishIndex}`);
+    }
+
+    for (const [articleIndex, rawArticle] of info.appmsg_info.entries()) {
+      const article = requireRecord(rawArticle, `appmsg_info[${articleIndex}]`);
+      for (const field of ['title', 'content_url', 'digest', 'author']) {
+        if (article[field] !== undefined && typeof article[field] !== 'string') {
+          throw commandError(`returned invalid ${field} at article index ${articleIndex}`);
+        }
+      }
+      articles.push({
+        title: article.title || '',
+        url: article.content_url || '',
+        isDeleted: article.is_deleted === true,
+        timestamp,
+        publishedAt: timestamp > 0 ? new Date(timestamp * 1000).toISOString() : null,
+        digest: article.digest || '',
+        author: article.author || '',
+      });
+    }
+  }
+
+  return { total: page.total_count, publishItemCount: page.publish_list.length, articles };
+}
+
+function buildReferer(token) {
+  const params = new URLSearchParams({
+    t: 'media/appmsg_edit_v2', action: 'edit', isNew: '1', type: '10',
+    token, lang: 'zh_CN',
+  });
+  return `https://${DOMAIN}/cgi-bin/appmsg?${params}`;
+}
+
+function transportError(error, credentials) {
+  const secrets = buildSecretSet(credentials);
+  const message = error instanceof Error ? error.message : String(error);
+  const hint = error && typeof error === 'object' && 'hint' in error
+    && typeof error.hint === 'string' ? error.hint : undefined;
+  const redactedMessage = redactText(message, secrets);
+  const redactedHint = hint ? redactText(hint, secrets) : undefined;
+  if (error instanceof AuthRequiredError && error.domain === DOMAIN
+    && redactedMessage === message && redactedHint === hint) return error;
+  return new CommandExecutionError(
+    `WeChat appmsgpublish request failed: ${redactedMessage}`,
+    redactedHint,
+  );
+}
+
+export function createArticleIndexFetcher({
+  page,
+  source,
+  credentials,
+  fetchImpl = fetch,
+  timeoutMs = 30_000,
+}) {
+  return async function fetchPage({ fakeid, begin = 0, count = 10 }) {
+    const params = new URLSearchParams({
+      sub: 'list', begin: String(begin), count: String(count), fakeid,
+      token: credentials.token, lang: 'zh_CN', f: 'json', ajax: '1',
+    });
+    const url = `${ENDPOINT}?${params}`;
+    const headers = {
+      Referer: buildReferer(credentials.token),
+      'X-Requested-With': 'XMLHttpRequest',
+    };
+
+    try {
+      let payload;
+      if (source === 'browser') {
+        if (!page || typeof page.fetchJson !== 'function') {
+          throw new CommandExecutionError('Browser page.fetchJson is unavailable');
+        }
+        payload = await page.fetchJson(url, { headers });
+      } else {
+        const response = await fetchImpl(url, {
+          headers: { ...headers, Cookie: credentials.cookie },
+          signal: AbortSignal.timeout(timeoutMs),
+        });
+        if (!response.ok) {
+          throw new CommandExecutionError(`HTTP ${String(response.status ?? 'unknown')}`);
+        }
+        payload = await response.json();
+      }
+      return mapArticleIndexPayload(payload);
+    } catch (error) {
+      throw transportError(error, credentials);
+    }
+  };
+}

package/clis/weixin/_wechat/auth-session.js

@@ -1,4 +1,4 @@
-import { AuthRequiredError, BrowserConnectError } from '@sovovs/bycli/errors';
+import { AuthRequiredError, BrowserConnectError, CommandExecutionError } from '@sovovs/bycli/errors';
 
 const DOMAIN = 'mp.weixin.qq.com';
 const LOGIN_URL = `https://${DOMAIN}/`;
@@ -46,6 +46,20 @@ export function isLoggedInPreflight(state) {
   }
 }
 
+/** @param {PreflightState} state @returns {boolean} */
+export function isLoggedInMiniProgramPreflight(state) {
+  if (state.url === null || state.hasLoginUi) return false;
+
+  try {
+    const url = new URL(state.url);
+    return url.origin === `https://${DOMAIN}`
+      && url.pathname.startsWith('/wxamp/')
+      && Boolean(url.searchParams.get('token')?.trim());
+  } catch {
+    return false;
+  }
+}
+
 /** @param {AuthPage} page @returns {Promise<PreflightState>} */
 async function readPreflight(page) {
   const result = await page.evaluate(() => {
@@ -100,6 +114,12 @@ export async function resolveBrowserCredentials(page, options = {}) {
     state = await readPreflight(page);
 
     if (!isLoggedInPreflight(state)) {
+      if (isLoggedInMiniProgramPreflight(state)) {
+        throw new CommandExecutionError(
+          'The connected WeChat session is authenticated as a Mini Program account',
+          'Switch to a WeChat Official Account in the same browser profile before running bycli weixin commands.',
+        );
+      }
       if (!page.focusWindow) {
         throw new BrowserConnectError(
           'The connected browser cannot be focused for WeChat login',

package/clis/weixin/articles.js

@@ -1,7 +1,8 @@
 import { ArgumentError, EmptyResultError } from '@sovovs/bycli/errors';
 import { cli, Strategy } from '@sovovs/bycli/registry';
 import { readEnvironmentCredentials, resolveBrowserCredentials } from './_wechat/auth-session.js';
-import { callCrawler, collectArticles, createWechatApi } from './_wechat/crawler-runtime.js';
+import { createArticleIndexFetcher } from './_wechat/article-index.js';
+import { callCrawler, collectArticles } from './_wechat/crawler-runtime.js';
 import { readAuthSource } from './_wechat/args.js';
 
 const DOMAIN = 'mp.weixin.qq.com';
@@ -23,10 +24,10 @@ export const articlesCommand = cli({
     const authSource = readAuthSource(args);
     const credentials = authSource === 'env'
       ? readEnvironmentCredentials(false) : await resolveBrowserCredentials(page);
-    const { articles } = await callCrawler(async () => {
-      const { fetchPage } = createWechatApi(credentials);
-      return collectArticles({ fakeid, fetchPage, limit: args.limit, maxPages: args['max-pages'] });
-    });
+    const fetchPage = createArticleIndexFetcher({ page, source: authSource, credentials });
+    const { articles } = await callCrawler(() => collectArticles({
+      fakeid, fetchPage, limit: args.limit, maxPages: args['max-pages'],
+    }));
     if (articles.length === 0) throw new EmptyResultError('weixin articles', `No published articles were found for ${fakeid}.`);
     return articles.map(article => ({
       title: article.title, author: article.author || null, digest: article.digest || null,

package/clis/weixin/download-publish-data.js

@@ -12,7 +12,10 @@ import {
   validatePublishDate,
 } from './_wechat/publish-records.js';
 
-const COLUMNS = ['title', 'publishedAt', 'url', 'status', 'markdownPath', 'dataPath', 'size', 'error'];
+const COLUMNS = [
+  'title', 'publishedAt', 'url', 'status',
+  'markdownPath', 'markdownSize', 'dataPath', 'dataSize', 'error',
+];
 
 function sanitizedError(error, secrets, fallback) {
   const message = error instanceof Error ? error.message : fallback;
@@ -25,14 +28,14 @@ export const downloadPublishDataCommand = cli({
   name: 'download-publish-data',
   access: 'write',
   domain: 'mp.weixin.qq.com',
-  description: 'Match a Weixin published article and save its content analysis as Markdown',
+  description: 'Match a Weixin published article and save its Excel data and Markdown analysis',
   strategy: Strategy.INTERCEPT,
   browser: true,
   navigateBefore: false,
   args: [
     { name: 'query', positional: true, required: true, help: 'Exact article URL or title text' },
     { name: 'date', help: 'Optional publication date in YYYY-MM-DD' },
-    { name: 'output', default: './weixin-publish-data', help: 'Directory for generated Markdown reports' },
+    { name: 'output', default: './weixin-publish-data', help: 'Directory for generated Excel data and Markdown reports' },
     { name: 'max-pages', type: 'int', default: 5, help: 'Maximum published-record pages to scan' },
     { name: 'timeout', type: 'int', default: 60, help: 'Maximum seconds for page capture' },
   ],
@@ -64,31 +67,35 @@ export const downloadPublishDataCommand = cli({
     };
     const secrets = buildSecretSet({ token, cookie });
 
+    let dataResult = null;
+    let markdownResult = null;
+    const errors = [];
     try {
-      const result = await downloadPublishData(page, commonOptions);
-      return [{ title: record.title, publishedAt: record.publishedAt, url: record.url,
-        status: 'saved', markdownPath: null, dataPath: result.path,
-        size: result.size, error: null }];
-    } catch (downloadError) {
-      const downloadMessage = sanitizedError(downloadError, secrets, 'Excel download failed');
-      try {
-        const result = await collectPublishAnalysis(page, {
-          ...commonOptions,
-          publishedAt: record.publishedAt,
-        });
-        return [{ title: record.title, publishedAt: record.publishedAt, url: record.url,
-          status: 'saved', markdownPath: result.path, dataPath: null,
-          size: result.size, error: downloadMessage }];
-      } catch (analysisError) {
-        const analysisMessage = sanitizedError(
-          analysisError,
-          secrets,
-          'Markdown fallback failed',
-        );
-        return [{ title: record.title, publishedAt: record.publishedAt, url: record.url,
-          status: 'failed', markdownPath: null, dataPath: null, size: null,
-          error: `Excel download failed: ${downloadMessage}; Markdown fallback failed: ${analysisMessage}` }];
-      }
+      dataResult = await downloadPublishData(page, commonOptions);
+    } catch (error) {
+      errors.push(`Excel download failed: ${sanitizedError(error, secrets, 'Excel download failed')}`);
     }
+    try {
+      markdownResult = await collectPublishAnalysis(page, {
+        ...commonOptions,
+        publishedAt: record.publishedAt,
+      });
+    } catch (error) {
+      errors.push(`Markdown analysis failed: ${sanitizedError(error, secrets, 'Markdown analysis failed')}`);
+    }
+
+    const status = dataResult && markdownResult ? 'downloaded'
+      : dataResult || markdownResult ? 'partial' : 'failed';
+    return [{
+      title: record.title,
+      publishedAt: record.publishedAt,
+      url: record.url,
+      status,
+      markdownPath: markdownResult?.path ?? null,
+      markdownSize: markdownResult?.size ?? null,
+      dataPath: dataResult?.path ?? null,
+      dataSize: dataResult?.size ?? null,
+      error: errors.length > 0 ? errors.join('; ') : null,
+    }];
   },
 });

package/clis/weixin/download.js

@@ -8,7 +8,7 @@
  */
 import { cli, Strategy } from '@sovovs/bycli/registry';
 import { downloadArticle } from '@sovovs/bycli/download/article-download';
-import { AuthRequiredError } from '@sovovs/bycli/errors';
+import { ArgumentError, AuthRequiredError, CommandExecutionError } from '@sovovs/bycli/errors';
 import { buildExtractWechatArticleContentJs } from './_wechat/article-content.js';
 export { extractWechatArticleContent } from './_wechat/article-content.js';
 // ============================================================
@@ -49,6 +49,66 @@ export function normalizeWechatUrl(raw) {
     }
     return s;
 }
+
+function isStrictHttpsUrl(raw, hostname, pathname) {
+    try {
+        const url = new URL(raw);
+        return url.protocol === 'https:' && url.hostname === hostname
+            && url.port === '' && url.username === '' && url.password === ''
+            && pathname(url.pathname);
+    }
+    catch {
+        return false;
+    }
+}
+
+export function isTrustedWechatArticleUrl(raw) {
+    return isStrictHttpsUrl(raw, 'mp.weixin.qq.com', path => path === '/s' || path.startsWith('/s/'));
+}
+
+export function isTrustedSogouRedirectUrl(raw) {
+    return isStrictHttpsUrl(raw, 'weixin.sogou.com', path => path === '/link');
+}
+
+export async function resolveWechatDownloadUrl(page, rawUrl) {
+    const sourceUrl = normalizeWechatUrl(rawUrl);
+    if (isTrustedWechatArticleUrl(sourceUrl)) {
+        return { sourceUrl, resolvedUrl: sourceUrl, alreadyNavigated: false };
+    }
+    if (!isTrustedSogouRedirectUrl(sourceUrl)) {
+        throw new ArgumentError(
+            'A trusted WeChat article or Sogou Weixin result URL is required.',
+            'Pass an https://mp.weixin.qq.com/s/... or https://weixin.sogou.com/link?... URL.',
+        );
+    }
+    try {
+        await page.goto(sourceUrl);
+        await page.wait(2);
+        const result = await page.evaluate(`(() => ({
+          finalUrl: window.location.href,
+          pageText: document.body ? document.body.innerText : '',
+          html: document.documentElement ? document.documentElement.innerHTML : '',
+        }))()`);
+        const text = `${result?.pageText || ''} ${result?.html || ''}`;
+        if (/验证码|安全验证|异常访问|访问过于频繁|请输入验证码/.test(text)) {
+            throw new AuthRequiredError(
+                'weixin.sogou.com',
+                'Sogou Weixin requires verification. Complete it in the open browser tab and run the command again.',
+            );
+        }
+        if (!isTrustedWechatArticleUrl(result?.finalUrl)) {
+            throw new CommandExecutionError(
+                'Sogou Weixin did not resolve to a trusted WeChat article URL',
+                'Open the search result in a browser and confirm it redirects to mp.weixin.qq.com/s/... before retrying.',
+            );
+        }
+        return { sourceUrl, resolvedUrl: new URL(result.finalUrl).href, alreadyNavigated: true };
+    }
+    catch (error) {
+        if (error instanceof AuthRequiredError || error instanceof CommandExecutionError) throw error;
+        throw new CommandExecutionError('Failed to resolve the Sogou Weixin result URL');
+    }
+}
 /**
  * Format a WeChat article timestamp as a UTC+8 datetime string.
  * Accepts either Unix seconds or milliseconds.
@@ -183,15 +243,12 @@ cli({
         { name: 'output', default: './weixin-articles', help: 'Output directory' },
         { name: 'download-images', type: 'boolean', default: true, help: 'Download images locally' },
     ],
-    columns: ['title', 'author', 'publish_time', 'status', 'size', 'saved'],
+    columns: ['title', 'author', 'publish_time', 'status', 'size', 'saved', 'source_url', 'resolved_url'],
     func: async (page, kwargs) => {
-        const rawUrl = kwargs.url;
-        const url = normalizeWechatUrl(rawUrl);
-        if (!url.startsWith('https://mp.weixin.qq.com/')) {
-            return [{ title: 'Error', author: '-', publish_time: '-', status: 'invalid URL', size: '-', saved: '-' }];
-        }
-        // Navigate and wait for content to load
-        await page.goto(url);
+        const { sourceUrl, resolvedUrl, alreadyNavigated } = await resolveWechatDownloadUrl(page, kwargs.url);
+        // Navigate and wait for content to load. Sogou resolution already lands on the article.
+        if (!alreadyNavigated)
+            await page.goto(resolvedUrl);
         await page.wait(5);
         // Extract article data in browser context
         const data = await page.evaluate(`
@@ -255,11 +312,11 @@ cli({
                 'WeChat article page requires environment verification. Complete it in the open browser tab and run the command again.',
             );
         }
-        return downloadArticle({
+        const rows = await downloadArticle({
             title: data?.title || '',
             author: data?.author,
             publishTime: data?.publishTime,
-            sourceUrl: url,
+            sourceUrl: resolvedUrl,
             contentHtml: data?.contentHtml || '',
             codeBlocks: data?.codeBlocks,
             imageUrls: data?.imageUrls,
@@ -274,5 +331,6 @@ cli({
             },
             secureMarkdown: true,
         });
+        return rows.map(row => ({ ...row, source_url: sourceUrl, resolved_url: resolvedUrl }));
     },
 });

package/clis/weixin/save-articles.js

@@ -2,8 +2,9 @@ import { ArgumentError, AuthRequiredError, CommandExecutionError } from '@sovovs
 import { MAX_WECHAT_HTML_BYTES } from '@sovovs/bycli/download/wechat-article';
 import { cli, Strategy } from '@sovovs/bycli/registry';
 import { readEnvironmentCredentials, resolveBrowserCredentials } from './_wechat/auth-session.js';
+import { createArticleIndexFetcher } from './_wechat/article-index.js';
 import {
-  callCrawler, collectArticles, createWechatApi, isTrustedWechatArticleUrl, saveArticles,
+  callCrawler, collectArticles, isTrustedWechatArticleUrl, saveArticles,
 } from './_wechat/crawler-runtime.js';
 import { readAuthSource } from './_wechat/args.js';
 import { wechatArticleToMarkdown } from './_wechat/markdown.js';
@@ -167,8 +168,8 @@ export const saveArticlesCommand = cli({
     const credentials = authSource === 'env'
       ? readEnvironmentCredentials(false) : await resolveBrowserCredentials(page);
     const articleHtmlDownloader = createArticleHtmlDownloader({ authSource, page });
+    const fetchPage = createArticleIndexFetcher({ page, source: authSource, credentials });
     const rows = await callCrawler(async () => {
-      const { fetchPage } = createWechatApi(credentials);
       const { articles } = await collectArticles({ fakeid, fetchPage, limit: args.limit, maxPages: args['max-pages'] });
       return saveArticles({
         articles, accountName: String(args.name ?? '').trim(),

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@sovovs/bycli",
-  "version": "2.1.23",
+  "version": "2.1.24",
   "publishConfig": {
     "access": "public"
   },