@sovovs/bycli 2.1.22 → 2.1.24
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/cli-manifest.json +9 -5
- package/clis/weixin/_wechat/article-index.js +163 -0
- package/clis/weixin/_wechat/auth-session.js +21 -1
- package/clis/weixin/articles.js +6 -5
- package/clis/weixin/download-publish-data.js +34 -27
- package/clis/weixin/download.js +69 -11
- package/clis/weixin/save-articles.js +3 -2
- package/clis/weixin/search.js +11 -8
- package/package.json +1 -1
package/cli-manifest.json
CHANGED
|
@@ -28084,7 +28084,9 @@
|
|
|
28084
28084
|
"publish_time",
|
|
28085
28085
|
"status",
|
|
28086
28086
|
"size",
|
|
28087
|
-
"saved"
|
|
28087
|
+
"saved",
|
|
28088
|
+
"source_url",
|
|
28089
|
+
"resolved_url"
|
|
28088
28090
|
],
|
|
28089
28091
|
"type": "js",
|
|
28090
28092
|
"modulePath": "weixin/download.js",
|
|
@@ -28094,7 +28096,7 @@
|
|
|
28094
28096
|
{
|
|
28095
28097
|
"site": "weixin",
|
|
28096
28098
|
"name": "download-publish-data",
|
|
28097
|
-
"description": "Match a Weixin published article and save its
|
|
28099
|
+
"description": "Match a Weixin published article and save its Excel data and Markdown analysis",
|
|
28098
28100
|
"access": "write",
|
|
28099
28101
|
"domain": "mp.weixin.qq.com",
|
|
28100
28102
|
"strategy": "intercept",
|
|
@@ -28118,7 +28120,7 @@
|
|
|
28118
28120
|
"type": "str",
|
|
28119
28121
|
"default": "./weixin-publish-data",
|
|
28120
28122
|
"required": false,
|
|
28121
|
-
"help": "Directory for generated Markdown reports"
|
|
28123
|
+
"help": "Directory for generated Excel data and Markdown reports"
|
|
28122
28124
|
},
|
|
28123
28125
|
{
|
|
28124
28126
|
"name": "max-pages",
|
|
@@ -28141,8 +28143,9 @@
|
|
|
28141
28143
|
"url",
|
|
28142
28144
|
"status",
|
|
28143
28145
|
"markdownPath",
|
|
28146
|
+
"markdownSize",
|
|
28144
28147
|
"dataPath",
|
|
28145
|
-
"
|
|
28148
|
+
"dataSize",
|
|
28146
28149
|
"error"
|
|
28147
28150
|
],
|
|
28148
28151
|
"type": "js",
|
|
@@ -28309,7 +28312,7 @@
|
|
|
28309
28312
|
},
|
|
28310
28313
|
{
|
|
28311
28314
|
"site": "weixin",
|
|
28312
|
-
"name": "
|
|
28315
|
+
"name": "sougousearch",
|
|
28313
28316
|
"description": "使用搜狗微信搜索公众号文章;如需导出正文 Markdown,请使用 weixin download 处理公众号文章链接",
|
|
28314
28317
|
"access": "read",
|
|
28315
28318
|
"domain": "weixin.sogou.com",
|
|
@@ -28342,6 +28345,7 @@
|
|
|
28342
28345
|
"rank",
|
|
28343
28346
|
"page",
|
|
28344
28347
|
"title",
|
|
28348
|
+
"account",
|
|
28345
28349
|
"url",
|
|
28346
28350
|
"summary",
|
|
28347
28351
|
"publish_time"
|
|
@@ -0,0 +1,163 @@
|
|
|
1
|
+
import { AuthRequiredError, CommandExecutionError } from '@sovovs/bycli/errors';
|
|
2
|
+
import { buildSecretSet, redactText } from './redact.js';
|
|
3
|
+
|
|
4
|
+
const DOMAIN = 'mp.weixin.qq.com';
|
|
5
|
+
const ENDPOINT = `https://${DOMAIN}/cgi-bin/appmsgpublish`;
|
|
6
|
+
|
|
7
|
+
function commandError(message, hint) {
|
|
8
|
+
return new CommandExecutionError(`WeChat appmsgpublish ${message}`, hint);
|
|
9
|
+
}
|
|
10
|
+
|
|
11
|
+
function parseNestedJson(value, field) {
|
|
12
|
+
if (typeof value !== 'string') return value;
|
|
13
|
+
try {
|
|
14
|
+
return JSON.parse(value);
|
|
15
|
+
} catch {
|
|
16
|
+
throw commandError(`returned invalid ${field} JSON`);
|
|
17
|
+
}
|
|
18
|
+
}
|
|
19
|
+
|
|
20
|
+
function requireRecord(value, field) {
|
|
21
|
+
if (!value || typeof value !== 'object' || Array.isArray(value)) {
|
|
22
|
+
throw commandError(`returned invalid ${field}`);
|
|
23
|
+
}
|
|
24
|
+
return value;
|
|
25
|
+
}
|
|
26
|
+
|
|
27
|
+
export function mapArticleIndexPayload(payload) {
|
|
28
|
+
const response = requireRecord(payload, 'response');
|
|
29
|
+
const baseResponse = requireRecord(response.base_resp, 'base_resp');
|
|
30
|
+
const ret = baseResponse.ret;
|
|
31
|
+
const normalizedMessage = typeof baseResponse.err_msg === 'string'
|
|
32
|
+
? baseResponse.err_msg.trim().toLowerCase().replace(/\s+/g, ' ') : '';
|
|
33
|
+
if (ret === 200013 && normalizedMessage === 'invalid credential') {
|
|
34
|
+
throw new AuthRequiredError(DOMAIN, 'WeChat article-index credentials have expired');
|
|
35
|
+
}
|
|
36
|
+
if (ret === 200013 && normalizedMessage === 'freq control') {
|
|
37
|
+
throw commandError(
|
|
38
|
+
'was rate limited (ret=200013)',
|
|
39
|
+
'Wait before retrying the WeChat article-index request; repeated retries may extend frequency control.',
|
|
40
|
+
);
|
|
41
|
+
}
|
|
42
|
+
if (!Number.isInteger(ret) || ret !== 0) {
|
|
43
|
+
throw commandError(`failed (ret=${String(ret ?? 'unknown')})`);
|
|
44
|
+
}
|
|
45
|
+
if (response.publish_page === undefined || response.publish_page === null
|
|
46
|
+
|| response.publish_page === '') {
|
|
47
|
+
return { total: 0, publishItemCount: 0, articles: [] };
|
|
48
|
+
}
|
|
49
|
+
|
|
50
|
+
const page = requireRecord(parseNestedJson(response.publish_page, 'publish_page'), 'publish_page');
|
|
51
|
+
if (!Number.isInteger(page.total_count) || page.total_count < 0) {
|
|
52
|
+
throw commandError('returned invalid total_count');
|
|
53
|
+
}
|
|
54
|
+
if (!Array.isArray(page.publish_list)) {
|
|
55
|
+
throw commandError('returned invalid publish_list');
|
|
56
|
+
}
|
|
57
|
+
|
|
58
|
+
const articles = [];
|
|
59
|
+
for (const [publishIndex, rawItem] of page.publish_list.entries()) {
|
|
60
|
+
const item = requireRecord(rawItem, `publish_list[${publishIndex}]`);
|
|
61
|
+
if (!Object.prototype.hasOwnProperty.call(item, 'publish_info')) {
|
|
62
|
+
throw commandError(`returned missing publish_info at index ${publishIndex}`);
|
|
63
|
+
}
|
|
64
|
+
const info = requireRecord(
|
|
65
|
+
parseNestedJson(item.publish_info, `publish_info at index ${publishIndex}`),
|
|
66
|
+
`publish_info at index ${publishIndex}`,
|
|
67
|
+
);
|
|
68
|
+
if (!Array.isArray(info.appmsg_info)) {
|
|
69
|
+
throw commandError(`returned invalid appmsg_info at index ${publishIndex}`);
|
|
70
|
+
}
|
|
71
|
+
if (info.sent_info !== undefined) requireRecord(info.sent_info, `sent_info at index ${publishIndex}`);
|
|
72
|
+
if (info.publish_info !== undefined) requireRecord(info.publish_info, `publish metadata at index ${publishIndex}`);
|
|
73
|
+
const timestamp = info.sent_info?.time ?? info.publish_info?.create_time ?? 0;
|
|
74
|
+
if (!Number.isInteger(timestamp) || timestamp < 0) {
|
|
75
|
+
throw commandError(`returned invalid timestamp at index ${publishIndex}`);
|
|
76
|
+
}
|
|
77
|
+
|
|
78
|
+
for (const [articleIndex, rawArticle] of info.appmsg_info.entries()) {
|
|
79
|
+
const article = requireRecord(rawArticle, `appmsg_info[${articleIndex}]`);
|
|
80
|
+
for (const field of ['title', 'content_url', 'digest', 'author']) {
|
|
81
|
+
if (article[field] !== undefined && typeof article[field] !== 'string') {
|
|
82
|
+
throw commandError(`returned invalid ${field} at article index ${articleIndex}`);
|
|
83
|
+
}
|
|
84
|
+
}
|
|
85
|
+
articles.push({
|
|
86
|
+
title: article.title || '',
|
|
87
|
+
url: article.content_url || '',
|
|
88
|
+
isDeleted: article.is_deleted === true,
|
|
89
|
+
timestamp,
|
|
90
|
+
publishedAt: timestamp > 0 ? new Date(timestamp * 1000).toISOString() : null,
|
|
91
|
+
digest: article.digest || '',
|
|
92
|
+
author: article.author || '',
|
|
93
|
+
});
|
|
94
|
+
}
|
|
95
|
+
}
|
|
96
|
+
|
|
97
|
+
return { total: page.total_count, publishItemCount: page.publish_list.length, articles };
|
|
98
|
+
}
|
|
99
|
+
|
|
100
|
+
function buildReferer(token) {
|
|
101
|
+
const params = new URLSearchParams({
|
|
102
|
+
t: 'media/appmsg_edit_v2', action: 'edit', isNew: '1', type: '10',
|
|
103
|
+
token, lang: 'zh_CN',
|
|
104
|
+
});
|
|
105
|
+
return `https://${DOMAIN}/cgi-bin/appmsg?${params}`;
|
|
106
|
+
}
|
|
107
|
+
|
|
108
|
+
function transportError(error, credentials) {
|
|
109
|
+
const secrets = buildSecretSet(credentials);
|
|
110
|
+
const message = error instanceof Error ? error.message : String(error);
|
|
111
|
+
const hint = error && typeof error === 'object' && 'hint' in error
|
|
112
|
+
&& typeof error.hint === 'string' ? error.hint : undefined;
|
|
113
|
+
const redactedMessage = redactText(message, secrets);
|
|
114
|
+
const redactedHint = hint ? redactText(hint, secrets) : undefined;
|
|
115
|
+
if (error instanceof AuthRequiredError && error.domain === DOMAIN
|
|
116
|
+
&& redactedMessage === message && redactedHint === hint) return error;
|
|
117
|
+
return new CommandExecutionError(
|
|
118
|
+
`WeChat appmsgpublish request failed: ${redactedMessage}`,
|
|
119
|
+
redactedHint,
|
|
120
|
+
);
|
|
121
|
+
}
|
|
122
|
+
|
|
123
|
+
export function createArticleIndexFetcher({
|
|
124
|
+
page,
|
|
125
|
+
source,
|
|
126
|
+
credentials,
|
|
127
|
+
fetchImpl = fetch,
|
|
128
|
+
timeoutMs = 30_000,
|
|
129
|
+
}) {
|
|
130
|
+
return async function fetchPage({ fakeid, begin = 0, count = 10 }) {
|
|
131
|
+
const params = new URLSearchParams({
|
|
132
|
+
sub: 'list', begin: String(begin), count: String(count), fakeid,
|
|
133
|
+
token: credentials.token, lang: 'zh_CN', f: 'json', ajax: '1',
|
|
134
|
+
});
|
|
135
|
+
const url = `${ENDPOINT}?${params}`;
|
|
136
|
+
const headers = {
|
|
137
|
+
Referer: buildReferer(credentials.token),
|
|
138
|
+
'X-Requested-With': 'XMLHttpRequest',
|
|
139
|
+
};
|
|
140
|
+
|
|
141
|
+
try {
|
|
142
|
+
let payload;
|
|
143
|
+
if (source === 'browser') {
|
|
144
|
+
if (!page || typeof page.fetchJson !== 'function') {
|
|
145
|
+
throw new CommandExecutionError('Browser page.fetchJson is unavailable');
|
|
146
|
+
}
|
|
147
|
+
payload = await page.fetchJson(url, { headers });
|
|
148
|
+
} else {
|
|
149
|
+
const response = await fetchImpl(url, {
|
|
150
|
+
headers: { ...headers, Cookie: credentials.cookie },
|
|
151
|
+
signal: AbortSignal.timeout(timeoutMs),
|
|
152
|
+
});
|
|
153
|
+
if (!response.ok) {
|
|
154
|
+
throw new CommandExecutionError(`HTTP ${String(response.status ?? 'unknown')}`);
|
|
155
|
+
}
|
|
156
|
+
payload = await response.json();
|
|
157
|
+
}
|
|
158
|
+
return mapArticleIndexPayload(payload);
|
|
159
|
+
} catch (error) {
|
|
160
|
+
throw transportError(error, credentials);
|
|
161
|
+
}
|
|
162
|
+
};
|
|
163
|
+
}
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { AuthRequiredError, BrowserConnectError } from '@sovovs/bycli/errors';
|
|
1
|
+
import { AuthRequiredError, BrowserConnectError, CommandExecutionError } from '@sovovs/bycli/errors';
|
|
2
2
|
|
|
3
3
|
const DOMAIN = 'mp.weixin.qq.com';
|
|
4
4
|
const LOGIN_URL = `https://${DOMAIN}/`;
|
|
@@ -46,6 +46,20 @@ export function isLoggedInPreflight(state) {
|
|
|
46
46
|
}
|
|
47
47
|
}
|
|
48
48
|
|
|
49
|
+
/** @param {PreflightState} state @returns {boolean} */
|
|
50
|
+
export function isLoggedInMiniProgramPreflight(state) {
|
|
51
|
+
if (state.url === null || state.hasLoginUi) return false;
|
|
52
|
+
|
|
53
|
+
try {
|
|
54
|
+
const url = new URL(state.url);
|
|
55
|
+
return url.origin === `https://${DOMAIN}`
|
|
56
|
+
&& url.pathname.startsWith('/wxamp/')
|
|
57
|
+
&& Boolean(url.searchParams.get('token')?.trim());
|
|
58
|
+
} catch {
|
|
59
|
+
return false;
|
|
60
|
+
}
|
|
61
|
+
}
|
|
62
|
+
|
|
49
63
|
/** @param {AuthPage} page @returns {Promise<PreflightState>} */
|
|
50
64
|
async function readPreflight(page) {
|
|
51
65
|
const result = await page.evaluate(() => {
|
|
@@ -100,6 +114,12 @@ export async function resolveBrowserCredentials(page, options = {}) {
|
|
|
100
114
|
state = await readPreflight(page);
|
|
101
115
|
|
|
102
116
|
if (!isLoggedInPreflight(state)) {
|
|
117
|
+
if (isLoggedInMiniProgramPreflight(state)) {
|
|
118
|
+
throw new CommandExecutionError(
|
|
119
|
+
'The connected WeChat session is authenticated as a Mini Program account',
|
|
120
|
+
'Switch to a WeChat Official Account in the same browser profile before running bycli weixin commands.',
|
|
121
|
+
);
|
|
122
|
+
}
|
|
103
123
|
if (!page.focusWindow) {
|
|
104
124
|
throw new BrowserConnectError(
|
|
105
125
|
'The connected browser cannot be focused for WeChat login',
|
package/clis/weixin/articles.js
CHANGED
|
@@ -1,7 +1,8 @@
|
|
|
1
1
|
import { ArgumentError, EmptyResultError } from '@sovovs/bycli/errors';
|
|
2
2
|
import { cli, Strategy } from '@sovovs/bycli/registry';
|
|
3
3
|
import { readEnvironmentCredentials, resolveBrowserCredentials } from './_wechat/auth-session.js';
|
|
4
|
-
import {
|
|
4
|
+
import { createArticleIndexFetcher } from './_wechat/article-index.js';
|
|
5
|
+
import { callCrawler, collectArticles } from './_wechat/crawler-runtime.js';
|
|
5
6
|
import { readAuthSource } from './_wechat/args.js';
|
|
6
7
|
|
|
7
8
|
const DOMAIN = 'mp.weixin.qq.com';
|
|
@@ -23,10 +24,10 @@ export const articlesCommand = cli({
|
|
|
23
24
|
const authSource = readAuthSource(args);
|
|
24
25
|
const credentials = authSource === 'env'
|
|
25
26
|
? readEnvironmentCredentials(false) : await resolveBrowserCredentials(page);
|
|
26
|
-
const {
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
});
|
|
27
|
+
const fetchPage = createArticleIndexFetcher({ page, source: authSource, credentials });
|
|
28
|
+
const { articles } = await callCrawler(() => collectArticles({
|
|
29
|
+
fakeid, fetchPage, limit: args.limit, maxPages: args['max-pages'],
|
|
30
|
+
}));
|
|
30
31
|
if (articles.length === 0) throw new EmptyResultError('weixin articles', `No published articles were found for ${fakeid}.`);
|
|
31
32
|
return articles.map(article => ({
|
|
32
33
|
title: article.title, author: article.author || null, digest: article.digest || null,
|
|
@@ -12,7 +12,10 @@ import {
|
|
|
12
12
|
validatePublishDate,
|
|
13
13
|
} from './_wechat/publish-records.js';
|
|
14
14
|
|
|
15
|
-
const COLUMNS = [
|
|
15
|
+
const COLUMNS = [
|
|
16
|
+
'title', 'publishedAt', 'url', 'status',
|
|
17
|
+
'markdownPath', 'markdownSize', 'dataPath', 'dataSize', 'error',
|
|
18
|
+
];
|
|
16
19
|
|
|
17
20
|
function sanitizedError(error, secrets, fallback) {
|
|
18
21
|
const message = error instanceof Error ? error.message : fallback;
|
|
@@ -25,14 +28,14 @@ export const downloadPublishDataCommand = cli({
|
|
|
25
28
|
name: 'download-publish-data',
|
|
26
29
|
access: 'write',
|
|
27
30
|
domain: 'mp.weixin.qq.com',
|
|
28
|
-
description: 'Match a Weixin published article and save its
|
|
31
|
+
description: 'Match a Weixin published article and save its Excel data and Markdown analysis',
|
|
29
32
|
strategy: Strategy.INTERCEPT,
|
|
30
33
|
browser: true,
|
|
31
34
|
navigateBefore: false,
|
|
32
35
|
args: [
|
|
33
36
|
{ name: 'query', positional: true, required: true, help: 'Exact article URL or title text' },
|
|
34
37
|
{ name: 'date', help: 'Optional publication date in YYYY-MM-DD' },
|
|
35
|
-
{ name: 'output', default: './weixin-publish-data', help: 'Directory for generated Markdown reports' },
|
|
38
|
+
{ name: 'output', default: './weixin-publish-data', help: 'Directory for generated Excel data and Markdown reports' },
|
|
36
39
|
{ name: 'max-pages', type: 'int', default: 5, help: 'Maximum published-record pages to scan' },
|
|
37
40
|
{ name: 'timeout', type: 'int', default: 60, help: 'Maximum seconds for page capture' },
|
|
38
41
|
],
|
|
@@ -64,31 +67,35 @@ export const downloadPublishDataCommand = cli({
|
|
|
64
67
|
};
|
|
65
68
|
const secrets = buildSecretSet({ token, cookie });
|
|
66
69
|
|
|
70
|
+
let dataResult = null;
|
|
71
|
+
let markdownResult = null;
|
|
72
|
+
const errors = [];
|
|
67
73
|
try {
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
size: result.size, error: null }];
|
|
72
|
-
} catch (downloadError) {
|
|
73
|
-
const downloadMessage = sanitizedError(downloadError, secrets, 'Excel download failed');
|
|
74
|
-
try {
|
|
75
|
-
const result = await collectPublishAnalysis(page, {
|
|
76
|
-
...commonOptions,
|
|
77
|
-
publishedAt: record.publishedAt,
|
|
78
|
-
});
|
|
79
|
-
return [{ title: record.title, publishedAt: record.publishedAt, url: record.url,
|
|
80
|
-
status: 'saved', markdownPath: result.path, dataPath: null,
|
|
81
|
-
size: result.size, error: downloadMessage }];
|
|
82
|
-
} catch (analysisError) {
|
|
83
|
-
const analysisMessage = sanitizedError(
|
|
84
|
-
analysisError,
|
|
85
|
-
secrets,
|
|
86
|
-
'Markdown fallback failed',
|
|
87
|
-
);
|
|
88
|
-
return [{ title: record.title, publishedAt: record.publishedAt, url: record.url,
|
|
89
|
-
status: 'failed', markdownPath: null, dataPath: null, size: null,
|
|
90
|
-
error: `Excel download failed: ${downloadMessage}; Markdown fallback failed: ${analysisMessage}` }];
|
|
91
|
-
}
|
|
74
|
+
dataResult = await downloadPublishData(page, commonOptions);
|
|
75
|
+
} catch (error) {
|
|
76
|
+
errors.push(`Excel download failed: ${sanitizedError(error, secrets, 'Excel download failed')}`);
|
|
92
77
|
}
|
|
78
|
+
try {
|
|
79
|
+
markdownResult = await collectPublishAnalysis(page, {
|
|
80
|
+
...commonOptions,
|
|
81
|
+
publishedAt: record.publishedAt,
|
|
82
|
+
});
|
|
83
|
+
} catch (error) {
|
|
84
|
+
errors.push(`Markdown analysis failed: ${sanitizedError(error, secrets, 'Markdown analysis failed')}`);
|
|
85
|
+
}
|
|
86
|
+
|
|
87
|
+
const status = dataResult && markdownResult ? 'downloaded'
|
|
88
|
+
: dataResult || markdownResult ? 'partial' : 'failed';
|
|
89
|
+
return [{
|
|
90
|
+
title: record.title,
|
|
91
|
+
publishedAt: record.publishedAt,
|
|
92
|
+
url: record.url,
|
|
93
|
+
status,
|
|
94
|
+
markdownPath: markdownResult?.path ?? null,
|
|
95
|
+
markdownSize: markdownResult?.size ?? null,
|
|
96
|
+
dataPath: dataResult?.path ?? null,
|
|
97
|
+
dataSize: dataResult?.size ?? null,
|
|
98
|
+
error: errors.length > 0 ? errors.join('; ') : null,
|
|
99
|
+
}];
|
|
93
100
|
},
|
|
94
101
|
});
|
package/clis/weixin/download.js
CHANGED
|
@@ -8,7 +8,7 @@
|
|
|
8
8
|
*/
|
|
9
9
|
import { cli, Strategy } from '@sovovs/bycli/registry';
|
|
10
10
|
import { downloadArticle } from '@sovovs/bycli/download/article-download';
|
|
11
|
-
import { AuthRequiredError } from '@sovovs/bycli/errors';
|
|
11
|
+
import { ArgumentError, AuthRequiredError, CommandExecutionError } from '@sovovs/bycli/errors';
|
|
12
12
|
import { buildExtractWechatArticleContentJs } from './_wechat/article-content.js';
|
|
13
13
|
export { extractWechatArticleContent } from './_wechat/article-content.js';
|
|
14
14
|
// ============================================================
|
|
@@ -49,6 +49,66 @@ export function normalizeWechatUrl(raw) {
|
|
|
49
49
|
}
|
|
50
50
|
return s;
|
|
51
51
|
}
|
|
52
|
+
|
|
53
|
+
function isStrictHttpsUrl(raw, hostname, pathname) {
|
|
54
|
+
try {
|
|
55
|
+
const url = new URL(raw);
|
|
56
|
+
return url.protocol === 'https:' && url.hostname === hostname
|
|
57
|
+
&& url.port === '' && url.username === '' && url.password === ''
|
|
58
|
+
&& pathname(url.pathname);
|
|
59
|
+
}
|
|
60
|
+
catch {
|
|
61
|
+
return false;
|
|
62
|
+
}
|
|
63
|
+
}
|
|
64
|
+
|
|
65
|
+
export function isTrustedWechatArticleUrl(raw) {
|
|
66
|
+
return isStrictHttpsUrl(raw, 'mp.weixin.qq.com', path => path === '/s' || path.startsWith('/s/'));
|
|
67
|
+
}
|
|
68
|
+
|
|
69
|
+
export function isTrustedSogouRedirectUrl(raw) {
|
|
70
|
+
return isStrictHttpsUrl(raw, 'weixin.sogou.com', path => path === '/link');
|
|
71
|
+
}
|
|
72
|
+
|
|
73
|
+
export async function resolveWechatDownloadUrl(page, rawUrl) {
|
|
74
|
+
const sourceUrl = normalizeWechatUrl(rawUrl);
|
|
75
|
+
if (isTrustedWechatArticleUrl(sourceUrl)) {
|
|
76
|
+
return { sourceUrl, resolvedUrl: sourceUrl, alreadyNavigated: false };
|
|
77
|
+
}
|
|
78
|
+
if (!isTrustedSogouRedirectUrl(sourceUrl)) {
|
|
79
|
+
throw new ArgumentError(
|
|
80
|
+
'A trusted WeChat article or Sogou Weixin result URL is required.',
|
|
81
|
+
'Pass an https://mp.weixin.qq.com/s/... or https://weixin.sogou.com/link?... URL.',
|
|
82
|
+
);
|
|
83
|
+
}
|
|
84
|
+
try {
|
|
85
|
+
await page.goto(sourceUrl);
|
|
86
|
+
await page.wait(2);
|
|
87
|
+
const result = await page.evaluate(`(() => ({
|
|
88
|
+
finalUrl: window.location.href,
|
|
89
|
+
pageText: document.body ? document.body.innerText : '',
|
|
90
|
+
html: document.documentElement ? document.documentElement.innerHTML : '',
|
|
91
|
+
}))()`);
|
|
92
|
+
const text = `${result?.pageText || ''} ${result?.html || ''}`;
|
|
93
|
+
if (/验证码|安全验证|异常访问|访问过于频繁|请输入验证码/.test(text)) {
|
|
94
|
+
throw new AuthRequiredError(
|
|
95
|
+
'weixin.sogou.com',
|
|
96
|
+
'Sogou Weixin requires verification. Complete it in the open browser tab and run the command again.',
|
|
97
|
+
);
|
|
98
|
+
}
|
|
99
|
+
if (!isTrustedWechatArticleUrl(result?.finalUrl)) {
|
|
100
|
+
throw new CommandExecutionError(
|
|
101
|
+
'Sogou Weixin did not resolve to a trusted WeChat article URL',
|
|
102
|
+
'Open the search result in a browser and confirm it redirects to mp.weixin.qq.com/s/... before retrying.',
|
|
103
|
+
);
|
|
104
|
+
}
|
|
105
|
+
return { sourceUrl, resolvedUrl: new URL(result.finalUrl).href, alreadyNavigated: true };
|
|
106
|
+
}
|
|
107
|
+
catch (error) {
|
|
108
|
+
if (error instanceof AuthRequiredError || error instanceof CommandExecutionError) throw error;
|
|
109
|
+
throw new CommandExecutionError('Failed to resolve the Sogou Weixin result URL');
|
|
110
|
+
}
|
|
111
|
+
}
|
|
52
112
|
/**
|
|
53
113
|
* Format a WeChat article timestamp as a UTC+8 datetime string.
|
|
54
114
|
* Accepts either Unix seconds or milliseconds.
|
|
@@ -183,15 +243,12 @@ cli({
|
|
|
183
243
|
{ name: 'output', default: './weixin-articles', help: 'Output directory' },
|
|
184
244
|
{ name: 'download-images', type: 'boolean', default: true, help: 'Download images locally' },
|
|
185
245
|
],
|
|
186
|
-
columns: ['title', 'author', 'publish_time', 'status', 'size', 'saved'],
|
|
246
|
+
columns: ['title', 'author', 'publish_time', 'status', 'size', 'saved', 'source_url', 'resolved_url'],
|
|
187
247
|
func: async (page, kwargs) => {
|
|
188
|
-
const
|
|
189
|
-
|
|
190
|
-
if (!
|
|
191
|
-
|
|
192
|
-
}
|
|
193
|
-
// Navigate and wait for content to load
|
|
194
|
-
await page.goto(url);
|
|
248
|
+
const { sourceUrl, resolvedUrl, alreadyNavigated } = await resolveWechatDownloadUrl(page, kwargs.url);
|
|
249
|
+
// Navigate and wait for content to load. Sogou resolution already lands on the article.
|
|
250
|
+
if (!alreadyNavigated)
|
|
251
|
+
await page.goto(resolvedUrl);
|
|
195
252
|
await page.wait(5);
|
|
196
253
|
// Extract article data in browser context
|
|
197
254
|
const data = await page.evaluate(`
|
|
@@ -255,11 +312,11 @@ cli({
|
|
|
255
312
|
'WeChat article page requires environment verification. Complete it in the open browser tab and run the command again.',
|
|
256
313
|
);
|
|
257
314
|
}
|
|
258
|
-
|
|
315
|
+
const rows = await downloadArticle({
|
|
259
316
|
title: data?.title || '',
|
|
260
317
|
author: data?.author,
|
|
261
318
|
publishTime: data?.publishTime,
|
|
262
|
-
sourceUrl:
|
|
319
|
+
sourceUrl: resolvedUrl,
|
|
263
320
|
contentHtml: data?.contentHtml || '',
|
|
264
321
|
codeBlocks: data?.codeBlocks,
|
|
265
322
|
imageUrls: data?.imageUrls,
|
|
@@ -274,5 +331,6 @@ cli({
|
|
|
274
331
|
},
|
|
275
332
|
secureMarkdown: true,
|
|
276
333
|
});
|
|
334
|
+
return rows.map(row => ({ ...row, source_url: sourceUrl, resolved_url: resolvedUrl }));
|
|
277
335
|
},
|
|
278
336
|
});
|
|
@@ -2,8 +2,9 @@ import { ArgumentError, AuthRequiredError, CommandExecutionError } from '@sovovs
|
|
|
2
2
|
import { MAX_WECHAT_HTML_BYTES } from '@sovovs/bycli/download/wechat-article';
|
|
3
3
|
import { cli, Strategy } from '@sovovs/bycli/registry';
|
|
4
4
|
import { readEnvironmentCredentials, resolveBrowserCredentials } from './_wechat/auth-session.js';
|
|
5
|
+
import { createArticleIndexFetcher } from './_wechat/article-index.js';
|
|
5
6
|
import {
|
|
6
|
-
callCrawler, collectArticles,
|
|
7
|
+
callCrawler, collectArticles, isTrustedWechatArticleUrl, saveArticles,
|
|
7
8
|
} from './_wechat/crawler-runtime.js';
|
|
8
9
|
import { readAuthSource } from './_wechat/args.js';
|
|
9
10
|
import { wechatArticleToMarkdown } from './_wechat/markdown.js';
|
|
@@ -167,8 +168,8 @@ export const saveArticlesCommand = cli({
|
|
|
167
168
|
const credentials = authSource === 'env'
|
|
168
169
|
? readEnvironmentCredentials(false) : await resolveBrowserCredentials(page);
|
|
169
170
|
const articleHtmlDownloader = createArticleHtmlDownloader({ authSource, page });
|
|
171
|
+
const fetchPage = createArticleIndexFetcher({ page, source: authSource, credentials });
|
|
170
172
|
const rows = await callCrawler(async () => {
|
|
171
|
-
const { fetchPage } = createWechatApi(credentials);
|
|
172
173
|
const { articles } = await collectArticles({ fakeid, fetchPage, limit: args.limit, maxPages: args['max-pages'] });
|
|
173
174
|
return saveArticles({
|
|
174
175
|
articles, accountName: String(args.name ?? '').trim(),
|
package/clis/weixin/search.js
CHANGED
|
@@ -11,14 +11,14 @@ function normalizePositiveInteger(value, name, defaultValue, maxValue) {
|
|
|
11
11
|
const text = String(value).trim();
|
|
12
12
|
if (!/^\d+$/.test(text)) {
|
|
13
13
|
throw new ArgumentError(
|
|
14
|
-
`weixin
|
|
14
|
+
`weixin sougousearch --${name} must be a positive integer`,
|
|
15
15
|
`Pass --${name} as a whole number${maxValue ? ` from 1 to ${maxValue}` : ' greater than 0'}.`,
|
|
16
16
|
);
|
|
17
17
|
}
|
|
18
18
|
const parsed = Number(text);
|
|
19
19
|
if (!Number.isSafeInteger(parsed) || parsed < 1 || (maxValue && parsed > maxValue)) {
|
|
20
20
|
throw new ArgumentError(
|
|
21
|
-
`weixin
|
|
21
|
+
`weixin sougousearch --${name} is out of range`,
|
|
22
22
|
`Pass --${name} as a whole number${maxValue ? ` from 1 to ${maxValue}` : ' greater than 0'}.`,
|
|
23
23
|
);
|
|
24
24
|
}
|
|
@@ -69,9 +69,11 @@ function buildExtractSearchResultsEvaluate() {
|
|
|
69
69
|
const extracted = cards.map((item) => {
|
|
70
70
|
const linkEl = item.querySelector('h3 a[href]');
|
|
71
71
|
const summaryEl = item.querySelector('p.txt-info');
|
|
72
|
+
const accountEl = item.querySelector('.s-p .all-time-y2');
|
|
72
73
|
const timeEl = item.querySelector('.s-p .s2');
|
|
73
74
|
return {
|
|
74
75
|
title: clean(linkEl && linkEl.textContent),
|
|
76
|
+
account: clean(accountEl && accountEl.textContent),
|
|
75
77
|
url: absolutize(linkEl && linkEl.getAttribute('href')),
|
|
76
78
|
summary: clean(summaryEl && summaryEl.textContent),
|
|
77
79
|
publish_time: clean(timeEl && timeEl.textContent),
|
|
@@ -90,7 +92,7 @@ function buildExtractSearchResultsEvaluate() {
|
|
|
90
92
|
|
|
91
93
|
export const weixinSearchCommand = cli({
|
|
92
94
|
site: 'weixin',
|
|
93
|
-
name: '
|
|
95
|
+
name: 'sougousearch',
|
|
94
96
|
access: 'read',
|
|
95
97
|
description: '使用搜狗微信搜索公众号文章;如需导出正文 Markdown,请使用 weixin download 处理公众号文章链接',
|
|
96
98
|
domain: SOGOU_WEIXIN_DOMAIN,
|
|
@@ -101,7 +103,7 @@ export const weixinSearchCommand = cli({
|
|
|
101
103
|
{ name: 'page', type: 'int', default: 1, help: '结果页码,从 1 开始' },
|
|
102
104
|
{ name: 'limit', type: 'int', default: 10, help: '返回条数,最大 10' },
|
|
103
105
|
],
|
|
104
|
-
columns: ['rank', 'page', 'title', 'url', 'summary', 'publish_time'],
|
|
106
|
+
columns: ['rank', 'page', 'title', 'account', 'url', 'summary', 'publish_time'],
|
|
105
107
|
func: async (page, kwargs) => {
|
|
106
108
|
const query = String(kwargs.query ?? '').trim();
|
|
107
109
|
if (!query) {
|
|
@@ -119,11 +121,11 @@ export const weixinSearchCommand = cli({
|
|
|
119
121
|
payload = await page.evaluate(buildExtractSearchResultsEvaluate());
|
|
120
122
|
} catch (error) {
|
|
121
123
|
const detail = error instanceof Error ? error.message : String(error);
|
|
122
|
-
throw new CommandExecutionError('weixin
|
|
124
|
+
throw new CommandExecutionError('weixin sougousearch failed while loading Sogou results', detail);
|
|
123
125
|
}
|
|
124
126
|
|
|
125
127
|
if (!payload || typeof payload !== 'object' || !Array.isArray(payload.rows)) {
|
|
126
|
-
throw new CommandExecutionError('weixin
|
|
128
|
+
throw new CommandExecutionError('weixin sougousearch returned an unreadable browser payload', 'Sogou Weixin may have changed its result page structure.');
|
|
127
129
|
}
|
|
128
130
|
if (payload.blocked) {
|
|
129
131
|
throw new CommandExecutionError('Sogou Weixin blocked this search request', 'Open weixin.sogou.com in Chrome and complete any verification before retrying.');
|
|
@@ -134,16 +136,17 @@ export const weixinSearchCommand = cli({
|
|
|
134
136
|
|
|
135
137
|
const rows = payload.rows;
|
|
136
138
|
if (rows.length === 0 && payload.empty) {
|
|
137
|
-
throw new EmptyResultError('weixin
|
|
139
|
+
throw new EmptyResultError('weixin sougousearch', 'Try a different keyword or a different page number.');
|
|
138
140
|
}
|
|
139
141
|
if (rows.length === 0) {
|
|
140
|
-
throw new CommandExecutionError('weixin
|
|
142
|
+
throw new CommandExecutionError('weixin sougousearch did not expose article result cards', 'Sogou Weixin may have changed its selectors or returned a transient shell page.');
|
|
141
143
|
}
|
|
142
144
|
|
|
143
145
|
return rows.slice(0, limit).map((row, index) => ({
|
|
144
146
|
rank: (pageNo - 1) * 10 + index + 1,
|
|
145
147
|
page: pageNo,
|
|
146
148
|
title: row.title,
|
|
149
|
+
account: row.account,
|
|
147
150
|
url: row.url,
|
|
148
151
|
summary: row.summary,
|
|
149
152
|
publish_time: row.publish_time,
|