@sovovs/bycli 2.1.24 → 2.1.26
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/cli-manifest.json +8 -4
- package/clis/weixin/_wechat/article-fallback-policy.js +46 -0
- package/clis/weixin/_wechat/article-index.js +14 -0
- package/clis/weixin/_wechat/article-link.js +88 -0
- package/clis/weixin/_wechat/publish-records.js +14 -3
- package/clis/weixin/_wechat/sogou-fallback.js +265 -0
- package/clis/weixin/_wechat/sogou-search.js +178 -0
- package/clis/weixin/articles.js +37 -7
- package/clis/weixin/create-draft.js +128 -43
- package/clis/weixin/download-publish-data.js +46 -4
- package/clis/weixin/download.js +14 -100
- package/clis/weixin/save-articles.js +53 -10
- package/clis/weixin/search.js +9 -113
- package/package.json +1 -1
package/clis/weixin/articles.js
CHANGED
|
@@ -1,9 +1,15 @@
|
|
|
1
1
|
import { ArgumentError, EmptyResultError } from '@sovovs/bycli/errors';
|
|
2
2
|
import { cli, Strategy } from '@sovovs/bycli/registry';
|
|
3
3
|
import { readEnvironmentCredentials, resolveBrowserCredentials } from './_wechat/auth-session.js';
|
|
4
|
+
import {
|
|
5
|
+
combineArticleFallbackErrors,
|
|
6
|
+
isEligibleArticleFallbackError,
|
|
7
|
+
withMissingFallbackName,
|
|
8
|
+
} from './_wechat/article-fallback-policy.js';
|
|
4
9
|
import { createArticleIndexFetcher } from './_wechat/article-index.js';
|
|
5
10
|
import { callCrawler, collectArticles } from './_wechat/crawler-runtime.js';
|
|
6
11
|
import { readAuthSource } from './_wechat/args.js';
|
|
12
|
+
import { collectSogouAccountArticles } from './_wechat/sogou-fallback.js';
|
|
7
13
|
|
|
8
14
|
const DOMAIN = 'mp.weixin.qq.com';
|
|
9
15
|
const browserRequired = args => readAuthSource(args) === 'browser';
|
|
@@ -14,10 +20,10 @@ export const articlesCommand = cli({
|
|
|
14
20
|
strategy: Strategy.COOKIE, browser: browserRequired,
|
|
15
21
|
args: [
|
|
16
22
|
{ name: 'fakeid', positional: true, required: true, help: 'Official-account fakeid returned by weixin accounts' },
|
|
17
|
-
{ name: 'name', help: '
|
|
23
|
+
{ name: 'name', help: 'Official-account name; exact case-insensitive match required for browser Sogou fallback' }, { name: 'limit', type: 'int', help: 'Maximum number of articles to return' }, { name: 'max-pages', type: 'int', help: 'Maximum number of history pages to scan' },
|
|
18
24
|
{ name: 'auth-source', default: 'browser', choices: ['browser', 'env'], help: 'Credential source: browser session or environment variables' },
|
|
19
25
|
],
|
|
20
|
-
columns: ['title', 'author', 'digest', 'publishedAt', 'url'],
|
|
26
|
+
columns: ['title', 'author', 'digest', 'publishedAt', 'url', 'source', 'coverage'],
|
|
21
27
|
func: async (page, args) => {
|
|
22
28
|
const fakeid = String(args.fakeid ?? '').trim();
|
|
23
29
|
if (!fakeid) throw new ArgumentError('fakeid is required');
|
|
@@ -25,13 +31,37 @@ export const articlesCommand = cli({
|
|
|
25
31
|
const credentials = authSource === 'env'
|
|
26
32
|
? readEnvironmentCredentials(false) : await resolveBrowserCredentials(page);
|
|
27
33
|
const fetchPage = createArticleIndexFetcher({ page, source: authSource, credentials });
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
34
|
+
let articles;
|
|
35
|
+
let source = 'wechat';
|
|
36
|
+
let coverage = null;
|
|
37
|
+
try {
|
|
38
|
+
const result = await callCrawler(() => collectArticles({
|
|
39
|
+
fakeid, fetchPage, limit: args.limit, maxPages: args['max-pages'],
|
|
40
|
+
}));
|
|
41
|
+
articles = result.articles;
|
|
42
|
+
if (articles.length === 0) {
|
|
43
|
+
throw new EmptyResultError('weixin articles', `No published articles were found for ${fakeid}.`);
|
|
44
|
+
}
|
|
45
|
+
} catch (primaryError) {
|
|
46
|
+
if (authSource !== 'browser' || !isEligibleArticleFallbackError(primaryError)) throw primaryError;
|
|
47
|
+
const accountName = String(args.name ?? '').trim();
|
|
48
|
+
if (!accountName) throw withMissingFallbackName('weixin articles', primaryError);
|
|
49
|
+
try {
|
|
50
|
+
const fallback = await collectSogouAccountArticles({
|
|
51
|
+
page, accountName, limit: args.limit, maxPages: args['max-pages'], freshPage: true,
|
|
52
|
+
});
|
|
53
|
+
articles = fallback.articles;
|
|
54
|
+
source = fallback.source;
|
|
55
|
+
coverage = fallback.coverage;
|
|
56
|
+
} catch (fallbackError) {
|
|
57
|
+
throw combineArticleFallbackErrors({
|
|
58
|
+
operation: 'weixin articles', primaryError, fallbackError, credentials,
|
|
59
|
+
});
|
|
60
|
+
}
|
|
61
|
+
}
|
|
32
62
|
return articles.map(article => ({
|
|
33
63
|
title: article.title, author: article.author || null, digest: article.digest || null,
|
|
34
|
-
publishedAt: article.publishedAt || null, url: article.url,
|
|
64
|
+
publishedAt: article.publishedAt || null, url: article.url, source, coverage,
|
|
35
65
|
}));
|
|
36
66
|
},
|
|
37
67
|
});
|
|
@@ -1,8 +1,61 @@
|
|
|
1
|
+
import * as nodeFs from 'node:fs';
|
|
2
|
+
import * as nodePath from 'node:path';
|
|
1
3
|
import { cli, Strategy } from '@sovovs/bycli/registry';
|
|
2
|
-
import { CommandExecutionError } from '@sovovs/bycli/errors';
|
|
4
|
+
import { ArgumentError, AuthRequiredError, CommandExecutionError } from '@sovovs/bycli/errors';
|
|
3
5
|
|
|
4
6
|
const WEIXIN_DOMAIN = 'mp.weixin.qq.com';
|
|
5
7
|
const WEIXIN_HOME = 'https://mp.weixin.qq.com/';
|
|
8
|
+
const MAX_TITLE_LENGTH = 64;
|
|
9
|
+
const MAX_AUTHOR_LENGTH = 8;
|
|
10
|
+
const SUPPORTED_COVER_EXTENSIONS = new Set(['.jpg', '.jpeg', '.png', '.gif', '.webp']);
|
|
11
|
+
|
|
12
|
+
function codePointLength(value) {
|
|
13
|
+
return [...value].length;
|
|
14
|
+
}
|
|
15
|
+
|
|
16
|
+
function requiredText(value, name) {
|
|
17
|
+
const text = String(value ?? '').trim();
|
|
18
|
+
if (!text) throw new ArgumentError(`${name} must not be empty`);
|
|
19
|
+
return text;
|
|
20
|
+
}
|
|
21
|
+
|
|
22
|
+
function validateCoverImage(value) {
|
|
23
|
+
if (value === undefined || value === null) return null;
|
|
24
|
+
const coverPath = nodePath.resolve(requiredText(value, 'cover-image'));
|
|
25
|
+
const extension = nodePath.extname(coverPath).toLowerCase();
|
|
26
|
+
if (!SUPPORTED_COVER_EXTENSIONS.has(extension)) {
|
|
27
|
+
throw new ArgumentError('cover-image must be a jpg, jpeg, png, gif, or webp file');
|
|
28
|
+
}
|
|
29
|
+
const info = nodeFs.statSync(coverPath, { throwIfNoEntry: false });
|
|
30
|
+
if (!info?.isFile() || info.size <= 0) {
|
|
31
|
+
throw new ArgumentError(`cover-image must be a readable non-empty file: ${coverPath}`);
|
|
32
|
+
}
|
|
33
|
+
try {
|
|
34
|
+
nodeFs.accessSync(coverPath, nodeFs.constants.R_OK);
|
|
35
|
+
} catch {
|
|
36
|
+
throw new ArgumentError(`cover-image must be readable: ${coverPath}`);
|
|
37
|
+
}
|
|
38
|
+
return coverPath;
|
|
39
|
+
}
|
|
40
|
+
|
|
41
|
+
function normalizeCreateDraftArgs(kwargs) {
|
|
42
|
+
const title = requiredText(kwargs.title, 'title');
|
|
43
|
+
requiredText(kwargs.content, 'content');
|
|
44
|
+
if (codePointLength(title) > MAX_TITLE_LENGTH) {
|
|
45
|
+
throw new ArgumentError(`title must be at most ${MAX_TITLE_LENGTH} characters`);
|
|
46
|
+
}
|
|
47
|
+
const author = kwargs.author == null ? null : requiredText(kwargs.author, 'author');
|
|
48
|
+
if (author && codePointLength(author) > MAX_AUTHOR_LENGTH) {
|
|
49
|
+
throw new ArgumentError(`author must be at most ${MAX_AUTHOR_LENGTH} characters`);
|
|
50
|
+
}
|
|
51
|
+
return {
|
|
52
|
+
title,
|
|
53
|
+
content: String(kwargs.content),
|
|
54
|
+
author,
|
|
55
|
+
summary: kwargs.summary == null ? null : String(kwargs.summary).trim(),
|
|
56
|
+
coverImage: validateCoverImage(kwargs['cover-image']),
|
|
57
|
+
};
|
|
58
|
+
}
|
|
6
59
|
|
|
7
60
|
async function getToken(page) {
|
|
8
61
|
return page.evaluate(`(window.location.href.match(/token=(\\d+)/)||[])[1]`);
|
|
@@ -13,13 +66,19 @@ async function navigateToEditor(page) {
|
|
|
13
66
|
await page.wait(3);
|
|
14
67
|
const token = await getToken(page);
|
|
15
68
|
if (!token) {
|
|
16
|
-
throw new
|
|
69
|
+
throw new AuthRequiredError(
|
|
70
|
+
WEIXIN_DOMAIN,
|
|
71
|
+
'Could not extract session token. Please log in to mp.weixin.qq.com',
|
|
72
|
+
);
|
|
17
73
|
}
|
|
18
74
|
await page.goto(`https://mp.weixin.qq.com/cgi-bin/appmsg?t=media/appmsg_edit_v2&action=edit&isNew=1&type=77&token=${token}&lang=zh_CN`);
|
|
19
75
|
await page.wait(4);
|
|
20
76
|
const hasTitle = await page.evaluate('!!document.querySelector("textarea#title")');
|
|
21
77
|
if (!hasTitle) {
|
|
22
|
-
throw new
|
|
78
|
+
throw new AuthRequiredError(
|
|
79
|
+
WEIXIN_DOMAIN,
|
|
80
|
+
'Article editor did not load. Session may have expired',
|
|
81
|
+
);
|
|
23
82
|
}
|
|
24
83
|
}
|
|
25
84
|
|
|
@@ -27,15 +86,18 @@ async function fillField(page, selector, value) {
|
|
|
27
86
|
return page.evaluate(`(() => {
|
|
28
87
|
var el = document.querySelector('${selector}');
|
|
29
88
|
if (!el) return { ok: false, reason: 'not found: ${selector}' };
|
|
89
|
+
var expected = ${JSON.stringify(value)};
|
|
30
90
|
el.focus();
|
|
31
91
|
var proto = el.tagName === 'TEXTAREA' ? HTMLTextAreaElement.prototype : HTMLInputElement.prototype;
|
|
32
92
|
var setter = Object.getOwnPropertyDescriptor(proto, 'value');
|
|
33
|
-
if (setter && setter.set) setter.set.call(el,
|
|
34
|
-
else el.value =
|
|
35
|
-
el.dispatchEvent(new InputEvent('input', { bubbles: true, data:
|
|
93
|
+
if (setter && setter.set) setter.set.call(el, expected);
|
|
94
|
+
else el.value = expected;
|
|
95
|
+
el.dispatchEvent(new InputEvent('input', { bubbles: true, data: expected }));
|
|
36
96
|
el.dispatchEvent(new Event('change', { bubbles: true }));
|
|
37
97
|
el.blur();
|
|
38
|
-
return
|
|
98
|
+
return el.value === expected
|
|
99
|
+
? { ok: true, value: el.value }
|
|
100
|
+
: { ok: false, reason: 'value mismatch', value: el.value };
|
|
39
101
|
})()`);
|
|
40
102
|
}
|
|
41
103
|
|
|
@@ -49,10 +111,21 @@ async function fillContent(page, text) {
|
|
|
49
111
|
document.execCommand('selectAll', false, null);
|
|
50
112
|
document.execCommand('insertText', false, ${JSON.stringify(text)});
|
|
51
113
|
editor.dispatchEvent(new InputEvent('input', { bubbles: true }));
|
|
52
|
-
|
|
114
|
+
var normalize = value => String(value ?? '').replace(/\\r\\n?/g, '\\n').trim();
|
|
115
|
+
var expected = normalize(${JSON.stringify(text)});
|
|
116
|
+
var actual = normalize(editor.innerText ?? editor.textContent ?? '');
|
|
117
|
+
return actual === expected
|
|
118
|
+
? { ok: true, value: actual }
|
|
119
|
+
: { ok: false, reason: 'value mismatch', value: actual };
|
|
53
120
|
})()`);
|
|
54
121
|
}
|
|
55
122
|
|
|
123
|
+
function requirePageResult(result, label) {
|
|
124
|
+
if (!result?.ok) {
|
|
125
|
+
throw new CommandExecutionError(`Failed to fill ${label}: ${result?.reason ?? 'unverified page state'}`);
|
|
126
|
+
}
|
|
127
|
+
}
|
|
128
|
+
|
|
56
129
|
async function uploadContentImage(page, imagePath) {
|
|
57
130
|
const fs = await import('node:fs');
|
|
58
131
|
const path = await import('node:path');
|
|
@@ -76,15 +149,20 @@ async function uploadContentImage(page, imagePath) {
|
|
|
76
149
|
await page.wait(1);
|
|
77
150
|
|
|
78
151
|
await page.setFileInput([absPath], 'input[type="file"][name="file"]');
|
|
79
|
-
await page.wait(8);
|
|
80
152
|
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
|
|
153
|
+
for (let attempt = 0; attempt < 15; attempt++) {
|
|
154
|
+
await page.wait(1);
|
|
155
|
+
const cdnCount = await page.evaluate(`(() => {
|
|
156
|
+
var editors = document.querySelectorAll('#ueditor_0, div[contenteditable="true"]');
|
|
157
|
+
var count = 0;
|
|
158
|
+
editors.forEach(function(editor) {
|
|
159
|
+
count += editor.querySelectorAll('img[src*="mmbiz"], img[data-src*="mmbiz"]').length;
|
|
160
|
+
});
|
|
161
|
+
return count;
|
|
162
|
+
})()`);
|
|
163
|
+
if (cdnCount > 0) return;
|
|
87
164
|
}
|
|
165
|
+
throw new CommandExecutionError('Image did not upload to WeChat CDN');
|
|
88
166
|
}
|
|
89
167
|
|
|
90
168
|
async function selectCoverFromContent(page) {
|
|
@@ -134,18 +212,23 @@ async function selectCoverFromContent(page) {
|
|
|
134
212
|
if (btns[i].textContent.trim() === '确认' && btns[i].offsetHeight > 0 && !btns[i].disabled) { btns[i].click(); return; }
|
|
135
213
|
}
|
|
136
214
|
})()`);
|
|
137
|
-
|
|
138
|
-
|
|
139
|
-
|
|
140
|
-
|
|
141
|
-
|
|
142
|
-
|
|
143
|
-
|
|
144
|
-
|
|
145
|
-
|
|
146
|
-
|
|
147
|
-
|
|
148
|
-
|
|
215
|
+
for (let attempt = 0; attempt < 15; attempt++) {
|
|
216
|
+
await page.wait(1);
|
|
217
|
+
const hasCover = await page.evaluate(`(() => {
|
|
218
|
+
var areas = document.querySelectorAll('#js_cover_area, #js_cover_description_area, #appmsgItem');
|
|
219
|
+
var found = false;
|
|
220
|
+
areas.forEach(function(area) {
|
|
221
|
+
if (area.querySelector('img[src*="mmbiz"], img[data-src*="mmbiz"]')) found = true;
|
|
222
|
+
[area].concat(Array.from(area.querySelectorAll('*'))).forEach(function(el) {
|
|
223
|
+
var bg = window.getComputedStyle(el).backgroundImage;
|
|
224
|
+
if (bg && bg.includes('mmbiz')) found = true;
|
|
225
|
+
});
|
|
226
|
+
});
|
|
227
|
+
return found;
|
|
228
|
+
})()`);
|
|
229
|
+
if (hasCover) return true;
|
|
230
|
+
}
|
|
231
|
+
return false;
|
|
149
232
|
}
|
|
150
233
|
|
|
151
234
|
async function clickSaveDraft(page) {
|
|
@@ -167,7 +250,7 @@ async function clickSaveDraft(page) {
|
|
|
167
250
|
})()`);
|
|
168
251
|
if (saved) return true;
|
|
169
252
|
}
|
|
170
|
-
|
|
253
|
+
throw new CommandExecutionError('Draft save could not be confirmed');
|
|
171
254
|
}
|
|
172
255
|
|
|
173
256
|
export const createDraftCommand = cli({
|
|
@@ -190,37 +273,39 @@ export const createDraftCommand = cli({
|
|
|
190
273
|
columns: ['status', 'detail'],
|
|
191
274
|
|
|
192
275
|
func: async (page, kwargs) => {
|
|
276
|
+
const args = normalizeCreateDraftArgs(kwargs);
|
|
193
277
|
await navigateToEditor(page);
|
|
194
278
|
|
|
195
|
-
const titleResult = await fillField(page, 'textarea#title',
|
|
196
|
-
|
|
279
|
+
const titleResult = await fillField(page, 'textarea#title', args.title);
|
|
280
|
+
requirePageResult(titleResult, 'title');
|
|
197
281
|
|
|
198
|
-
if (
|
|
199
|
-
const authorResult = await fillField(page, 'input#author',
|
|
200
|
-
|
|
282
|
+
if (args.author) {
|
|
283
|
+
const authorResult = await fillField(page, 'input#author', args.author);
|
|
284
|
+
requirePageResult(authorResult, 'author');
|
|
201
285
|
}
|
|
202
286
|
|
|
203
|
-
const contentResult = await fillContent(page,
|
|
204
|
-
|
|
287
|
+
const contentResult = await fillContent(page, args.content);
|
|
288
|
+
requirePageResult(contentResult, 'content');
|
|
205
289
|
|
|
206
|
-
if (
|
|
207
|
-
await uploadContentImage(page,
|
|
290
|
+
if (args.coverImage) {
|
|
291
|
+
await uploadContentImage(page, args.coverImage);
|
|
208
292
|
const coverSet = await selectCoverFromContent(page);
|
|
209
293
|
if (!coverSet) {
|
|
210
|
-
|
|
294
|
+
throw new CommandExecutionError('Failed to set the requested cover image');
|
|
211
295
|
}
|
|
212
296
|
}
|
|
213
297
|
|
|
214
|
-
if (
|
|
215
|
-
await fillField(page, 'textarea#js_description',
|
|
298
|
+
if (args.summary) {
|
|
299
|
+
const summaryResult = await fillField(page, 'textarea#js_description', args.summary);
|
|
300
|
+
requirePageResult(summaryResult, 'summary');
|
|
216
301
|
}
|
|
217
302
|
|
|
218
303
|
await page.wait(1);
|
|
219
|
-
|
|
304
|
+
await clickSaveDraft(page);
|
|
220
305
|
|
|
221
306
|
return [{
|
|
222
|
-
status:
|
|
223
|
-
detail: `"${
|
|
307
|
+
status: 'draft saved',
|
|
308
|
+
detail: `"${args.title}"${args.author ? ` by ${args.author}` : ''}${args.coverImage ? ' (with cover)' : ''}`,
|
|
224
309
|
}];
|
|
225
310
|
},
|
|
226
311
|
});
|
|
@@ -1,4 +1,7 @@
|
|
|
1
|
-
import {
|
|
1
|
+
import { constants } from 'node:fs';
|
|
2
|
+
import { access, stat } from 'node:fs/promises';
|
|
3
|
+
import { extname, resolve } from 'node:path';
|
|
4
|
+
import { ArgumentError, CommandExecutionError } from '@sovovs/bycli/errors';
|
|
2
5
|
import { cli, Strategy } from '@sovovs/bycli/registry';
|
|
3
6
|
import { resolveBrowserCredentials } from './_wechat/auth-session.js';
|
|
4
7
|
import { buildSecretSet, redactText } from './_wechat/redact.js';
|
|
@@ -10,6 +13,7 @@ import {
|
|
|
10
13
|
matchPublishedRecord,
|
|
11
14
|
positiveSafeInteger,
|
|
12
15
|
validatePublishDate,
|
|
16
|
+
validatePublishedQuery,
|
|
13
17
|
} from './_wechat/publish-records.js';
|
|
14
18
|
|
|
15
19
|
const COLUMNS = [
|
|
@@ -23,6 +27,33 @@ function sanitizedError(error, secrets, fallback) {
|
|
|
23
27
|
.replace(/https?:\/\/mp\.weixin\.qq\.com\/\S*/giu, '[REDACTED]');
|
|
24
28
|
}
|
|
25
29
|
|
|
30
|
+
async function validateArtifact(result, { label, expectedStatus, expectedExtension }) {
|
|
31
|
+
if (!result || result.status !== expectedStatus) {
|
|
32
|
+
throw new CommandExecutionError(`${label} returned an invalid status`);
|
|
33
|
+
}
|
|
34
|
+
if (typeof result.path !== 'string' || !result.path.trim()) {
|
|
35
|
+
throw new CommandExecutionError(`${label} returned no output path`);
|
|
36
|
+
}
|
|
37
|
+
if (!Number.isSafeInteger(result.size) || result.size <= 0) {
|
|
38
|
+
throw new CommandExecutionError(`${label} returned an invalid size`);
|
|
39
|
+
}
|
|
40
|
+
const path = resolve(result.path);
|
|
41
|
+
if (extname(path).toLowerCase() !== expectedExtension) {
|
|
42
|
+
throw new CommandExecutionError(`${label} returned an unexpected file type`);
|
|
43
|
+
}
|
|
44
|
+
let info;
|
|
45
|
+
try {
|
|
46
|
+
await access(path, constants.R_OK);
|
|
47
|
+
info = await stat(path);
|
|
48
|
+
} catch {
|
|
49
|
+
throw new CommandExecutionError(`${label} returned an unreadable file`);
|
|
50
|
+
}
|
|
51
|
+
if (!info.isFile() || info.size <= 0 || info.size !== result.size) {
|
|
52
|
+
throw new CommandExecutionError(`${label} returned an unreadable or mismatched file`);
|
|
53
|
+
}
|
|
54
|
+
return { ...result, path, size: info.size };
|
|
55
|
+
}
|
|
56
|
+
|
|
26
57
|
export const downloadPublishDataCommand = cli({
|
|
27
58
|
site: 'weixin',
|
|
28
59
|
name: 'download-publish-data',
|
|
@@ -43,6 +74,7 @@ export const downloadPublishDataCommand = cli({
|
|
|
43
74
|
func: async (page, args) => {
|
|
44
75
|
const query = String(args.query ?? '').trim();
|
|
45
76
|
if (!query) throw new ArgumentError('query required');
|
|
77
|
+
const validatedQuery = validatePublishedQuery(query);
|
|
46
78
|
|
|
47
79
|
const timeoutSeconds = positiveSafeInteger(args.timeout, 'timeout', 60);
|
|
48
80
|
const maxPages = positiveSafeInteger(args['max-pages'], 'max-pages', 5);
|
|
@@ -56,7 +88,7 @@ export const downloadPublishDataCommand = cli({
|
|
|
56
88
|
maxPages,
|
|
57
89
|
timeout: timeoutSeconds,
|
|
58
90
|
});
|
|
59
|
-
const record = matchPublishedRecord(rows,
|
|
91
|
+
const record = matchPublishedRecord(rows, validatedQuery, validatedDate);
|
|
60
92
|
const detailUrl = buildDetailUrl(record, token);
|
|
61
93
|
const outputDir = args.output ?? './weixin-publish-data';
|
|
62
94
|
const commonOptions = {
|
|
@@ -71,15 +103,25 @@ export const downloadPublishDataCommand = cli({
|
|
|
71
103
|
let markdownResult = null;
|
|
72
104
|
const errors = [];
|
|
73
105
|
try {
|
|
74
|
-
|
|
106
|
+
const result = await downloadPublishData(page, commonOptions);
|
|
107
|
+
dataResult = await validateArtifact(result, {
|
|
108
|
+
label: 'Excel artifact',
|
|
109
|
+
expectedStatus: 'downloaded',
|
|
110
|
+
expectedExtension: '.xls',
|
|
111
|
+
});
|
|
75
112
|
} catch (error) {
|
|
76
113
|
errors.push(`Excel download failed: ${sanitizedError(error, secrets, 'Excel download failed')}`);
|
|
77
114
|
}
|
|
78
115
|
try {
|
|
79
|
-
|
|
116
|
+
const result = await collectPublishAnalysis(page, {
|
|
80
117
|
...commonOptions,
|
|
81
118
|
publishedAt: record.publishedAt,
|
|
82
119
|
});
|
|
120
|
+
markdownResult = await validateArtifact(result, {
|
|
121
|
+
label: 'Markdown artifact',
|
|
122
|
+
expectedStatus: 'saved',
|
|
123
|
+
expectedExtension: '.md',
|
|
124
|
+
});
|
|
83
125
|
} catch (error) {
|
|
84
126
|
errors.push(`Markdown analysis failed: ${sanitizedError(error, secrets, 'Markdown analysis failed')}`);
|
|
85
127
|
}
|
package/clis/weixin/download.js
CHANGED
|
@@ -8,107 +8,21 @@
|
|
|
8
8
|
*/
|
|
9
9
|
import { cli, Strategy } from '@sovovs/bycli/registry';
|
|
10
10
|
import { downloadArticle } from '@sovovs/bycli/download/article-download';
|
|
11
|
-
import {
|
|
11
|
+
import { AuthRequiredError } from '@sovovs/bycli/errors';
|
|
12
12
|
import { buildExtractWechatArticleContentJs } from './_wechat/article-content.js';
|
|
13
|
+
import {
|
|
14
|
+
isTrustedSogouRedirectUrl,
|
|
15
|
+
isTrustedWechatArticleUrl,
|
|
16
|
+
normalizeWechatUrl,
|
|
17
|
+
resolveWechatArticleUrl,
|
|
18
|
+
} from './_wechat/article-link.js';
|
|
13
19
|
export { extractWechatArticleContent } from './_wechat/article-content.js';
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
export function normalizeWechatUrl(raw) {
|
|
21
|
-
let s = (raw || '').trim();
|
|
22
|
-
if (!s)
|
|
23
|
-
return s;
|
|
24
|
-
// Strip wrapping quotes / angle brackets
|
|
25
|
-
if ((s.startsWith('"') && s.endsWith('"')) || (s.startsWith("'") && s.endsWith("'"))) {
|
|
26
|
-
s = s.slice(1, -1).trim();
|
|
27
|
-
}
|
|
28
|
-
if (s.startsWith('<') && s.endsWith('>')) {
|
|
29
|
-
s = s.slice(1, -1).trim();
|
|
30
|
-
}
|
|
31
|
-
// Remove backslash escapes before URL-significant characters
|
|
32
|
-
s = s.replace(/\\+([:/&?=#%])/g, '$1');
|
|
33
|
-
// Decode HTML entities
|
|
34
|
-
s = s.replace(/&/g, '&').replace(/</g, '<').replace(/>/g, '>').replace(/"/g, '"');
|
|
35
|
-
// Allow bare hostnames
|
|
36
|
-
if (s.startsWith('mp.weixin.qq.com/') || s.startsWith('//mp.weixin.qq.com/')) {
|
|
37
|
-
s = 'https://' + s.replace(/^\/+/, '');
|
|
38
|
-
}
|
|
39
|
-
// Force https for mp.weixin.qq.com
|
|
40
|
-
try {
|
|
41
|
-
const parsed = new URL(s);
|
|
42
|
-
if (['http:', 'https:'].includes(parsed.protocol) && parsed.hostname.toLowerCase() === 'mp.weixin.qq.com') {
|
|
43
|
-
parsed.protocol = 'https:';
|
|
44
|
-
s = parsed.toString();
|
|
45
|
-
}
|
|
46
|
-
}
|
|
47
|
-
catch {
|
|
48
|
-
// Ignore parse errors
|
|
49
|
-
}
|
|
50
|
-
return s;
|
|
51
|
-
}
|
|
52
|
-
|
|
53
|
-
function isStrictHttpsUrl(raw, hostname, pathname) {
|
|
54
|
-
try {
|
|
55
|
-
const url = new URL(raw);
|
|
56
|
-
return url.protocol === 'https:' && url.hostname === hostname
|
|
57
|
-
&& url.port === '' && url.username === '' && url.password === ''
|
|
58
|
-
&& pathname(url.pathname);
|
|
59
|
-
}
|
|
60
|
-
catch {
|
|
61
|
-
return false;
|
|
62
|
-
}
|
|
63
|
-
}
|
|
64
|
-
|
|
65
|
-
export function isTrustedWechatArticleUrl(raw) {
|
|
66
|
-
return isStrictHttpsUrl(raw, 'mp.weixin.qq.com', path => path === '/s' || path.startsWith('/s/'));
|
|
67
|
-
}
|
|
68
|
-
|
|
69
|
-
export function isTrustedSogouRedirectUrl(raw) {
|
|
70
|
-
return isStrictHttpsUrl(raw, 'weixin.sogou.com', path => path === '/link');
|
|
71
|
-
}
|
|
72
|
-
|
|
73
|
-
export async function resolveWechatDownloadUrl(page, rawUrl) {
|
|
74
|
-
const sourceUrl = normalizeWechatUrl(rawUrl);
|
|
75
|
-
if (isTrustedWechatArticleUrl(sourceUrl)) {
|
|
76
|
-
return { sourceUrl, resolvedUrl: sourceUrl, alreadyNavigated: false };
|
|
77
|
-
}
|
|
78
|
-
if (!isTrustedSogouRedirectUrl(sourceUrl)) {
|
|
79
|
-
throw new ArgumentError(
|
|
80
|
-
'A trusted WeChat article or Sogou Weixin result URL is required.',
|
|
81
|
-
'Pass an https://mp.weixin.qq.com/s/... or https://weixin.sogou.com/link?... URL.',
|
|
82
|
-
);
|
|
83
|
-
}
|
|
84
|
-
try {
|
|
85
|
-
await page.goto(sourceUrl);
|
|
86
|
-
await page.wait(2);
|
|
87
|
-
const result = await page.evaluate(`(() => ({
|
|
88
|
-
finalUrl: window.location.href,
|
|
89
|
-
pageText: document.body ? document.body.innerText : '',
|
|
90
|
-
html: document.documentElement ? document.documentElement.innerHTML : '',
|
|
91
|
-
}))()`);
|
|
92
|
-
const text = `${result?.pageText || ''} ${result?.html || ''}`;
|
|
93
|
-
if (/验证码|安全验证|异常访问|访问过于频繁|请输入验证码/.test(text)) {
|
|
94
|
-
throw new AuthRequiredError(
|
|
95
|
-
'weixin.sogou.com',
|
|
96
|
-
'Sogou Weixin requires verification. Complete it in the open browser tab and run the command again.',
|
|
97
|
-
);
|
|
98
|
-
}
|
|
99
|
-
if (!isTrustedWechatArticleUrl(result?.finalUrl)) {
|
|
100
|
-
throw new CommandExecutionError(
|
|
101
|
-
'Sogou Weixin did not resolve to a trusted WeChat article URL',
|
|
102
|
-
'Open the search result in a browser and confirm it redirects to mp.weixin.qq.com/s/... before retrying.',
|
|
103
|
-
);
|
|
104
|
-
}
|
|
105
|
-
return { sourceUrl, resolvedUrl: new URL(result.finalUrl).href, alreadyNavigated: true };
|
|
106
|
-
}
|
|
107
|
-
catch (error) {
|
|
108
|
-
if (error instanceof AuthRequiredError || error instanceof CommandExecutionError) throw error;
|
|
109
|
-
throw new CommandExecutionError('Failed to resolve the Sogou Weixin result URL');
|
|
110
|
-
}
|
|
111
|
-
}
|
|
20
|
+
export {
|
|
21
|
+
isTrustedSogouRedirectUrl,
|
|
22
|
+
isTrustedWechatArticleUrl,
|
|
23
|
+
normalizeWechatUrl,
|
|
24
|
+
resolveWechatArticleUrl as resolveWechatDownloadUrl,
|
|
25
|
+
};
|
|
112
26
|
/**
|
|
113
27
|
* Format a WeChat article timestamp as a UTC+8 datetime string.
|
|
114
28
|
* Accepts either Unix seconds or milliseconds.
|
|
@@ -245,7 +159,7 @@ cli({
|
|
|
245
159
|
],
|
|
246
160
|
columns: ['title', 'author', 'publish_time', 'status', 'size', 'saved', 'source_url', 'resolved_url'],
|
|
247
161
|
func: async (page, kwargs) => {
|
|
248
|
-
const { sourceUrl, resolvedUrl, alreadyNavigated } = await
|
|
162
|
+
const { sourceUrl, resolvedUrl, alreadyNavigated } = await resolveWechatArticleUrl(page, kwargs.url);
|
|
249
163
|
// Navigate and wait for content to load. Sogou resolution already lands on the article.
|
|
250
164
|
if (!alreadyNavigated)
|
|
251
165
|
await page.goto(resolvedUrl);
|
|
@@ -1,13 +1,21 @@
|
|
|
1
|
-
import {
|
|
1
|
+
import {
|
|
2
|
+
ArgumentError, AuthRequiredError, CommandExecutionError, EmptyResultError,
|
|
3
|
+
} from '@sovovs/bycli/errors';
|
|
2
4
|
import { MAX_WECHAT_HTML_BYTES } from '@sovovs/bycli/download/wechat-article';
|
|
3
5
|
import { cli, Strategy } from '@sovovs/bycli/registry';
|
|
4
6
|
import { readEnvironmentCredentials, resolveBrowserCredentials } from './_wechat/auth-session.js';
|
|
7
|
+
import {
|
|
8
|
+
combineArticleFallbackErrors,
|
|
9
|
+
isEligibleArticleFallbackError,
|
|
10
|
+
withMissingFallbackName,
|
|
11
|
+
} from './_wechat/article-fallback-policy.js';
|
|
5
12
|
import { createArticleIndexFetcher } from './_wechat/article-index.js';
|
|
6
13
|
import {
|
|
7
14
|
callCrawler, collectArticles, isTrustedWechatArticleUrl, saveArticles,
|
|
8
15
|
} from './_wechat/crawler-runtime.js';
|
|
9
16
|
import { readAuthSource } from './_wechat/args.js';
|
|
10
17
|
import { wechatArticleToMarkdown } from './_wechat/markdown.js';
|
|
18
|
+
import { collectSogouAccountArticles } from './_wechat/sogou-fallback.js';
|
|
11
19
|
|
|
12
20
|
const DOMAIN = 'mp.weixin.qq.com';
|
|
13
21
|
const browserRequired = args => readAuthSource(args) === 'browser';
|
|
@@ -156,11 +164,11 @@ export const saveArticlesCommand = cli({
|
|
|
156
164
|
description: 'Download WeChat official-account articles as Markdown files',
|
|
157
165
|
strategy: Strategy.COOKIE, browser: browserRequired,
|
|
158
166
|
args: [
|
|
159
|
-
{ name: 'fakeid', positional: true, required: true, help: 'Official-account fakeid returned by weixin accounts' }, { name: 'name', help: 'Official-account name
|
|
167
|
+
{ name: 'fakeid', positional: true, required: true, help: 'Official-account fakeid returned by weixin accounts' }, { name: 'name', help: 'Official-account name; exact case-insensitive match required for browser Sogou fallback' },
|
|
160
168
|
{ name: 'output', default: './weixin-articles', help: 'Directory for saved Markdown files' }, { name: 'limit', type: 'int', help: 'Maximum number of articles to save' },
|
|
161
169
|
{ name: 'max-pages', type: 'int', help: 'Maximum number of history pages to scan' }, { name: 'auth-source', default: 'browser', choices: ['browser', 'env'], help: 'Credential source: browser session or environment variables' },
|
|
162
170
|
],
|
|
163
|
-
columns: ['title', 'status', 'stage', 'path', 'error', 'url'],
|
|
171
|
+
columns: ['title', 'status', 'stage', 'path', 'error', 'url', 'source', 'coverage'],
|
|
164
172
|
func: async (page, args) => {
|
|
165
173
|
const fakeid = String(args.fakeid ?? '').trim();
|
|
166
174
|
if (!fakeid) throw new ArgumentError('fakeid is required');
|
|
@@ -169,20 +177,55 @@ export const saveArticlesCommand = cli({
|
|
|
169
177
|
? readEnvironmentCredentials(false) : await resolveBrowserCredentials(page);
|
|
170
178
|
const articleHtmlDownloader = createArticleHtmlDownloader({ authSource, page });
|
|
171
179
|
const fetchPage = createArticleIndexFetcher({ page, source: authSource, credentials });
|
|
172
|
-
|
|
173
|
-
|
|
174
|
-
|
|
180
|
+
let articles;
|
|
181
|
+
let resolutionFailures = [];
|
|
182
|
+
let source = 'wechat';
|
|
183
|
+
let coverage = null;
|
|
184
|
+
try {
|
|
185
|
+
const result = await callCrawler(() => collectArticles({
|
|
186
|
+
fakeid, fetchPage, limit: args.limit, maxPages: args['max-pages'],
|
|
187
|
+
}));
|
|
188
|
+
articles = result.articles;
|
|
189
|
+
if (articles.length === 0) {
|
|
190
|
+
throw new EmptyResultError('weixin save-articles', `No published articles were found for ${fakeid}.`);
|
|
191
|
+
}
|
|
192
|
+
} catch (primaryError) {
|
|
193
|
+
if (authSource !== 'browser' || !isEligibleArticleFallbackError(primaryError)) throw primaryError;
|
|
194
|
+
const accountName = String(args.name ?? '').trim();
|
|
195
|
+
if (!accountName) throw withMissingFallbackName('weixin save-articles', primaryError);
|
|
196
|
+
try {
|
|
197
|
+
const fallback = await collectSogouAccountArticles({
|
|
198
|
+
page, accountName, limit: args.limit, maxPages: args['max-pages'],
|
|
199
|
+
resolutionPolicy: 'rows', freshPage: true,
|
|
200
|
+
});
|
|
201
|
+
articles = fallback.articles;
|
|
202
|
+
resolutionFailures = fallback.resolutionFailures;
|
|
203
|
+
source = fallback.source;
|
|
204
|
+
coverage = fallback.coverage;
|
|
205
|
+
} catch (fallbackError) {
|
|
206
|
+
throw combineArticleFallbackErrors({
|
|
207
|
+
operation: 'weixin save-articles', primaryError, fallbackError, credentials,
|
|
208
|
+
});
|
|
209
|
+
}
|
|
210
|
+
}
|
|
211
|
+
|
|
212
|
+
const savedRows = articles.length === 0 ? [] : await callCrawler(() => saveArticles({
|
|
175
213
|
articles, accountName: String(args.name ?? '').trim(),
|
|
176
214
|
outputDir: args.output ?? './weixin-articles', fetchArticleHtml: articleHtmlDownloader,
|
|
177
215
|
buildMarkdown: (article, html) => wechatArticleToMarkdown({
|
|
178
216
|
html, title: article.title, accountName: String(args.name ?? '').trim(), author: article.author,
|
|
179
217
|
publishedAt: article.publishedAt, digest: article.digest, url: article.url,
|
|
180
218
|
}), existingFilePolicy: 'suffix',
|
|
181
|
-
});
|
|
182
|
-
|
|
183
|
-
|
|
219
|
+
}));
|
|
220
|
+
const orderedRows = source === 'sogou'
|
|
221
|
+
? [
|
|
222
|
+
...savedRows.map((row, index) => ({ ...row, order: articles[index]?.order ?? index })),
|
|
223
|
+
...resolutionFailures,
|
|
224
|
+
].sort((left, right) => left.order - right.order)
|
|
225
|
+
: savedRows;
|
|
226
|
+
return orderedRows.map(row => ({
|
|
184
227
|
title: row.title, status: row.status, stage: row.stage || null, path: row.saved || null,
|
|
185
|
-
error: row.error || null, url: row.url,
|
|
228
|
+
error: row.error || null, url: row.url, source, coverage,
|
|
186
229
|
}));
|
|
187
230
|
},
|
|
188
231
|
});
|