@sovovs/bycli 2.1.0 → 2.1.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/cli-manifest.json +169 -0
- package/clis/weixin/_wechat/args.js +48 -0
- package/clis/weixin/_wechat/article-content.js +53 -0
- package/clis/weixin/_wechat/article-service.js +124 -0
- package/clis/weixin/_wechat/auth-session.js +142 -0
- package/clis/weixin/_wechat/fingerprint.js +443 -0
- package/clis/weixin/_wechat/fixtures/articles-auth-expired.json +3 -0
- package/clis/weixin/_wechat/fixtures/articles-page.json +4 -0
- package/clis/weixin/_wechat/fixtures/search-auth-expired.json +4 -0
- package/clis/weixin/_wechat/fixtures/search-success.json +7 -0
- package/clis/weixin/_wechat/markdown.js +29 -0
- package/clis/weixin/_wechat/redact.js +405 -0
- package/clis/weixin/_wechat/save-service.js +175 -0
- package/clis/weixin/_wechat/search-biz.js +102 -0
- package/clis/weixin/_wechat/wechat-api.js +133 -0
- package/clis/weixin/accounts.js +38 -0
- package/clis/weixin/articles.js +35 -0
- package/clis/weixin/download.js +5 -47
- package/clis/weixin/save-articles.js +175 -0
- package/dist/src/daemon-config.d.ts +2 -0
- package/dist/src/daemon-config.js +11 -0
- package/dist/src/daemon-config.test.d.ts +1 -0
- package/dist/src/daemon.d.ts +1 -1
- package/dist/src/daemon.js +5 -3
- package/dist/src/download/article-download.d.ts +6 -0
- package/dist/src/download/article-download.js +78 -17
- package/dist/src/download/wechat-article.d.ts +8 -0
- package/dist/src/download/wechat-article.js +137 -0
- package/dist/src/download/wechat-article.test.d.ts +1 -0
- package/dist/src/recorder/highlevel/verify.d.ts +3 -0
- package/dist/src/recorder/highlevel/verify.js +4 -0
- package/dist/src/recorder/highlevel/verify.test.d.ts +1 -0
- package/dist/src/recorder/http/handlers.js +27 -1
- package/dist/src/recorder/runner/runner-port.js +1 -0
- package/dist/src/recorder/runner/verify-runner-main.d.ts +17 -2
- package/dist/src/recorder/runner/verify-runner-main.js +70 -14
- package/dist/src/release-workflow.test.d.ts +1 -0
- package/dist/src/weixin-built-in-docs.test.d.ts +1 -0
- package/package.json +7 -3
- package/scripts/check-package-install.mjs +71 -0
|
@@ -0,0 +1,133 @@
|
|
|
1
|
+
import { AuthRequiredError, CommandExecutionError } from '@sovovs/bycli/errors';
|
|
2
|
+
import { buildSecretSet, redactText } from './redact.js';
|
|
3
|
+
|
|
4
|
+
const DOMAIN = 'mp.weixin.qq.com';
|
|
5
|
+
const ENDPOINT = `https://${DOMAIN}/cgi-bin/appmsgpublish`;
|
|
6
|
+
|
|
7
|
+
function normalizedMessage(value) {
|
|
8
|
+
return String(value ?? '').trim().toLowerCase().replace(/\s+/g, ' ');
|
|
9
|
+
}
|
|
10
|
+
|
|
11
|
+
function commandError(message) {
|
|
12
|
+
return new CommandExecutionError(redactText(message, []));
|
|
13
|
+
}
|
|
14
|
+
|
|
15
|
+
function parseNestedJson(value, label) {
|
|
16
|
+
if (typeof value !== 'string') return value;
|
|
17
|
+
try {
|
|
18
|
+
return JSON.parse(value);
|
|
19
|
+
} catch (error) {
|
|
20
|
+
const detail = error instanceof Error ? error.message : String(error);
|
|
21
|
+
throw commandError(`WeChat ${label} is malformed: ${detail}`);
|
|
22
|
+
}
|
|
23
|
+
}
|
|
24
|
+
|
|
25
|
+
/** @param {unknown} data */
|
|
26
|
+
export function parsePublishData(data) {
|
|
27
|
+
if (!data || typeof data !== 'object') {
|
|
28
|
+
throw new CommandExecutionError('WeChat article history returned an unreadable response');
|
|
29
|
+
}
|
|
30
|
+
const response = /** @type {Record<string, any>} */ (data);
|
|
31
|
+
const ret = response.base_resp?.ret;
|
|
32
|
+
const message = response.base_resp?.err_msg ?? response.base_resp?.err_msg_en ?? '';
|
|
33
|
+
if (ret === 200013 && normalizedMessage(message) === 'invalid credential') {
|
|
34
|
+
throw new AuthRequiredError(DOMAIN, 'WeChat article-history credentials have expired');
|
|
35
|
+
}
|
|
36
|
+
if (ret !== undefined && ret !== 0) {
|
|
37
|
+
throw new CommandExecutionError(`WeChat article history failed (ret=${String(ret)})`);
|
|
38
|
+
}
|
|
39
|
+
if (response.publish_page === undefined || response.publish_page === null || response.publish_page === '') {
|
|
40
|
+
return { total: 0, publishItemCount: 0, articles: [] };
|
|
41
|
+
}
|
|
42
|
+
const page = parseNestedJson(response.publish_page, 'publish_page');
|
|
43
|
+
if (!page || typeof page !== 'object' || !Array.isArray(page.publish_list)) {
|
|
44
|
+
throw new CommandExecutionError('WeChat article history returned an invalid publish page');
|
|
45
|
+
}
|
|
46
|
+
const total = page.total_count === undefined ? 0 : page.total_count;
|
|
47
|
+
if (!Number.isSafeInteger(total) || total < 0) {
|
|
48
|
+
throw new CommandExecutionError('WeChat article history returned invalid total metadata');
|
|
49
|
+
}
|
|
50
|
+
const articles = [];
|
|
51
|
+
for (const item of page.publish_list) {
|
|
52
|
+
const info = parseNestedJson(item?.publish_info ?? {}, 'publish_info');
|
|
53
|
+
if (!info || typeof info !== 'object' || !Array.isArray(info.appmsg_info)) {
|
|
54
|
+
throw new CommandExecutionError('WeChat article history returned invalid publish information');
|
|
55
|
+
}
|
|
56
|
+
const timestamp = info.sent_info?.time ?? info.publish_info?.create_time ?? 0;
|
|
57
|
+
let publishedAt = null;
|
|
58
|
+
if (timestamp !== 0) {
|
|
59
|
+
if (typeof timestamp !== 'number' || !Number.isFinite(timestamp) || timestamp <= 0) {
|
|
60
|
+
throw new CommandExecutionError('WeChat article history returned an invalid publish timestamp');
|
|
61
|
+
}
|
|
62
|
+
const date = new Date(timestamp * 1000);
|
|
63
|
+
if (!Number.isFinite(date.getTime())) {
|
|
64
|
+
throw new CommandExecutionError('WeChat article history returned an invalid publish timestamp');
|
|
65
|
+
}
|
|
66
|
+
publishedAt = date.toISOString();
|
|
67
|
+
}
|
|
68
|
+
for (const messageItem of info.appmsg_info) {
|
|
69
|
+
const article = messageItem && typeof messageItem === 'object' ? messageItem : {};
|
|
70
|
+
articles.push({
|
|
71
|
+
title: typeof article.title === 'string' ? article.title : '',
|
|
72
|
+
url: typeof article.content_url === 'string' ? article.content_url : '',
|
|
73
|
+
isDeleted: article.is_deleted === true,
|
|
74
|
+
timestamp,
|
|
75
|
+
publishedAt,
|
|
76
|
+
digest: typeof article.digest === 'string' ? article.digest : '',
|
|
77
|
+
author: typeof article.author === 'string' ? article.author : '',
|
|
78
|
+
});
|
|
79
|
+
}
|
|
80
|
+
}
|
|
81
|
+
return {
|
|
82
|
+
total,
|
|
83
|
+
publishItemCount: page.publish_list.length,
|
|
84
|
+
articles,
|
|
85
|
+
};
|
|
86
|
+
}
|
|
87
|
+
|
|
88
|
+
export function requestHeaders(cookie, token) {
|
|
89
|
+
return {
|
|
90
|
+
Accept: 'application/json, text/javascript, */*; q=0.01',
|
|
91
|
+
Cookie: cookie,
|
|
92
|
+
Origin: `https://${DOMAIN}`,
|
|
93
|
+
Referer: `https://${DOMAIN}/cgi-bin/appmsg?t=media/appmsg_edit_v2&action=edit&isNew=1&type=10&token=${encodeURIComponent(token)}&lang=zh_CN`,
|
|
94
|
+
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 Chrome/143 Safari/537.36',
|
|
95
|
+
'X-Requested-With': 'XMLHttpRequest',
|
|
96
|
+
};
|
|
97
|
+
}
|
|
98
|
+
|
|
99
|
+
/**
|
|
100
|
+
* @param {{token:string,cookie:string,timeoutMs?:number,fetchImpl?:typeof fetch}} options
|
|
101
|
+
*/
|
|
102
|
+
export function createWechatApi({ token, cookie, timeoutMs = 30_000, fetchImpl = fetch }) {
|
|
103
|
+
const headers = requestHeaders(cookie, token);
|
|
104
|
+
const secrets = buildSecretSet({ token, cookie });
|
|
105
|
+
|
|
106
|
+
return {
|
|
107
|
+
async fetchPage({ fakeid, begin = 0, count = 10 }) {
|
|
108
|
+
const query = new URLSearchParams({
|
|
109
|
+
sub: 'list', begin: String(begin), count: String(count), fakeid, token,
|
|
110
|
+
lang: 'zh_CN', f: 'json', ajax: '1',
|
|
111
|
+
});
|
|
112
|
+
try {
|
|
113
|
+
const response = await fetchImpl(`${ENDPOINT}?${query}`, {
|
|
114
|
+
headers,
|
|
115
|
+
signal: AbortSignal.timeout(timeoutMs),
|
|
116
|
+
});
|
|
117
|
+
if (!response.ok) {
|
|
118
|
+
throw new CommandExecutionError(`WeChat article history request failed: HTTP ${response.status} ${response.statusText ?? ''}`.trim());
|
|
119
|
+
}
|
|
120
|
+
return parsePublishData(await response.json());
|
|
121
|
+
} catch (error) {
|
|
122
|
+
if (error instanceof AuthRequiredError && error.domain === DOMAIN) throw error;
|
|
123
|
+
const message = error instanceof Error ? error.message : String(error);
|
|
124
|
+
const hint = error && typeof error === 'object' && 'hint' in error && typeof error.hint === 'string'
|
|
125
|
+
? error.hint : undefined;
|
|
126
|
+
throw new CommandExecutionError(
|
|
127
|
+
`WeChat article history request failed: ${redactText(message, secrets)}`,
|
|
128
|
+
hint ? redactText(hint, secrets) : undefined,
|
|
129
|
+
);
|
|
130
|
+
}
|
|
131
|
+
},
|
|
132
|
+
};
|
|
133
|
+
}
|
|
@@ -0,0 +1,38 @@
|
|
|
1
|
+
import { ArgumentError, EmptyResultError } from '@sovovs/bycli/errors';
|
|
2
|
+
import { cli, Strategy } from '@sovovs/bycli/registry';
|
|
3
|
+
import { readEnvironmentCredentials, resolveBrowserCredentials } from './_wechat/auth-session.js';
|
|
4
|
+
import { captureSearchBizFingerprint } from './_wechat/fingerprint.js';
|
|
5
|
+
import { executeSearchBiz } from './_wechat/search-biz.js';
|
|
6
|
+
import { readAuthSource } from './_wechat/args.js';
|
|
7
|
+
|
|
8
|
+
const DOMAIN = 'mp.weixin.qq.com';
|
|
9
|
+
const browserRequired = args => readAuthSource(args) === 'browser';
|
|
10
|
+
|
|
11
|
+
export const accountsCommand = cli({
|
|
12
|
+
site: 'weixin', name: 'accounts', access: 'read', domain: DOMAIN,
|
|
13
|
+
description: 'Search WeChat official accounts and return their fakeids',
|
|
14
|
+
strategy: Strategy.INTERCEPT, browser: browserRequired,
|
|
15
|
+
args: [
|
|
16
|
+
{ name: 'query', positional: true, required: true, help: 'Official-account name to search for' },
|
|
17
|
+
{ name: 'limit', type: 'int', default: 10, help: 'Maximum number of matching accounts to return' },
|
|
18
|
+
{ name: 'auth-source', default: 'browser', choices: ['browser', 'env'], help: 'Credential source: browser session or environment variables' },
|
|
19
|
+
],
|
|
20
|
+
columns: ['nickname', 'fakeid', 'alias'],
|
|
21
|
+
func: async (page, args) => {
|
|
22
|
+
const query = String(args.query ?? '').trim();
|
|
23
|
+
if (!query) throw new ArgumentError('query is required');
|
|
24
|
+
const limit = args.limit ?? 10;
|
|
25
|
+
if (!Number.isSafeInteger(limit) || limit <= 0) throw new ArgumentError('limit must be a positive safe integer');
|
|
26
|
+
const authSource = readAuthSource(args);
|
|
27
|
+
let credentials;
|
|
28
|
+
if (authSource === 'env') {
|
|
29
|
+
credentials = readEnvironmentCredentials(true);
|
|
30
|
+
} else {
|
|
31
|
+
credentials = await resolveBrowserCredentials(page);
|
|
32
|
+
credentials = { ...credentials, fingerprint: await captureSearchBizFingerprint(page, query) };
|
|
33
|
+
}
|
|
34
|
+
const rows = await executeSearchBiz({ page, source: authSource, credentials, query, limit });
|
|
35
|
+
if (rows.length === 0) throw new EmptyResultError('weixin accounts', `No official accounts matched "${query}".`);
|
|
36
|
+
return rows.map(row => ({ nickname: row.nickname, fakeid: row.fakeid, alias: row.alias || null }));
|
|
37
|
+
},
|
|
38
|
+
});
|
|
@@ -0,0 +1,35 @@
|
|
|
1
|
+
import { ArgumentError, EmptyResultError } from '@sovovs/bycli/errors';
|
|
2
|
+
import { cli, Strategy } from '@sovovs/bycli/registry';
|
|
3
|
+
import { readEnvironmentCredentials, resolveBrowserCredentials } from './_wechat/auth-session.js';
|
|
4
|
+
import { collectArticles } from './_wechat/article-service.js';
|
|
5
|
+
import { createWechatApi } from './_wechat/wechat-api.js';
|
|
6
|
+
import { readAuthSource } from './_wechat/args.js';
|
|
7
|
+
|
|
8
|
+
const DOMAIN = 'mp.weixin.qq.com';
|
|
9
|
+
const browserRequired = args => readAuthSource(args) === 'browser';
|
|
10
|
+
|
|
11
|
+
export const articlesCommand = cli({
|
|
12
|
+
site: 'weixin', name: 'articles', access: 'read', domain: DOMAIN,
|
|
13
|
+
description: 'List published articles from a WeChat official account',
|
|
14
|
+
strategy: Strategy.COOKIE, browser: browserRequired,
|
|
15
|
+
args: [
|
|
16
|
+
{ name: 'fakeid', positional: true, required: true, help: 'Official-account fakeid returned by weixin accounts' },
|
|
17
|
+
{ name: 'name', help: 'Optional official-account name for display context' }, { name: 'limit', type: 'int', help: 'Maximum number of articles to return' }, { name: 'max-pages', type: 'int', help: 'Maximum number of history pages to scan' },
|
|
18
|
+
{ name: 'auth-source', default: 'browser', choices: ['browser', 'env'], help: 'Credential source: browser session or environment variables' },
|
|
19
|
+
],
|
|
20
|
+
columns: ['title', 'author', 'digest', 'publishedAt', 'url'],
|
|
21
|
+
func: async (page, args) => {
|
|
22
|
+
const fakeid = String(args.fakeid ?? '').trim();
|
|
23
|
+
if (!fakeid) throw new ArgumentError('fakeid is required');
|
|
24
|
+
const authSource = readAuthSource(args);
|
|
25
|
+
const credentials = authSource === 'env'
|
|
26
|
+
? readEnvironmentCredentials(false) : await resolveBrowserCredentials(page);
|
|
27
|
+
const { fetchPage } = createWechatApi(credentials);
|
|
28
|
+
const { articles } = await collectArticles({ fakeid, fetchPage, limit: args.limit, maxPages: args['max-pages'] });
|
|
29
|
+
if (articles.length === 0) throw new EmptyResultError('weixin articles', `No published articles were found for ${fakeid}.`);
|
|
30
|
+
return articles.map(article => ({
|
|
31
|
+
title: article.title, author: article.author || null, digest: article.digest || null,
|
|
32
|
+
publishedAt: article.publishedAt || null, url: article.url,
|
|
33
|
+
}));
|
|
34
|
+
},
|
|
35
|
+
});
|
package/clis/weixin/download.js
CHANGED
|
@@ -8,6 +8,8 @@
|
|
|
8
8
|
*/
|
|
9
9
|
import { cli, Strategy } from '@sovovs/bycli/registry';
|
|
10
10
|
import { downloadArticle } from '@sovovs/bycli/download/article-download';
|
|
11
|
+
import { buildExtractWechatArticleContentJs } from './_wechat/article-content.js';
|
|
12
|
+
export { extractWechatArticleContent } from './_wechat/article-content.js';
|
|
11
13
|
// ============================================================
|
|
12
14
|
// URL Normalization
|
|
13
15
|
// ============================================================
|
|
@@ -241,53 +243,8 @@ cli({
|
|
|
241
243
|
);
|
|
242
244
|
if (result.errorHint) return result;
|
|
243
245
|
|
|
244
|
-
|
|
245
|
-
|
|
246
|
-
if (!contentEl) return result;
|
|
247
|
-
|
|
248
|
-
// Fix lazy-loaded images: data-src -> src
|
|
249
|
-
contentEl.querySelectorAll('img').forEach(img => {
|
|
250
|
-
const dataSrc = img.getAttribute('data-src');
|
|
251
|
-
if (dataSrc) img.setAttribute('src', dataSrc);
|
|
252
|
-
});
|
|
253
|
-
|
|
254
|
-
// Extract code blocks with placeholder replacement
|
|
255
|
-
const codeBlocks = [];
|
|
256
|
-
contentEl.querySelectorAll('.code-snippet__fix').forEach(el => {
|
|
257
|
-
el.querySelectorAll('.code-snippet__line-index').forEach(li => li.remove());
|
|
258
|
-
const pre = el.querySelector('pre[data-lang]');
|
|
259
|
-
const lang = pre ? (pre.getAttribute('data-lang') || '') : '';
|
|
260
|
-
const lines = [];
|
|
261
|
-
el.querySelectorAll('code').forEach(codeTag => {
|
|
262
|
-
const text = codeTag.textContent;
|
|
263
|
-
if (/^[ce]?ounter\\(line/.test(text)) return;
|
|
264
|
-
lines.push(text);
|
|
265
|
-
});
|
|
266
|
-
if (lines.length === 0) lines.push(el.textContent);
|
|
267
|
-
const placeholder = 'CODEBLOCK-PLACEHOLDER-' + codeBlocks.length;
|
|
268
|
-
codeBlocks.push({ lang, code: lines.join('\\n') });
|
|
269
|
-
const p = document.createElement('p');
|
|
270
|
-
p.textContent = placeholder;
|
|
271
|
-
el.replaceWith(p);
|
|
272
|
-
});
|
|
273
|
-
result.codeBlocks = codeBlocks;
|
|
274
|
-
|
|
275
|
-
// Remove noise elements
|
|
276
|
-
['script', 'style', '.qr_code_pc', '.reward_area'].forEach(sel => {
|
|
277
|
-
contentEl.querySelectorAll(sel).forEach(tag => tag.remove());
|
|
278
|
-
});
|
|
279
|
-
|
|
280
|
-
// Collect image URLs (deduplicated)
|
|
281
|
-
const seen = new Set();
|
|
282
|
-
contentEl.querySelectorAll('img[src]').forEach(img => {
|
|
283
|
-
const src = img.getAttribute('src');
|
|
284
|
-
if (src && !seen.has(src)) {
|
|
285
|
-
seen.add(src);
|
|
286
|
-
result.imageUrls.push(src);
|
|
287
|
-
}
|
|
288
|
-
});
|
|
289
|
-
|
|
290
|
-
result.contentHtml = contentEl.innerHTML;
|
|
246
|
+
const extractWechatArticleContent = ${buildExtractWechatArticleContentJs()};
|
|
247
|
+
Object.assign(result, extractWechatArticleContent(document));
|
|
291
248
|
return result;
|
|
292
249
|
})()
|
|
293
250
|
`);
|
|
@@ -318,6 +275,7 @@ cli({
|
|
|
318
275
|
const m = url.match(/wx_fmt=(\w+)/) || url.match(/\.(\w{3,4})(?:\?|$)/);
|
|
319
276
|
return m ? m[1] : 'png';
|
|
320
277
|
},
|
|
278
|
+
secureMarkdown: true,
|
|
321
279
|
});
|
|
322
280
|
},
|
|
323
281
|
});
|
|
@@ -0,0 +1,175 @@
|
|
|
1
|
+
import { ArgumentError, CommandExecutionError } from '@sovovs/bycli/errors';
|
|
2
|
+
import { MAX_WECHAT_HTML_BYTES } from '@sovovs/bycli/download/wechat-article';
|
|
3
|
+
import { cli, Strategy } from '@sovovs/bycli/registry';
|
|
4
|
+
import { readEnvironmentCredentials, resolveBrowserCredentials } from './_wechat/auth-session.js';
|
|
5
|
+
import { collectArticles, isTrustedWechatArticleUrl } from './_wechat/article-service.js';
|
|
6
|
+
import { saveArticles } from './_wechat/save-service.js';
|
|
7
|
+
import { createWechatApi } from './_wechat/wechat-api.js';
|
|
8
|
+
import { readAuthSource } from './_wechat/args.js';
|
|
9
|
+
|
|
10
|
+
const DOMAIN = 'mp.weixin.qq.com';
|
|
11
|
+
const browserRequired = args => readAuthSource(args) === 'browser';
|
|
12
|
+
|
|
13
|
+
const MAX_REDIRECTS = 5;
|
|
14
|
+
|
|
15
|
+
async function readBoundedHtml(response) {
|
|
16
|
+
const lengthValue = response.headers?.get?.('content-length');
|
|
17
|
+
if (lengthValue !== null && lengthValue !== undefined && lengthValue !== '') {
|
|
18
|
+
const length = Number(lengthValue);
|
|
19
|
+
if (!Number.isSafeInteger(length) || length < 0 || length > MAX_WECHAT_HTML_BYTES) {
|
|
20
|
+
throw new CommandExecutionError('Article response exceeds the allowed size');
|
|
21
|
+
}
|
|
22
|
+
}
|
|
23
|
+
if (!response.body?.getReader) {
|
|
24
|
+
if (typeof response.text !== 'function') throw new CommandExecutionError('Article response has no readable body');
|
|
25
|
+
const text = await response.text();
|
|
26
|
+
if (new TextEncoder().encode(text).byteLength > MAX_WECHAT_HTML_BYTES) {
|
|
27
|
+
throw new CommandExecutionError('Article response exceeds the allowed size');
|
|
28
|
+
}
|
|
29
|
+
return text;
|
|
30
|
+
}
|
|
31
|
+
const reader = response.body.getReader();
|
|
32
|
+
const chunks = [];
|
|
33
|
+
let total = 0;
|
|
34
|
+
while (true) {
|
|
35
|
+
const { done, value } = await reader.read();
|
|
36
|
+
if (done) break;
|
|
37
|
+
if (!(value instanceof Uint8Array)) throw new CommandExecutionError('Article response returned invalid body data');
|
|
38
|
+
total += value.byteLength;
|
|
39
|
+
if (total > MAX_WECHAT_HTML_BYTES) {
|
|
40
|
+
try { await reader.cancel(); } catch { /* best-effort stream cleanup */ }
|
|
41
|
+
throw new CommandExecutionError('Article response exceeds the allowed size');
|
|
42
|
+
}
|
|
43
|
+
chunks.push(value);
|
|
44
|
+
}
|
|
45
|
+
const bytes = new Uint8Array(total);
|
|
46
|
+
let offset = 0;
|
|
47
|
+
for (const chunk of chunks) { bytes.set(chunk, offset); offset += chunk.byteLength; }
|
|
48
|
+
return new TextDecoder().decode(bytes);
|
|
49
|
+
}
|
|
50
|
+
|
|
51
|
+
export async function fetchArticleHtml(article, { fetchImpl = fetch, timeoutMs = 30_000 } = {}) {
|
|
52
|
+
try {
|
|
53
|
+
if (!isTrustedWechatArticleUrl(article?.url)) throw new CommandExecutionError('Article request rejected an untrusted URL');
|
|
54
|
+
let current = new URL(article.url).href;
|
|
55
|
+
const seen = new Set([current]);
|
|
56
|
+
for (let redirects = 0; ; redirects += 1) {
|
|
57
|
+
const response = await fetchImpl(current, {
|
|
58
|
+
signal: AbortSignal.timeout(timeoutMs), redirect: 'manual',
|
|
59
|
+
});
|
|
60
|
+
if (response.status >= 300 && response.status < 400) {
|
|
61
|
+
if (redirects >= MAX_REDIRECTS) throw new CommandExecutionError('Article request exceeded the redirect limit');
|
|
62
|
+
const location = response.headers?.get?.('location');
|
|
63
|
+
if (!location) throw new CommandExecutionError('Article redirect was missing a destination');
|
|
64
|
+
let next;
|
|
65
|
+
try { next = new URL(location, current).href; } catch { throw new CommandExecutionError('Article redirect was invalid'); }
|
|
66
|
+
if (!isTrustedWechatArticleUrl(next)) throw new CommandExecutionError('Article redirect was rejected');
|
|
67
|
+
if (seen.has(next)) throw new CommandExecutionError('Article redirect loop was rejected');
|
|
68
|
+
seen.add(next);
|
|
69
|
+
current = next;
|
|
70
|
+
continue;
|
|
71
|
+
}
|
|
72
|
+
if (!response.ok) throw new CommandExecutionError(`Article request failed: HTTP ${response.status}`);
|
|
73
|
+
return await readBoundedHtml(response);
|
|
74
|
+
}
|
|
75
|
+
} catch (error) {
|
|
76
|
+
if (error instanceof CommandExecutionError) throw error;
|
|
77
|
+
throw new CommandExecutionError('Article request failed');
|
|
78
|
+
}
|
|
79
|
+
}
|
|
80
|
+
|
|
81
|
+
export async function fetchArticleHtmlInBrowser(article, page) {
|
|
82
|
+
try {
|
|
83
|
+
if (!isTrustedWechatArticleUrl(article?.url)) {
|
|
84
|
+
throw new CommandExecutionError('Article browser request rejected an untrusted URL');
|
|
85
|
+
}
|
|
86
|
+
if (!page || typeof page.goto !== 'function' || typeof page.evaluate !== 'function') {
|
|
87
|
+
throw new CommandExecutionError('Article browser fallback is unavailable');
|
|
88
|
+
}
|
|
89
|
+
await page.goto(article.url);
|
|
90
|
+
await page.wait(5);
|
|
91
|
+
const result = await page.evaluate(({ maxBytes }) => {
|
|
92
|
+
const html = document.documentElement?.outerHTML ?? '';
|
|
93
|
+
const pageText = document.body?.innerText?.replace(/\s+/g, ' ').trim() ?? '';
|
|
94
|
+
const finalUrl = window.location.href;
|
|
95
|
+
const pathname = window.location.pathname;
|
|
96
|
+
const accessIssue = pathname.includes('/mp/wappoc_appmsgcaptcha')
|
|
97
|
+
|| (/环境异常/.test(pageText) && /(完成验证后即可继续访问|去验证)/.test(pageText))
|
|
98
|
+
|| /secitptpage\/verify\.html/.test(html)
|
|
99
|
+
|| /id=["']js_verify["']/.test(html)
|
|
100
|
+
? 'environment verification required' : '';
|
|
101
|
+
const byteLength = new TextEncoder().encode(html).byteLength;
|
|
102
|
+
return {
|
|
103
|
+
finalUrl,
|
|
104
|
+
accessIssue,
|
|
105
|
+
byteLength,
|
|
106
|
+
tooLarge: byteLength > maxBytes,
|
|
107
|
+
html: byteLength > maxBytes ? '' : html,
|
|
108
|
+
};
|
|
109
|
+
}, { maxBytes: MAX_WECHAT_HTML_BYTES });
|
|
110
|
+
if (result?.accessIssue) throw new CommandExecutionError('Article browser page requires environment verification');
|
|
111
|
+
if (!isTrustedWechatArticleUrl(result?.finalUrl)) {
|
|
112
|
+
throw new CommandExecutionError('Article browser navigation left the trusted article path');
|
|
113
|
+
}
|
|
114
|
+
if (result?.tooLarge || !Number.isSafeInteger(result?.byteLength)
|
|
115
|
+
|| result.byteLength < 0 || result.byteLength > MAX_WECHAT_HTML_BYTES) {
|
|
116
|
+
throw new CommandExecutionError('Article response exceeds the allowed size');
|
|
117
|
+
}
|
|
118
|
+
if (typeof result?.html !== 'string' || result.html.length === 0) {
|
|
119
|
+
throw new CommandExecutionError('Article browser page returned no HTML');
|
|
120
|
+
}
|
|
121
|
+
if (new TextEncoder().encode(result.html).byteLength > MAX_WECHAT_HTML_BYTES) {
|
|
122
|
+
throw new CommandExecutionError('Article response exceeds the allowed size');
|
|
123
|
+
}
|
|
124
|
+
return result.html;
|
|
125
|
+
} catch (error) {
|
|
126
|
+
if (error instanceof CommandExecutionError) throw error;
|
|
127
|
+
throw new CommandExecutionError('Article browser request failed');
|
|
128
|
+
}
|
|
129
|
+
}
|
|
130
|
+
|
|
131
|
+
export function createArticleHtmlDownloader({
|
|
132
|
+
authSource,
|
|
133
|
+
page,
|
|
134
|
+
nodeFetcher = fetchArticleHtml,
|
|
135
|
+
browserFetcher = fetchArticleHtmlInBrowser,
|
|
136
|
+
}) {
|
|
137
|
+
return async article => {
|
|
138
|
+
try {
|
|
139
|
+
return await nodeFetcher(article);
|
|
140
|
+
} catch (error) {
|
|
141
|
+
if (authSource !== 'browser') throw error;
|
|
142
|
+
return browserFetcher(article, page);
|
|
143
|
+
}
|
|
144
|
+
};
|
|
145
|
+
}
|
|
146
|
+
|
|
147
|
+
export const saveArticlesCommand = cli({
|
|
148
|
+
site: 'weixin', name: 'save-articles', access: 'write', domain: DOMAIN,
|
|
149
|
+
description: 'Download WeChat official-account articles as Markdown files',
|
|
150
|
+
strategy: Strategy.COOKIE, browser: browserRequired,
|
|
151
|
+
args: [
|
|
152
|
+
{ name: 'fakeid', positional: true, required: true, help: 'Official-account fakeid returned by weixin accounts' }, { name: 'name', help: 'Official-account name used in Markdown metadata' },
|
|
153
|
+
{ name: 'output', default: './weixin-articles', help: 'Directory for saved Markdown files' }, { name: 'limit', type: 'int', help: 'Maximum number of articles to save' },
|
|
154
|
+
{ name: 'max-pages', type: 'int', help: 'Maximum number of history pages to scan' }, { name: 'auth-source', default: 'browser', choices: ['browser', 'env'], help: 'Credential source: browser session or environment variables' },
|
|
155
|
+
],
|
|
156
|
+
columns: ['title', 'status', 'stage', 'path', 'error', 'url'],
|
|
157
|
+
func: async (page, args) => {
|
|
158
|
+
const fakeid = String(args.fakeid ?? '').trim();
|
|
159
|
+
if (!fakeid) throw new ArgumentError('fakeid is required');
|
|
160
|
+
const authSource = readAuthSource(args);
|
|
161
|
+
const credentials = authSource === 'env'
|
|
162
|
+
? readEnvironmentCredentials(false) : await resolveBrowserCredentials(page);
|
|
163
|
+
const { fetchPage } = createWechatApi(credentials);
|
|
164
|
+
const { articles } = await collectArticles({ fakeid, fetchPage, limit: args.limit, maxPages: args['max-pages'] });
|
|
165
|
+
const articleHtmlDownloader = createArticleHtmlDownloader({ authSource, page });
|
|
166
|
+
const rows = await saveArticles({
|
|
167
|
+
articles, accountName: String(args.name ?? '').trim(),
|
|
168
|
+
outputDir: args.output ?? './weixin-articles', fetchArticleHtml: articleHtmlDownloader,
|
|
169
|
+
});
|
|
170
|
+
return rows.map(row => ({
|
|
171
|
+
title: row.title, status: row.status, stage: row.stage || null, path: row.saved || null,
|
|
172
|
+
error: row.error || null, url: row.url,
|
|
173
|
+
}));
|
|
174
|
+
},
|
|
175
|
+
});
|
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
import { ConfigError } from './errors.js';
|
|
2
|
+
export function resolveDaemonHost(env = process.env) {
|
|
3
|
+
const raw = env.BYCLI_DAEMON_HOST;
|
|
4
|
+
if (raw === undefined || raw === '') {
|
|
5
|
+
return '127.0.0.1';
|
|
6
|
+
}
|
|
7
|
+
if (raw === '127.0.0.1' || raw === '0.0.0.0') {
|
|
8
|
+
return raw;
|
|
9
|
+
}
|
|
10
|
+
throw new ConfigError(`config_invalid: BYCLI_DAEMON_HOST=${raw} is not allowed`, 'Use 127.0.0.1 for local use or 0.0.0.0 for an isolated sandbox.');
|
|
11
|
+
}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
export {};
|
package/dist/src/daemon.d.ts
CHANGED
|
@@ -17,6 +17,6 @@
|
|
|
17
17
|
* Lifecycle:
|
|
18
18
|
* - Auto-spawned by bycli on first browser command
|
|
19
19
|
* - Persistent — stays alive until explicit shutdown, SIGTERM, or uninstall
|
|
20
|
-
* - Listens on
|
|
20
|
+
* - Listens on 127.0.0.1:19825 by default; isolated sandboxes may opt into 0.0.0.0
|
|
21
21
|
*/
|
|
22
22
|
export {};
|
package/dist/src/daemon.js
CHANGED
|
@@ -17,7 +17,7 @@
|
|
|
17
17
|
* Lifecycle:
|
|
18
18
|
* - Auto-spawned by bycli on first browser command
|
|
19
19
|
* - Persistent — stays alive until explicit shutdown, SIGTERM, or uninstall
|
|
20
|
-
* - Listens on
|
|
20
|
+
* - Listens on 127.0.0.1:19825 by default; isolated sandboxes may opt into 0.0.0.0
|
|
21
21
|
*/
|
|
22
22
|
import { createServer } from 'node:http';
|
|
23
23
|
import { WebSocketServer, WebSocket } from 'ws';
|
|
@@ -37,7 +37,9 @@ import { defaultSessionKeyRegistry } from './recorder/runner/session-keys.js';
|
|
|
37
37
|
import { recordExtensionVersion } from './update-check.js';
|
|
38
38
|
import { EXTENSION_CAPABILITY_MISSING_ERROR_CODE, EXTENSION_CAPABILITY_MISSING_HTTP_STATUS, extensionCapabilityHint, missingRequiredExtensionCapability, normalizeExtensionCapabilities, } from './browser/extension-capabilities.js';
|
|
39
39
|
import { buildCommandDispatchFailure, buildExtensionDisconnectFailure, getResponseCorsHeaders, } from './daemon-utils.js';
|
|
40
|
+
import { resolveDaemonHost } from './daemon-config.js';
|
|
40
41
|
const PORT = parseInt(process.env.BYCLI_DAEMON_PORT ?? String(DEFAULT_DAEMON_PORT), 10);
|
|
42
|
+
const HOST = resolveDaemonHost();
|
|
41
43
|
// The verify runner (M6b) spawns child processes that connect back to THIS daemon for a
|
|
42
44
|
// browser Page. Hand them our port (→ BYCLI_DAEMON_PORT in the child env) so the child's
|
|
43
45
|
// Page reaches us, not a freshly-spawned daemon. Must run before the first /v1/verify
|
|
@@ -585,8 +587,8 @@ wss.on('connection', (ws) => {
|
|
|
585
587
|
});
|
|
586
588
|
});
|
|
587
589
|
// ─── Start ───────────────────────────────────────────────────────────
|
|
588
|
-
httpServer.listen(PORT,
|
|
589
|
-
log.info(`[daemon] Listening on http
|
|
590
|
+
httpServer.listen(PORT, HOST, () => {
|
|
591
|
+
log.info(`[daemon] Listening on http://${HOST}:${PORT}`);
|
|
590
592
|
// Temp-store reap policy (M7b · 09:27-29). Resolved once at startup; out-of-range env → throws,
|
|
591
593
|
// but we keep the daemon alive by falling back to the (validated-elsewhere) defaults on error.
|
|
592
594
|
let tempPolicy;
|
|
@@ -6,6 +6,7 @@
|
|
|
6
6
|
* Flow: ArticleData → TurndownService → image download → frontmatter → .md file
|
|
7
7
|
*/
|
|
8
8
|
import TurndownService from 'turndown';
|
|
9
|
+
export { extractWechatArticleHtml } from './wechat-article.js';
|
|
9
10
|
export interface ArticleData {
|
|
10
11
|
title: string;
|
|
11
12
|
author?: string;
|
|
@@ -49,6 +50,8 @@ export interface ArticleDownloadOptions {
|
|
|
49
50
|
* as-is so the output is self-contained when piped.
|
|
50
51
|
*/
|
|
51
52
|
stdout?: boolean;
|
|
53
|
+
/** Opt-in hardened Markdown rules used by HTML-focused adapters. */
|
|
54
|
+
secureMarkdown?: boolean;
|
|
52
55
|
}
|
|
53
56
|
export interface ArticleDownloadResult {
|
|
54
57
|
title: string;
|
|
@@ -58,6 +61,9 @@ export interface ArticleDownloadResult {
|
|
|
58
61
|
size: string;
|
|
59
62
|
saved: string;
|
|
60
63
|
}
|
|
64
|
+
export declare function convertArticleHtmlToMarkdown(contentHtml: string, options?: {
|
|
65
|
+
safeFencedCodeBlocks?: boolean;
|
|
66
|
+
}): string;
|
|
61
67
|
/**
|
|
62
68
|
* Download an article to Markdown with optional image localization.
|
|
63
69
|
*
|