@sovovs/bycli 2.1.45 → 2.1.47
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/cli-manifest.json +18 -3
- package/clis/weixin/_wechat/api-draft.js +55 -30
- package/clis/weixin/_wechat/draft-content.js +39 -7
- package/clis/weixin/_wechat/draft-image-stage.js +51 -0
- package/clis/weixin/_wechat/remote-image.js +369 -0
- package/clis/weixin/_wechat/user-analysis.js +3 -0
- package/clis/weixin/_wechat/user-growth-download.js +152 -0
- package/clis/weixin/create-draft.js +67 -51
- package/clis/weixin/user-growth.js +16 -3
- package/dist/src/node-network.d.ts +5 -0
- package/dist/src/node-network.js +29 -0
- package/package.json +2 -1
package/cli-manifest.json
CHANGED
|
@@ -29020,6 +29020,13 @@
|
|
|
29020
29020
|
"required": false,
|
|
29021
29021
|
"help": "浏览器模式:填充并验证正文后停止,不会保存草稿"
|
|
29022
29022
|
},
|
|
29023
|
+
{
|
|
29024
|
+
"name": "allow-private-image-hosts",
|
|
29025
|
+
"type": "boolean",
|
|
29026
|
+
"default": false,
|
|
29027
|
+
"required": false,
|
|
29028
|
+
"help": "允许下载 localhost/内网 HTTP(S) 正文图片;云元数据地址始终禁止"
|
|
29029
|
+
},
|
|
29023
29030
|
{
|
|
29024
29031
|
"name": "timeout",
|
|
29025
29032
|
"type": "int",
|
|
@@ -29457,8 +29464,8 @@
|
|
|
29457
29464
|
{
|
|
29458
29465
|
"site": "weixin",
|
|
29459
29466
|
"name": "user-growth",
|
|
29460
|
-
"description": "
|
|
29461
|
-
"access": "
|
|
29467
|
+
"description": "读取公众号用户增长趋势,可选返回全部渠道并下载官方“全部来源”XLS",
|
|
29468
|
+
"access": "write",
|
|
29462
29469
|
"domain": "mp.weixin.qq.com",
|
|
29463
29470
|
"strategy": "cookie",
|
|
29464
29471
|
"browser": true,
|
|
@@ -29481,6 +29488,12 @@
|
|
|
29481
29488
|
"default": "all",
|
|
29482
29489
|
"required": false,
|
|
29483
29490
|
"help": "传播渠道名称或代码;多个值用逗号分隔"
|
|
29491
|
+
},
|
|
29492
|
+
{
|
|
29493
|
+
"name": "output",
|
|
29494
|
+
"type": "str",
|
|
29495
|
+
"required": false,
|
|
29496
|
+
"help": "可选的官方“全部来源”XLS 保存目录;不传则不下载"
|
|
29484
29497
|
}
|
|
29485
29498
|
],
|
|
29486
29499
|
"columns": [
|
|
@@ -29490,7 +29503,9 @@
|
|
|
29490
29503
|
"new_followers",
|
|
29491
29504
|
"unfollows",
|
|
29492
29505
|
"net_new_followers",
|
|
29493
|
-
"cumulative_followers"
|
|
29506
|
+
"cumulative_followers",
|
|
29507
|
+
"official_xls_path",
|
|
29508
|
+
"official_xls_size"
|
|
29494
29509
|
],
|
|
29495
29510
|
"type": "js",
|
|
29496
29511
|
"modulePath": "weixin/user-growth.js",
|
|
@@ -2,6 +2,7 @@ import * as nodeFs from 'node:fs/promises';
|
|
|
2
2
|
import * as nodePath from 'node:path';
|
|
3
3
|
import { CommandExecutionError } from '@sovovs/bycli/errors';
|
|
4
4
|
import { prepareHtmlContent } from './draft-content.js';
|
|
5
|
+
import { stageDraftHtmlImages } from './draft-image-stage.js';
|
|
5
6
|
|
|
6
7
|
const API_BASE = 'https://api.weixin.qq.com/cgi-bin';
|
|
7
8
|
|
|
@@ -55,6 +56,18 @@ async function uploadImage(filePath, token, fetchImpl) {
|
|
|
55
56
|
return payload;
|
|
56
57
|
}
|
|
57
58
|
|
|
59
|
+
async function uploadContentImage(filePath, token, fetchImpl) {
|
|
60
|
+
const data = await nodeFs.readFile(filePath);
|
|
61
|
+
const form = new FormData();
|
|
62
|
+
form.append('media', new Blob([data], { type: mimeType(filePath) }), nodePath.basename(filePath));
|
|
63
|
+
const url = new URL(`${API_BASE}/media/uploadimg`);
|
|
64
|
+
url.searchParams.set('access_token', token);
|
|
65
|
+
const response = await fetchImpl(url.toString(), { method: 'POST', body: form });
|
|
66
|
+
const payload = await readJsonResponse(response, '上传正文图片');
|
|
67
|
+
if (!payload.url) throw new CommandExecutionError('上传正文图片 failed: response did not contain url');
|
|
68
|
+
return payload.url;
|
|
69
|
+
}
|
|
70
|
+
|
|
58
71
|
function removeCoverImage(html) {
|
|
59
72
|
return String(html ?? '').replace(/<img\b[^>]*(?:alt|title)=["'][^"']*封面[^"']*["'][^>]*>\s*/giu, '');
|
|
60
73
|
}
|
|
@@ -69,6 +82,9 @@ export async function createDraftViaApi({
|
|
|
69
82
|
html,
|
|
70
83
|
baseDir = process.cwd(),
|
|
71
84
|
fetchImpl = globalThis.fetch,
|
|
85
|
+
imageFetchImpl = globalThis.fetch,
|
|
86
|
+
lookupImpl,
|
|
87
|
+
allowPrivateImageHosts = false,
|
|
72
88
|
} = {}) {
|
|
73
89
|
if (!String(appid ?? '').trim() || !String(appsecret ?? '').trim()) {
|
|
74
90
|
throw new CommandExecutionError('API mode requires both appid and appsecret');
|
|
@@ -76,39 +92,48 @@ export async function createDraftViaApi({
|
|
|
76
92
|
if (typeof fetchImpl !== 'function') throw new CommandExecutionError('API mode requires fetch support');
|
|
77
93
|
if (!coverImage) throw new CommandExecutionError('API mode requires cover-image');
|
|
78
94
|
|
|
79
|
-
const
|
|
80
|
-
const
|
|
81
|
-
const prepared = await prepareHtmlContent(removeCoverImage(html), {
|
|
95
|
+
const bodyHtml = removeCoverImage(html);
|
|
96
|
+
const staged = await stageDraftHtmlImages(bodyHtml, {
|
|
82
97
|
baseDir,
|
|
83
|
-
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
},
|
|
98
|
+
allowPrivateHosts: allowPrivateImageHosts,
|
|
99
|
+
fetchImpl: imageFetchImpl,
|
|
100
|
+
...(lookupImpl ? { lookupImpl } : {}),
|
|
87
101
|
});
|
|
102
|
+
try {
|
|
103
|
+
const token = await getAccessToken(String(appid).trim(), String(appsecret).trim(), fetchImpl);
|
|
104
|
+
const cover = await uploadImage(nodePath.resolve(coverImage), token, fetchImpl);
|
|
105
|
+
const prepared = await prepareHtmlContent(staged.html, {
|
|
106
|
+
baseDir,
|
|
107
|
+
allowRemoteImages: false,
|
|
108
|
+
resolveImage: imagePath => uploadContentImage(imagePath, token, fetchImpl),
|
|
109
|
+
});
|
|
88
110
|
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
|
|
104
|
-
|
|
105
|
-
|
|
106
|
-
|
|
107
|
-
|
|
108
|
-
|
|
109
|
-
|
|
110
|
-
|
|
111
|
-
|
|
111
|
+
const url = new URL(`${API_BASE}/draft/add`);
|
|
112
|
+
url.searchParams.set('access_token', token);
|
|
113
|
+
const body = {
|
|
114
|
+
articles: [{
|
|
115
|
+
title: String(title ?? ''),
|
|
116
|
+
author: String(author ?? ''),
|
|
117
|
+
digest: String(digest || title || ''),
|
|
118
|
+
content: prepared.html,
|
|
119
|
+
content_source_url: '',
|
|
120
|
+
thumb_media_id: cover.media_id,
|
|
121
|
+
show_cover_pic: 1,
|
|
122
|
+
need_open_comment: 0,
|
|
123
|
+
only_fans_can_comment: 0,
|
|
124
|
+
}],
|
|
125
|
+
};
|
|
126
|
+
const response = await fetchImpl(url.toString(), {
|
|
127
|
+
method: 'POST',
|
|
128
|
+
headers: { 'Content-Type': 'application/json; charset=utf-8' },
|
|
129
|
+
body: JSON.stringify(body),
|
|
130
|
+
});
|
|
131
|
+
const payload = await readJsonResponse(response, '创建草稿');
|
|
132
|
+
if (!payload.media_id) throw new CommandExecutionError('创建草稿 failed: response did not contain media_id');
|
|
133
|
+
return { mediaId: payload.media_id };
|
|
134
|
+
} finally {
|
|
135
|
+
await staged.cleanup();
|
|
136
|
+
}
|
|
112
137
|
}
|
|
113
138
|
|
|
114
139
|
export { removeCoverImage };
|
|
@@ -101,6 +101,31 @@ function safeUrl(value, { image = false } = {}) {
|
|
|
101
101
|
return raw;
|
|
102
102
|
}
|
|
103
103
|
|
|
104
|
+
function validatedImageSource(node, { allowRemoteImages = false } = {}) {
|
|
105
|
+
const source = attributes(node).find(attr => attr.name === 'src')?.value;
|
|
106
|
+
const url = safeUrl(source, { image: true });
|
|
107
|
+
if (!url) throw new CommandExecutionError('HTML contains an unsupported image source');
|
|
108
|
+
const isHttpsRemote = /^https:\/\//iu.test(url);
|
|
109
|
+
const isHttpRemote = /^http:\/\//iu.test(url);
|
|
110
|
+
const isRemote = isHttpsRemote || isHttpRemote;
|
|
111
|
+
if (isRemote && !allowRemoteImages) {
|
|
112
|
+
throw new CommandExecutionError('API mode requires HTML images to be local files');
|
|
113
|
+
}
|
|
114
|
+
return { url, isRemote };
|
|
115
|
+
}
|
|
116
|
+
|
|
117
|
+
function validateImageSources(node, options) {
|
|
118
|
+
if (!node || typeof node !== 'object') return;
|
|
119
|
+
const tag = String(node.tagName ?? node.nodeName ?? '').toLowerCase();
|
|
120
|
+
if (tag === 'img') validatedImageSource(node, options);
|
|
121
|
+
for (const child of node.childNodes ?? []) validateImageSources(child, options);
|
|
122
|
+
}
|
|
123
|
+
|
|
124
|
+
export function validateHtmlImageSources(html, { allowRemoteImages = false } = {}) {
|
|
125
|
+
const fragment = parseWechatHtmlFragment(String(html ?? ''));
|
|
126
|
+
for (const node of fragment.childNodes ?? []) validateImageSources(node, { allowRemoteImages });
|
|
127
|
+
}
|
|
128
|
+
|
|
104
129
|
function sanitizeAttributes(node) {
|
|
105
130
|
for (const attr of [...attributes(node)]) {
|
|
106
131
|
const name = attr.name.toLowerCase();
|
|
@@ -157,11 +182,8 @@ async function sanitizeNode(node, options) {
|
|
|
157
182
|
convertBackgroundSection(node);
|
|
158
183
|
|
|
159
184
|
if (tag === 'img') {
|
|
160
|
-
const
|
|
161
|
-
const
|
|
162
|
-
if (!url) throw new CommandExecutionError('HTML contains an unsupported image source');
|
|
163
|
-
const isRemote = /^https?:\/\//iu.test(url);
|
|
164
|
-
const resolved = isRemote ? url : await options.resolveImage(url);
|
|
185
|
+
const { url } = validatedImageSource(node, options);
|
|
186
|
+
const resolved = await options.resolveImage(url);
|
|
165
187
|
if (!resolved) throw new CommandExecutionError(`Could not upload HTML image: ${url}`);
|
|
166
188
|
setAttribute(node, 'src', resolved);
|
|
167
189
|
}
|
|
@@ -196,13 +218,23 @@ export function loadDraftContent({ content, contentFile, contentFormat = 'text'
|
|
|
196
218
|
};
|
|
197
219
|
}
|
|
198
220
|
|
|
199
|
-
export async function prepareHtmlContent(html, {
|
|
221
|
+
export async function prepareHtmlContent(html, {
|
|
222
|
+
baseDir = process.cwd(),
|
|
223
|
+
resolveImage,
|
|
224
|
+
allowRemoteImages = false,
|
|
225
|
+
} = {}) {
|
|
200
226
|
if (typeof resolveImage !== 'function') throw new ArgumentError('resolveImage is required for HTML content');
|
|
201
227
|
const fragment = parseWechatHtmlFragment(String(html ?? ''));
|
|
228
|
+
for (const node of fragment.childNodes ?? []) validateImageSources(node, { allowRemoteImages });
|
|
202
229
|
const imageResolver = async source => {
|
|
203
230
|
const absolute = /^https?:\/\//iu.test(source) ? source : nodePath.resolve(baseDir, source);
|
|
204
231
|
return resolveImage(absolute);
|
|
205
232
|
};
|
|
206
|
-
|
|
233
|
+
for (const node of fragment.childNodes ?? []) {
|
|
234
|
+
await sanitizeNode(node, {
|
|
235
|
+
resolveImage: imageResolver,
|
|
236
|
+
allowRemoteImages,
|
|
237
|
+
});
|
|
238
|
+
}
|
|
207
239
|
return { html: serializeWechatHtml(fragment) };
|
|
208
240
|
}
|
|
@@ -0,0 +1,51 @@
|
|
|
1
|
+
import { prepareHtmlContent } from './draft-content.js';
|
|
2
|
+
import { downloadRemoteImage } from './remote-image.js';
|
|
3
|
+
import { CommandExecutionError } from '@sovovs/bycli/errors';
|
|
4
|
+
|
|
5
|
+
function isRemoteImageSource(source) {
|
|
6
|
+
return /^https?:\/\//iu.test(String(source || ''));
|
|
7
|
+
}
|
|
8
|
+
|
|
9
|
+
export async function stageDraftHtmlImages(html, {
|
|
10
|
+
baseDir = process.cwd(),
|
|
11
|
+
allowPrivateHosts = false,
|
|
12
|
+
fetchImpl = globalThis.fetch,
|
|
13
|
+
lookupImpl,
|
|
14
|
+
downloadImpl = downloadRemoteImage,
|
|
15
|
+
} = {}) {
|
|
16
|
+
const downloads = [];
|
|
17
|
+
let cleaned = false;
|
|
18
|
+
const cleanup = async () => {
|
|
19
|
+
if (cleaned) return;
|
|
20
|
+
cleaned = true;
|
|
21
|
+
const results = await Promise.allSettled(downloads.map(downloaded => downloaded.cleanup()));
|
|
22
|
+
const failure = results.find(result => result.status === 'rejected');
|
|
23
|
+
if (failure) {
|
|
24
|
+
throw new CommandExecutionError(`Failed to clean up a temporary Weixin image: ${failure.reason?.message ?? failure.reason}`);
|
|
25
|
+
}
|
|
26
|
+
};
|
|
27
|
+
try {
|
|
28
|
+
const prepared = await prepareHtmlContent(html, {
|
|
29
|
+
baseDir,
|
|
30
|
+
allowRemoteImages: true,
|
|
31
|
+
resolveImage: async source => {
|
|
32
|
+
if (!isRemoteImageSource(source)) return source;
|
|
33
|
+
const downloaded = await downloadImpl(source, {
|
|
34
|
+
allowPrivateHosts,
|
|
35
|
+
fetchImpl,
|
|
36
|
+
...(lookupImpl ? { lookupImpl } : {}),
|
|
37
|
+
});
|
|
38
|
+
downloads.push(downloaded);
|
|
39
|
+
return downloaded.path;
|
|
40
|
+
},
|
|
41
|
+
});
|
|
42
|
+
return { html: prepared.html, cleanup };
|
|
43
|
+
} catch (error) {
|
|
44
|
+
try {
|
|
45
|
+
await cleanup();
|
|
46
|
+
} catch (cleanupError) {
|
|
47
|
+
throw new CommandExecutionError(`${error?.message ?? error}; ${cleanupError.message}`);
|
|
48
|
+
}
|
|
49
|
+
throw error;
|
|
50
|
+
}
|
|
51
|
+
}
|
|
@@ -0,0 +1,369 @@
|
|
|
1
|
+
import { mkdtemp, rm, writeFile } from 'node:fs/promises';
|
|
2
|
+
import { lookup as dnsLookup } from 'node:dns/promises';
|
|
3
|
+
import { isIP } from 'node:net';
|
|
4
|
+
import { tmpdir } from 'node:os';
|
|
5
|
+
import { join } from 'node:path';
|
|
6
|
+
import { ArgumentError } from '@sovovs/bycli/errors';
|
|
7
|
+
import { createPinnedDispatcher } from '@sovovs/bycli/node-network';
|
|
8
|
+
|
|
9
|
+
const DEFAULT_MAX_BYTES = 10 * 1024 * 1024;
|
|
10
|
+
const DEFAULT_TIMEOUT_MS = 20_000;
|
|
11
|
+
const DEFAULT_MAX_REDIRECTS = 5;
|
|
12
|
+
const CLOUD_METADATA_HOSTS = new Set([
|
|
13
|
+
'instance-data.ec2.internal',
|
|
14
|
+
'metadata.google.internal',
|
|
15
|
+
'metadata.goog',
|
|
16
|
+
]);
|
|
17
|
+
const CLOUD_METADATA_ADDRESSES = new Set([
|
|
18
|
+
'100.100.100.200',
|
|
19
|
+
'169.254.169.254',
|
|
20
|
+
'169.254.170.2',
|
|
21
|
+
'169.254.170.23',
|
|
22
|
+
'fd00:ec2::254',
|
|
23
|
+
'fd00:ec2::23',
|
|
24
|
+
]);
|
|
25
|
+
const IMAGE_FORMATS = new Map([
|
|
26
|
+
['image/jpeg', { extension: '.jpg', matches: bytes => bytes[0] === 0xff && bytes[1] === 0xd8 && bytes[2] === 0xff }],
|
|
27
|
+
['image/png', { extension: '.png', matches: bytes => isPng(bytes) }],
|
|
28
|
+
['image/gif', { extension: '.gif', matches: bytes => {
|
|
29
|
+
const header = String.fromCharCode(...bytes.slice(0, 6));
|
|
30
|
+
return header === 'GIF87a' || header === 'GIF89a';
|
|
31
|
+
} }],
|
|
32
|
+
['image/webp', { extension: '.webp', matches: bytes => {
|
|
33
|
+
const riff = String.fromCharCode(...bytes.slice(0, 4));
|
|
34
|
+
const webp = String.fromCharCode(...bytes.slice(8, 12));
|
|
35
|
+
return riff === 'RIFF' && webp === 'WEBP';
|
|
36
|
+
} }],
|
|
37
|
+
]);
|
|
38
|
+
|
|
39
|
+
function isPng(bytes) {
|
|
40
|
+
const signature = [0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a];
|
|
41
|
+
return signature.every((value, index) => bytes[index] === value);
|
|
42
|
+
}
|
|
43
|
+
|
|
44
|
+
function normalizeHostname(value) {
|
|
45
|
+
return String(value || '').replace(/^\[|\]$/gu, '').replace(/\.$/u, '').toLowerCase();
|
|
46
|
+
}
|
|
47
|
+
|
|
48
|
+
function parseIpv4(address) {
|
|
49
|
+
const parts = String(address).split('.');
|
|
50
|
+
if (parts.length !== 4 || parts.some(part => !/^\d{1,3}$/u.test(part))) return null;
|
|
51
|
+
const numbers = parts.map(Number);
|
|
52
|
+
return numbers.some(value => value > 255) ? null : numbers;
|
|
53
|
+
}
|
|
54
|
+
|
|
55
|
+
function isPrivateIpv4(address) {
|
|
56
|
+
const parts = parseIpv4(address);
|
|
57
|
+
if (!parts) return false;
|
|
58
|
+
const [a, b, c] = parts;
|
|
59
|
+
return a === 0
|
|
60
|
+
|| a === 10
|
|
61
|
+
|| a === 127
|
|
62
|
+
|| (a === 100 && b >= 64 && b <= 127)
|
|
63
|
+
|| (a === 169 && b === 254)
|
|
64
|
+
|| (a === 172 && b >= 16 && b <= 31)
|
|
65
|
+
|| (a === 192 && ((b === 0 && (c === 0 || c === 2)) || (b === 88 && c === 99) || b === 168))
|
|
66
|
+
|| (a === 198 && (b === 18 || b === 19 || (b === 51 && c === 100)))
|
|
67
|
+
|| (a === 203 && b === 0 && c === 113)
|
|
68
|
+
|| a >= 224;
|
|
69
|
+
}
|
|
70
|
+
|
|
71
|
+
function parseIpv6Groups(address) {
|
|
72
|
+
let normalized = normalizeHostname(address).split('%')[0];
|
|
73
|
+
if (normalized.includes('.')) {
|
|
74
|
+
const lastColon = normalized.lastIndexOf(':');
|
|
75
|
+
const ipv4 = parseIpv4(normalized.slice(lastColon + 1));
|
|
76
|
+
if (!ipv4) return null;
|
|
77
|
+
normalized = `${normalized.slice(0, lastColon)}:${((ipv4[0] << 8) | ipv4[1]).toString(16)}:${((ipv4[2] << 8) | ipv4[3]).toString(16)}`;
|
|
78
|
+
}
|
|
79
|
+
const halves = normalized.split('::');
|
|
80
|
+
if (halves.length > 2) return null;
|
|
81
|
+
const parseHalf = half => half
|
|
82
|
+
? half.split(':').map(part => (/^[0-9a-f]{1,4}$/u.test(part) ? Number.parseInt(part, 16) : NaN))
|
|
83
|
+
: [];
|
|
84
|
+
const left = parseHalf(halves[0]);
|
|
85
|
+
const right = parseHalf(halves[1] ?? '');
|
|
86
|
+
if ([...left, ...right].some(Number.isNaN)) return null;
|
|
87
|
+
const omitted = 8 - left.length - right.length;
|
|
88
|
+
if ((halves.length === 1 && omitted !== 0) || omitted < 0) return null;
|
|
89
|
+
return [...left, ...Array.from({ length: omitted }, () => 0), ...right];
|
|
90
|
+
}
|
|
91
|
+
|
|
92
|
+
function mappedIpv4Address(address) {
|
|
93
|
+
const groups = parseIpv6Groups(address);
|
|
94
|
+
if (!groups || groups.length !== 8) return null;
|
|
95
|
+
if (!groups.slice(0, 5).every(group => group === 0) || groups[5] !== 0xffff) return null;
|
|
96
|
+
return [groups[6] >> 8, groups[6] & 0xff, groups[7] >> 8, groups[7] & 0xff].join('.');
|
|
97
|
+
}
|
|
98
|
+
|
|
99
|
+
function translatedIpv4Address(address) {
|
|
100
|
+
const groups = parseIpv6Groups(address);
|
|
101
|
+
if (!groups || groups.length !== 8) return null;
|
|
102
|
+
const isWellKnownNat64 = groups[0] === 0x0064
|
|
103
|
+
&& groups[1] === 0xff9b
|
|
104
|
+
&& groups.slice(2, 6).every(group => group === 0);
|
|
105
|
+
const isSixToFour = groups[0] === 0x2002;
|
|
106
|
+
if (!isWellKnownNat64 && !isSixToFour) return null;
|
|
107
|
+
const high = isSixToFour ? groups[1] : groups[6];
|
|
108
|
+
const low = isSixToFour ? groups[2] : groups[7];
|
|
109
|
+
return [high >> 8, high & 0xff, low >> 8, low & 0xff].join('.');
|
|
110
|
+
}
|
|
111
|
+
|
|
112
|
+
function isPrivateIpv6(address) {
|
|
113
|
+
const normalized = normalizeHostname(address).split('%')[0];
|
|
114
|
+
const mappedIpv4 = mappedIpv4Address(normalized);
|
|
115
|
+
if (mappedIpv4) return isPrivateIpv4(mappedIpv4);
|
|
116
|
+
const translatedIpv4 = translatedIpv4Address(normalized);
|
|
117
|
+
if (translatedIpv4 && isPrivateIpv4(translatedIpv4)) return true;
|
|
118
|
+
const groups = parseIpv6Groups(normalized);
|
|
119
|
+
if (!groups) return true;
|
|
120
|
+
const [a, b, c, d] = groups;
|
|
121
|
+
const protocolAssignments = a === 0x2001 && b <= 0x01ff;
|
|
122
|
+
const globallyReachableProtocolAssignment = (b === 0x0001
|
|
123
|
+
&& groups.slice(2, 7).every(group => group === 0)
|
|
124
|
+
&& [1, 2, 3].includes(groups[7]))
|
|
125
|
+
|| b === 0x0003
|
|
126
|
+
|| (b === 0x0004 && c === 0x0112)
|
|
127
|
+
|| (b & 0xfff0) === 0x0020
|
|
128
|
+
|| (b & 0xfff0) === 0x0030;
|
|
129
|
+
return groups.slice(0, 6).every(group => group === 0)
|
|
130
|
+
|| (a & 0xfe00) === 0xfc00
|
|
131
|
+
|| (a & 0xffc0) === 0xfe80
|
|
132
|
+
|| (a & 0xffc0) === 0xfec0
|
|
133
|
+
|| (a & 0xff00) === 0xff00
|
|
134
|
+
|| (a === 0x0064 && b === 0xff9b && c === 0x0001)
|
|
135
|
+
|| (a === 0x0100 && b === 0 && c === 0 && (d === 0 || d === 1))
|
|
136
|
+
|| (protocolAssignments && !globallyReachableProtocolAssignment)
|
|
137
|
+
|| (a === 0x2001 && b === 0x0db8)
|
|
138
|
+
|| a === 0x2002
|
|
139
|
+
|| (a === 0x3fff && (b & 0xf000) === 0)
|
|
140
|
+
|| a === 0x5f00;
|
|
141
|
+
}
|
|
142
|
+
|
|
143
|
+
function isPrivateAddress(address) {
|
|
144
|
+
const normalized = normalizeHostname(address);
|
|
145
|
+
if (isIP(normalized) === 4) return isPrivateIpv4(normalized);
|
|
146
|
+
if (isIP(normalized) === 6) return isPrivateIpv6(normalized);
|
|
147
|
+
return false;
|
|
148
|
+
}
|
|
149
|
+
|
|
150
|
+
function isCloudMetadataAddress(address) {
|
|
151
|
+
const normalized = normalizeHostname(address);
|
|
152
|
+
const mappedIpv4 = mappedIpv4Address(normalized);
|
|
153
|
+
const translatedIpv4 = translatedIpv4Address(normalized);
|
|
154
|
+
return CLOUD_METADATA_ADDRESSES.has(normalized)
|
|
155
|
+
|| Boolean(mappedIpv4 && CLOUD_METADATA_ADDRESSES.has(mappedIpv4))
|
|
156
|
+
|| Boolean(translatedIpv4 && CLOUD_METADATA_ADDRESSES.has(translatedIpv4));
|
|
157
|
+
}
|
|
158
|
+
|
|
159
|
+
async function defaultLookup(hostname) {
|
|
160
|
+
return dnsLookup(hostname, { all: true, verbatim: true });
|
|
161
|
+
}
|
|
162
|
+
|
|
163
|
+
function remainingTime(deadline) {
|
|
164
|
+
const remaining = deadline - Date.now();
|
|
165
|
+
if (remaining <= 0) throw new ArgumentError('Remote image download timed out');
|
|
166
|
+
return remaining;
|
|
167
|
+
}
|
|
168
|
+
|
|
169
|
+
async function lookupBeforeDeadline(lookupImpl, hostname, deadline) {
|
|
170
|
+
const remaining = remainingTime(deadline);
|
|
171
|
+
let timer;
|
|
172
|
+
try {
|
|
173
|
+
return await Promise.race([
|
|
174
|
+
lookupImpl(hostname),
|
|
175
|
+
new Promise((_, reject) => {
|
|
176
|
+
timer = setTimeout(() => reject(new ArgumentError('Remote image download timed out')), remaining);
|
|
177
|
+
}),
|
|
178
|
+
]);
|
|
179
|
+
} finally {
|
|
180
|
+
clearTimeout(timer);
|
|
181
|
+
}
|
|
182
|
+
}
|
|
183
|
+
|
|
184
|
+
async function assertAllowedTarget(url, { allowPrivateHosts, lookupImpl, deadline }) {
|
|
185
|
+
const hostname = normalizeHostname(url.hostname);
|
|
186
|
+
if (CLOUD_METADATA_HOSTS.has(hostname) || isCloudMetadataAddress(hostname)) {
|
|
187
|
+
throw new ArgumentError('Cloud metadata addresses are not allowed');
|
|
188
|
+
}
|
|
189
|
+
let addresses;
|
|
190
|
+
if (isIP(hostname)) {
|
|
191
|
+
addresses = [{ address: hostname, family: isIP(hostname) }];
|
|
192
|
+
} else if (hostname === 'localhost' || hostname.endsWith('.localhost')) {
|
|
193
|
+
addresses = [{ address: '127.0.0.1', family: 4 }];
|
|
194
|
+
} else {
|
|
195
|
+
try {
|
|
196
|
+
addresses = await lookupBeforeDeadline(lookupImpl, hostname, deadline);
|
|
197
|
+
} catch (error) {
|
|
198
|
+
if (error instanceof ArgumentError && error.message.includes('timed out')) throw error;
|
|
199
|
+
throw new ArgumentError(`Remote image host lookup failed: ${error?.message ?? error}`);
|
|
200
|
+
}
|
|
201
|
+
}
|
|
202
|
+
if (!Array.isArray(addresses) || addresses.length === 0) {
|
|
203
|
+
throw new ArgumentError('Remote image host did not resolve to an address');
|
|
204
|
+
}
|
|
205
|
+
if (addresses.some(item => !item || !isIP(normalizeHostname(item.address)))) {
|
|
206
|
+
throw new ArgumentError('Remote image host resolved to an invalid address');
|
|
207
|
+
}
|
|
208
|
+
if (addresses.some(item => isCloudMetadataAddress(item.address))) {
|
|
209
|
+
throw new ArgumentError('Cloud metadata addresses are not allowed');
|
|
210
|
+
}
|
|
211
|
+
if (!allowPrivateHosts && addresses.some(item => isPrivateAddress(item.address))) {
|
|
212
|
+
throw new ArgumentError('Private remote image hosts require --allow-private-image-hosts true');
|
|
213
|
+
}
|
|
214
|
+
return addresses.map(item => ({ address: normalizeHostname(item.address), family: isIP(normalizeHostname(item.address)) }));
|
|
215
|
+
}
|
|
216
|
+
|
|
217
|
+
async function startTimedFetch(fetchImpl, url, timeoutMs, addresses) {
|
|
218
|
+
const controller = new AbortController();
|
|
219
|
+
const timer = setTimeout(() => controller.abort(), timeoutMs);
|
|
220
|
+
const dispatcher = createPinnedDispatcher(addresses);
|
|
221
|
+
try {
|
|
222
|
+
const response = await fetchImpl(url, {
|
|
223
|
+
redirect: 'manual',
|
|
224
|
+
signal: controller.signal,
|
|
225
|
+
dispatcher,
|
|
226
|
+
});
|
|
227
|
+
let disposed = false;
|
|
228
|
+
return {
|
|
229
|
+
response,
|
|
230
|
+
signal: controller.signal,
|
|
231
|
+
dispose: async () => {
|
|
232
|
+
if (disposed) return;
|
|
233
|
+
disposed = true;
|
|
234
|
+
clearTimeout(timer);
|
|
235
|
+
await dispatcher.close();
|
|
236
|
+
},
|
|
237
|
+
};
|
|
238
|
+
} catch (error) {
|
|
239
|
+
clearTimeout(timer);
|
|
240
|
+
await dispatcher.close();
|
|
241
|
+
if (controller.signal.aborted) throw new ArgumentError('Remote image download timed out');
|
|
242
|
+
throw new ArgumentError(`Remote image download failed: ${error?.message ?? error}`);
|
|
243
|
+
}
|
|
244
|
+
}
|
|
245
|
+
|
|
246
|
+
async function readWithAbort(reader, signal) {
|
|
247
|
+
if (signal.aborted) throw new ArgumentError('Remote image download timed out');
|
|
248
|
+
let onAbort;
|
|
249
|
+
const aborted = new Promise((_, reject) => {
|
|
250
|
+
onAbort = () => reject(new ArgumentError('Remote image download timed out'));
|
|
251
|
+
signal.addEventListener('abort', onAbort, { once: true });
|
|
252
|
+
});
|
|
253
|
+
try {
|
|
254
|
+
return await Promise.race([reader.read(), aborted]);
|
|
255
|
+
} finally {
|
|
256
|
+
signal.removeEventListener('abort', onAbort);
|
|
257
|
+
}
|
|
258
|
+
}
|
|
259
|
+
|
|
260
|
+
async function readBoundedBody(response, maxBytes, signal) {
|
|
261
|
+
if (!response.body || typeof response.body.getReader !== 'function') {
|
|
262
|
+
throw new ArgumentError('Remote image response body was empty');
|
|
263
|
+
}
|
|
264
|
+
const reader = response.body.getReader();
|
|
265
|
+
const chunks = [];
|
|
266
|
+
let size = 0;
|
|
267
|
+
try {
|
|
268
|
+
for (;;) {
|
|
269
|
+
const { done, value } = await readWithAbort(reader, signal);
|
|
270
|
+
if (done) break;
|
|
271
|
+
const chunk = value instanceof Uint8Array ? value : new Uint8Array(value);
|
|
272
|
+
size += chunk.byteLength;
|
|
273
|
+
if (size > maxBytes) {
|
|
274
|
+
throw new ArgumentError(`Remote image exceeds ${maxBytes} bytes`);
|
|
275
|
+
}
|
|
276
|
+
chunks.push(chunk);
|
|
277
|
+
}
|
|
278
|
+
} catch (error) {
|
|
279
|
+
await reader.cancel().catch(() => {});
|
|
280
|
+
throw error;
|
|
281
|
+
}
|
|
282
|
+
const bytes = new Uint8Array(size);
|
|
283
|
+
let offset = 0;
|
|
284
|
+
for (const chunk of chunks) {
|
|
285
|
+
bytes.set(chunk, offset);
|
|
286
|
+
offset += chunk.byteLength;
|
|
287
|
+
}
|
|
288
|
+
return bytes;
|
|
289
|
+
}
|
|
290
|
+
|
|
291
|
+
export async function downloadRemoteImage(sourceUrl, {
|
|
292
|
+
allowPrivateHosts = false,
|
|
293
|
+
fetchImpl = globalThis.fetch,
|
|
294
|
+
lookupImpl = defaultLookup,
|
|
295
|
+
maxBytes = DEFAULT_MAX_BYTES,
|
|
296
|
+
timeoutMs = DEFAULT_TIMEOUT_MS,
|
|
297
|
+
maxRedirects = DEFAULT_MAX_REDIRECTS,
|
|
298
|
+
mkdtempImpl = mkdtemp,
|
|
299
|
+
writeFileImpl = writeFile,
|
|
300
|
+
rmImpl = rm,
|
|
301
|
+
} = {}) {
|
|
302
|
+
let url;
|
|
303
|
+
try {
|
|
304
|
+
url = new URL(sourceUrl);
|
|
305
|
+
} catch {
|
|
306
|
+
throw new ArgumentError(`Invalid remote image URL: ${sourceUrl}`);
|
|
307
|
+
}
|
|
308
|
+
if (!['http:', 'https:'].includes(url.protocol)) {
|
|
309
|
+
throw new ArgumentError(`Unsupported remote image protocol: ${url.protocol}`);
|
|
310
|
+
}
|
|
311
|
+
const deadline = Date.now() + timeoutMs;
|
|
312
|
+
let timedFetch;
|
|
313
|
+
for (let redirects = 0; ; redirects += 1) {
|
|
314
|
+
const addresses = await assertAllowedTarget(url, {
|
|
315
|
+
allowPrivateHosts,
|
|
316
|
+
lookupImpl,
|
|
317
|
+
deadline,
|
|
318
|
+
});
|
|
319
|
+
timedFetch = await startTimedFetch(fetchImpl, url.href, remainingTime(deadline), addresses);
|
|
320
|
+
const { response } = timedFetch;
|
|
321
|
+
if (response.status < 300 || response.status >= 400) break;
|
|
322
|
+
await response.body?.cancel().catch(() => {});
|
|
323
|
+
await timedFetch.dispose();
|
|
324
|
+
if (redirects >= maxRedirects) throw new ArgumentError(`Remote image exceeded ${maxRedirects} redirects`);
|
|
325
|
+
const location = response.headers.get('location');
|
|
326
|
+
if (!location) throw new ArgumentError('Remote image redirect was missing a destination');
|
|
327
|
+
try {
|
|
328
|
+
url = new URL(location, url);
|
|
329
|
+
} catch {
|
|
330
|
+
throw new ArgumentError('Remote image redirect destination was invalid');
|
|
331
|
+
}
|
|
332
|
+
if (!['http:', 'https:'].includes(url.protocol)) {
|
|
333
|
+
throw new ArgumentError(`Unsupported remote image protocol: ${url.protocol}`);
|
|
334
|
+
}
|
|
335
|
+
}
|
|
336
|
+
try {
|
|
337
|
+
const { response, signal } = timedFetch;
|
|
338
|
+
if (!response.ok) throw new ArgumentError(`Remote image download failed: HTTP ${response.status}`);
|
|
339
|
+
const contentType = String(response.headers.get('content-type') || '').split(';')[0].trim().toLowerCase();
|
|
340
|
+
const format = IMAGE_FORMATS.get(contentType === 'image/jpg' ? 'image/jpeg' : contentType);
|
|
341
|
+
if (!format) throw new ArgumentError(`Unsupported remote image content type: ${contentType || 'missing'}`);
|
|
342
|
+
const contentLength = Number(response.headers.get('content-length') || 0);
|
|
343
|
+
if (Number.isFinite(contentLength) && contentLength > maxBytes) {
|
|
344
|
+
throw new ArgumentError(`Remote image exceeds ${maxBytes} bytes`);
|
|
345
|
+
}
|
|
346
|
+
const bytes = await readBoundedBody(response, maxBytes, signal);
|
|
347
|
+
if (!format.matches(bytes)) throw new ArgumentError(`Remote image content does not match ${contentType}`);
|
|
348
|
+
const directory = await mkdtempImpl(join(tmpdir(), 'bycli-weixin-image-'));
|
|
349
|
+
const path = join(directory, `image${format.extension}`);
|
|
350
|
+
try {
|
|
351
|
+
await writeFileImpl(path, bytes, { mode: 0o600 });
|
|
352
|
+
return {
|
|
353
|
+
path,
|
|
354
|
+
extension: format.extension,
|
|
355
|
+
size: bytes.byteLength,
|
|
356
|
+
resolvedUrl: url.href,
|
|
357
|
+
cleanup: () => rmImpl(directory, { recursive: true, force: true }),
|
|
358
|
+
};
|
|
359
|
+
} catch (error) {
|
|
360
|
+
await rmImpl(directory, { recursive: true, force: true });
|
|
361
|
+
throw error;
|
|
362
|
+
}
|
|
363
|
+
} catch (error) {
|
|
364
|
+
await timedFetch.response.body?.cancel().catch(() => {});
|
|
365
|
+
throw error;
|
|
366
|
+
} finally {
|
|
367
|
+
await timedFetch.dispose();
|
|
368
|
+
}
|
|
369
|
+
}
|
|
@@ -79,6 +79,9 @@ export function resolveAttributeDate(value, options = {}) {
|
|
|
79
79
|
}
|
|
80
80
|
|
|
81
81
|
export function parseGrowthSources(value = 'all') {
|
|
82
|
+
if (String(value).trim() === 'all-sources') {
|
|
83
|
+
return SOURCE_ENTRIES.map(([name, code]) => ({ name, code }));
|
|
84
|
+
}
|
|
82
85
|
const rawItems = String(value).split(',').map(item => item.trim()).filter(Boolean);
|
|
83
86
|
argument(rawItems.length > 0, 'source must not be empty');
|
|
84
87
|
const seen = new Set();
|
|
@@ -0,0 +1,152 @@
|
|
|
1
|
+
import { constants } from 'node:fs';
|
|
2
|
+
import { randomUUID } from 'node:crypto';
|
|
3
|
+
import { copyFile, link, mkdir, stat, unlink } from 'node:fs/promises';
|
|
4
|
+
import { extname, resolve } from 'node:path';
|
|
5
|
+
import { ArgumentError, CommandExecutionError, TimeoutError } from '@sovovs/bycli/errors';
|
|
6
|
+
import { buildGrowthUrl } from './user-analysis.js';
|
|
7
|
+
|
|
8
|
+
const DOWNLOAD_TIMEOUT_MS = 60_000;
|
|
9
|
+
|
|
10
|
+
function commandError(message) {
|
|
11
|
+
return new CommandExecutionError(`WeChat user-growth XLS download ${message}`);
|
|
12
|
+
}
|
|
13
|
+
|
|
14
|
+
export function buildUserGrowthDownloadUrl({ token, begin, end }) {
|
|
15
|
+
const url = new URL(buildGrowthUrl({
|
|
16
|
+
token,
|
|
17
|
+
begin,
|
|
18
|
+
end,
|
|
19
|
+
sourceCodes: [99999999],
|
|
20
|
+
}));
|
|
21
|
+
url.searchParams.delete('f');
|
|
22
|
+
url.searchParams.delete('ajax');
|
|
23
|
+
url.searchParams.set('download', '1');
|
|
24
|
+
return url.toString();
|
|
25
|
+
}
|
|
26
|
+
|
|
27
|
+
export function isTrustedUserGrowthDownloadUrl(candidateValue, expectedValue) {
|
|
28
|
+
let candidate;
|
|
29
|
+
let expected;
|
|
30
|
+
try {
|
|
31
|
+
candidate = new URL(candidateValue);
|
|
32
|
+
expected = new URL(expectedValue);
|
|
33
|
+
} catch {
|
|
34
|
+
return false;
|
|
35
|
+
}
|
|
36
|
+
const exactParams = ['download', 'begin_date', 'end_date', 'source', 'token'];
|
|
37
|
+
return candidate.protocol === 'https:'
|
|
38
|
+
&& candidate.hostname === 'mp.weixin.qq.com'
|
|
39
|
+
&& candidate.port === ''
|
|
40
|
+
&& candidate.pathname === '/misc/useranalysis'
|
|
41
|
+
&& exactParams.every(name => candidate.searchParams.get(name) === expected.searchParams.get(name))
|
|
42
|
+
&& candidate.searchParams.get('download') === '1'
|
|
43
|
+
&& candidate.searchParams.get('source') === '99999999';
|
|
44
|
+
}
|
|
45
|
+
|
|
46
|
+
async function publishExclusively(source, outputDir, filename, beforePublish) {
|
|
47
|
+
const extension = extname(filename);
|
|
48
|
+
const stem = filename.slice(0, -extension.length);
|
|
49
|
+
const staged = resolve(outputDir, `.bycli-user-growth-${randomUUID()}.tmp`);
|
|
50
|
+
let stagedCreated = false;
|
|
51
|
+
try {
|
|
52
|
+
try {
|
|
53
|
+
await copyFile(source, staged, constants.COPYFILE_EXCL);
|
|
54
|
+
stagedCreated = true;
|
|
55
|
+
} catch {
|
|
56
|
+
try {
|
|
57
|
+
await unlink(staged);
|
|
58
|
+
} catch {
|
|
59
|
+
// COPYFILE_EXCL may fail before creating its destination.
|
|
60
|
+
}
|
|
61
|
+
throw commandError('could not stage the downloaded file');
|
|
62
|
+
}
|
|
63
|
+
for (let index = 0; index <= 9999; index += 1) {
|
|
64
|
+
const candidate = resolve(outputDir, index === 0 ? filename : `${stem}-${index}${extension}`);
|
|
65
|
+
try {
|
|
66
|
+
await beforePublish?.();
|
|
67
|
+
await link(staged, candidate);
|
|
68
|
+
return candidate;
|
|
69
|
+
} catch (error) {
|
|
70
|
+
if (error?.code === 'EEXIST') continue;
|
|
71
|
+
throw commandError('could not publish the downloaded file');
|
|
72
|
+
}
|
|
73
|
+
}
|
|
74
|
+
throw commandError('could not allocate a destination filename');
|
|
75
|
+
} finally {
|
|
76
|
+
if (stagedCreated) {
|
|
77
|
+
try {
|
|
78
|
+
await unlink(staged);
|
|
79
|
+
} catch {
|
|
80
|
+
// The final hard link remains a complete file.
|
|
81
|
+
}
|
|
82
|
+
}
|
|
83
|
+
}
|
|
84
|
+
}
|
|
85
|
+
|
|
86
|
+
export async function downloadUserGrowthXls(page, options) {
|
|
87
|
+
if (typeof options?.outputDir !== 'string' || !options.outputDir.trim()) {
|
|
88
|
+
throw new ArgumentError('output must be a non-empty directory');
|
|
89
|
+
}
|
|
90
|
+
if (typeof page?.waitForDownload !== 'function') {
|
|
91
|
+
throw commandError('requires browser download support');
|
|
92
|
+
}
|
|
93
|
+
|
|
94
|
+
const expectedUrl = buildUserGrowthDownloadUrl(options);
|
|
95
|
+
const startedAfterMs = Date.now();
|
|
96
|
+
let downloaded;
|
|
97
|
+
try {
|
|
98
|
+
await page.goto(expectedUrl, { waitUntil: 'none' });
|
|
99
|
+
downloaded = await page.waitForDownload('download=1', DOWNLOAD_TIMEOUT_MS, {
|
|
100
|
+
includeRecent: true,
|
|
101
|
+
startedAfterMs,
|
|
102
|
+
});
|
|
103
|
+
} catch (error) {
|
|
104
|
+
if (error instanceof TimeoutError) throw error;
|
|
105
|
+
throw commandError('could not complete the browser download');
|
|
106
|
+
}
|
|
107
|
+
|
|
108
|
+
if (!downloaded || downloaded.downloaded !== true) {
|
|
109
|
+
throw new TimeoutError('Weixin user-growth XLS download', DOWNLOAD_TIMEOUT_MS / 1000);
|
|
110
|
+
}
|
|
111
|
+
if (typeof downloaded.filename !== 'string'
|
|
112
|
+
|| !downloaded.filename
|
|
113
|
+
|| extname(downloaded.filename).toLowerCase() !== '.xls'
|
|
114
|
+
|| downloaded.state !== 'complete'
|
|
115
|
+
|| !['safe', 'accepted'].includes(downloaded.danger)) {
|
|
116
|
+
throw commandError('returned incomplete or unsafe download metadata');
|
|
117
|
+
}
|
|
118
|
+
if (![downloaded.url, downloaded.finalUrl]
|
|
119
|
+
.some(value => isTrustedUserGrowthDownloadUrl(value, expectedUrl))) {
|
|
120
|
+
throw commandError('rejected an unrelated downloaded file');
|
|
121
|
+
}
|
|
122
|
+
|
|
123
|
+
let sourceInfo;
|
|
124
|
+
try {
|
|
125
|
+
sourceInfo = await stat(downloaded.filename);
|
|
126
|
+
} catch {
|
|
127
|
+
throw commandError('could not read the downloaded file');
|
|
128
|
+
}
|
|
129
|
+
if (!sourceInfo.isFile() || sourceInfo.size <= 0) {
|
|
130
|
+
throw commandError('returned an empty downloaded file');
|
|
131
|
+
}
|
|
132
|
+
|
|
133
|
+
const outputDir = resolve(options.outputDir);
|
|
134
|
+
try {
|
|
135
|
+
await mkdir(outputDir, { recursive: true });
|
|
136
|
+
} catch {
|
|
137
|
+
throw commandError('could not create the output directory');
|
|
138
|
+
}
|
|
139
|
+
const filename = `weixin-user-growth-${options.begin}-${options.end}-all.xls`;
|
|
140
|
+
const target = await publishExclusively(
|
|
141
|
+
downloaded.filename,
|
|
142
|
+
outputDir,
|
|
143
|
+
filename,
|
|
144
|
+
options.beforePublish,
|
|
145
|
+
);
|
|
146
|
+
try {
|
|
147
|
+
await unlink(downloaded.filename);
|
|
148
|
+
} catch {
|
|
149
|
+
// Destination publication succeeded; browser temporary cleanup is best effort.
|
|
150
|
+
}
|
|
151
|
+
return { status: 'downloaded', path: target, size: sourceInfo.size };
|
|
152
|
+
}
|
|
@@ -5,6 +5,7 @@ import { ArgumentError, AuthRequiredError, CommandExecutionError } from '@sovovs
|
|
|
5
5
|
import { loadDraftContent, prepareHtmlContent } from './_wechat/draft-content.js';
|
|
6
6
|
import { pasteHtmlThroughClipboard } from './_wechat/html-clipboard.js';
|
|
7
7
|
import { createDraftViaApi } from './_wechat/api-draft.js';
|
|
8
|
+
import { stageDraftHtmlImages } from './_wechat/draft-image-stage.js';
|
|
8
9
|
|
|
9
10
|
const WEIXIN_DOMAIN = 'mp.weixin.qq.com';
|
|
10
11
|
const WEIXIN_HOME = 'https://mp.weixin.qq.com/';
|
|
@@ -52,6 +53,7 @@ function readApiCredentials(kwargs) {
|
|
|
52
53
|
|
|
53
54
|
function requiresBrowser(kwargs) {
|
|
54
55
|
const { appid, appsecret } = readApiCredentials(kwargs);
|
|
56
|
+
if (kwargs['dry-run'] === true) return true;
|
|
55
57
|
return !(appid && appsecret);
|
|
56
58
|
}
|
|
57
59
|
|
|
@@ -77,6 +79,7 @@ function normalizeCreateDraftArgs(kwargs) {
|
|
|
77
79
|
summary: kwargs.summary == null ? null : String(kwargs.summary).trim(),
|
|
78
80
|
coverImage: validateCoverImage(kwargs['cover-image']),
|
|
79
81
|
dryRun: kwargs['dry-run'] === true,
|
|
82
|
+
allowPrivateImageHosts: kwargs['allow-private-image-hosts'] === true,
|
|
80
83
|
appid,
|
|
81
84
|
appsecret,
|
|
82
85
|
};
|
|
@@ -254,7 +257,7 @@ async function uploadContentImage(page, imagePath) {
|
|
|
254
257
|
var editors = document.querySelectorAll('#ueditor_0, div[contenteditable="true"]');
|
|
255
258
|
var sources = [];
|
|
256
259
|
editors.forEach(function(editor) {
|
|
257
|
-
editor.querySelectorAll('img[src*="
|
|
260
|
+
editor.querySelectorAll('img[src*=".qpic.cn"], img[data-src*=".qpic.cn"]').forEach(function(image) {
|
|
258
261
|
var src = image.getAttribute('src') || image.getAttribute('data-src') || '';
|
|
259
262
|
if (src && !sources.includes(src)) sources.push(src);
|
|
260
263
|
});
|
|
@@ -328,10 +331,10 @@ async function selectCoverFromContent(page) {
|
|
|
328
331
|
var areas = document.querySelectorAll('#js_cover_area, #js_cover_description_area, #appmsgItem');
|
|
329
332
|
var found = false;
|
|
330
333
|
areas.forEach(function(area) {
|
|
331
|
-
if (area.querySelector('img[src*="
|
|
334
|
+
if (area.querySelector('img[src*=".qpic.cn"], img[data-src*=".qpic.cn"]')) found = true;
|
|
332
335
|
[area].concat(Array.from(area.querySelectorAll('*'))).forEach(function(el) {
|
|
333
336
|
var bg = window.getComputedStyle(el).backgroundImage;
|
|
334
|
-
if (bg && bg.includes('
|
|
337
|
+
if (bg && bg.includes('.qpic.cn')) found = true;
|
|
335
338
|
});
|
|
336
339
|
});
|
|
337
340
|
return found;
|
|
@@ -384,13 +387,14 @@ export const createDraftCommand = cli({
|
|
|
384
387
|
{ name: 'appid', help: '公众号 AppID;与 --appsecret 同传时走官方 API,不打开浏览器' },
|
|
385
388
|
{ name: 'appsecret', help: '公众号 AppSecret;请勿提交到 shell 历史或日志' },
|
|
386
389
|
{ name: 'dry-run', type: 'boolean', default: false, help: '浏览器模式:填充并验证正文后停止,不会保存草稿' },
|
|
390
|
+
{ name: 'allow-private-image-hosts', type: 'boolean', default: false, help: '允许下载 localhost/内网 HTTP(S) 正文图片;云元数据地址始终禁止' },
|
|
387
391
|
{ name: 'timeout', type: 'int', required: false, default: 180, help: '命令总超时时间(秒,默认 180)' },
|
|
388
392
|
],
|
|
389
393
|
columns: ['status', 'detail'],
|
|
390
394
|
|
|
391
395
|
func: async (page, kwargs) => {
|
|
392
396
|
const args = normalizeCreateDraftArgs(kwargs);
|
|
393
|
-
if (args.appid && args.appsecret) {
|
|
397
|
+
if (!args.dryRun && args.appid && args.appsecret) {
|
|
394
398
|
const baseDir = args.filePath ? nodePath.dirname(args.filePath) : process.cwd();
|
|
395
399
|
const result = await createDraftViaApi({
|
|
396
400
|
appid: args.appid,
|
|
@@ -401,68 +405,80 @@ export const createDraftCommand = cli({
|
|
|
401
405
|
coverImage: args.coverImage,
|
|
402
406
|
html: args.content,
|
|
403
407
|
baseDir,
|
|
408
|
+
allowPrivateImageHosts: args.allowPrivateImageHosts,
|
|
404
409
|
});
|
|
405
410
|
return [{
|
|
406
411
|
status: 'draft created',
|
|
407
412
|
detail: `"${args.title}" (media_id: ${result.mediaId})`,
|
|
408
413
|
}];
|
|
409
414
|
}
|
|
410
|
-
|
|
411
|
-
|
|
415
|
+
const baseDir = args.filePath ? nodePath.dirname(args.filePath) : process.cwd();
|
|
416
|
+
const staged = args.format === 'html'
|
|
417
|
+
? await stageDraftHtmlImages(args.content, {
|
|
418
|
+
baseDir,
|
|
419
|
+
allowPrivateHosts: args.allowPrivateImageHosts,
|
|
420
|
+
})
|
|
421
|
+
: null;
|
|
422
|
+
try {
|
|
423
|
+
await navigateToEditor(page);
|
|
412
424
|
|
|
413
|
-
const titleResult = await fillField(page, 'textarea#title', args.title);
|
|
414
|
-
requirePageResult(titleResult, 'title');
|
|
415
425
|
|
|
416
|
-
|
|
417
|
-
|
|
418
|
-
requirePageResult(authorResult, 'author');
|
|
419
|
-
}
|
|
426
|
+
const titleResult = await fillField(page, 'textarea#title', args.title);
|
|
427
|
+
requirePageResult(titleResult, 'title');
|
|
420
428
|
|
|
421
|
-
|
|
429
|
+
if (args.author) {
|
|
430
|
+
const authorResult = await fillField(page, 'input#author', args.author);
|
|
431
|
+
requirePageResult(authorResult, 'author');
|
|
432
|
+
}
|
|
422
433
|
|
|
423
|
-
|
|
424
|
-
|
|
425
|
-
const
|
|
426
|
-
|
|
427
|
-
|
|
428
|
-
|
|
429
|
-
|
|
430
|
-
|
|
431
|
-
|
|
432
|
-
|
|
433
|
-
|
|
434
|
-
|
|
435
|
-
|
|
436
|
-
|
|
437
|
-
|
|
438
|
-
|
|
434
|
+
await page.wait(10);
|
|
435
|
+
|
|
436
|
+
const content = staged?.html ?? args.content;
|
|
437
|
+
if (args.format === 'html') {
|
|
438
|
+
const prepared = await prepareHtmlContent(content, {
|
|
439
|
+
baseDir,
|
|
440
|
+
allowRemoteImages: false,
|
|
441
|
+
resolveImage: async imagePath => {
|
|
442
|
+
const uploaded = await uploadContentImage(page, imagePath);
|
|
443
|
+
await removeTemporaryInsertedImage(page);
|
|
444
|
+
return uploaded;
|
|
445
|
+
},
|
|
446
|
+
});
|
|
447
|
+
await pasteHtmlThroughClipboard(page, prepared.html, { origin: `https://${WEIXIN_DOMAIN}` });
|
|
448
|
+
} else {
|
|
449
|
+
const contentResult = await fillContent(page, content);
|
|
450
|
+
requirePageResult(contentResult, 'content');
|
|
451
|
+
}
|
|
439
452
|
|
|
440
|
-
|
|
441
|
-
|
|
442
|
-
|
|
443
|
-
|
|
444
|
-
|
|
445
|
-
|
|
453
|
+
if (args.dryRun) {
|
|
454
|
+
return [{
|
|
455
|
+
status: 'draft ready',
|
|
456
|
+
detail: `"${args.title}" (dry-run)`,
|
|
457
|
+
}];
|
|
458
|
+
}
|
|
446
459
|
|
|
447
|
-
|
|
448
|
-
|
|
449
|
-
|
|
450
|
-
|
|
451
|
-
|
|
460
|
+
if (args.coverImage) {
|
|
461
|
+
await uploadContentImage(page, args.coverImage);
|
|
462
|
+
const coverSet = await selectCoverFromContent(page);
|
|
463
|
+
if (!coverSet) {
|
|
464
|
+
throw new CommandExecutionError('Failed to set the requested cover image');
|
|
465
|
+
}
|
|
452
466
|
}
|
|
453
|
-
}
|
|
454
467
|
|
|
455
|
-
|
|
456
|
-
|
|
457
|
-
|
|
458
|
-
|
|
468
|
+
if (args.summary) {
|
|
469
|
+
const summaryResult = await fillField(page, 'textarea#js_description', args.summary);
|
|
470
|
+
requirePageResult(summaryResult, 'summary');
|
|
471
|
+
}
|
|
459
472
|
|
|
460
|
-
|
|
461
|
-
|
|
473
|
+
await page.wait(1);
|
|
474
|
+
await clickSaveDraft(page);
|
|
462
475
|
|
|
463
|
-
|
|
464
|
-
|
|
465
|
-
|
|
466
|
-
|
|
476
|
+
return [{
|
|
477
|
+
status: 'draft saved',
|
|
478
|
+
detail: `"${args.title}"${args.author ? ` by ${args.author}` : ''}${args.coverImage ? ' (with cover)' : ''}`,
|
|
479
|
+
}];
|
|
480
|
+
} finally {
|
|
481
|
+
await staged?.cleanup();
|
|
482
|
+
}
|
|
467
483
|
},
|
|
468
484
|
});
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { EmptyResultError } from '@sovovs/bycli/errors';
|
|
1
|
+
import { ArgumentError, EmptyResultError } from '@sovovs/bycli/errors';
|
|
2
2
|
import { cli, Strategy } from '@sovovs/bycli/registry';
|
|
3
3
|
import { resolveBrowserCredentials } from './_wechat/auth-session.js';
|
|
4
4
|
import {
|
|
@@ -6,6 +6,7 @@ import {
|
|
|
6
6
|
parseGrowthSources,
|
|
7
7
|
resolveGrowthRange,
|
|
8
8
|
} from './_wechat/user-analysis.js';
|
|
9
|
+
import { downloadUserGrowthXls } from './_wechat/user-growth-download.js';
|
|
9
10
|
|
|
10
11
|
const COLUMNS = [
|
|
11
12
|
'date',
|
|
@@ -15,14 +16,16 @@ const COLUMNS = [
|
|
|
15
16
|
'unfollows',
|
|
16
17
|
'net_new_followers',
|
|
17
18
|
'cumulative_followers',
|
|
19
|
+
'official_xls_path',
|
|
20
|
+
'official_xls_size',
|
|
18
21
|
];
|
|
19
22
|
|
|
20
23
|
export const userGrowthCommand = cli({
|
|
21
24
|
site: 'weixin',
|
|
22
25
|
name: 'user-growth',
|
|
23
|
-
access: '
|
|
26
|
+
access: 'write',
|
|
24
27
|
domain: 'mp.weixin.qq.com',
|
|
25
|
-
description: '
|
|
28
|
+
description: '读取公众号用户增长趋势,可选返回全部渠道并下载官方“全部来源”XLS',
|
|
26
29
|
strategy: Strategy.COOKIE,
|
|
27
30
|
browser: true,
|
|
28
31
|
navigateBefore: false,
|
|
@@ -30,9 +33,14 @@ export const userGrowthCommand = cli({
|
|
|
30
33
|
{ name: 'begin', help: '开始日期(YYYY-MM-DD);默认 30 天窗口的第一天' },
|
|
31
34
|
{ name: 'end', help: '结束日期(YYYY-MM-DD);默认昨天' },
|
|
32
35
|
{ name: 'source', default: 'all', help: '传播渠道名称或代码;多个值用逗号分隔' },
|
|
36
|
+
{ name: 'output', help: '可选的官方“全部来源”XLS 保存目录;不传则不下载' },
|
|
33
37
|
],
|
|
34
38
|
columns: COLUMNS,
|
|
35
39
|
func: async (page, args) => {
|
|
40
|
+
const outputDir = typeof args.output === 'string' ? args.output.trim() : null;
|
|
41
|
+
if (args.output !== undefined && (!outputDir || typeof args.output !== 'string')) {
|
|
42
|
+
throw new ArgumentError('output must be a non-empty directory');
|
|
43
|
+
}
|
|
36
44
|
const { token } = await resolveBrowserCredentials(page);
|
|
37
45
|
const { begin, end } = resolveGrowthRange(args);
|
|
38
46
|
const sources = parseGrowthSources(args.source);
|
|
@@ -40,6 +48,9 @@ export const userGrowthCommand = cli({
|
|
|
40
48
|
if (rows.length === 0) {
|
|
41
49
|
throw new EmptyResultError('weixin user-growth', `No user growth rows are available from ${begin} through ${end}.`);
|
|
42
50
|
}
|
|
51
|
+
const artifact = outputDir
|
|
52
|
+
? await downloadUserGrowthXls(page, { token, begin, end, outputDir })
|
|
53
|
+
: null;
|
|
43
54
|
return rows.map(row => ({
|
|
44
55
|
date: row.date,
|
|
45
56
|
source: row.source,
|
|
@@ -48,6 +59,8 @@ export const userGrowthCommand = cli({
|
|
|
48
59
|
unfollows: row.unfollows,
|
|
49
60
|
net_new_followers: row.netNewFollowers,
|
|
50
61
|
cumulative_followers: row.cumulativeFollowers,
|
|
62
|
+
official_xls_path: artifact?.path ?? null,
|
|
63
|
+
official_xls_size: artifact?.size ?? null,
|
|
51
64
|
}));
|
|
52
65
|
},
|
|
53
66
|
});
|
|
@@ -3,8 +3,13 @@ export interface ProxyDecision {
|
|
|
3
3
|
mode: 'direct' | 'proxy';
|
|
4
4
|
proxyUrl?: string;
|
|
5
5
|
}
|
|
6
|
+
export interface PinnedNetworkAddress {
|
|
7
|
+
address: string;
|
|
8
|
+
family: 4 | 6;
|
|
9
|
+
}
|
|
6
10
|
export declare function hasProxyEnv(env?: NodeJS.ProcessEnv): boolean;
|
|
7
11
|
export declare function decideProxy(url: URL, env?: NodeJS.ProcessEnv): ProxyDecision;
|
|
8
12
|
export declare function getDispatcherForUrl(url: URL, env?: NodeJS.ProcessEnv): Dispatcher;
|
|
13
|
+
export declare function createPinnedDispatcher(addresses: PinnedNetworkAddress[]): Dispatcher;
|
|
9
14
|
export declare function fetchWithNodeNetwork(input: RequestInfo | URL, init?: RequestInit): Promise<Response>;
|
|
10
15
|
export declare function installNodeNetwork(): void;
|
package/dist/src/node-network.js
CHANGED
|
@@ -156,7 +156,36 @@ export function getDispatcherForUrl(url, env = process.env) {
|
|
|
156
156
|
return directDispatcher;
|
|
157
157
|
return createProxyDispatcher(config);
|
|
158
158
|
}
|
|
159
|
+
export function createPinnedDispatcher(addresses) {
|
|
160
|
+
if (addresses.length === 0)
|
|
161
|
+
throw new Error('At least one pinned network address is required');
|
|
162
|
+
return new Agent({
|
|
163
|
+
connect: {
|
|
164
|
+
lookup(_hostname, options, callback) {
|
|
165
|
+
const requestedFamily = Number(options?.family) || 0;
|
|
166
|
+
const candidates = requestedFamily
|
|
167
|
+
? addresses.filter(item => item.family === requestedFamily)
|
|
168
|
+
: addresses;
|
|
169
|
+
if (candidates.length === 0) {
|
|
170
|
+
callback(new Error(`No pinned address for IPv${requestedFamily}`), []);
|
|
171
|
+
return;
|
|
172
|
+
}
|
|
173
|
+
if (options?.all)
|
|
174
|
+
callback(null, candidates);
|
|
175
|
+
else
|
|
176
|
+
callback(null, candidates[0].address, candidates[0].family);
|
|
177
|
+
},
|
|
178
|
+
},
|
|
179
|
+
});
|
|
180
|
+
}
|
|
159
181
|
export async function fetchWithNodeNetwork(input, init = {}) {
|
|
182
|
+
const explicitDispatcher = init.dispatcher;
|
|
183
|
+
if (explicitDispatcher) {
|
|
184
|
+
return (await undiciFetch(input, {
|
|
185
|
+
...init,
|
|
186
|
+
dispatcher: explicitDispatcher,
|
|
187
|
+
}));
|
|
188
|
+
}
|
|
160
189
|
const url = resolveUrl(input);
|
|
161
190
|
if (!url || !hasProxyEnv()) {
|
|
162
191
|
return nativeFetch(input, init);
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@sovovs/bycli",
|
|
3
|
-
"version": "2.1.
|
|
3
|
+
"version": "2.1.47",
|
|
4
4
|
"publishConfig": {
|
|
5
5
|
"access": "public"
|
|
6
6
|
},
|
|
@@ -24,6 +24,7 @@
|
|
|
24
24
|
"./types": "./dist/src/types.js",
|
|
25
25
|
"./utils": "./dist/src/utils.js",
|
|
26
26
|
"./logger": "./dist/src/logger.js",
|
|
27
|
+
"./node-network": "./dist/src/node-network.js",
|
|
27
28
|
"./launcher": "./dist/src/launcher.js",
|
|
28
29
|
"./browser/cdp": "./dist/src/browser/cdp.js",
|
|
29
30
|
"./browser/page": "./dist/src/browser/page.js",
|