@sovovs/bycli 2.1.0 → 2.1.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/cli-manifest.json +169 -0
- package/clis/weixin/_wechat/args.js +48 -0
- package/clis/weixin/_wechat/article-content.js +53 -0
- package/clis/weixin/_wechat/article-service.js +124 -0
- package/clis/weixin/_wechat/auth-session.js +142 -0
- package/clis/weixin/_wechat/fingerprint.js +443 -0
- package/clis/weixin/_wechat/fixtures/articles-auth-expired.json +3 -0
- package/clis/weixin/_wechat/fixtures/articles-page.json +4 -0
- package/clis/weixin/_wechat/fixtures/search-auth-expired.json +4 -0
- package/clis/weixin/_wechat/fixtures/search-success.json +7 -0
- package/clis/weixin/_wechat/markdown.js +29 -0
- package/clis/weixin/_wechat/redact.js +405 -0
- package/clis/weixin/_wechat/save-service.js +175 -0
- package/clis/weixin/_wechat/search-biz.js +102 -0
- package/clis/weixin/_wechat/wechat-api.js +133 -0
- package/clis/weixin/accounts.js +38 -0
- package/clis/weixin/articles.js +35 -0
- package/clis/weixin/download.js +5 -47
- package/clis/weixin/save-articles.js +175 -0
- package/dist/src/daemon-config.d.ts +2 -0
- package/dist/src/daemon-config.js +11 -0
- package/dist/src/daemon-config.test.d.ts +1 -0
- package/dist/src/daemon.d.ts +1 -1
- package/dist/src/daemon.js +5 -3
- package/dist/src/download/article-download.d.ts +6 -0
- package/dist/src/download/article-download.js +78 -17
- package/dist/src/download/wechat-article.d.ts +8 -0
- package/dist/src/download/wechat-article.js +137 -0
- package/dist/src/download/wechat-article.test.d.ts +1 -0
- package/dist/src/recorder/highlevel/verify.d.ts +3 -0
- package/dist/src/recorder/highlevel/verify.js +4 -0
- package/dist/src/recorder/highlevel/verify.test.d.ts +1 -0
- package/dist/src/recorder/http/handlers.js +27 -1
- package/dist/src/recorder/runner/runner-port.js +1 -0
- package/dist/src/recorder/runner/verify-runner-main.d.ts +17 -2
- package/dist/src/recorder/runner/verify-runner-main.js +70 -14
- package/dist/src/release-workflow.test.d.ts +1 -0
- package/dist/src/weixin-built-in-docs.test.d.ts +1 -0
- package/package.json +7 -3
- package/scripts/check-package-install.mjs +71 -0
|
@@ -11,12 +11,29 @@ import TurndownService from 'turndown';
|
|
|
11
11
|
import { gfm } from 'turndown-plugin-gfm';
|
|
12
12
|
import { httpDownload, sanitizeFilename } from './index.js';
|
|
13
13
|
import { formatBytes } from './progress.js';
|
|
14
|
+
export { extractWechatArticleHtml } from './wechat-article.js';
|
|
14
15
|
const IMAGE_CONCURRENCY = 5;
|
|
15
16
|
const DEFAULT_LABELS = {
|
|
16
17
|
author: '作者',
|
|
17
18
|
publishTime: '发布时间',
|
|
18
19
|
sourceUrl: '原文链接',
|
|
19
20
|
};
|
|
21
|
+
function escapeMarkdownText(value) {
|
|
22
|
+
return value.replace(/\s+/g, ' ').trim()
|
|
23
|
+
.replace(/&/g, '&').replace(/</g, '<').replace(/>/g, '>')
|
|
24
|
+
.replace(/"/g, '"').replace(/'/g, ''')
|
|
25
|
+
.replace(/([\\`*_[\]{}()#+.!|>~-])/g, '\\$1');
|
|
26
|
+
}
|
|
27
|
+
function safeHttpUrl(value) {
|
|
28
|
+
const normalized = value.trim().startsWith('//') ? `https:${value.trim()}` : value.trim();
|
|
29
|
+
try {
|
|
30
|
+
const url = new URL(normalized);
|
|
31
|
+
return url.protocol === 'http:' || url.protocol === 'https:' ? url.href : '';
|
|
32
|
+
}
|
|
33
|
+
catch {
|
|
34
|
+
return '';
|
|
35
|
+
}
|
|
36
|
+
}
|
|
20
37
|
// ============================================================
|
|
21
38
|
// Markdown Conversion
|
|
22
39
|
// ============================================================
|
|
@@ -36,13 +53,48 @@ const STRIPPED_TAGS = [
|
|
|
36
53
|
'form', 'button', 'dialog',
|
|
37
54
|
'header', 'footer', 'nav', 'aside',
|
|
38
55
|
];
|
|
39
|
-
function createTurndown(configure, cleanSelectors) {
|
|
56
|
+
function createTurndown(configure, cleanSelectors, secureMarkdown = false) {
|
|
40
57
|
const td = new TurndownService({
|
|
41
58
|
headingStyle: 'atx',
|
|
42
59
|
codeBlockStyle: 'fenced',
|
|
43
60
|
bulletListMarker: '-',
|
|
44
61
|
});
|
|
45
62
|
td.use(gfm);
|
|
63
|
+
if (secureMarkdown) {
|
|
64
|
+
const escapeDestination = (value) => value.replace(/([\\()])/g, '\\$1');
|
|
65
|
+
td.addRule('safeFencedCodeBlock', {
|
|
66
|
+
filter: 'pre',
|
|
67
|
+
replacement: (_content, node) => {
|
|
68
|
+
const element = node;
|
|
69
|
+
const code = element.textContent || '';
|
|
70
|
+
const longest = Math.max(0, ...[...code.matchAll(/`+/g)].map(match => match[0].length));
|
|
71
|
+
const fence = '`'.repeat(Math.max(3, longest + 1));
|
|
72
|
+
const className = element.querySelector('code')?.getAttribute('class') || '';
|
|
73
|
+
const language = element.getAttribute('data-lang')
|
|
74
|
+
|| /(?:^|\s)language-([^\s]+)/.exec(className)?.[1]
|
|
75
|
+
|| '';
|
|
76
|
+
return `\n${fence}${language}\n${code.replace(/\n$/, '')}\n${fence}\n`;
|
|
77
|
+
},
|
|
78
|
+
});
|
|
79
|
+
td.addRule('safeImage', {
|
|
80
|
+
filter: 'img',
|
|
81
|
+
replacement: (_content, node) => {
|
|
82
|
+
const element = node;
|
|
83
|
+
const alt = escapeMarkdownText(element.getAttribute('alt') || '');
|
|
84
|
+
const src = element.getAttribute('src') || '';
|
|
85
|
+
return src ? `})` : alt;
|
|
86
|
+
},
|
|
87
|
+
});
|
|
88
|
+
td.addRule('safeLink', {
|
|
89
|
+
filter: 'a',
|
|
90
|
+
replacement: (_content, node) => {
|
|
91
|
+
const element = node;
|
|
92
|
+
const label = escapeMarkdownText(element.textContent || '');
|
|
93
|
+
const href = element.getAttribute('href') || '';
|
|
94
|
+
return href ? `[${label}](${escapeDestination(href)})` : label;
|
|
95
|
+
},
|
|
96
|
+
});
|
|
97
|
+
}
|
|
46
98
|
td.remove(STRIPPED_TAGS);
|
|
47
99
|
// turndown-plugin-gfm@1.0.2 emits single-tilde strikethrough (`~x~`), which
|
|
48
100
|
// is not the canonical GFM form. Override it so exported markdown is
|
|
@@ -104,11 +156,14 @@ function createTurndown(configure, cleanSelectors) {
|
|
|
104
156
|
filter: (node) => node.nodeName === 'IFRAME',
|
|
105
157
|
replacement: (_content, node) => {
|
|
106
158
|
const el = node;
|
|
107
|
-
const
|
|
159
|
+
const rawSrc = el.getAttribute('src') || '';
|
|
160
|
+
const src = secureMarkdown ? safeHttpUrl(rawSrc) : rawSrc;
|
|
108
161
|
if (!src)
|
|
109
162
|
return '';
|
|
110
|
-
const
|
|
111
|
-
|
|
163
|
+
const rawTitle = el.getAttribute('title') || 'Embedded content';
|
|
164
|
+
const title = secureMarkdown ? escapeMarkdownText(rawTitle) : rawTitle;
|
|
165
|
+
const destination = secureMarkdown ? src.replace(/([\\()])/g, '\\$1') : src;
|
|
166
|
+
return `\n[${title}](${destination})\n`;
|
|
112
167
|
},
|
|
113
168
|
});
|
|
114
169
|
// Per-adapter dirty-node removal. Adapters know their site's specific noise
|
|
@@ -139,14 +194,16 @@ function createTurndown(configure, cleanSelectors) {
|
|
|
139
194
|
configure(td);
|
|
140
195
|
return td;
|
|
141
196
|
}
|
|
142
|
-
function convertToMarkdown(contentHtml, codeBlocks, configure, cleanSelectors) {
|
|
143
|
-
const td = createTurndown(configure, cleanSelectors);
|
|
197
|
+
function convertToMarkdown(contentHtml, codeBlocks, configure, cleanSelectors, secureMarkdown = false) {
|
|
198
|
+
const td = createTurndown(configure, cleanSelectors, secureMarkdown);
|
|
144
199
|
let md = td.turndown(contentHtml);
|
|
145
|
-
//
|
|
146
|
-
|
|
147
|
-
|
|
148
|
-
|
|
149
|
-
|
|
200
|
+
// Legacy callers may still supply extracted blocks. New callers preserve
|
|
201
|
+
// real <pre>/<code> nodes so Turndown owns fencing without collision-prone
|
|
202
|
+
// magic text; legacy blocks are appended with safe dynamic fences.
|
|
203
|
+
codeBlocks.forEach((block) => {
|
|
204
|
+
const longest = Math.max(0, ...[...block.code.matchAll(/`+/g)].map(match => match[0].length));
|
|
205
|
+
const fence = '`'.repeat(Math.max(3, longest + 1));
|
|
206
|
+
md += `\n\n${fence}${block.lang}\n${block.code}\n${fence}`;
|
|
150
207
|
});
|
|
151
208
|
// Clean up
|
|
152
209
|
md = md.replace(/\u00a0/g, ' ');
|
|
@@ -158,6 +215,9 @@ function convertToMarkdown(contentHtml, codeBlocks, configure, cleanSelectors) {
|
|
|
158
215
|
md = md.replace(/\n{3,}/g, '\n\n');
|
|
159
216
|
return md;
|
|
160
217
|
}
|
|
218
|
+
export function convertArticleHtmlToMarkdown(contentHtml, options = {}) {
|
|
219
|
+
return convertToMarkdown(contentHtml, [], undefined, undefined, options.safeFencedCodeBlocks === true);
|
|
220
|
+
}
|
|
161
221
|
function replaceImageUrls(md, urlMap) {
|
|
162
222
|
return md.replace(/!\[([^\]]*)\]\(([^)]+)\)/g, (match, alt, imgUrl) => {
|
|
163
223
|
const local = urlMap[imgUrl];
|
|
@@ -230,7 +290,7 @@ async function downloadImages(imgUrls, imgDir, headers, detectExt) {
|
|
|
230
290
|
* 6. File write
|
|
231
291
|
*/
|
|
232
292
|
export async function downloadArticle(data, options) {
|
|
233
|
-
const { output, downloadImages: shouldDownloadImages = true, imageHeaders, maxTitleLength = 80, configureTurndown, detectImageExt, frontmatterLabels, cleanSelectors, stdout = false, } = options;
|
|
293
|
+
const { output, downloadImages: shouldDownloadImages = true, imageHeaders, maxTitleLength = 80, configureTurndown, detectImageExt, frontmatterLabels, cleanSelectors, stdout = false, secureMarkdown = false, } = options;
|
|
234
294
|
const labels = { ...DEFAULT_LABELS, ...frontmatterLabels };
|
|
235
295
|
if (!data.title) {
|
|
236
296
|
return [{
|
|
@@ -253,7 +313,7 @@ export async function downloadArticle(data, options) {
|
|
|
253
313
|
}];
|
|
254
314
|
}
|
|
255
315
|
// Convert HTML to Markdown
|
|
256
|
-
let markdown = convertToMarkdown(data.contentHtml, data.codeBlocks || [], configureTurndown, cleanSelectors);
|
|
316
|
+
let markdown = convertToMarkdown(data.contentHtml, data.codeBlocks || [], configureTurndown, cleanSelectors, secureMarkdown);
|
|
257
317
|
const safeTitle = sanitizeFilename(data.title, maxTitleLength);
|
|
258
318
|
// Download images only when writing to disk. In stdout mode remote URLs
|
|
259
319
|
// stay intact so the piped output is self-contained.
|
|
@@ -268,13 +328,14 @@ export async function downloadArticle(data, options) {
|
|
|
268
328
|
// Build frontmatter with customizable labels.
|
|
269
329
|
// Shape: `# Title\n[> meta\n...]\n---\n\n<markdown>` — exactly one blank
|
|
270
330
|
// line separates every section, so we never produce ≥3 consecutive newlines.
|
|
271
|
-
const
|
|
331
|
+
const headerValue = (value) => secureMarkdown ? escapeMarkdownText(value) : value;
|
|
332
|
+
const headerLines = [`# ${headerValue(data.title)}`];
|
|
272
333
|
if (data.author)
|
|
273
|
-
headerLines.push(`> ${labels.author}: ${data.author}`);
|
|
334
|
+
headerLines.push(`> ${labels.author}: ${headerValue(data.author)}`);
|
|
274
335
|
if (data.publishTime)
|
|
275
|
-
headerLines.push(`> ${labels.publishTime}: ${data.publishTime}`);
|
|
336
|
+
headerLines.push(`> ${labels.publishTime}: ${headerValue(data.publishTime)}`);
|
|
276
337
|
if (data.sourceUrl)
|
|
277
|
-
headerLines.push(`> ${labels.sourceUrl}: ${data.sourceUrl}`);
|
|
338
|
+
headerLines.push(`> ${labels.sourceUrl}: ${headerValue(data.sourceUrl)}`);
|
|
278
339
|
const frontmatter = headerLines.join('\n') + '\n\n---\n\n';
|
|
279
340
|
const fullContent = frontmatter + markdown;
|
|
280
341
|
const size = Buffer.byteLength(fullContent, 'utf-8');
|
|
@@ -0,0 +1,8 @@
|
|
|
1
|
+
export declare const MAX_WECHAT_HTML_BYTES: number;
|
|
2
|
+
export declare const MAX_WECHAT_NODES = 100000;
|
|
3
|
+
export declare const MAX_WECHAT_CODE_BLOCKS = 1000;
|
|
4
|
+
export interface ExtractedWechatArticle {
|
|
5
|
+
contentHtml: string;
|
|
6
|
+
imageUrls: string[];
|
|
7
|
+
}
|
|
8
|
+
export declare function extractWechatArticleHtml(html: string): ExtractedWechatArticle;
|
|
@@ -0,0 +1,137 @@
|
|
|
1
|
+
import { parse, serialize } from 'parse5';
|
|
2
|
+
import { CommandExecutionError } from '../errors.js';
|
|
3
|
+
export const MAX_WECHAT_HTML_BYTES = 10 * 1024 * 1024;
|
|
4
|
+
export const MAX_WECHAT_NODES = 100_000;
|
|
5
|
+
export const MAX_WECHAT_CODE_BLOCKS = 1_000;
|
|
6
|
+
function isElement(node) {
|
|
7
|
+
return 'tagName' in node;
|
|
8
|
+
}
|
|
9
|
+
function attr(node, name) {
|
|
10
|
+
return node.attrs.find(item => item.name === name)?.value;
|
|
11
|
+
}
|
|
12
|
+
function hasClass(node, name) {
|
|
13
|
+
return (attr(node, 'class') || '').split(/\s+/).includes(name);
|
|
14
|
+
}
|
|
15
|
+
function safeUrl(value) {
|
|
16
|
+
const normalized = value.trim().startsWith('//') ? `https:${value.trim()}` : value.trim();
|
|
17
|
+
try {
|
|
18
|
+
const url = new URL(normalized);
|
|
19
|
+
return url.protocol === 'http:' || url.protocol === 'https:' ? url.href : undefined;
|
|
20
|
+
}
|
|
21
|
+
catch {
|
|
22
|
+
return undefined;
|
|
23
|
+
}
|
|
24
|
+
}
|
|
25
|
+
function textContent(node) {
|
|
26
|
+
if ('value' in node)
|
|
27
|
+
return node.value;
|
|
28
|
+
if (!('childNodes' in node))
|
|
29
|
+
return '';
|
|
30
|
+
return node.childNodes.map(textContent).join('');
|
|
31
|
+
}
|
|
32
|
+
export function extractWechatArticleHtml(html) {
|
|
33
|
+
if (Buffer.byteLength(html, 'utf8') > MAX_WECHAT_HTML_BYTES) {
|
|
34
|
+
throw new CommandExecutionError('WeChat article HTML exceeds the 10 MiB limit');
|
|
35
|
+
}
|
|
36
|
+
const document = parse(html);
|
|
37
|
+
let content;
|
|
38
|
+
let nodes = 0;
|
|
39
|
+
let codeBlocks = 0;
|
|
40
|
+
const stack = [document];
|
|
41
|
+
while (stack.length > 0) {
|
|
42
|
+
const node = stack.pop();
|
|
43
|
+
nodes += 1;
|
|
44
|
+
if (nodes > MAX_WECHAT_NODES)
|
|
45
|
+
throw new CommandExecutionError('WeChat article HTML exceeds the DOM node limit');
|
|
46
|
+
if (isElement(node)) {
|
|
47
|
+
if (attr(node, 'id') === 'js_content')
|
|
48
|
+
content = node;
|
|
49
|
+
if (node.tagName === 'pre') {
|
|
50
|
+
codeBlocks += 1;
|
|
51
|
+
if (codeBlocks > MAX_WECHAT_CODE_BLOCKS)
|
|
52
|
+
throw new CommandExecutionError('WeChat article HTML exceeds the code block limit');
|
|
53
|
+
}
|
|
54
|
+
}
|
|
55
|
+
if ('childNodes' in node) {
|
|
56
|
+
for (let i = node.childNodes.length - 1; i >= 0; i -= 1)
|
|
57
|
+
stack.push(node.childNodes[i]);
|
|
58
|
+
}
|
|
59
|
+
}
|
|
60
|
+
if (!content)
|
|
61
|
+
throw new CommandExecutionError('WeChat article has no #js_content');
|
|
62
|
+
const parents = [content];
|
|
63
|
+
while (parents.length > 0) {
|
|
64
|
+
const parent = parents.pop();
|
|
65
|
+
parent.childNodes = parent.childNodes.filter(node => {
|
|
66
|
+
if (!isElement(node))
|
|
67
|
+
return true;
|
|
68
|
+
return !['script', 'style'].includes(node.tagName)
|
|
69
|
+
&& !hasClass(node, 'qr_code_pc') && !hasClass(node, 'reward_area')
|
|
70
|
+
&& !hasClass(node, 'code-snippet__line-index');
|
|
71
|
+
});
|
|
72
|
+
for (const node of parent.childNodes) {
|
|
73
|
+
if (isElement(node)) {
|
|
74
|
+
node.attrs = node.attrs.flatMap(item => {
|
|
75
|
+
if (!['href', 'src', 'poster', 'data-src'].includes(item.name))
|
|
76
|
+
return [item];
|
|
77
|
+
const value = safeUrl(item.value);
|
|
78
|
+
return value ? [{ ...item, value }] : [];
|
|
79
|
+
});
|
|
80
|
+
if (node.tagName === 'img') {
|
|
81
|
+
const lazy = attr(node, 'data-src');
|
|
82
|
+
if (lazy) {
|
|
83
|
+
node.attrs = node.attrs.filter(item => !['src', 'data-src'].includes(item.name));
|
|
84
|
+
node.attrs.push({ name: 'src', value: lazy });
|
|
85
|
+
}
|
|
86
|
+
}
|
|
87
|
+
if (hasClass(node, 'code-snippet__fix')) {
|
|
88
|
+
const descendants = [...node.childNodes].reverse();
|
|
89
|
+
let pre;
|
|
90
|
+
const lines = [];
|
|
91
|
+
while (descendants.length > 0) {
|
|
92
|
+
const descendant = descendants.pop();
|
|
93
|
+
if (isElement(descendant)) {
|
|
94
|
+
if (descendant.tagName === 'pre')
|
|
95
|
+
pre = descendant;
|
|
96
|
+
if (descendant.tagName === 'code') {
|
|
97
|
+
const line = textContent(descendant);
|
|
98
|
+
if (!/^[ce]?ounter\(line/.test(line))
|
|
99
|
+
lines.push(line);
|
|
100
|
+
continue;
|
|
101
|
+
}
|
|
102
|
+
}
|
|
103
|
+
if ('childNodes' in descendant) {
|
|
104
|
+
for (let i = descendant.childNodes.length - 1; i >= 0; i -= 1)
|
|
105
|
+
descendants.push(descendant.childNodes[i]);
|
|
106
|
+
}
|
|
107
|
+
}
|
|
108
|
+
if (pre && lines.length > 0) {
|
|
109
|
+
pre.childNodes = [{ nodeName: '#text', value: lines.join('\n'), parentNode: pre }];
|
|
110
|
+
}
|
|
111
|
+
}
|
|
112
|
+
parents.push(node);
|
|
113
|
+
}
|
|
114
|
+
}
|
|
115
|
+
}
|
|
116
|
+
const imageUrls = [];
|
|
117
|
+
const seenImages = new Set();
|
|
118
|
+
const imageStack = [content];
|
|
119
|
+
while (imageStack.length > 0) {
|
|
120
|
+
const node = imageStack.pop();
|
|
121
|
+
if (isElement(node) && node.tagName === 'img') {
|
|
122
|
+
const src = attr(node, 'src');
|
|
123
|
+
if (src && !seenImages.has(src)) {
|
|
124
|
+
seenImages.add(src);
|
|
125
|
+
imageUrls.push(src);
|
|
126
|
+
}
|
|
127
|
+
}
|
|
128
|
+
if ('childNodes' in node) {
|
|
129
|
+
for (let i = node.childNodes.length - 1; i >= 0; i -= 1)
|
|
130
|
+
imageStack.push(node.childNodes[i]);
|
|
131
|
+
}
|
|
132
|
+
}
|
|
133
|
+
const fragment = {
|
|
134
|
+
nodeName: '#document-fragment', childNodes: content.childNodes,
|
|
135
|
+
};
|
|
136
|
+
return { contentHtml: serialize(fragment), imageUrls };
|
|
137
|
+
}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
export {};
|
|
@@ -29,6 +29,8 @@ export interface VerifyInput {
|
|
|
29
29
|
trace?: 'off' | 'retain-on-failure' | 'always';
|
|
30
30
|
/** N3:显式 adapter 路径 override —— verify 录制器 LLM 生成的临时草稿(不在 clis/),缺省按 name 派生。 */
|
|
31
31
|
adapterPath?: string;
|
|
32
|
+
/** Optional lowercase SHA-256 expected for the exact adapter bytes the runner will execute. */
|
|
33
|
+
expectedSourceSha256?: string;
|
|
32
34
|
}
|
|
33
35
|
/** The runner boundary (08). M6 provides the real child-process implementation. */
|
|
34
36
|
export interface RunnerPort {
|
|
@@ -42,6 +44,7 @@ export interface RunnerPort {
|
|
|
42
44
|
trace: string;
|
|
43
45
|
/** N3: explicit adapter path override (recorder draft verify); default = name→clis path. */
|
|
44
46
|
adapterPath?: string;
|
|
47
|
+
expectedSourceSha256?: string;
|
|
45
48
|
}): Promise<{
|
|
46
49
|
requestId: string;
|
|
47
50
|
}>;
|
|
@@ -49,6 +49,9 @@ export async function verifyAdapter(input, sessionHmacKey, runner) {
|
|
|
49
49
|
}
|
|
50
50
|
adapterPath = abs;
|
|
51
51
|
}
|
|
52
|
+
if (input.expectedSourceSha256 !== undefined && !/^[0-9a-f]{64}$/.test(input.expectedSourceSha256)) {
|
|
53
|
+
return { ok: false, errorCode: 'validation_failed', reason: 'expectedSourceSha256 must be 64 lowercase hex characters' };
|
|
54
|
+
}
|
|
52
55
|
const port = runner ?? defaultRunnerPort();
|
|
53
56
|
const rawSeedArgs = input.executionSeedArgs ?? {};
|
|
54
57
|
const evidenceSeedArgs = deriveEvidenceSeedArgs(rawSeedArgs, sessionHmacKey);
|
|
@@ -61,6 +64,7 @@ export async function verifyAdapter(input, sessionHmacKey, runner) {
|
|
|
61
64
|
fixture: input.fixture ?? 'ignore',
|
|
62
65
|
trace: input.trace ?? 'retain-on-failure',
|
|
63
66
|
adapterPath, // N3: validated draft path override (undefined → name→clis)
|
|
67
|
+
expectedSourceSha256: input.expectedSourceSha256,
|
|
64
68
|
});
|
|
65
69
|
return { ok: true, requestId };
|
|
66
70
|
}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
export {};
|
|
@@ -142,7 +142,33 @@ export async function handleVerify(ctx, body) {
|
|
|
142
142
|
const executionSeedArgs = body.executionSeedArgs && typeof body.executionSeedArgs === 'object' && !Array.isArray(body.executionSeedArgs)
|
|
143
143
|
? body.executionSeedArgs
|
|
144
144
|
: undefined;
|
|
145
|
-
const
|
|
145
|
+
const adapterPathRaw = body.adapterPath;
|
|
146
|
+
const expectedSourceSha256Raw = body.expectedSourceSha256;
|
|
147
|
+
const invalidAdapterPath = adapterPathRaw !== undefined
|
|
148
|
+
&& (typeof adapterPathRaw !== 'string' || adapterPathRaw.trim().length === 0);
|
|
149
|
+
const invalidExpectedSourceHash = expectedSourceSha256Raw !== undefined
|
|
150
|
+
&& (typeof expectedSourceSha256Raw !== 'string' || !/^[0-9a-f]{64}$/.test(expectedSourceSha256Raw));
|
|
151
|
+
if (invalidAdapterPath || invalidExpectedSourceHash) {
|
|
152
|
+
ctx.registry.finalizeRequest(requestId, {
|
|
153
|
+
status: 'failed',
|
|
154
|
+
error: errorBody('validation_failed', 'adapterPath or expectedSourceSha256 is invalid'),
|
|
155
|
+
});
|
|
156
|
+
return { status: 202, body: accepted(requestId) };
|
|
157
|
+
}
|
|
158
|
+
const adapterPath = typeof adapterPathRaw === 'string' ? adapterPathRaw : undefined;
|
|
159
|
+
const expectedSourceSha256 = typeof expectedSourceSha256Raw === 'string'
|
|
160
|
+
? expectedSourceSha256Raw
|
|
161
|
+
: undefined;
|
|
162
|
+
const input = {
|
|
163
|
+
name,
|
|
164
|
+
requestId,
|
|
165
|
+
sessionId,
|
|
166
|
+
executionSeedArgs,
|
|
167
|
+
fixture,
|
|
168
|
+
trace,
|
|
169
|
+
adapterPath,
|
|
170
|
+
expectedSourceSha256,
|
|
171
|
+
};
|
|
146
172
|
const result = await verifyAdapter(input, sessionHmacKey, ctx.runner);
|
|
147
173
|
if (!result.ok) {
|
|
148
174
|
ctx.registry.finalizeRequest(requestId, { status: 'failed', error: errorBody(result.errorCode, result.reason) });
|
|
@@ -252,6 +252,7 @@ export function createRunnerPort(opts = {}) {
|
|
|
252
252
|
requestId,
|
|
253
253
|
name: input.name,
|
|
254
254
|
adapterPath: input.adapterPath ?? resolveAdapterPath(input.name),
|
|
255
|
+
expectedSourceSha256: input.expectedSourceSha256,
|
|
255
256
|
executionSeedArgs: input.rawSeedArgs, // raw → input.json only
|
|
256
257
|
fixture: input.fixture,
|
|
257
258
|
trace: input.trace,
|
|
@@ -23,6 +23,8 @@ export interface RunnerInput {
|
|
|
23
23
|
name: string;
|
|
24
24
|
/** Resolved adapter module file to import. */
|
|
25
25
|
adapterPath: string;
|
|
26
|
+
/** Expected hash supplied by the caller; mismatch means the captured module is never loaded. */
|
|
27
|
+
expectedSourceSha256?: string;
|
|
26
28
|
/** Browser profile contextId for browser adapters (M6b). Omitted → daemon default profile. */
|
|
27
29
|
contextId?: string;
|
|
28
30
|
/** Raw seed args — prepared before resolver/adapter calls and never echoed into events. */
|
|
@@ -47,7 +49,20 @@ export declare function installRunnerBackstops(maxRuntimeMs: number): void;
|
|
|
47
49
|
* Load an adapter by importing its module (which registers via `cli()`), then look it up
|
|
48
50
|
* in the registry by name. Mirrors execution.ts's lazy-import pattern (118-135).
|
|
49
51
|
*/
|
|
50
|
-
export
|
|
52
|
+
export interface AdapterSourceSnapshot {
|
|
53
|
+
canonicalUrl: string;
|
|
54
|
+
source: ArrayBuffer;
|
|
55
|
+
sourceSha256: string;
|
|
56
|
+
}
|
|
57
|
+
/** Read the main module once. The same exact bytes are hashed and transferred to the ESM loader. */
|
|
58
|
+
export declare function captureAdapterSource(adapterPath: string): AdapterSourceSnapshot;
|
|
59
|
+
/** Import exactly the captured main-module bytes while preserving its canonical URL as import base. */
|
|
60
|
+
export declare function loadAdapterSnapshot(snapshot: AdapterSourceSnapshot, name: string): Promise<CliCommand | undefined>;
|
|
61
|
+
export interface VerifyRunnerDependencies {
|
|
62
|
+
capture?: (adapterPath: string) => AdapterSourceSnapshot | Promise<AdapterSourceSnapshot>;
|
|
63
|
+
load?: (snapshot: AdapterSourceSnapshot, name: string) => Promise<CliCommand | undefined>;
|
|
64
|
+
browserRunner?: BrowserAdapterRunner;
|
|
65
|
+
}
|
|
51
66
|
/**
|
|
52
67
|
* The browser-adapter execution seam (M6b). A browser adapter's `func` needs an IPage; the
|
|
53
68
|
* default implementation connects BACK to the running daemon for one. Injectable so unit
|
|
@@ -85,7 +100,7 @@ export declare function executeAdapterForVerify(command: CliCommand | undefined,
|
|
|
85
100
|
* tests); `load` is injected so unit tests can supply an in-memory command. Never throws —
|
|
86
101
|
* any failure becomes a terminal result so the parent always sees one and only one.
|
|
87
102
|
*/
|
|
88
|
-
export declare function runVerifyRunner(input: RunnerInput, emit: (event: RunnerEvent) => void,
|
|
103
|
+
export declare function runVerifyRunner(input: RunnerInput, emit: (event: RunnerEvent) => void, dependencies?: VerifyRunnerDependencies): Promise<void>;
|
|
89
104
|
/**
|
|
90
105
|
* Entry point for `bycli internal verify-runner --jsonl --request-id … --name … --input …`.
|
|
91
106
|
* Writes JSONL events to the dedicated protocol fd (`--protocol-fd`, set by the parent RunnerPort
|
|
@@ -16,7 +16,8 @@
|
|
|
16
16
|
* are echoed into the emitted result/started events.
|
|
17
17
|
*/
|
|
18
18
|
import * as fs from 'node:fs';
|
|
19
|
-
import { randomUUID } from 'node:crypto';
|
|
19
|
+
import { createHash, randomUUID } from 'node:crypto';
|
|
20
|
+
import { register } from 'node:module';
|
|
20
21
|
import { pathToFileURL } from 'node:url';
|
|
21
22
|
import { getRegistry, } from '../../registry.js';
|
|
22
23
|
import { prepareCommandArgsOrThrowArgumentError } from '../../execution.js';
|
|
@@ -78,14 +79,48 @@ function fieldCountOf(rows) {
|
|
|
78
79
|
}
|
|
79
80
|
return undefined;
|
|
80
81
|
}
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
|
|
85
|
-
export
|
|
86
|
-
|
|
82
|
+
const SNAPSHOT_LOADER_URL = `data:text/javascript,${encodeURIComponent(`
|
|
83
|
+
let targetSpecifier = '';
|
|
84
|
+
let targetUrl = '';
|
|
85
|
+
let source = new ArrayBuffer(0);
|
|
86
|
+
export function initialize(data) {
|
|
87
|
+
targetSpecifier = data.targetSpecifier;
|
|
88
|
+
targetUrl = data.targetUrl;
|
|
89
|
+
source = data.source;
|
|
90
|
+
}
|
|
91
|
+
export async function resolve(specifier, context, nextResolve) {
|
|
92
|
+
if (specifier === targetSpecifier) {
|
|
93
|
+
return { url: targetUrl, shortCircuit: true };
|
|
94
|
+
}
|
|
95
|
+
return nextResolve(specifier, context);
|
|
96
|
+
}
|
|
97
|
+
export async function load(url, context, nextLoad) {
|
|
98
|
+
if (url === targetUrl) {
|
|
99
|
+
return { format: 'module', shortCircuit: true, source: new Uint8Array(source) };
|
|
100
|
+
}
|
|
101
|
+
return nextLoad(url, context);
|
|
102
|
+
}
|
|
103
|
+
`)}`;
|
|
104
|
+
/** Read the main module once. The same exact bytes are hashed and transferred to the ESM loader. */
|
|
105
|
+
export function captureAdapterSource(adapterPath) {
|
|
106
|
+
const canonicalPath = fs.realpathSync(adapterPath);
|
|
107
|
+
const bytes = fs.readFileSync(canonicalPath);
|
|
108
|
+
const source = Uint8Array.from(bytes).buffer;
|
|
109
|
+
return {
|
|
110
|
+
canonicalUrl: pathToFileURL(canonicalPath).href,
|
|
111
|
+
source,
|
|
112
|
+
sourceSha256: createHash('sha256').update(new Uint8Array(source)).digest('hex'),
|
|
113
|
+
};
|
|
114
|
+
}
|
|
115
|
+
/** Import exactly the captured main-module bytes while preserving its canonical URL as import base. */
|
|
116
|
+
export async function loadAdapterSnapshot(snapshot, name) {
|
|
87
117
|
try {
|
|
88
|
-
|
|
118
|
+
const targetSpecifier = `bycli-verify-snapshot:${randomUUID()}`;
|
|
119
|
+
register(SNAPSHOT_LOADER_URL, import.meta.url, {
|
|
120
|
+
data: { targetSpecifier, targetUrl: snapshot.canonicalUrl, source: snapshot.source },
|
|
121
|
+
transferList: [snapshot.source],
|
|
122
|
+
});
|
|
123
|
+
await import(targetSpecifier);
|
|
89
124
|
}
|
|
90
125
|
catch (e) {
|
|
91
126
|
// The adapter module's top-level code threw during evaluation (a SyntaxError, or a deliberate
|
|
@@ -189,29 +224,50 @@ export async function executeAdapterForVerify(command, opts) {
|
|
|
189
224
|
* tests); `load` is injected so unit tests can supply an in-memory command. Never throws —
|
|
190
225
|
* any failure becomes a terminal result so the parent always sees one and only one.
|
|
191
226
|
*/
|
|
192
|
-
export async function runVerifyRunner(input, emit,
|
|
227
|
+
export async function runVerifyRunner(input, emit, dependencies = {}) {
|
|
193
228
|
emit({ type: 'started', requestId: input.requestId, pid: process.pid, stage: 'load' });
|
|
229
|
+
let sourceSha256;
|
|
194
230
|
try {
|
|
195
|
-
const
|
|
231
|
+
const capture = dependencies.capture ?? captureAdapterSource;
|
|
232
|
+
const load = dependencies.load ?? loadAdapterSnapshot;
|
|
233
|
+
const snapshot = await capture(input.adapterPath);
|
|
234
|
+
sourceSha256 = snapshot.sourceSha256;
|
|
235
|
+
if (input.expectedSourceSha256 !== undefined && input.expectedSourceSha256 !== sourceSha256) {
|
|
236
|
+
emit({
|
|
237
|
+
type: 'result', requestId: input.requestId, ok: false,
|
|
238
|
+
data: { stage: 'load', sourceSha256 },
|
|
239
|
+
error: { code: 'validation_failed', message: 'adapter source hash does not match expected source' },
|
|
240
|
+
});
|
|
241
|
+
return;
|
|
242
|
+
}
|
|
243
|
+
const command = await load(snapshot, input.name);
|
|
196
244
|
const r = await executeAdapterForVerify(command, {
|
|
197
245
|
name: input.name,
|
|
198
246
|
fixture: input.fixture,
|
|
199
247
|
trace: input.trace,
|
|
200
248
|
seedArgs: input.executionSeedArgs ?? {},
|
|
201
249
|
contextId: input.contextId,
|
|
202
|
-
browserRunner,
|
|
250
|
+
browserRunner: dependencies.browserRunner,
|
|
251
|
+
});
|
|
252
|
+
emit({
|
|
253
|
+
type: 'result', requestId: input.requestId, ok: r.ok,
|
|
254
|
+
data: { ...r.data, sourceSha256 },
|
|
255
|
+
error: r.ok ? null : r.error,
|
|
203
256
|
});
|
|
204
|
-
emit({ type: 'result', requestId: input.requestId, ok: r.ok, data: r.data, error: r.ok ? null : r.error });
|
|
205
257
|
}
|
|
206
258
|
catch (e) {
|
|
207
|
-
// Load failure → single terminal result. An adapter-evaluation error (tagged by
|
|
259
|
+
// Load failure → single terminal result. An adapter-evaluation error (tagged by loadAdapterSnapshot)
|
|
208
260
|
// is adapter-controlled and may echo adapter-file contents, so its message is redacted; a
|
|
209
261
|
// runner-side failure (bad path / resolve) is runner-generated and surfaces verbatim (Codex M7c).
|
|
210
262
|
const adapterEval = e?.adapterEvaluation === true;
|
|
211
263
|
const message = adapterEval ? REDACTED_ADAPTER_LOAD_MESSAGE : (e instanceof Error ? e.message : String(e));
|
|
212
264
|
emit({
|
|
213
265
|
type: 'result', requestId: input.requestId, ok: false,
|
|
214
|
-
data: {
|
|
266
|
+
data: {
|
|
267
|
+
stage: 'load',
|
|
268
|
+
...(sourceSha256 === undefined ? {} : { sourceSha256 }),
|
|
269
|
+
trace: { policy: input.trace ?? 'retain-on-failure', retained: false, path: null },
|
|
270
|
+
},
|
|
215
271
|
error: { code: 'adapter_runtime_error', message, hint: 'adapter failed to load' },
|
|
216
272
|
});
|
|
217
273
|
}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
export {};
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
export {};
|
package/package.json
CHANGED
|
@@ -1,12 +1,12 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@sovovs/bycli",
|
|
3
|
-
"version": "2.1.
|
|
3
|
+
"version": "2.1.2",
|
|
4
4
|
"publishConfig": {
|
|
5
5
|
"access": "public"
|
|
6
6
|
},
|
|
7
7
|
"description": "Make any website or Electron App your CLI. AI-powered.",
|
|
8
8
|
"engines": {
|
|
9
|
-
"node": ">=20.
|
|
9
|
+
"node": ">=20.6.0"
|
|
10
10
|
},
|
|
11
11
|
"type": "module",
|
|
12
12
|
"workspaces": [
|
|
@@ -32,6 +32,7 @@
|
|
|
32
32
|
"./download/article-download": "./dist/src/download/article-download.js",
|
|
33
33
|
"./download/media-download": "./dist/src/download/media-download.js",
|
|
34
34
|
"./download/progress": "./dist/src/download/progress.js",
|
|
35
|
+
"./download/wechat-article": "./dist/src/download/wechat-article.js",
|
|
35
36
|
"./pipeline": "./dist/src/pipeline/index.js"
|
|
36
37
|
},
|
|
37
38
|
"files": [
|
|
@@ -66,6 +67,7 @@
|
|
|
66
67
|
"advise:listing-id-pairing": "node scripts/check-listing-id-pairing.mjs",
|
|
67
68
|
"check:silent-column-drop": "node scripts/check-silent-column-drop.mjs",
|
|
68
69
|
"check:typed-error-lint": "node scripts/check-typed-error-lint.mjs",
|
|
70
|
+
"check:package-install": "node scripts/check-package-install.mjs",
|
|
69
71
|
"docs:dev": "vitepress dev docs",
|
|
70
72
|
"docs:build": "vitepress build docs",
|
|
71
73
|
"docs:preview": "vitepress preview docs"
|
|
@@ -87,17 +89,19 @@
|
|
|
87
89
|
},
|
|
88
90
|
"dependencies": {
|
|
89
91
|
"@mozilla/readability": "^0.6.0",
|
|
92
|
+
"@sovovs/bycli-recorder-core": "^0.1.0",
|
|
90
93
|
"cli-table3": "^0.6.5",
|
|
91
94
|
"commander": "^14.0.3",
|
|
92
95
|
"js-yaml": "^4.1.0",
|
|
96
|
+
"parse5": "^7.3.0",
|
|
93
97
|
"turndown": "^7.2.2",
|
|
94
98
|
"turndown-plugin-gfm": "^1.0.2",
|
|
95
99
|
"undici": "^6.25.0",
|
|
96
100
|
"ws": "^8.18.0"
|
|
97
101
|
},
|
|
98
102
|
"devDependencies": {
|
|
99
|
-
"@types/jsdom": "^27.0.0",
|
|
100
103
|
"@types/js-yaml": "^4.0.9",
|
|
104
|
+
"@types/jsdom": "^27.0.0",
|
|
101
105
|
"@types/node": "^25.5.2",
|
|
102
106
|
"@types/turndown": "^5.0.6",
|
|
103
107
|
"@types/ws": "^8.5.13",
|