@sovovs/bycli 2.0.0 → 2.1.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/cli-manifest.json +169 -0
- package/clis/twitter/search.js +3 -3
- package/clis/weixin/_wechat/args.js +48 -0
- package/clis/weixin/_wechat/article-content.js +53 -0
- package/clis/weixin/_wechat/article-service.js +124 -0
- package/clis/weixin/_wechat/auth-session.js +142 -0
- package/clis/weixin/_wechat/fingerprint.js +443 -0
- package/clis/weixin/_wechat/fixtures/articles-auth-expired.json +3 -0
- package/clis/weixin/_wechat/fixtures/articles-page.json +4 -0
- package/clis/weixin/_wechat/fixtures/search-auth-expired.json +4 -0
- package/clis/weixin/_wechat/fixtures/search-success.json +7 -0
- package/clis/weixin/_wechat/markdown.js +29 -0
- package/clis/weixin/_wechat/redact.js +405 -0
- package/clis/weixin/_wechat/save-service.js +175 -0
- package/clis/weixin/_wechat/search-biz.js +102 -0
- package/clis/weixin/_wechat/wechat-api.js +133 -0
- package/clis/weixin/accounts.js +38 -0
- package/clis/weixin/articles.js +35 -0
- package/clis/weixin/download.js +5 -47
- package/clis/weixin/save-articles.js +175 -0
- package/dist/src/browser/cdp.js +3 -0
- package/dist/src/browser/daemon-client.d.ts +2 -0
- package/dist/src/browser/daemon-client.js +1 -1
- package/dist/src/browser/extension-capabilities.d.ts +13 -0
- package/dist/src/browser/extension-capabilities.js +22 -0
- package/dist/src/browser/extension-capabilities.test.d.ts +1 -0
- package/dist/src/browser/extension-version-metadata.test.d.ts +1 -0
- package/dist/src/browser/page.d.ts +1 -0
- package/dist/src/browser/page.js +20 -1
- package/dist/src/build-manifest.js +4 -2
- package/dist/src/capabilityRouting.d.ts +3 -2
- package/dist/src/capabilityRouting.js +10 -2
- package/dist/src/cli.js +1 -1
- package/dist/src/commanderAdapter.js +5 -5
- package/dist/src/daemon.js +16 -0
- package/dist/src/discovery.d.ts +5 -0
- package/dist/src/discovery.js +12 -4
- package/dist/src/discovery.test.d.ts +1 -0
- package/dist/src/download/article-download.d.ts +6 -0
- package/dist/src/download/article-download.js +78 -17
- package/dist/src/download/wechat-article.d.ts +8 -0
- package/dist/src/download/wechat-article.js +137 -0
- package/dist/src/download/wechat-article.test.d.ts +1 -0
- package/dist/src/execution.d.ts +5 -0
- package/dist/src/execution.js +269 -50
- package/dist/src/help.js +8 -8
- package/dist/src/manifest-schema.d.ts +9 -0
- package/dist/src/manifest-schema.js +162 -0
- package/dist/src/manifest-schema.test.d.ts +1 -0
- package/dist/src/manifest-types.d.ts +1 -1
- package/dist/src/observation/redaction.js +10 -4
- package/dist/src/recorder/highlevel/verify.d.ts +3 -0
- package/dist/src/recorder/highlevel/verify.js +4 -0
- package/dist/src/recorder/highlevel/verify.test.d.ts +1 -0
- package/dist/src/recorder/runner/runner-port.js +1 -0
- package/dist/src/recorder/runner/verify-runner-main.d.ts +23 -7
- package/dist/src/recorder/runner/verify-runner-main.js +92 -19
- package/dist/src/registry-api.d.ts +1 -1
- package/dist/src/registry-api.types.test.d.ts +1 -0
- package/dist/src/registry-transaction.d.ts +42 -0
- package/dist/src/registry-transaction.js +194 -0
- package/dist/src/registry-transaction.test.d.ts +1 -0
- package/dist/src/registry.d.ts +58 -16
- package/dist/src/registry.js +131 -15
- package/dist/src/serialization.d.ts +1 -1
- package/dist/src/serialization.js +3 -3
- package/dist/src/types.d.ts +2 -0
- package/dist/src/weixin-built-in-docs.test.d.ts +1 -0
- package/package.json +7 -3
- package/scripts/check-package-install.mjs +71 -0
- package/scripts/recorder.sh +0 -186
package/dist/src/cli.js
CHANGED
|
@@ -506,7 +506,7 @@ export function createProgram(BUILTIN_CLIS, USER_CLIS) {
|
|
|
506
506
|
description: c.description,
|
|
507
507
|
access: c.access,
|
|
508
508
|
strategy: strategyLabel(c),
|
|
509
|
-
browser:
|
|
509
|
+
browser: c.browser,
|
|
510
510
|
args: formatArgSummary(c.args),
|
|
511
511
|
}));
|
|
512
512
|
renderOutput(rows, {
|
|
@@ -11,7 +11,7 @@
|
|
|
11
11
|
*/
|
|
12
12
|
import { log } from './logger.js';
|
|
13
13
|
import yaml from 'js-yaml';
|
|
14
|
-
import { fullName, getRegistry } from './registry.js';
|
|
14
|
+
import { fullName, getRegistry, hasBrowserCapability } from './registry.js';
|
|
15
15
|
import { render as renderOutput } from './output.js';
|
|
16
16
|
import { executeCommand, prepareCommandArgs } from './execution.js';
|
|
17
17
|
import { commandHelpData, formatCommandHelpText, formatCommandListTerm, formatSiteCommandDescription, formatSiteHelpText, getRequestedHelpFormat, renderStructuredHelp, siteHelpData, } from './help.js';
|
|
@@ -48,7 +48,7 @@ export function registerCommandToProgram(siteCmd, cmd) {
|
|
|
48
48
|
.option('-f, --format <fmt>', 'Output format: table, plain, json, yaml, md, csv', 'table')
|
|
49
49
|
.option('--trace <mode>', 'Trace capture: off, on, retain-on-failure', 'off')
|
|
50
50
|
.option('-v, --verbose', 'Debug output', false);
|
|
51
|
-
if (cmd
|
|
51
|
+
if (hasBrowserCapability(cmd)) {
|
|
52
52
|
subCmd
|
|
53
53
|
.option('--window <mode>', 'Browser window mode: foreground or background')
|
|
54
54
|
.option('--site-session <mode>', 'Adapter site session lifecycle: ephemeral or persistent')
|
|
@@ -108,9 +108,9 @@ export function registerCommandToProgram(siteCmd, cmd) {
|
|
|
108
108
|
prepared: true,
|
|
109
109
|
...(typeof globals.profile === 'string' && globals.profile.trim() ? { profile: globals.profile.trim() } : {}),
|
|
110
110
|
...(typeof optionsRecord.trace === 'string' && optionsRecord.trace !== 'off' ? { trace: optionsRecord.trace } : {}),
|
|
111
|
-
...(cmd
|
|
112
|
-
...(cmd
|
|
113
|
-
...(cmd
|
|
111
|
+
...(hasBrowserCapability(cmd) && typeof optionsRecord.window === 'string' ? { windowMode: optionsRecord.window } : {}),
|
|
112
|
+
...(hasBrowserCapability(cmd) && typeof optionsRecord.siteSession === 'string' ? { siteSession: optionsRecord.siteSession } : {}),
|
|
113
|
+
...(hasBrowserCapability(cmd) && typeof optionsRecord.keepTab === 'string' ? { keepTab: optionsRecord.keepTab } : {}),
|
|
114
114
|
});
|
|
115
115
|
if (result === null || result === undefined) {
|
|
116
116
|
return;
|
package/dist/src/daemon.js
CHANGED
|
@@ -35,6 +35,7 @@ import { reapOrphanedVerifyRuns } from './recorder/runner/reap.js';
|
|
|
35
35
|
import { resolveTempPolicy, resolveRunnerConfig, resolveTempCapacity } from './recorder/runner/config.js';
|
|
36
36
|
import { defaultSessionKeyRegistry } from './recorder/runner/session-keys.js';
|
|
37
37
|
import { recordExtensionVersion } from './update-check.js';
|
|
38
|
+
import { EXTENSION_CAPABILITY_MISSING_ERROR_CODE, EXTENSION_CAPABILITY_MISSING_HTTP_STATUS, extensionCapabilityHint, missingRequiredExtensionCapability, normalizeExtensionCapabilities, } from './browser/extension-capabilities.js';
|
|
38
39
|
import { buildCommandDispatchFailure, buildExtensionDisconnectFailure, getResponseCorsHeaders, } from './daemon-utils.js';
|
|
39
40
|
const PORT = parseInt(process.env.BYCLI_DAEMON_PORT ?? String(DEFAULT_DAEMON_PORT), 10);
|
|
40
41
|
// The verify runner (M6b) spawns child processes that connect back to THIS daemon for a
|
|
@@ -123,6 +124,7 @@ function registerExtensionConnection(ws, rawContextId) {
|
|
|
123
124
|
ws,
|
|
124
125
|
extensionVersion: current?.ws === ws ? current.extensionVersion : null,
|
|
125
126
|
extensionCompatRange: current?.ws === ws ? current.extensionCompatRange : null,
|
|
127
|
+
extensionCapabilities: current?.ws === ws ? current.extensionCapabilities : [],
|
|
126
128
|
lastSeenAt: Date.now(),
|
|
127
129
|
};
|
|
128
130
|
extensionProfiles.set(contextId, connection);
|
|
@@ -357,6 +359,7 @@ async function handleRequest(req, res) {
|
|
|
357
359
|
extensionConnected: true,
|
|
358
360
|
extensionVersion: profile.extensionVersion ?? undefined,
|
|
359
361
|
extensionCompatRange: profile.extensionCompatRange ?? undefined,
|
|
362
|
+
extensionCapabilities: profile.extensionCapabilities,
|
|
360
363
|
pending: [...pending.values()].filter((entry) => entry.contextId === profile.contextId).length,
|
|
361
364
|
lastSeenAt: profile.lastSeenAt,
|
|
362
365
|
}));
|
|
@@ -368,6 +371,7 @@ async function handleRequest(req, res) {
|
|
|
368
371
|
extensionConnected: !!route.connection,
|
|
369
372
|
extensionVersion: route.connection?.extensionVersion ?? undefined,
|
|
370
373
|
extensionCompatRange: route.connection?.extensionCompatRange ?? undefined,
|
|
374
|
+
extensionCapabilities: route.connection?.extensionCapabilities,
|
|
371
375
|
contextId: route.connection?.contextId ?? requestedContextId,
|
|
372
376
|
profileRequired: route.errorCode === 'profile_required',
|
|
373
377
|
profileDisconnected: route.errorCode === 'profile_disconnected',
|
|
@@ -423,6 +427,17 @@ async function handleRequest(req, res) {
|
|
|
423
427
|
});
|
|
424
428
|
return;
|
|
425
429
|
}
|
|
430
|
+
const requiredCapability = missingRequiredExtensionCapability(body, route.connection.extensionCapabilities);
|
|
431
|
+
if (requiredCapability) {
|
|
432
|
+
jsonResponse(res, EXTENSION_CAPABILITY_MISSING_HTTP_STATUS, {
|
|
433
|
+
id: body.id,
|
|
434
|
+
ok: false,
|
|
435
|
+
errorCode: EXTENSION_CAPABILITY_MISSING_ERROR_CODE,
|
|
436
|
+
error: `Connected Browser Bridge does not advertise ${requiredCapability}.`,
|
|
437
|
+
errorHint: extensionCapabilityHint(requiredCapability),
|
|
438
|
+
});
|
|
439
|
+
return;
|
|
440
|
+
}
|
|
426
441
|
const timeoutMs = typeof body.timeout === 'number' && body.timeout > 0
|
|
427
442
|
? body.timeout * 1000
|
|
428
443
|
: 120000;
|
|
@@ -529,6 +544,7 @@ wss.on('connection', (ws) => {
|
|
|
529
544
|
const connection = registerExtensionConnection(ws, msg.contextId);
|
|
530
545
|
connection.extensionVersion = typeof msg.version === 'string' ? msg.version : null;
|
|
531
546
|
connection.extensionCompatRange = typeof msg.compatRange === 'string' ? msg.compatRange : null;
|
|
547
|
+
connection.extensionCapabilities = normalizeExtensionCapabilities(msg.capabilities);
|
|
532
548
|
connection.lastSeenAt = Date.now();
|
|
533
549
|
if (connection.extensionVersion)
|
|
534
550
|
recordExtensionVersion(connection.extensionVersion);
|
package/dist/src/discovery.d.ts
CHANGED
|
@@ -37,6 +37,11 @@ export declare function ensureUserAdapters(): Promise<void>;
|
|
|
37
37
|
* Uses pre-compiled manifest when available for instant startup.
|
|
38
38
|
*/
|
|
39
39
|
export declare function discoverClis(...dirs: string[]): Promise<void>;
|
|
40
|
+
/**
|
|
41
|
+
* Fast-path: register commands from pre-compiled manifest.
|
|
42
|
+
* TS modules are deferred — loaded lazily on first execution.
|
|
43
|
+
*/
|
|
44
|
+
export declare function loadFromManifest(manifestPath: string, clisDir: string): Promise<boolean>;
|
|
40
45
|
/**
|
|
41
46
|
* Discover and register plugins from ~/.bycli/plugins/.
|
|
42
47
|
* Each subdirectory is treated as a plugin (site = directory name).
|
package/dist/src/discovery.js
CHANGED
|
@@ -113,7 +113,7 @@ export async function discoverClis(...dirs) {
|
|
|
113
113
|
* Fast-path: register commands from pre-compiled manifest.
|
|
114
114
|
* TS modules are deferred — loaded lazily on first execution.
|
|
115
115
|
*/
|
|
116
|
-
async function loadFromManifest(manifestPath, clisDir) {
|
|
116
|
+
export async function loadFromManifest(manifestPath, clisDir) {
|
|
117
117
|
try {
|
|
118
118
|
const raw = await fs.promises.readFile(manifestPath, 'utf-8');
|
|
119
119
|
const manifest = JSON.parse(raw);
|
|
@@ -121,7 +121,7 @@ async function loadFromManifest(manifestPath, clisDir) {
|
|
|
121
121
|
if (!entry.modulePath)
|
|
122
122
|
continue;
|
|
123
123
|
const modulePath = path.resolve(clisDir, entry.modulePath);
|
|
124
|
-
const
|
|
124
|
+
const common = {
|
|
125
125
|
site: entry.site,
|
|
126
126
|
name: entry.name,
|
|
127
127
|
aliases: entry.aliases,
|
|
@@ -130,7 +130,6 @@ async function loadFromManifest(manifestPath, clisDir) {
|
|
|
130
130
|
example: entry.example,
|
|
131
131
|
domain: entry.domain,
|
|
132
132
|
strategy: parseStrategy(entry.strategy),
|
|
133
|
-
browser: entry.browser,
|
|
134
133
|
args: entry.args ?? [],
|
|
135
134
|
columns: entry.columns,
|
|
136
135
|
defaultFormat: entry.defaultFormat,
|
|
@@ -141,7 +140,16 @@ async function loadFromManifest(manifestPath, clisDir) {
|
|
|
141
140
|
_lazy: true,
|
|
142
141
|
_modulePath: modulePath,
|
|
143
142
|
};
|
|
144
|
-
|
|
143
|
+
const cmd = entry.browser === 'conditional'
|
|
144
|
+
? {
|
|
145
|
+
...common,
|
|
146
|
+
browser: 'conditional',
|
|
147
|
+
requiresBrowser: () => {
|
|
148
|
+
throw new Error(`Conditional manifest placeholder ${entry.site}/${entry.name} was not hydrated`);
|
|
149
|
+
},
|
|
150
|
+
_hydrateBeforeBrowserRouting: true,
|
|
151
|
+
}
|
|
152
|
+
: { ...common, browser: entry.browser };
|
|
145
153
|
registerCommand(cmd);
|
|
146
154
|
}
|
|
147
155
|
return true;
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
export {};
|
|
@@ -6,6 +6,7 @@
|
|
|
6
6
|
* Flow: ArticleData → TurndownService → image download → frontmatter → .md file
|
|
7
7
|
*/
|
|
8
8
|
import TurndownService from 'turndown';
|
|
9
|
+
export { extractWechatArticleHtml } from './wechat-article.js';
|
|
9
10
|
export interface ArticleData {
|
|
10
11
|
title: string;
|
|
11
12
|
author?: string;
|
|
@@ -49,6 +50,8 @@ export interface ArticleDownloadOptions {
|
|
|
49
50
|
* as-is so the output is self-contained when piped.
|
|
50
51
|
*/
|
|
51
52
|
stdout?: boolean;
|
|
53
|
+
/** Opt-in hardened Markdown rules used by HTML-focused adapters. */
|
|
54
|
+
secureMarkdown?: boolean;
|
|
52
55
|
}
|
|
53
56
|
export interface ArticleDownloadResult {
|
|
54
57
|
title: string;
|
|
@@ -58,6 +61,9 @@ export interface ArticleDownloadResult {
|
|
|
58
61
|
size: string;
|
|
59
62
|
saved: string;
|
|
60
63
|
}
|
|
64
|
+
export declare function convertArticleHtmlToMarkdown(contentHtml: string, options?: {
|
|
65
|
+
safeFencedCodeBlocks?: boolean;
|
|
66
|
+
}): string;
|
|
61
67
|
/**
|
|
62
68
|
* Download an article to Markdown with optional image localization.
|
|
63
69
|
*
|
|
@@ -11,12 +11,29 @@ import TurndownService from 'turndown';
|
|
|
11
11
|
import { gfm } from 'turndown-plugin-gfm';
|
|
12
12
|
import { httpDownload, sanitizeFilename } from './index.js';
|
|
13
13
|
import { formatBytes } from './progress.js';
|
|
14
|
+
export { extractWechatArticleHtml } from './wechat-article.js';
|
|
14
15
|
const IMAGE_CONCURRENCY = 5;
|
|
15
16
|
const DEFAULT_LABELS = {
|
|
16
17
|
author: '作者',
|
|
17
18
|
publishTime: '发布时间',
|
|
18
19
|
sourceUrl: '原文链接',
|
|
19
20
|
};
|
|
21
|
+
function escapeMarkdownText(value) {
|
|
22
|
+
return value.replace(/\s+/g, ' ').trim()
|
|
23
|
+
.replace(/&/g, '&').replace(/</g, '<').replace(/>/g, '>')
|
|
24
|
+
.replace(/"/g, '"').replace(/'/g, ''')
|
|
25
|
+
.replace(/([\\`*_[\]{}()#+.!|>~-])/g, '\\$1');
|
|
26
|
+
}
|
|
27
|
+
function safeHttpUrl(value) {
|
|
28
|
+
const normalized = value.trim().startsWith('//') ? `https:${value.trim()}` : value.trim();
|
|
29
|
+
try {
|
|
30
|
+
const url = new URL(normalized);
|
|
31
|
+
return url.protocol === 'http:' || url.protocol === 'https:' ? url.href : '';
|
|
32
|
+
}
|
|
33
|
+
catch {
|
|
34
|
+
return '';
|
|
35
|
+
}
|
|
36
|
+
}
|
|
20
37
|
// ============================================================
|
|
21
38
|
// Markdown Conversion
|
|
22
39
|
// ============================================================
|
|
@@ -36,13 +53,48 @@ const STRIPPED_TAGS = [
|
|
|
36
53
|
'form', 'button', 'dialog',
|
|
37
54
|
'header', 'footer', 'nav', 'aside',
|
|
38
55
|
];
|
|
39
|
-
function createTurndown(configure, cleanSelectors) {
|
|
56
|
+
function createTurndown(configure, cleanSelectors, secureMarkdown = false) {
|
|
40
57
|
const td = new TurndownService({
|
|
41
58
|
headingStyle: 'atx',
|
|
42
59
|
codeBlockStyle: 'fenced',
|
|
43
60
|
bulletListMarker: '-',
|
|
44
61
|
});
|
|
45
62
|
td.use(gfm);
|
|
63
|
+
if (secureMarkdown) {
|
|
64
|
+
const escapeDestination = (value) => value.replace(/([\\()])/g, '\\$1');
|
|
65
|
+
td.addRule('safeFencedCodeBlock', {
|
|
66
|
+
filter: 'pre',
|
|
67
|
+
replacement: (_content, node) => {
|
|
68
|
+
const element = node;
|
|
69
|
+
const code = element.textContent || '';
|
|
70
|
+
const longest = Math.max(0, ...[...code.matchAll(/`+/g)].map(match => match[0].length));
|
|
71
|
+
const fence = '`'.repeat(Math.max(3, longest + 1));
|
|
72
|
+
const className = element.querySelector('code')?.getAttribute('class') || '';
|
|
73
|
+
const language = element.getAttribute('data-lang')
|
|
74
|
+
|| /(?:^|\s)language-([^\s]+)/.exec(className)?.[1]
|
|
75
|
+
|| '';
|
|
76
|
+
return `\n${fence}${language}\n${code.replace(/\n$/, '')}\n${fence}\n`;
|
|
77
|
+
},
|
|
78
|
+
});
|
|
79
|
+
td.addRule('safeImage', {
|
|
80
|
+
filter: 'img',
|
|
81
|
+
replacement: (_content, node) => {
|
|
82
|
+
const element = node;
|
|
83
|
+
const alt = escapeMarkdownText(element.getAttribute('alt') || '');
|
|
84
|
+
const src = element.getAttribute('src') || '';
|
|
85
|
+
return src ? `})` : alt;
|
|
86
|
+
},
|
|
87
|
+
});
|
|
88
|
+
td.addRule('safeLink', {
|
|
89
|
+
filter: 'a',
|
|
90
|
+
replacement: (_content, node) => {
|
|
91
|
+
const element = node;
|
|
92
|
+
const label = escapeMarkdownText(element.textContent || '');
|
|
93
|
+
const href = element.getAttribute('href') || '';
|
|
94
|
+
return href ? `[${label}](${escapeDestination(href)})` : label;
|
|
95
|
+
},
|
|
96
|
+
});
|
|
97
|
+
}
|
|
46
98
|
td.remove(STRIPPED_TAGS);
|
|
47
99
|
// turndown-plugin-gfm@1.0.2 emits single-tilde strikethrough (`~x~`), which
|
|
48
100
|
// is not the canonical GFM form. Override it so exported markdown is
|
|
@@ -104,11 +156,14 @@ function createTurndown(configure, cleanSelectors) {
|
|
|
104
156
|
filter: (node) => node.nodeName === 'IFRAME',
|
|
105
157
|
replacement: (_content, node) => {
|
|
106
158
|
const el = node;
|
|
107
|
-
const
|
|
159
|
+
const rawSrc = el.getAttribute('src') || '';
|
|
160
|
+
const src = secureMarkdown ? safeHttpUrl(rawSrc) : rawSrc;
|
|
108
161
|
if (!src)
|
|
109
162
|
return '';
|
|
110
|
-
const
|
|
111
|
-
|
|
163
|
+
const rawTitle = el.getAttribute('title') || 'Embedded content';
|
|
164
|
+
const title = secureMarkdown ? escapeMarkdownText(rawTitle) : rawTitle;
|
|
165
|
+
const destination = secureMarkdown ? src.replace(/([\\()])/g, '\\$1') : src;
|
|
166
|
+
return `\n[${title}](${destination})\n`;
|
|
112
167
|
},
|
|
113
168
|
});
|
|
114
169
|
// Per-adapter dirty-node removal. Adapters know their site's specific noise
|
|
@@ -139,14 +194,16 @@ function createTurndown(configure, cleanSelectors) {
|
|
|
139
194
|
configure(td);
|
|
140
195
|
return td;
|
|
141
196
|
}
|
|
142
|
-
function convertToMarkdown(contentHtml, codeBlocks, configure, cleanSelectors) {
|
|
143
|
-
const td = createTurndown(configure, cleanSelectors);
|
|
197
|
+
function convertToMarkdown(contentHtml, codeBlocks, configure, cleanSelectors, secureMarkdown = false) {
|
|
198
|
+
const td = createTurndown(configure, cleanSelectors, secureMarkdown);
|
|
144
199
|
let md = td.turndown(contentHtml);
|
|
145
|
-
//
|
|
146
|
-
|
|
147
|
-
|
|
148
|
-
|
|
149
|
-
|
|
200
|
+
// Legacy callers may still supply extracted blocks. New callers preserve
|
|
201
|
+
// real <pre>/<code> nodes so Turndown owns fencing without collision-prone
|
|
202
|
+
// magic text; legacy blocks are appended with safe dynamic fences.
|
|
203
|
+
codeBlocks.forEach((block) => {
|
|
204
|
+
const longest = Math.max(0, ...[...block.code.matchAll(/`+/g)].map(match => match[0].length));
|
|
205
|
+
const fence = '`'.repeat(Math.max(3, longest + 1));
|
|
206
|
+
md += `\n\n${fence}${block.lang}\n${block.code}\n${fence}`;
|
|
150
207
|
});
|
|
151
208
|
// Clean up
|
|
152
209
|
md = md.replace(/\u00a0/g, ' ');
|
|
@@ -158,6 +215,9 @@ function convertToMarkdown(contentHtml, codeBlocks, configure, cleanSelectors) {
|
|
|
158
215
|
md = md.replace(/\n{3,}/g, '\n\n');
|
|
159
216
|
return md;
|
|
160
217
|
}
|
|
218
|
+
export function convertArticleHtmlToMarkdown(contentHtml, options = {}) {
|
|
219
|
+
return convertToMarkdown(contentHtml, [], undefined, undefined, options.safeFencedCodeBlocks === true);
|
|
220
|
+
}
|
|
161
221
|
function replaceImageUrls(md, urlMap) {
|
|
162
222
|
return md.replace(/!\[([^\]]*)\]\(([^)]+)\)/g, (match, alt, imgUrl) => {
|
|
163
223
|
const local = urlMap[imgUrl];
|
|
@@ -230,7 +290,7 @@ async function downloadImages(imgUrls, imgDir, headers, detectExt) {
|
|
|
230
290
|
* 6. File write
|
|
231
291
|
*/
|
|
232
292
|
export async function downloadArticle(data, options) {
|
|
233
|
-
const { output, downloadImages: shouldDownloadImages = true, imageHeaders, maxTitleLength = 80, configureTurndown, detectImageExt, frontmatterLabels, cleanSelectors, stdout = false, } = options;
|
|
293
|
+
const { output, downloadImages: shouldDownloadImages = true, imageHeaders, maxTitleLength = 80, configureTurndown, detectImageExt, frontmatterLabels, cleanSelectors, stdout = false, secureMarkdown = false, } = options;
|
|
234
294
|
const labels = { ...DEFAULT_LABELS, ...frontmatterLabels };
|
|
235
295
|
if (!data.title) {
|
|
236
296
|
return [{
|
|
@@ -253,7 +313,7 @@ export async function downloadArticle(data, options) {
|
|
|
253
313
|
}];
|
|
254
314
|
}
|
|
255
315
|
// Convert HTML to Markdown
|
|
256
|
-
let markdown = convertToMarkdown(data.contentHtml, data.codeBlocks || [], configureTurndown, cleanSelectors);
|
|
316
|
+
let markdown = convertToMarkdown(data.contentHtml, data.codeBlocks || [], configureTurndown, cleanSelectors, secureMarkdown);
|
|
257
317
|
const safeTitle = sanitizeFilename(data.title, maxTitleLength);
|
|
258
318
|
// Download images only when writing to disk. In stdout mode remote URLs
|
|
259
319
|
// stay intact so the piped output is self-contained.
|
|
@@ -268,13 +328,14 @@ export async function downloadArticle(data, options) {
|
|
|
268
328
|
// Build frontmatter with customizable labels.
|
|
269
329
|
// Shape: `# Title\n[> meta\n...]\n---\n\n<markdown>` — exactly one blank
|
|
270
330
|
// line separates every section, so we never produce ≥3 consecutive newlines.
|
|
271
|
-
const
|
|
331
|
+
const headerValue = (value) => secureMarkdown ? escapeMarkdownText(value) : value;
|
|
332
|
+
const headerLines = [`# ${headerValue(data.title)}`];
|
|
272
333
|
if (data.author)
|
|
273
|
-
headerLines.push(`> ${labels.author}: ${data.author}`);
|
|
334
|
+
headerLines.push(`> ${labels.author}: ${headerValue(data.author)}`);
|
|
274
335
|
if (data.publishTime)
|
|
275
|
-
headerLines.push(`> ${labels.publishTime}: ${data.publishTime}`);
|
|
336
|
+
headerLines.push(`> ${labels.publishTime}: ${headerValue(data.publishTime)}`);
|
|
276
337
|
if (data.sourceUrl)
|
|
277
|
-
headerLines.push(`> ${labels.sourceUrl}: ${data.sourceUrl}`);
|
|
338
|
+
headerLines.push(`> ${labels.sourceUrl}: ${headerValue(data.sourceUrl)}`);
|
|
278
339
|
const frontmatter = headerLines.join('\n') + '\n\n---\n\n';
|
|
279
340
|
const fullContent = frontmatter + markdown;
|
|
280
341
|
const size = Buffer.byteLength(fullContent, 'utf-8');
|
|
@@ -0,0 +1,8 @@
|
|
|
1
|
+
export declare const MAX_WECHAT_HTML_BYTES: number;
|
|
2
|
+
export declare const MAX_WECHAT_NODES = 100000;
|
|
3
|
+
export declare const MAX_WECHAT_CODE_BLOCKS = 1000;
|
|
4
|
+
export interface ExtractedWechatArticle {
|
|
5
|
+
contentHtml: string;
|
|
6
|
+
imageUrls: string[];
|
|
7
|
+
}
|
|
8
|
+
export declare function extractWechatArticleHtml(html: string): ExtractedWechatArticle;
|
|
@@ -0,0 +1,137 @@
|
|
|
1
|
+
import { parse, serialize } from 'parse5';
|
|
2
|
+
import { CommandExecutionError } from '../errors.js';
|
|
3
|
+
export const MAX_WECHAT_HTML_BYTES = 10 * 1024 * 1024;
|
|
4
|
+
export const MAX_WECHAT_NODES = 100_000;
|
|
5
|
+
export const MAX_WECHAT_CODE_BLOCKS = 1_000;
|
|
6
|
+
function isElement(node) {
|
|
7
|
+
return 'tagName' in node;
|
|
8
|
+
}
|
|
9
|
+
function attr(node, name) {
|
|
10
|
+
return node.attrs.find(item => item.name === name)?.value;
|
|
11
|
+
}
|
|
12
|
+
function hasClass(node, name) {
|
|
13
|
+
return (attr(node, 'class') || '').split(/\s+/).includes(name);
|
|
14
|
+
}
|
|
15
|
+
function safeUrl(value) {
|
|
16
|
+
const normalized = value.trim().startsWith('//') ? `https:${value.trim()}` : value.trim();
|
|
17
|
+
try {
|
|
18
|
+
const url = new URL(normalized);
|
|
19
|
+
return url.protocol === 'http:' || url.protocol === 'https:' ? url.href : undefined;
|
|
20
|
+
}
|
|
21
|
+
catch {
|
|
22
|
+
return undefined;
|
|
23
|
+
}
|
|
24
|
+
}
|
|
25
|
+
function textContent(node) {
|
|
26
|
+
if ('value' in node)
|
|
27
|
+
return node.value;
|
|
28
|
+
if (!('childNodes' in node))
|
|
29
|
+
return '';
|
|
30
|
+
return node.childNodes.map(textContent).join('');
|
|
31
|
+
}
|
|
32
|
+
export function extractWechatArticleHtml(html) {
|
|
33
|
+
if (Buffer.byteLength(html, 'utf8') > MAX_WECHAT_HTML_BYTES) {
|
|
34
|
+
throw new CommandExecutionError('WeChat article HTML exceeds the 10 MiB limit');
|
|
35
|
+
}
|
|
36
|
+
const document = parse(html);
|
|
37
|
+
let content;
|
|
38
|
+
let nodes = 0;
|
|
39
|
+
let codeBlocks = 0;
|
|
40
|
+
const stack = [document];
|
|
41
|
+
while (stack.length > 0) {
|
|
42
|
+
const node = stack.pop();
|
|
43
|
+
nodes += 1;
|
|
44
|
+
if (nodes > MAX_WECHAT_NODES)
|
|
45
|
+
throw new CommandExecutionError('WeChat article HTML exceeds the DOM node limit');
|
|
46
|
+
if (isElement(node)) {
|
|
47
|
+
if (attr(node, 'id') === 'js_content')
|
|
48
|
+
content = node;
|
|
49
|
+
if (node.tagName === 'pre') {
|
|
50
|
+
codeBlocks += 1;
|
|
51
|
+
if (codeBlocks > MAX_WECHAT_CODE_BLOCKS)
|
|
52
|
+
throw new CommandExecutionError('WeChat article HTML exceeds the code block limit');
|
|
53
|
+
}
|
|
54
|
+
}
|
|
55
|
+
if ('childNodes' in node) {
|
|
56
|
+
for (let i = node.childNodes.length - 1; i >= 0; i -= 1)
|
|
57
|
+
stack.push(node.childNodes[i]);
|
|
58
|
+
}
|
|
59
|
+
}
|
|
60
|
+
if (!content)
|
|
61
|
+
throw new CommandExecutionError('WeChat article has no #js_content');
|
|
62
|
+
const parents = [content];
|
|
63
|
+
while (parents.length > 0) {
|
|
64
|
+
const parent = parents.pop();
|
|
65
|
+
parent.childNodes = parent.childNodes.filter(node => {
|
|
66
|
+
if (!isElement(node))
|
|
67
|
+
return true;
|
|
68
|
+
return !['script', 'style'].includes(node.tagName)
|
|
69
|
+
&& !hasClass(node, 'qr_code_pc') && !hasClass(node, 'reward_area')
|
|
70
|
+
&& !hasClass(node, 'code-snippet__line-index');
|
|
71
|
+
});
|
|
72
|
+
for (const node of parent.childNodes) {
|
|
73
|
+
if (isElement(node)) {
|
|
74
|
+
node.attrs = node.attrs.flatMap(item => {
|
|
75
|
+
if (!['href', 'src', 'poster', 'data-src'].includes(item.name))
|
|
76
|
+
return [item];
|
|
77
|
+
const value = safeUrl(item.value);
|
|
78
|
+
return value ? [{ ...item, value }] : [];
|
|
79
|
+
});
|
|
80
|
+
if (node.tagName === 'img') {
|
|
81
|
+
const lazy = attr(node, 'data-src');
|
|
82
|
+
if (lazy) {
|
|
83
|
+
node.attrs = node.attrs.filter(item => !['src', 'data-src'].includes(item.name));
|
|
84
|
+
node.attrs.push({ name: 'src', value: lazy });
|
|
85
|
+
}
|
|
86
|
+
}
|
|
87
|
+
if (hasClass(node, 'code-snippet__fix')) {
|
|
88
|
+
const descendants = [...node.childNodes].reverse();
|
|
89
|
+
let pre;
|
|
90
|
+
const lines = [];
|
|
91
|
+
while (descendants.length > 0) {
|
|
92
|
+
const descendant = descendants.pop();
|
|
93
|
+
if (isElement(descendant)) {
|
|
94
|
+
if (descendant.tagName === 'pre')
|
|
95
|
+
pre = descendant;
|
|
96
|
+
if (descendant.tagName === 'code') {
|
|
97
|
+
const line = textContent(descendant);
|
|
98
|
+
if (!/^[ce]?ounter\(line/.test(line))
|
|
99
|
+
lines.push(line);
|
|
100
|
+
continue;
|
|
101
|
+
}
|
|
102
|
+
}
|
|
103
|
+
if ('childNodes' in descendant) {
|
|
104
|
+
for (let i = descendant.childNodes.length - 1; i >= 0; i -= 1)
|
|
105
|
+
descendants.push(descendant.childNodes[i]);
|
|
106
|
+
}
|
|
107
|
+
}
|
|
108
|
+
if (pre && lines.length > 0) {
|
|
109
|
+
pre.childNodes = [{ nodeName: '#text', value: lines.join('\n'), parentNode: pre }];
|
|
110
|
+
}
|
|
111
|
+
}
|
|
112
|
+
parents.push(node);
|
|
113
|
+
}
|
|
114
|
+
}
|
|
115
|
+
}
|
|
116
|
+
const imageUrls = [];
|
|
117
|
+
const seenImages = new Set();
|
|
118
|
+
const imageStack = [content];
|
|
119
|
+
while (imageStack.length > 0) {
|
|
120
|
+
const node = imageStack.pop();
|
|
121
|
+
if (isElement(node) && node.tagName === 'img') {
|
|
122
|
+
const src = attr(node, 'src');
|
|
123
|
+
if (src && !seenImages.has(src)) {
|
|
124
|
+
seenImages.add(src);
|
|
125
|
+
imageUrls.push(src);
|
|
126
|
+
}
|
|
127
|
+
}
|
|
128
|
+
if ('childNodes' in node) {
|
|
129
|
+
for (let i = node.childNodes.length - 1; i >= 0; i -= 1)
|
|
130
|
+
imageStack.push(node.childNodes[i]);
|
|
131
|
+
}
|
|
132
|
+
}
|
|
133
|
+
const fragment = {
|
|
134
|
+
nodeName: '#document-fragment', childNodes: content.childNodes,
|
|
135
|
+
};
|
|
136
|
+
return { contentHtml: serialize(fragment), imageUrls };
|
|
137
|
+
}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
export {};
|
package/dist/src/execution.d.ts
CHANGED
|
@@ -11,6 +11,9 @@
|
|
|
11
11
|
*/
|
|
12
12
|
import { type CliCommand, type Arg, type CommandArgs } from './registry.js';
|
|
13
13
|
import { type ObservationExportResult } from './observation/index.js';
|
|
14
|
+
/** Internal test-only reset. Never use this to recover a production process. */
|
|
15
|
+
export declare function _resetLazyModuleStateForTests(): void;
|
|
16
|
+
export declare function _getLazyModuleFingerprintReadCountForTests(): number;
|
|
14
17
|
export declare function coerceAndValidateArgs(cmdArgs: Arg[], kwargs: CommandArgs): CommandArgs;
|
|
15
18
|
export declare function executeCommand(cmd: CliCommand, rawKwargs: CommandArgs, debug?: boolean, opts?: {
|
|
16
19
|
prepared?: boolean;
|
|
@@ -22,3 +25,5 @@ export declare function executeCommand(cmd: CliCommand, rawKwargs: CommandArgs,
|
|
|
22
25
|
onTraceExport?: (trace: ObservationExportResult) => void;
|
|
23
26
|
}): Promise<unknown>;
|
|
24
27
|
export declare function prepareCommandArgs(cmd: CliCommand, rawKwargs: CommandArgs): CommandArgs;
|
|
28
|
+
/** Prepare adapter arguments using the execution boundary's public error contract. */
|
|
29
|
+
export declare function prepareCommandArgsOrThrowArgumentError(cmd: CliCommand, rawKwargs: CommandArgs): CommandArgs;
|