@jackwener/opencli 1.3.0 → 1.3.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +128 -0
- package/README.md +44 -5
- package/README.zh-CN.md +44 -5
- package/SKILL.md +317 -5
- package/TESTING.md +4 -4
- package/dist/browser/errors.d.ts +2 -1
- package/dist/browser/errors.js +9 -10
- package/dist/build-manifest.js +1 -3
- package/dist/cli-manifest.json +2573 -989
- package/dist/cli.js +42 -2
- package/dist/clis/bilibili/download.js +20 -65
- package/dist/clis/bilibili/utils.js +2 -1
- package/dist/clis/chaoxing/assignments.js +2 -1
- package/dist/clis/doubao/ask.d.ts +1 -0
- package/dist/clis/doubao/ask.js +35 -0
- package/dist/clis/doubao/common.d.ts +23 -0
- package/dist/clis/doubao/common.js +564 -0
- package/dist/clis/doubao/new.d.ts +1 -0
- package/dist/clis/doubao/new.js +20 -0
- package/dist/clis/doubao/read.d.ts +1 -0
- package/dist/clis/doubao/read.js +19 -0
- package/dist/clis/doubao/send.d.ts +1 -0
- package/dist/clis/doubao/send.js +22 -0
- package/dist/clis/doubao/status.d.ts +1 -0
- package/dist/clis/doubao/status.js +24 -0
- package/dist/clis/doubao-app/ask.d.ts +1 -0
- package/dist/clis/doubao-app/ask.js +53 -0
- package/dist/clis/doubao-app/common.d.ts +37 -0
- package/dist/clis/doubao-app/common.js +110 -0
- package/dist/clis/doubao-app/dump.d.ts +1 -0
- package/dist/clis/doubao-app/dump.js +24 -0
- package/dist/clis/doubao-app/new.d.ts +1 -0
- package/dist/clis/doubao-app/new.js +20 -0
- package/dist/clis/doubao-app/read.d.ts +1 -0
- package/dist/clis/doubao-app/read.js +18 -0
- package/dist/clis/doubao-app/screenshot.d.ts +1 -0
- package/dist/clis/doubao-app/screenshot.js +18 -0
- package/dist/clis/doubao-app/send.d.ts +1 -0
- package/dist/clis/doubao-app/send.js +27 -0
- package/dist/clis/doubao-app/status.d.ts +1 -0
- package/dist/clis/doubao-app/status.js +16 -0
- package/dist/clis/hackernews/ask.yaml +38 -0
- package/dist/clis/hackernews/best.yaml +38 -0
- package/dist/clis/hackernews/jobs.yaml +36 -0
- package/dist/clis/hackernews/new.yaml +38 -0
- package/dist/clis/hackernews/search.yaml +44 -0
- package/dist/clis/hackernews/show.yaml +38 -0
- package/dist/clis/hackernews/top.yaml +3 -1
- package/dist/clis/hackernews/user.yaml +25 -0
- package/dist/clis/twitter/download.js +13 -97
- package/dist/clis/twitter/thread.js +2 -1
- package/dist/clis/v2ex/member.yaml +29 -0
- package/dist/clis/v2ex/node.yaml +34 -0
- package/dist/clis/v2ex/nodes.yaml +31 -0
- package/dist/clis/v2ex/replies.yaml +32 -0
- package/dist/clis/v2ex/user.yaml +34 -0
- package/dist/clis/weibo/search.d.ts +1 -0
- package/dist/clis/weibo/search.js +73 -0
- package/dist/clis/weixin/download.d.ts +12 -0
- package/dist/clis/weixin/download.js +183 -0
- package/dist/clis/xiaohongshu/download.js +12 -60
- package/dist/clis/xiaohongshu/publish.d.ts +18 -0
- package/dist/clis/xiaohongshu/publish.js +352 -0
- package/dist/clis/xiaohongshu/search.js +47 -15
- package/dist/clis/xiaohongshu/search.test.d.ts +1 -0
- package/dist/clis/xiaohongshu/search.test.js +114 -0
- package/dist/clis/yollomi/background.d.ts +4 -0
- package/dist/clis/yollomi/background.js +45 -0
- package/dist/clis/yollomi/edit.d.ts +5 -0
- package/dist/clis/yollomi/edit.js +56 -0
- package/dist/clis/yollomi/face-swap.d.ts +5 -0
- package/dist/clis/yollomi/face-swap.js +43 -0
- package/dist/clis/yollomi/generate.d.ts +9 -0
- package/dist/clis/yollomi/generate.js +100 -0
- package/dist/clis/yollomi/models.d.ts +1 -0
- package/dist/clis/yollomi/models.js +33 -0
- package/dist/clis/yollomi/object-remover.d.ts +4 -0
- package/dist/clis/yollomi/object-remover.js +42 -0
- package/dist/clis/yollomi/remove-bg.d.ts +4 -0
- package/dist/clis/yollomi/remove-bg.js +38 -0
- package/dist/clis/yollomi/restore.d.ts +4 -0
- package/dist/clis/yollomi/restore.js +38 -0
- package/dist/clis/yollomi/try-on.d.ts +4 -0
- package/dist/clis/yollomi/try-on.js +46 -0
- package/dist/clis/yollomi/upload.d.ts +7 -0
- package/dist/clis/yollomi/upload.js +71 -0
- package/dist/clis/yollomi/upscale.d.ts +4 -0
- package/dist/clis/yollomi/upscale.js +53 -0
- package/dist/clis/yollomi/utils.d.ts +45 -0
- package/dist/clis/yollomi/utils.js +180 -0
- package/dist/clis/yollomi/video.d.ts +5 -0
- package/dist/clis/yollomi/video.js +56 -0
- package/dist/clis/zhihu/download.d.ts +1 -5
- package/dist/clis/zhihu/download.js +20 -126
- package/dist/clis/zhihu/download.test.js +7 -5
- package/dist/clis/zhihu/question.js +2 -1
- package/dist/commanderAdapter.js +4 -6
- package/dist/daemon.js +5 -2
- package/dist/discovery.js +10 -10
- package/dist/download/article-download.d.ts +59 -0
- package/dist/download/article-download.js +178 -0
- package/dist/download/media-download.d.ts +49 -0
- package/dist/download/media-download.js +112 -0
- package/dist/errors.d.ts +23 -2
- package/dist/errors.js +58 -2
- package/dist/errors.test.d.ts +1 -0
- package/dist/errors.test.js +59 -0
- package/dist/execution.js +9 -10
- package/dist/explore.js +4 -2
- package/dist/external.d.ts +15 -0
- package/dist/external.js +48 -2
- package/dist/external.test.d.ts +1 -0
- package/dist/external.test.js +64 -0
- package/dist/main.js +10 -0
- package/dist/plugin.d.ts +4 -0
- package/dist/plugin.js +45 -23
- package/dist/plugin.test.js +6 -1
- package/dist/record.d.ts +47 -0
- package/dist/record.js +545 -0
- package/dist/registry.d.ts +7 -2
- package/dist/registry.js +2 -6
- package/dist/runtime.d.ts +3 -1
- package/dist/runtime.js +10 -3
- package/dist/validate.js +1 -3
- package/docs/.vitepress/config.mts +1 -0
- package/docs/adapters/browser/doubao.md +35 -0
- package/docs/adapters/browser/hackernews.md +20 -4
- package/docs/adapters/browser/tiktok.md +1 -1
- package/docs/adapters/browser/v2ex.md +31 -10
- package/docs/adapters/browser/weibo.md +4 -0
- package/docs/adapters/browser/weixin.md +33 -0
- package/docs/adapters/browser/xiaohongshu.md +8 -6
- package/docs/adapters/browser/yollomi.md +69 -0
- package/docs/adapters/desktop/doubao-app.md +35 -0
- package/docs/adapters/index.md +16 -5
- package/docs/advanced/download.md +4 -0
- package/package.json +3 -1
- package/src/browser/errors.ts +17 -11
- package/src/build-manifest.ts +2 -3
- package/src/cli.ts +45 -2
- package/src/clis/bilibili/download.ts +25 -83
- package/src/clis/bilibili/utils.ts +2 -1
- package/src/clis/chaoxing/assignments.ts +2 -1
- package/src/clis/doubao/ask.ts +40 -0
- package/src/clis/doubao/common.ts +619 -0
- package/src/clis/doubao/new.ts +22 -0
- package/src/clis/doubao/read.ts +20 -0
- package/src/clis/doubao/send.ts +25 -0
- package/src/clis/doubao/status.ts +27 -0
- package/src/clis/doubao-app/ask.ts +60 -0
- package/src/clis/doubao-app/common.ts +116 -0
- package/src/clis/doubao-app/dump.ts +28 -0
- package/src/clis/doubao-app/new.ts +21 -0
- package/src/clis/doubao-app/read.ts +21 -0
- package/src/clis/doubao-app/screenshot.ts +19 -0
- package/src/clis/doubao-app/send.ts +30 -0
- package/src/clis/doubao-app/status.ts +17 -0
- package/src/clis/hackernews/ask.yaml +38 -0
- package/src/clis/hackernews/best.yaml +38 -0
- package/src/clis/hackernews/jobs.yaml +36 -0
- package/src/clis/hackernews/new.yaml +38 -0
- package/src/clis/hackernews/search.yaml +44 -0
- package/src/clis/hackernews/show.yaml +38 -0
- package/src/clis/hackernews/top.yaml +3 -1
- package/src/clis/hackernews/user.yaml +25 -0
- package/src/clis/twitter/download.ts +13 -111
- package/src/clis/twitter/thread.ts +2 -1
- package/src/clis/v2ex/member.yaml +29 -0
- package/src/clis/v2ex/node.yaml +34 -0
- package/src/clis/v2ex/nodes.yaml +31 -0
- package/src/clis/v2ex/replies.yaml +32 -0
- package/src/clis/v2ex/user.yaml +34 -0
- package/src/clis/weibo/search.ts +78 -0
- package/src/clis/weixin/download.ts +199 -0
- package/src/clis/xiaohongshu/download.ts +12 -71
- package/src/clis/xiaohongshu/publish.ts +392 -0
- package/src/clis/xiaohongshu/search.test.ts +134 -0
- package/src/clis/xiaohongshu/search.ts +49 -15
- package/src/clis/yollomi/background.ts +48 -0
- package/src/clis/yollomi/edit.ts +58 -0
- package/src/clis/yollomi/face-swap.ts +45 -0
- package/src/clis/yollomi/generate.ts +95 -0
- package/src/clis/yollomi/models.ts +38 -0
- package/src/clis/yollomi/object-remover.ts +44 -0
- package/src/clis/yollomi/remove-bg.ts +40 -0
- package/src/clis/yollomi/restore.ts +40 -0
- package/src/clis/yollomi/try-on.ts +48 -0
- package/src/clis/yollomi/upload.ts +78 -0
- package/src/clis/yollomi/upscale.ts +49 -0
- package/src/clis/yollomi/utils.ts +202 -0
- package/src/clis/yollomi/video.ts +61 -0
- package/src/clis/zhihu/download.test.ts +7 -5
- package/src/clis/zhihu/download.ts +23 -158
- package/src/clis/zhihu/question.ts +2 -1
- package/src/commanderAdapter.ts +4 -7
- package/src/daemon.ts +5 -2
- package/src/discovery.ts +26 -26
- package/src/download/article-download.ts +272 -0
- package/src/download/media-download.ts +178 -0
- package/src/errors.test.ts +79 -0
- package/src/errors.ts +92 -2
- package/src/execution.ts +14 -10
- package/src/explore.ts +4 -2
- package/src/external.test.ts +88 -0
- package/src/external.ts +56 -2
- package/src/generate.ts +2 -1
- package/src/main.ts +10 -0
- package/src/plugin.test.ts +7 -1
- package/src/plugin.ts +49 -25
- package/src/record.ts +617 -0
- package/src/registry.ts +9 -5
- package/src/runtime.ts +16 -4
- package/src/validate.ts +2 -3
- package/tests/e2e/browser-auth.test.ts +10 -1
- package/tests/e2e/browser-public.test.ts +13 -8
- package/tests/e2e/public-commands.test.ts +209 -21
- package/tests/smoke/api-health.test.ts +65 -6
- package/.github/workflows/release-please.yml +0 -25
package/dist/commanderAdapter.js
CHANGED
|
@@ -14,10 +14,7 @@ import { fullName, getRegistry } from './registry.js';
|
|
|
14
14
|
import { formatRegistryHelpText } from './serialization.js';
|
|
15
15
|
import { render as renderOutput } from './output.js';
|
|
16
16
|
import { executeCommand } from './execution.js';
|
|
17
|
-
import { CliError } from './errors.js';
|
|
18
|
-
function getErrorMessage(error) {
|
|
19
|
-
return error instanceof Error ? error.message : String(error);
|
|
20
|
-
}
|
|
17
|
+
import { CliError, ERROR_ICONS, getErrorMessage } from './errors.js';
|
|
21
18
|
/**
|
|
22
19
|
* Register a single CliCommand as a Commander subcommand.
|
|
23
20
|
*/
|
|
@@ -88,9 +85,10 @@ export function registerCommandToProgram(siteCmd, cmd) {
|
|
|
88
85
|
}
|
|
89
86
|
catch (err) {
|
|
90
87
|
if (err instanceof CliError) {
|
|
91
|
-
|
|
88
|
+
const icon = ERROR_ICONS[err.code] ?? '⚠️';
|
|
89
|
+
console.error(chalk.red(`${icon} ${err.message}`));
|
|
92
90
|
if (err.hint)
|
|
93
|
-
console.error(chalk.yellow(
|
|
91
|
+
console.error(chalk.yellow(`→ ${err.hint}`));
|
|
94
92
|
}
|
|
95
93
|
else if (optionsRecord.verbose === true && err instanceof Error && err.stack) {
|
|
96
94
|
console.error(chalk.red(err.stack));
|
package/dist/daemon.js
CHANGED
|
@@ -128,11 +128,14 @@ async function handleRequest(req, res) {
|
|
|
128
128
|
jsonResponse(res, 503, { id: body.id, ok: false, error: 'Extension not connected. Please install the opencli Browser Bridge extension.' });
|
|
129
129
|
return;
|
|
130
130
|
}
|
|
131
|
+
const timeoutMs = typeof body.timeout === 'number' && body.timeout > 0
|
|
132
|
+
? body.timeout * 1000
|
|
133
|
+
: 120000;
|
|
131
134
|
const result = await new Promise((resolve, reject) => {
|
|
132
135
|
const timer = setTimeout(() => {
|
|
133
136
|
pending.delete(body.id);
|
|
134
|
-
reject(new Error(
|
|
135
|
-
},
|
|
137
|
+
reject(new Error(`Command timeout (${timeoutMs / 1000}s)`));
|
|
138
|
+
}, timeoutMs);
|
|
136
139
|
pending.set(body.id, { resolve, reject, timer });
|
|
137
140
|
extensionWs.send(JSON.stringify(body));
|
|
138
141
|
});
|
package/dist/discovery.js
CHANGED
|
@@ -13,13 +13,11 @@ import * as path from 'node:path';
|
|
|
13
13
|
import { pathToFileURL } from 'node:url';
|
|
14
14
|
import yaml from 'js-yaml';
|
|
15
15
|
import { Strategy, registerCommand } from './registry.js';
|
|
16
|
+
import { getErrorMessage } from './errors.js';
|
|
16
17
|
import { log } from './logger.js';
|
|
17
18
|
/** Plugins directory: ~/.opencli/plugins/ */
|
|
18
19
|
export const PLUGINS_DIR = path.join(os.homedir(), '.opencli', 'plugins');
|
|
19
20
|
const CLI_MODULE_PATTERN = /\bcli\s*\(/;
|
|
20
|
-
function getErrorMessage(error) {
|
|
21
|
-
return error instanceof Error ? error.message : String(error);
|
|
22
|
-
}
|
|
23
21
|
function parseStrategy(rawStrategy, fallback = Strategy.COOKIE) {
|
|
24
22
|
if (!rawStrategy)
|
|
25
23
|
return fallback;
|
|
@@ -117,28 +115,30 @@ async function discoverClisFromFs(dir) {
|
|
|
117
115
|
}
|
|
118
116
|
const promises = [];
|
|
119
117
|
const entries = await fs.promises.readdir(dir, { withFileTypes: true });
|
|
120
|
-
|
|
121
|
-
|
|
122
|
-
|
|
118
|
+
const sitePromises = entries
|
|
119
|
+
.filter(entry => entry.isDirectory())
|
|
120
|
+
.map(async (entry) => {
|
|
123
121
|
const site = entry.name;
|
|
124
122
|
const siteDir = path.join(dir, site);
|
|
125
123
|
const files = await fs.promises.readdir(siteDir);
|
|
124
|
+
const filePromises = [];
|
|
126
125
|
for (const file of files) {
|
|
127
126
|
const filePath = path.join(siteDir, file);
|
|
128
127
|
if (file.endsWith('.yaml') || file.endsWith('.yml')) {
|
|
129
|
-
|
|
128
|
+
filePromises.push(registerYamlCli(filePath, site));
|
|
130
129
|
}
|
|
131
130
|
else if ((file.endsWith('.js') && !file.endsWith('.d.js')) ||
|
|
132
131
|
(file.endsWith('.ts') && !file.endsWith('.d.ts') && !file.endsWith('.test.ts'))) {
|
|
133
132
|
if (!(await isCliModule(filePath)))
|
|
134
133
|
continue;
|
|
135
|
-
|
|
134
|
+
filePromises.push(import(pathToFileURL(filePath).href).catch((err) => {
|
|
136
135
|
log.warn(`Failed to load module ${filePath}: ${getErrorMessage(err)}`);
|
|
137
136
|
}));
|
|
138
137
|
}
|
|
139
138
|
}
|
|
140
|
-
|
|
141
|
-
|
|
139
|
+
await Promise.all(filePromises);
|
|
140
|
+
});
|
|
141
|
+
await Promise.all(sitePromises);
|
|
142
142
|
}
|
|
143
143
|
async function registerYamlCli(filePath, defaultSite) {
|
|
144
144
|
try {
|
|
@@ -0,0 +1,59 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Article download helper — shared logic for downloading articles as Markdown.
|
|
3
|
+
*
|
|
4
|
+
* Used by: zhihu/download, weixin/download, and future article adapters.
|
|
5
|
+
*
|
|
6
|
+
* Flow: ArticleData → TurndownService → image download → frontmatter → .md file
|
|
7
|
+
*/
|
|
8
|
+
import TurndownService from 'turndown';
|
|
9
|
+
export interface ArticleData {
|
|
10
|
+
title: string;
|
|
11
|
+
author?: string;
|
|
12
|
+
publishTime?: string;
|
|
13
|
+
sourceUrl?: string;
|
|
14
|
+
contentHtml: string;
|
|
15
|
+
/** Pre-extracted code blocks to restore after Markdown conversion */
|
|
16
|
+
codeBlocks?: Array<{
|
|
17
|
+
lang: string;
|
|
18
|
+
code: string;
|
|
19
|
+
}>;
|
|
20
|
+
/** Image URLs found in the article (pre-collected from DOM) */
|
|
21
|
+
imageUrls?: string[];
|
|
22
|
+
}
|
|
23
|
+
export interface FrontmatterLabels {
|
|
24
|
+
author?: string;
|
|
25
|
+
publishTime?: string;
|
|
26
|
+
sourceUrl?: string;
|
|
27
|
+
}
|
|
28
|
+
export interface ArticleDownloadOptions {
|
|
29
|
+
output: string;
|
|
30
|
+
downloadImages?: boolean;
|
|
31
|
+
/** Extra headers for image downloads (e.g. { Referer: '...' }) */
|
|
32
|
+
imageHeaders?: Record<string, string>;
|
|
33
|
+
maxTitleLength?: number;
|
|
34
|
+
/** Custom TurndownService configuration callback */
|
|
35
|
+
configureTurndown?: (td: TurndownService) => void;
|
|
36
|
+
/** Custom image extension detector (default: infer from URL extension) */
|
|
37
|
+
detectImageExt?: (url: string) => string;
|
|
38
|
+
/** Custom frontmatter labels (default: Chinese labels) */
|
|
39
|
+
frontmatterLabels?: FrontmatterLabels;
|
|
40
|
+
}
|
|
41
|
+
export interface ArticleDownloadResult {
|
|
42
|
+
title: string;
|
|
43
|
+
author: string;
|
|
44
|
+
publish_time: string;
|
|
45
|
+
status: string;
|
|
46
|
+
size: string;
|
|
47
|
+
}
|
|
48
|
+
/**
|
|
49
|
+
* Download an article to Markdown with optional image localization.
|
|
50
|
+
*
|
|
51
|
+
* Handles the full pipeline:
|
|
52
|
+
* 1. HTML → Markdown (via TurndownService)
|
|
53
|
+
* 2. Code block placeholder restoration
|
|
54
|
+
* 3. Batch image downloading with concurrency + deduplication
|
|
55
|
+
* 4. Image URL replacement in Markdown
|
|
56
|
+
* 5. Frontmatter generation (customizable labels)
|
|
57
|
+
* 6. File write
|
|
58
|
+
*/
|
|
59
|
+
export declare function downloadArticle(data: ArticleData, options: ArticleDownloadOptions): Promise<ArticleDownloadResult[]>;
|
|
@@ -0,0 +1,178 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Article download helper — shared logic for downloading articles as Markdown.
|
|
3
|
+
*
|
|
4
|
+
* Used by: zhihu/download, weixin/download, and future article adapters.
|
|
5
|
+
*
|
|
6
|
+
* Flow: ArticleData → TurndownService → image download → frontmatter → .md file
|
|
7
|
+
*/
|
|
8
|
+
import * as fs from 'node:fs';
|
|
9
|
+
import * as path from 'node:path';
|
|
10
|
+
import TurndownService from 'turndown';
|
|
11
|
+
import { httpDownload, sanitizeFilename } from './index.js';
|
|
12
|
+
import { formatBytes } from './progress.js';
|
|
13
|
+
const IMAGE_CONCURRENCY = 5;
|
|
14
|
+
const DEFAULT_LABELS = {
|
|
15
|
+
author: '作者',
|
|
16
|
+
publishTime: '发布时间',
|
|
17
|
+
sourceUrl: '原文链接',
|
|
18
|
+
};
|
|
19
|
+
// ============================================================
|
|
20
|
+
// Markdown Conversion
|
|
21
|
+
// ============================================================
|
|
22
|
+
function createTurndown(configure) {
|
|
23
|
+
const td = new TurndownService({
|
|
24
|
+
headingStyle: 'atx',
|
|
25
|
+
codeBlockStyle: 'fenced',
|
|
26
|
+
bulletListMarker: '-',
|
|
27
|
+
});
|
|
28
|
+
td.addRule('linebreak', {
|
|
29
|
+
filter: 'br',
|
|
30
|
+
replacement: () => '\n',
|
|
31
|
+
});
|
|
32
|
+
if (configure)
|
|
33
|
+
configure(td);
|
|
34
|
+
return td;
|
|
35
|
+
}
|
|
36
|
+
function convertToMarkdown(contentHtml, codeBlocks, configure) {
|
|
37
|
+
const td = createTurndown(configure);
|
|
38
|
+
let md = td.turndown(contentHtml);
|
|
39
|
+
// Restore code block placeholders
|
|
40
|
+
codeBlocks.forEach((block, i) => {
|
|
41
|
+
const placeholder = `CODEBLOCK-PLACEHOLDER-${i}`;
|
|
42
|
+
const fenced = `\n\`\`\`${block.lang}\n${block.code}\n\`\`\`\n`;
|
|
43
|
+
md = md.replace(placeholder, fenced);
|
|
44
|
+
});
|
|
45
|
+
// Clean up
|
|
46
|
+
md = md.replace(/\u00a0/g, ' ');
|
|
47
|
+
md = md.replace(/\n{4,}/g, '\n\n\n');
|
|
48
|
+
md = md.replace(/[ \t]+$/gm, '');
|
|
49
|
+
return md;
|
|
50
|
+
}
|
|
51
|
+
function replaceImageUrls(md, urlMap) {
|
|
52
|
+
return md.replace(/!\[([^\]]*)\]\(([^)]+)\)/g, (match, alt, imgUrl) => {
|
|
53
|
+
const local = urlMap[imgUrl];
|
|
54
|
+
return local ? `` : match;
|
|
55
|
+
});
|
|
56
|
+
}
|
|
57
|
+
// ============================================================
|
|
58
|
+
// Image Downloading
|
|
59
|
+
// ============================================================
|
|
60
|
+
function defaultDetectImageExt(url) {
|
|
61
|
+
const extMatch = url.match(/\.(\w{3,4})(?:\?|$)/);
|
|
62
|
+
return extMatch ? extMatch[1] : 'jpg';
|
|
63
|
+
}
|
|
64
|
+
async function downloadImages(imgUrls, imgDir, headers, detectExt) {
|
|
65
|
+
const urlMap = {};
|
|
66
|
+
if (imgUrls.length === 0)
|
|
67
|
+
return urlMap;
|
|
68
|
+
const detect = detectExt || defaultDetectImageExt;
|
|
69
|
+
// Deduplicate image URLs
|
|
70
|
+
const seen = new Set();
|
|
71
|
+
const uniqueUrls = imgUrls.filter(url => {
|
|
72
|
+
if (seen.has(url))
|
|
73
|
+
return false;
|
|
74
|
+
seen.add(url);
|
|
75
|
+
return true;
|
|
76
|
+
});
|
|
77
|
+
for (let i = 0; i < uniqueUrls.length; i += IMAGE_CONCURRENCY) {
|
|
78
|
+
const batch = uniqueUrls.slice(i, i + IMAGE_CONCURRENCY);
|
|
79
|
+
const results = await Promise.all(batch.map(async (rawUrl, j) => {
|
|
80
|
+
const index = i + j + 1;
|
|
81
|
+
let imgUrl = rawUrl;
|
|
82
|
+
if (imgUrl.startsWith('//'))
|
|
83
|
+
imgUrl = `https:${imgUrl}`;
|
|
84
|
+
const ext = detect(imgUrl);
|
|
85
|
+
const filename = `img_${String(index).padStart(3, '0')}.${ext}`;
|
|
86
|
+
const filepath = path.join(imgDir, filename);
|
|
87
|
+
try {
|
|
88
|
+
const result = await httpDownload(imgUrl, filepath, {
|
|
89
|
+
headers,
|
|
90
|
+
timeout: 15000,
|
|
91
|
+
});
|
|
92
|
+
if (result.success) {
|
|
93
|
+
return { remoteUrl: rawUrl, localPath: `images/${filename}` };
|
|
94
|
+
}
|
|
95
|
+
}
|
|
96
|
+
catch {
|
|
97
|
+
// Skip failed downloads
|
|
98
|
+
}
|
|
99
|
+
return null;
|
|
100
|
+
}));
|
|
101
|
+
for (const r of results) {
|
|
102
|
+
if (r)
|
|
103
|
+
urlMap[r.remoteUrl] = r.localPath;
|
|
104
|
+
}
|
|
105
|
+
}
|
|
106
|
+
return urlMap;
|
|
107
|
+
}
|
|
108
|
+
// ============================================================
|
|
109
|
+
// Main API
|
|
110
|
+
// ============================================================
|
|
111
|
+
/**
|
|
112
|
+
* Download an article to Markdown with optional image localization.
|
|
113
|
+
*
|
|
114
|
+
* Handles the full pipeline:
|
|
115
|
+
* 1. HTML → Markdown (via TurndownService)
|
|
116
|
+
* 2. Code block placeholder restoration
|
|
117
|
+
* 3. Batch image downloading with concurrency + deduplication
|
|
118
|
+
* 4. Image URL replacement in Markdown
|
|
119
|
+
* 5. Frontmatter generation (customizable labels)
|
|
120
|
+
* 6. File write
|
|
121
|
+
*/
|
|
122
|
+
export async function downloadArticle(data, options) {
|
|
123
|
+
const { output, downloadImages: shouldDownloadImages = true, imageHeaders, maxTitleLength = 80, configureTurndown, detectImageExt, frontmatterLabels, } = options;
|
|
124
|
+
const labels = { ...DEFAULT_LABELS, ...frontmatterLabels };
|
|
125
|
+
if (!data.title) {
|
|
126
|
+
return [{
|
|
127
|
+
title: 'Error',
|
|
128
|
+
author: '-',
|
|
129
|
+
publish_time: '-',
|
|
130
|
+
status: 'failed — no title',
|
|
131
|
+
size: '-',
|
|
132
|
+
}];
|
|
133
|
+
}
|
|
134
|
+
if (!data.contentHtml) {
|
|
135
|
+
return [{
|
|
136
|
+
title: data.title,
|
|
137
|
+
author: data.author || '-',
|
|
138
|
+
publish_time: data.publishTime || '-',
|
|
139
|
+
status: 'failed — no content',
|
|
140
|
+
size: '-',
|
|
141
|
+
}];
|
|
142
|
+
}
|
|
143
|
+
// Convert HTML to Markdown
|
|
144
|
+
let markdown = convertToMarkdown(data.contentHtml, data.codeBlocks || [], configureTurndown);
|
|
145
|
+
// Prepare output directory
|
|
146
|
+
const safeTitle = sanitizeFilename(data.title, maxTitleLength);
|
|
147
|
+
const articleDir = path.join(output, safeTitle);
|
|
148
|
+
fs.mkdirSync(articleDir, { recursive: true });
|
|
149
|
+
// Download images
|
|
150
|
+
if (shouldDownloadImages && data.imageUrls && data.imageUrls.length > 0) {
|
|
151
|
+
const imagesDir = path.join(articleDir, 'images');
|
|
152
|
+
fs.mkdirSync(imagesDir, { recursive: true });
|
|
153
|
+
const urlMap = await downloadImages(data.imageUrls, imagesDir, imageHeaders, detectImageExt);
|
|
154
|
+
markdown = replaceImageUrls(markdown, urlMap);
|
|
155
|
+
}
|
|
156
|
+
// Build frontmatter with customizable labels
|
|
157
|
+
const headerLines = [`# ${data.title}`, ''];
|
|
158
|
+
if (data.author)
|
|
159
|
+
headerLines.push(`> ${labels.author}: ${data.author}`);
|
|
160
|
+
if (data.publishTime)
|
|
161
|
+
headerLines.push(`> ${labels.publishTime}: ${data.publishTime}`);
|
|
162
|
+
if (data.sourceUrl)
|
|
163
|
+
headerLines.push(`> ${labels.sourceUrl}: ${data.sourceUrl}`);
|
|
164
|
+
headerLines.push('', '---', '');
|
|
165
|
+
const fullContent = headerLines.join('\n') + markdown;
|
|
166
|
+
// Write file
|
|
167
|
+
const filename = `${safeTitle}.md`;
|
|
168
|
+
const filePath = path.join(articleDir, filename);
|
|
169
|
+
fs.writeFileSync(filePath, fullContent, 'utf-8');
|
|
170
|
+
const size = Buffer.byteLength(fullContent, 'utf-8');
|
|
171
|
+
return [{
|
|
172
|
+
title: data.title,
|
|
173
|
+
author: data.author || '-',
|
|
174
|
+
publish_time: data.publishTime || '-',
|
|
175
|
+
status: 'success',
|
|
176
|
+
size: formatBytes(size),
|
|
177
|
+
}];
|
|
178
|
+
}
|
|
@@ -0,0 +1,49 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Media download helper — shared logic for batch downloading images/videos.
|
|
3
|
+
*
|
|
4
|
+
* Used by: xiaohongshu/download, twitter/download, bilibili/download,
|
|
5
|
+
* and future media adapters.
|
|
6
|
+
*
|
|
7
|
+
* Flow: MediaItem[] → DownloadProgressTracker → httpDownload/ytdlpDownload → results
|
|
8
|
+
*/
|
|
9
|
+
import type { BrowserCookie } from '../types.js';
|
|
10
|
+
export interface MediaItem {
|
|
11
|
+
type: 'image' | 'video' | 'video-tweet' | 'video-ytdlp';
|
|
12
|
+
url: string;
|
|
13
|
+
/** Optional custom filename (without directory) */
|
|
14
|
+
filename?: string;
|
|
15
|
+
}
|
|
16
|
+
export interface MediaDownloadOptions {
|
|
17
|
+
output: string;
|
|
18
|
+
/** Subdirectory inside output */
|
|
19
|
+
subdir?: string;
|
|
20
|
+
/** Cookie string for HTTP downloads */
|
|
21
|
+
cookies?: string;
|
|
22
|
+
/** Raw browser cookies — auto-exported to Netscape for yt-dlp, auto-cleaned up */
|
|
23
|
+
browserCookies?: BrowserCookie[];
|
|
24
|
+
/** Timeout in ms (default: 30000 for images, 60000 for videos) */
|
|
25
|
+
timeout?: number;
|
|
26
|
+
/** File name prefix (default: 'download') */
|
|
27
|
+
filenamePrefix?: string;
|
|
28
|
+
/** Extra yt-dlp args */
|
|
29
|
+
ytdlpExtraArgs?: string[];
|
|
30
|
+
/** Whether to show progress (default: true) */
|
|
31
|
+
verbose?: boolean;
|
|
32
|
+
}
|
|
33
|
+
export interface MediaDownloadResult {
|
|
34
|
+
index: number;
|
|
35
|
+
type: string;
|
|
36
|
+
status: string;
|
|
37
|
+
size: string;
|
|
38
|
+
}
|
|
39
|
+
/**
|
|
40
|
+
* Batch download media files with progress tracking.
|
|
41
|
+
*
|
|
42
|
+
* Handles:
|
|
43
|
+
* - DownloadProgressTracker for terminal UX
|
|
44
|
+
* - Automatic httpDownload vs ytdlpDownload routing via MediaItem.type
|
|
45
|
+
* - Cookie export to Netscape format for yt-dlp (auto-cleanup)
|
|
46
|
+
* - Directory creation
|
|
47
|
+
* - Error handling with per-file results
|
|
48
|
+
*/
|
|
49
|
+
export declare function downloadMedia(items: MediaItem[], options: MediaDownloadOptions): Promise<MediaDownloadResult[]>;
|
|
@@ -0,0 +1,112 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Media download helper — shared logic for batch downloading images/videos.
|
|
3
|
+
*
|
|
4
|
+
* Used by: xiaohongshu/download, twitter/download, bilibili/download,
|
|
5
|
+
* and future media adapters.
|
|
6
|
+
*
|
|
7
|
+
* Flow: MediaItem[] → DownloadProgressTracker → httpDownload/ytdlpDownload → results
|
|
8
|
+
*/
|
|
9
|
+
import * as fs from 'node:fs';
|
|
10
|
+
import * as path from 'node:path';
|
|
11
|
+
import { httpDownload, ytdlpDownload, checkYtdlp, getTempDir, exportCookiesToNetscape, } from './index.js';
|
|
12
|
+
import { DownloadProgressTracker, formatBytes } from './progress.js';
|
|
13
|
+
// ============================================================
|
|
14
|
+
// Main API
|
|
15
|
+
// ============================================================
|
|
16
|
+
/**
|
|
17
|
+
* Batch download media files with progress tracking.
|
|
18
|
+
*
|
|
19
|
+
* Handles:
|
|
20
|
+
* - DownloadProgressTracker for terminal UX
|
|
21
|
+
* - Automatic httpDownload vs ytdlpDownload routing via MediaItem.type
|
|
22
|
+
* - Cookie export to Netscape format for yt-dlp (auto-cleanup)
|
|
23
|
+
* - Directory creation
|
|
24
|
+
* - Error handling with per-file results
|
|
25
|
+
*/
|
|
26
|
+
export async function downloadMedia(items, options) {
|
|
27
|
+
const { output, subdir, cookies, browserCookies, timeout, filenamePrefix = 'download', ytdlpExtraArgs = [], verbose = true, } = options;
|
|
28
|
+
if (!items || items.length === 0) {
|
|
29
|
+
return [{ index: 0, type: '-', status: 'failed', size: 'No media found' }];
|
|
30
|
+
}
|
|
31
|
+
// Create output directory
|
|
32
|
+
const outputDir = subdir ? path.join(output, subdir) : output;
|
|
33
|
+
fs.mkdirSync(outputDir, { recursive: true });
|
|
34
|
+
// Pre-check yt-dlp availability (once, not per-item)
|
|
35
|
+
const hasYtdlp = checkYtdlp();
|
|
36
|
+
// Auto-export browser cookies to Netscape format for yt-dlp
|
|
37
|
+
let cookiesFile;
|
|
38
|
+
const needsYtdlp = items.some(m => m.type === 'video-tweet' || m.type === 'video-ytdlp');
|
|
39
|
+
if (needsYtdlp && browserCookies && browserCookies.length > 0) {
|
|
40
|
+
const tempDir = getTempDir();
|
|
41
|
+
fs.mkdirSync(tempDir, { recursive: true });
|
|
42
|
+
cookiesFile = path.join(tempDir, `media_cookies_${Date.now()}.txt`);
|
|
43
|
+
exportCookiesToNetscape(browserCookies, cookiesFile);
|
|
44
|
+
}
|
|
45
|
+
const tracker = new DownloadProgressTracker(items.length, verbose);
|
|
46
|
+
const results = [];
|
|
47
|
+
try {
|
|
48
|
+
for (let i = 0; i < items.length; i++) {
|
|
49
|
+
const media = items[i];
|
|
50
|
+
const isVideo = media.type !== 'image';
|
|
51
|
+
const ext = isVideo ? 'mp4' : 'jpg';
|
|
52
|
+
const filename = media.filename || `${filenamePrefix}_${i + 1}.${ext}`;
|
|
53
|
+
const destPath = path.join(outputDir, filename);
|
|
54
|
+
const progressBar = tracker.onFileStart(filename, i);
|
|
55
|
+
try {
|
|
56
|
+
let result;
|
|
57
|
+
const useYtdlp = (media.type === 'video-tweet' || media.type === 'video-ytdlp') && hasYtdlp;
|
|
58
|
+
if (useYtdlp) {
|
|
59
|
+
result = await ytdlpDownload(media.url, destPath, {
|
|
60
|
+
cookiesFile,
|
|
61
|
+
extraArgs: ytdlpExtraArgs,
|
|
62
|
+
onProgress: (percent) => {
|
|
63
|
+
if (progressBar)
|
|
64
|
+
progressBar.update(percent, 100);
|
|
65
|
+
},
|
|
66
|
+
});
|
|
67
|
+
}
|
|
68
|
+
else {
|
|
69
|
+
// Direct HTTP download for images and direct video URLs
|
|
70
|
+
const dlTimeout = timeout || (isVideo ? 60000 : 30000);
|
|
71
|
+
result = await httpDownload(media.url, destPath, {
|
|
72
|
+
cookies,
|
|
73
|
+
timeout: dlTimeout,
|
|
74
|
+
onProgress: (received, total) => {
|
|
75
|
+
if (progressBar)
|
|
76
|
+
progressBar.update(received, total);
|
|
77
|
+
},
|
|
78
|
+
});
|
|
79
|
+
}
|
|
80
|
+
if (progressBar) {
|
|
81
|
+
progressBar.complete(result.success, result.success ? formatBytes(result.size) : undefined);
|
|
82
|
+
}
|
|
83
|
+
tracker.onFileComplete(result.success);
|
|
84
|
+
results.push({
|
|
85
|
+
index: i + 1,
|
|
86
|
+
type: media.type === 'video-tweet' || media.type === 'video-ytdlp' ? 'video' : media.type,
|
|
87
|
+
status: result.success ? 'success' : 'failed',
|
|
88
|
+
size: result.success ? formatBytes(result.size) : (result.error || 'unknown error'),
|
|
89
|
+
});
|
|
90
|
+
}
|
|
91
|
+
catch (err) {
|
|
92
|
+
if (progressBar)
|
|
93
|
+
progressBar.fail(err.message);
|
|
94
|
+
tracker.onFileComplete(false);
|
|
95
|
+
results.push({
|
|
96
|
+
index: i + 1,
|
|
97
|
+
type: media.type,
|
|
98
|
+
status: 'failed',
|
|
99
|
+
size: err.message,
|
|
100
|
+
});
|
|
101
|
+
}
|
|
102
|
+
}
|
|
103
|
+
}
|
|
104
|
+
finally {
|
|
105
|
+
tracker.finish();
|
|
106
|
+
// Auto-cleanup exported cookies file
|
|
107
|
+
if (cookiesFile && fs.existsSync(cookiesFile)) {
|
|
108
|
+
fs.unlinkSync(cookiesFile);
|
|
109
|
+
}
|
|
110
|
+
}
|
|
111
|
+
return results;
|
|
112
|
+
}
|
package/dist/errors.d.ts
CHANGED
|
@@ -2,10 +2,11 @@
|
|
|
2
2
|
* Unified error types for opencli.
|
|
3
3
|
*
|
|
4
4
|
* All errors thrown by the framework should extend CliError so that
|
|
5
|
-
* the top-level handler in
|
|
5
|
+
* the top-level handler in commanderAdapter.ts can render consistent,
|
|
6
|
+
* helpful output with emoji-coded severity and actionable hints.
|
|
6
7
|
*/
|
|
7
8
|
export declare class CliError extends Error {
|
|
8
|
-
/** Machine-readable error code (e.g. 'BROWSER_CONNECT', '
|
|
9
|
+
/** Machine-readable error code (e.g. 'BROWSER_CONNECT', 'AUTH_REQUIRED') */
|
|
9
10
|
readonly code: string;
|
|
10
11
|
/** Human-readable hint on how to fix the problem */
|
|
11
12
|
readonly hint?: string;
|
|
@@ -23,3 +24,23 @@ export declare class CommandExecutionError extends CliError {
|
|
|
23
24
|
export declare class ConfigError extends CliError {
|
|
24
25
|
constructor(message: string, hint?: string);
|
|
25
26
|
}
|
|
27
|
+
export declare class AuthRequiredError extends CliError {
|
|
28
|
+
readonly domain: string;
|
|
29
|
+
constructor(domain: string, message?: string);
|
|
30
|
+
}
|
|
31
|
+
export declare class TimeoutError extends CliError {
|
|
32
|
+
constructor(label: string, seconds: number);
|
|
33
|
+
}
|
|
34
|
+
export declare class ArgumentError extends CliError {
|
|
35
|
+
constructor(message: string, hint?: string);
|
|
36
|
+
}
|
|
37
|
+
export declare class EmptyResultError extends CliError {
|
|
38
|
+
constructor(command: string, hint?: string);
|
|
39
|
+
}
|
|
40
|
+
export declare class SelectorError extends CliError {
|
|
41
|
+
constructor(selector: string, hint?: string);
|
|
42
|
+
}
|
|
43
|
+
/** Extract a human-readable message from an unknown caught value. */
|
|
44
|
+
export declare function getErrorMessage(error: unknown): string;
|
|
45
|
+
/** Error code → emoji mapping for CLI output rendering. */
|
|
46
|
+
export declare const ERROR_ICONS: Record<string, string>;
|
package/dist/errors.js
CHANGED
|
@@ -2,10 +2,11 @@
|
|
|
2
2
|
* Unified error types for opencli.
|
|
3
3
|
*
|
|
4
4
|
* All errors thrown by the framework should extend CliError so that
|
|
5
|
-
* the top-level handler in
|
|
5
|
+
* the top-level handler in commanderAdapter.ts can render consistent,
|
|
6
|
+
* helpful output with emoji-coded severity and actionable hints.
|
|
6
7
|
*/
|
|
7
8
|
export class CliError extends Error {
|
|
8
|
-
/** Machine-readable error code (e.g. 'BROWSER_CONNECT', '
|
|
9
|
+
/** Machine-readable error code (e.g. 'BROWSER_CONNECT', 'AUTH_REQUIRED') */
|
|
9
10
|
code;
|
|
10
11
|
/** Human-readable hint on how to fix the problem */
|
|
11
12
|
hint;
|
|
@@ -16,27 +17,82 @@ export class CliError extends Error {
|
|
|
16
17
|
this.hint = hint;
|
|
17
18
|
}
|
|
18
19
|
}
|
|
20
|
+
// ── Browser / Connection ────────────────────────────────────────────────────
|
|
19
21
|
export class BrowserConnectError extends CliError {
|
|
20
22
|
constructor(message, hint) {
|
|
21
23
|
super('BROWSER_CONNECT', message, hint);
|
|
22
24
|
this.name = 'BrowserConnectError';
|
|
23
25
|
}
|
|
24
26
|
}
|
|
27
|
+
// ── Adapter loading ─────────────────────────────────────────────────────────
|
|
25
28
|
export class AdapterLoadError extends CliError {
|
|
26
29
|
constructor(message, hint) {
|
|
27
30
|
super('ADAPTER_LOAD', message, hint);
|
|
28
31
|
this.name = 'AdapterLoadError';
|
|
29
32
|
}
|
|
30
33
|
}
|
|
34
|
+
// ── Command execution ───────────────────────────────────────────────────────
|
|
31
35
|
export class CommandExecutionError extends CliError {
|
|
32
36
|
constructor(message, hint) {
|
|
33
37
|
super('COMMAND_EXEC', message, hint);
|
|
34
38
|
this.name = 'CommandExecutionError';
|
|
35
39
|
}
|
|
36
40
|
}
|
|
41
|
+
// ── Configuration ───────────────────────────────────────────────────────────
|
|
37
42
|
export class ConfigError extends CliError {
|
|
38
43
|
constructor(message, hint) {
|
|
39
44
|
super('CONFIG', message, hint);
|
|
40
45
|
this.name = 'ConfigError';
|
|
41
46
|
}
|
|
42
47
|
}
|
|
48
|
+
// ── Authentication / Login ──────────────────────────────────────────────────
|
|
49
|
+
export class AuthRequiredError extends CliError {
|
|
50
|
+
domain;
|
|
51
|
+
constructor(domain, message) {
|
|
52
|
+
super('AUTH_REQUIRED', message ?? `Not logged in to ${domain}`, `Please open Chrome and log in to https://${domain}`);
|
|
53
|
+
this.name = 'AuthRequiredError';
|
|
54
|
+
this.domain = domain;
|
|
55
|
+
}
|
|
56
|
+
}
|
|
57
|
+
// ── Timeout ─────────────────────────────────────────────────────────────────
|
|
58
|
+
export class TimeoutError extends CliError {
|
|
59
|
+
constructor(label, seconds) {
|
|
60
|
+
super('TIMEOUT', `${label} timed out after ${seconds}s`, 'Try again, or increase timeout with OPENCLI_BROWSER_COMMAND_TIMEOUT env var');
|
|
61
|
+
this.name = 'TimeoutError';
|
|
62
|
+
}
|
|
63
|
+
}
|
|
64
|
+
// ── Argument validation ─────────────────────────────────────────────────────
|
|
65
|
+
export class ArgumentError extends CliError {
|
|
66
|
+
constructor(message, hint) {
|
|
67
|
+
super('ARGUMENT', message, hint);
|
|
68
|
+
this.name = 'ArgumentError';
|
|
69
|
+
}
|
|
70
|
+
}
|
|
71
|
+
// ── Empty result ────────────────────────────────────────────────────────────
|
|
72
|
+
export class EmptyResultError extends CliError {
|
|
73
|
+
constructor(command, hint) {
|
|
74
|
+
super('EMPTY_RESULT', `${command} returned no data`, hint ?? 'The page structure may have changed, or you may need to log in');
|
|
75
|
+
this.name = 'EmptyResultError';
|
|
76
|
+
}
|
|
77
|
+
}
|
|
78
|
+
// ── Selector / DOM ──────────────────────────────────────────────────────────
|
|
79
|
+
export class SelectorError extends CliError {
|
|
80
|
+
constructor(selector, hint) {
|
|
81
|
+
super('SELECTOR', `Could not find element: ${selector}`, hint ?? 'The page UI may have changed. Please report this issue.');
|
|
82
|
+
this.name = 'SelectorError';
|
|
83
|
+
}
|
|
84
|
+
}
|
|
85
|
+
// ── Utilities ───────────────────────────────────────────────────────────
|
|
86
|
+
/** Extract a human-readable message from an unknown caught value. */
|
|
87
|
+
export function getErrorMessage(error) {
|
|
88
|
+
return error instanceof Error ? error.message : String(error);
|
|
89
|
+
}
|
|
90
|
+
/** Error code → emoji mapping for CLI output rendering. */
|
|
91
|
+
export const ERROR_ICONS = {
|
|
92
|
+
AUTH_REQUIRED: '🔒',
|
|
93
|
+
BROWSER_CONNECT: '🔌',
|
|
94
|
+
TIMEOUT: '⏱ ',
|
|
95
|
+
ARGUMENT: '❌',
|
|
96
|
+
EMPTY_RESULT: '📭',
|
|
97
|
+
SELECTOR: '🔍',
|
|
98
|
+
};
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
export {};
|