@jackwener/opencli 0.9.6 → 1.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.github/ISSUE_TEMPLATE/bug_report.yml +83 -0
- package/.github/ISSUE_TEMPLATE/config.yml +8 -0
- package/.github/ISSUE_TEMPLATE/feature_request.yml +42 -0
- package/.github/ISSUE_TEMPLATE/new_site_adapter.yml +57 -0
- package/.github/dependabot.yml +27 -0
- package/.github/pull_request_template.md +24 -0
- package/.github/workflows/ci.yml +14 -8
- package/.github/workflows/e2e-headed.yml +6 -2
- package/.github/workflows/pkg-pr-new.yml +2 -2
- package/.github/workflows/release-please.yml +25 -0
- package/.github/workflows/release.yml +2 -2
- package/.github/workflows/security.yml +36 -0
- package/CDP.md +1 -1
- package/CDP.zh-CN.md +1 -1
- package/CLI-ELECTRON.md +89 -36
- package/CLI-EXPLORER.md +4 -4
- package/CONTRIBUTING.md +167 -0
- package/README.md +113 -89
- package/README.zh-CN.md +114 -91
- package/SKILL.md +10 -8
- package/TESTING.md +7 -7
- package/dist/browser/daemon-client.d.ts +37 -0
- package/dist/browser/daemon-client.js +82 -0
- package/dist/browser/discover.d.ts +11 -34
- package/dist/browser/discover.js +15 -190
- package/dist/browser/errors.d.ts +6 -20
- package/dist/browser/errors.js +24 -63
- package/dist/browser/index.d.ts +2 -11
- package/dist/browser/index.js +5 -11
- package/dist/browser/mcp.d.ts +9 -18
- package/dist/browser/mcp.js +70 -284
- package/dist/browser/page.d.ts +28 -6
- package/dist/browser/page.js +210 -85
- package/dist/browser.test.js +4 -202
- package/dist/build-manifest.d.ts +26 -0
- package/dist/build-manifest.js +132 -60
- package/dist/build-manifest.test.d.ts +1 -0
- package/dist/build-manifest.test.js +26 -0
- package/dist/cli-manifest.json +1582 -29
- package/dist/clis/bilibili/download.d.ts +10 -0
- package/dist/clis/bilibili/download.js +135 -0
- package/dist/clis/chatwise/ask.d.ts +1 -0
- package/dist/clis/chatwise/ask.js +76 -0
- package/dist/clis/chatwise/export.d.ts +1 -0
- package/dist/clis/chatwise/export.js +46 -0
- package/dist/clis/chatwise/history.d.ts +1 -0
- package/dist/clis/chatwise/history.js +43 -0
- package/dist/clis/chatwise/model.d.ts +1 -0
- package/dist/clis/chatwise/model.js +81 -0
- package/dist/clis/chatwise/new.d.ts +1 -0
- package/dist/clis/chatwise/new.js +18 -0
- package/dist/clis/chatwise/read.d.ts +1 -0
- package/dist/clis/chatwise/read.js +39 -0
- package/dist/clis/chatwise/screenshot.d.ts +1 -0
- package/dist/clis/chatwise/screenshot.js +27 -0
- package/dist/clis/chatwise/send.d.ts +1 -0
- package/dist/clis/chatwise/send.js +45 -0
- package/dist/clis/chatwise/status.d.ts +1 -0
- package/dist/clis/chatwise/status.js +22 -0
- package/dist/clis/discord-app/channels.d.ts +1 -0
- package/dist/clis/discord-app/channels.js +45 -0
- package/dist/clis/discord-app/members.d.ts +1 -0
- package/dist/clis/discord-app/members.js +38 -0
- package/dist/clis/discord-app/read.d.ts +1 -0
- package/dist/clis/discord-app/read.js +45 -0
- package/dist/clis/discord-app/search.d.ts +1 -0
- package/dist/clis/discord-app/search.js +56 -0
- package/dist/clis/discord-app/send.d.ts +1 -0
- package/dist/clis/discord-app/send.js +27 -0
- package/dist/clis/discord-app/servers.d.ts +1 -0
- package/dist/clis/discord-app/servers.js +36 -0
- package/dist/clis/discord-app/status.d.ts +1 -0
- package/dist/clis/discord-app/status.js +16 -0
- package/dist/clis/feishu/new.d.ts +1 -0
- package/dist/clis/feishu/new.js +27 -0
- package/dist/clis/feishu/read.d.ts +1 -0
- package/dist/clis/feishu/read.js +40 -0
- package/dist/clis/feishu/search.d.ts +1 -0
- package/dist/clis/feishu/search.js +30 -0
- package/dist/clis/feishu/send.d.ts +1 -0
- package/dist/clis/feishu/send.js +39 -0
- package/dist/clis/feishu/status.d.ts +1 -0
- package/dist/clis/feishu/status.js +28 -0
- package/dist/clis/grok/ask.d.ts +1 -0
- package/dist/clis/grok/ask.js +82 -0
- package/dist/clis/grok/debug.d.ts +1 -0
- package/dist/clis/grok/debug.js +45 -0
- package/dist/clis/jimeng/generate.yaml +84 -0
- package/dist/clis/jimeng/history.yaml +47 -0
- package/dist/clis/linux-do/categories.yaml +41 -0
- package/dist/clis/linux-do/category.yaml +49 -0
- package/dist/clis/linux-do/hot.yaml +50 -0
- package/dist/clis/linux-do/latest.yaml +40 -0
- package/dist/clis/linux-do/search.yaml +45 -0
- package/dist/clis/linux-do/topic.yaml +38 -0
- package/dist/clis/neteasemusic/like.d.ts +1 -0
- package/dist/clis/neteasemusic/like.js +25 -0
- package/dist/clis/neteasemusic/lyrics.d.ts +1 -0
- package/dist/clis/neteasemusic/lyrics.js +47 -0
- package/dist/clis/neteasemusic/next.d.ts +1 -0
- package/dist/clis/neteasemusic/next.js +26 -0
- package/dist/clis/neteasemusic/play.d.ts +1 -0
- package/dist/clis/neteasemusic/play.js +26 -0
- package/dist/clis/neteasemusic/playing.d.ts +1 -0
- package/dist/clis/neteasemusic/playing.js +59 -0
- package/dist/clis/neteasemusic/playlist.d.ts +1 -0
- package/dist/clis/neteasemusic/playlist.js +46 -0
- package/dist/clis/neteasemusic/prev.d.ts +1 -0
- package/dist/clis/neteasemusic/prev.js +25 -0
- package/dist/clis/neteasemusic/search.d.ts +1 -0
- package/dist/clis/neteasemusic/search.js +52 -0
- package/dist/clis/neteasemusic/status.d.ts +1 -0
- package/dist/clis/neteasemusic/status.js +16 -0
- package/dist/clis/neteasemusic/volume.d.ts +1 -0
- package/dist/clis/neteasemusic/volume.js +54 -0
- package/dist/clis/notion/export.d.ts +1 -0
- package/dist/clis/notion/export.js +31 -0
- package/dist/clis/notion/favorites.d.ts +1 -0
- package/dist/clis/notion/favorites.js +84 -0
- package/dist/clis/notion/new.d.ts +1 -0
- package/dist/clis/notion/new.js +34 -0
- package/dist/clis/notion/read.d.ts +1 -0
- package/dist/clis/notion/read.js +30 -0
- package/dist/clis/notion/search.d.ts +1 -0
- package/dist/clis/notion/search.js +46 -0
- package/dist/clis/notion/sidebar.d.ts +1 -0
- package/dist/clis/notion/sidebar.js +41 -0
- package/dist/clis/notion/status.d.ts +1 -0
- package/dist/clis/notion/status.js +16 -0
- package/dist/clis/notion/write.d.ts +1 -0
- package/dist/clis/notion/write.js +40 -0
- package/dist/clis/twitter/download.d.ts +8 -0
- package/dist/clis/twitter/download.js +204 -0
- package/dist/clis/wechat/chats.d.ts +1 -0
- package/dist/clis/wechat/chats.js +28 -0
- package/dist/clis/wechat/contacts.d.ts +1 -0
- package/dist/clis/wechat/contacts.js +28 -0
- package/dist/clis/wechat/read.d.ts +1 -0
- package/dist/clis/wechat/read.js +58 -0
- package/dist/clis/wechat/search.d.ts +1 -0
- package/dist/clis/wechat/search.js +31 -0
- package/dist/clis/wechat/send.d.ts +1 -0
- package/dist/clis/wechat/send.js +42 -0
- package/dist/clis/wechat/status.d.ts +1 -0
- package/dist/clis/wechat/status.js +29 -0
- package/dist/clis/xiaohongshu/creator-note-detail.d.ts +10 -0
- package/dist/clis/xiaohongshu/creator-note-detail.js +88 -0
- package/dist/clis/xiaohongshu/creator-notes.d.ts +11 -0
- package/dist/clis/xiaohongshu/creator-notes.js +109 -0
- package/dist/clis/xiaohongshu/creator-profile.d.ts +10 -0
- package/dist/clis/xiaohongshu/creator-profile.js +54 -0
- package/dist/clis/xiaohongshu/creator-stats.d.ts +10 -0
- package/dist/clis/xiaohongshu/creator-stats.js +74 -0
- package/dist/clis/xiaohongshu/download.d.ts +7 -0
- package/dist/clis/xiaohongshu/download.js +155 -0
- package/dist/clis/xiaohongshu/search.js +1 -1
- package/dist/clis/xiaohongshu/user-helpers.d.ts +15 -0
- package/dist/clis/xiaohongshu/user-helpers.js +67 -0
- package/dist/clis/xiaohongshu/user-helpers.test.d.ts +1 -0
- package/dist/clis/xiaohongshu/user-helpers.test.js +81 -0
- package/dist/clis/xiaohongshu/user.js +46 -29
- package/dist/clis/zhihu/download.d.ts +11 -0
- package/dist/clis/zhihu/download.js +186 -0
- package/dist/clis/zhihu/download.test.d.ts +1 -0
- package/dist/clis/zhihu/download.test.js +10 -0
- package/dist/daemon.d.ts +13 -0
- package/dist/daemon.js +187 -0
- package/dist/doctor.d.ts +27 -61
- package/dist/doctor.js +70 -601
- package/dist/doctor.test.js +30 -170
- package/dist/download/index.d.ts +79 -0
- package/dist/download/index.js +325 -0
- package/dist/download/progress.d.ts +36 -0
- package/dist/download/progress.js +111 -0
- package/dist/engine.test.js +15 -0
- package/dist/main.js +22 -28
- package/dist/pipeline/executor.test.js +1 -0
- package/dist/pipeline/registry.js +2 -0
- package/dist/pipeline/steps/browser.js +2 -2
- package/dist/pipeline/steps/download.d.ts +34 -0
- package/dist/pipeline/steps/download.js +251 -0
- package/dist/pipeline/steps/intercept.js +1 -2
- package/dist/pipeline/template.js +28 -0
- package/dist/setup.d.ts +6 -0
- package/dist/setup.js +46 -160
- package/dist/types.d.ts +6 -0
- package/extension/icons/icon-128.png +0 -0
- package/extension/icons/icon-16.png +0 -0
- package/extension/icons/icon-32.png +0 -0
- package/extension/icons/icon-48.png +0 -0
- package/extension/manifest.json +31 -0
- package/extension/package.json +16 -0
- package/extension/src/background.ts +293 -0
- package/extension/src/cdp.ts +125 -0
- package/extension/src/protocol.ts +57 -0
- package/extension/store-assets/screenshot-1280x800.png +0 -0
- package/extension/tsconfig.json +15 -0
- package/extension/vite.config.ts +18 -0
- package/package.json +8 -7
- package/scripts/test-site.mjs +70 -0
- package/src/browser/daemon-client.ts +113 -0
- package/src/browser/discover.ts +18 -216
- package/src/browser/errors.ts +30 -100
- package/src/browser/index.ts +6 -12
- package/src/browser/mcp.ts +78 -278
- package/src/browser/page.ts +222 -88
- package/src/browser.test.ts +3 -210
- package/src/build-manifest.test.ts +28 -0
- package/src/build-manifest.ts +147 -57
- package/src/clis/bilibili/download.ts +161 -0
- package/src/clis/chatgpt/README.md +1 -1
- package/src/clis/chatgpt/README.zh-CN.md +1 -1
- package/src/clis/chatwise/README.md +38 -0
- package/src/clis/chatwise/README.zh-CN.md +38 -0
- package/src/clis/chatwise/ask.ts +87 -0
- package/src/clis/chatwise/export.ts +51 -0
- package/src/clis/chatwise/history.ts +47 -0
- package/src/clis/chatwise/model.ts +87 -0
- package/src/clis/chatwise/new.ts +21 -0
- package/src/clis/chatwise/read.ts +42 -0
- package/src/clis/chatwise/screenshot.ts +33 -0
- package/src/clis/chatwise/send.ts +50 -0
- package/src/clis/chatwise/status.ts +25 -0
- package/src/clis/discord-app/README.md +28 -0
- package/src/clis/discord-app/README.zh-CN.md +28 -0
- package/src/clis/discord-app/channels.ts +48 -0
- package/src/clis/discord-app/members.ts +41 -0
- package/src/clis/discord-app/read.ts +49 -0
- package/src/clis/discord-app/search.ts +64 -0
- package/src/clis/discord-app/send.ts +32 -0
- package/src/clis/discord-app/servers.ts +39 -0
- package/src/clis/discord-app/status.ts +18 -0
- package/src/clis/feishu/README.md +20 -0
- package/src/clis/feishu/README.zh-CN.md +20 -0
- package/src/clis/feishu/new.ts +32 -0
- package/src/clis/feishu/read.ts +48 -0
- package/src/clis/feishu/search.ts +35 -0
- package/src/clis/feishu/send.ts +46 -0
- package/src/clis/feishu/status.ts +34 -0
- package/src/clis/grok/ask.ts +90 -0
- package/src/clis/grok/debug.ts +49 -0
- package/src/clis/jimeng/generate.yaml +84 -0
- package/src/clis/jimeng/history.yaml +47 -0
- package/src/clis/linux-do/categories.yaml +41 -0
- package/src/clis/linux-do/category.yaml +49 -0
- package/src/clis/linux-do/hot.yaml +50 -0
- package/src/clis/linux-do/latest.yaml +40 -0
- package/src/clis/linux-do/search.yaml +45 -0
- package/src/clis/linux-do/topic.yaml +38 -0
- package/src/clis/neteasemusic/README.md +31 -0
- package/src/clis/neteasemusic/README.zh-CN.md +31 -0
- package/src/clis/neteasemusic/like.ts +28 -0
- package/src/clis/neteasemusic/lyrics.ts +53 -0
- package/src/clis/neteasemusic/next.ts +30 -0
- package/src/clis/neteasemusic/play.ts +30 -0
- package/src/clis/neteasemusic/playing.ts +62 -0
- package/src/clis/neteasemusic/playlist.ts +51 -0
- package/src/clis/neteasemusic/prev.ts +29 -0
- package/src/clis/neteasemusic/search.ts +58 -0
- package/src/clis/neteasemusic/status.ts +18 -0
- package/src/clis/neteasemusic/volume.ts +61 -0
- package/src/clis/notion/README.md +29 -0
- package/src/clis/notion/README.zh-CN.md +29 -0
- package/src/clis/notion/export.ts +36 -0
- package/src/clis/notion/favorites.ts +87 -0
- package/src/clis/notion/new.ts +39 -0
- package/src/clis/notion/read.ts +33 -0
- package/src/clis/notion/search.ts +54 -0
- package/src/clis/notion/sidebar.ts +44 -0
- package/src/clis/notion/status.ts +18 -0
- package/src/clis/notion/write.ts +45 -0
- package/src/clis/twitter/download.ts +227 -0
- package/src/clis/wechat/README.md +28 -0
- package/src/clis/wechat/README.zh-CN.md +28 -0
- package/src/clis/wechat/chats.ts +33 -0
- package/src/clis/wechat/contacts.ts +33 -0
- package/src/clis/wechat/read.ts +72 -0
- package/src/clis/wechat/search.ts +36 -0
- package/src/clis/wechat/send.ts +49 -0
- package/src/clis/wechat/status.ts +35 -0
- package/src/clis/xiaohongshu/creator-note-detail.ts +95 -0
- package/src/clis/xiaohongshu/creator-notes.ts +116 -0
- package/src/clis/xiaohongshu/creator-profile.ts +60 -0
- package/src/clis/xiaohongshu/creator-stats.ts +81 -0
- package/src/clis/xiaohongshu/download.ts +173 -0
- package/src/clis/xiaohongshu/search.ts +1 -1
- package/src/clis/xiaohongshu/user-helpers.test.ts +106 -0
- package/src/clis/xiaohongshu/user-helpers.ts +85 -0
- package/src/clis/xiaohongshu/user.ts +52 -32
- package/src/clis/zhihu/download.test.ts +12 -0
- package/src/clis/zhihu/download.ts +223 -0
- package/src/daemon.ts +217 -0
- package/src/doctor.test.ts +32 -193
- package/src/doctor.ts +74 -668
- package/src/download/index.ts +395 -0
- package/src/download/progress.ts +125 -0
- package/src/engine.test.ts +17 -0
- package/src/main.ts +18 -26
- package/src/pipeline/executor.test.ts +1 -0
- package/src/pipeline/registry.ts +2 -0
- package/src/pipeline/steps/browser.ts +2 -2
- package/src/pipeline/steps/download.ts +310 -0
- package/src/pipeline/steps/intercept.ts +1 -2
- package/src/pipeline/template.ts +26 -0
- package/src/setup.ts +47 -183
- package/src/types.ts +1 -0
- package/tests/e2e/browser-auth.test.ts +25 -0
|
@@ -0,0 +1,155 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Xiaohongshu download — download images and videos from a note.
|
|
3
|
+
*
|
|
4
|
+
* Usage:
|
|
5
|
+
* opencli xiaohongshu download --note-id abc123 --output ./xhs
|
|
6
|
+
*/
|
|
7
|
+
import * as fs from 'node:fs';
|
|
8
|
+
import * as path from 'node:path';
|
|
9
|
+
import { cli, Strategy } from '../../registry.js';
|
|
10
|
+
import { httpDownload, } from '../../download/index.js';
|
|
11
|
+
import { DownloadProgressTracker, formatBytes } from '../../download/progress.js';
|
|
12
|
+
cli({
|
|
13
|
+
site: 'xiaohongshu',
|
|
14
|
+
name: 'download',
|
|
15
|
+
description: '下载小红书笔记中的图片和视频',
|
|
16
|
+
domain: 'www.xiaohongshu.com',
|
|
17
|
+
strategy: Strategy.COOKIE,
|
|
18
|
+
args: [
|
|
19
|
+
{ name: 'note_id', required: true, help: 'Note ID (from URL)' },
|
|
20
|
+
{ name: 'output', default: './xiaohongshu-downloads', help: 'Output directory' },
|
|
21
|
+
],
|
|
22
|
+
columns: ['index', 'type', 'status', 'size'],
|
|
23
|
+
func: async (page, kwargs) => {
|
|
24
|
+
const noteId = kwargs.note_id;
|
|
25
|
+
const output = kwargs.output;
|
|
26
|
+
// Navigate to note page
|
|
27
|
+
await page.goto(`https://www.xiaohongshu.com/explore/${noteId}`);
|
|
28
|
+
await page.wait(3);
|
|
29
|
+
// Extract note info and media URLs
|
|
30
|
+
const data = await page.evaluate(`
|
|
31
|
+
(() => {
|
|
32
|
+
const result = {
|
|
33
|
+
noteId: '${noteId}',
|
|
34
|
+
title: '',
|
|
35
|
+
author: '',
|
|
36
|
+
media: []
|
|
37
|
+
};
|
|
38
|
+
|
|
39
|
+
// Get title
|
|
40
|
+
const titleEl = document.querySelector('.title, #detail-title, .note-content .title');
|
|
41
|
+
result.title = titleEl?.textContent?.trim() || 'untitled';
|
|
42
|
+
|
|
43
|
+
// Get author
|
|
44
|
+
const authorEl = document.querySelector('.username, .author-name, .name');
|
|
45
|
+
result.author = authorEl?.textContent?.trim() || 'unknown';
|
|
46
|
+
|
|
47
|
+
// Get images - try multiple selectors
|
|
48
|
+
const imageSelectors = [
|
|
49
|
+
'.swiper-slide img',
|
|
50
|
+
'.carousel-image img',
|
|
51
|
+
'.note-slider img',
|
|
52
|
+
'.note-image img',
|
|
53
|
+
'.image-wrapper img',
|
|
54
|
+
'#noteContainer img[src*="xhscdn"]',
|
|
55
|
+
'img[src*="ci.xiaohongshu.com"]'
|
|
56
|
+
];
|
|
57
|
+
|
|
58
|
+
const imageUrls = new Set();
|
|
59
|
+
for (const selector of imageSelectors) {
|
|
60
|
+
document.querySelectorAll(selector).forEach(img => {
|
|
61
|
+
let src = img.src || img.getAttribute('data-src') || '';
|
|
62
|
+
if (src && (src.includes('xhscdn') || src.includes('xiaohongshu'))) {
|
|
63
|
+
// Convert to high quality URL (remove resize parameters)
|
|
64
|
+
src = src.split('?')[0];
|
|
65
|
+
// Try to get original size
|
|
66
|
+
src = src.replace(/\\/imageView\\d+\\/\\d+\\/w\\/\\d+/, '');
|
|
67
|
+
imageUrls.add(src);
|
|
68
|
+
}
|
|
69
|
+
});
|
|
70
|
+
}
|
|
71
|
+
|
|
72
|
+
// Get video if exists
|
|
73
|
+
const videoSelectors = [
|
|
74
|
+
'video source',
|
|
75
|
+
'video[src]',
|
|
76
|
+
'.player video',
|
|
77
|
+
'.video-player video'
|
|
78
|
+
];
|
|
79
|
+
|
|
80
|
+
for (const selector of videoSelectors) {
|
|
81
|
+
document.querySelectorAll(selector).forEach(v => {
|
|
82
|
+
const src = v.src || v.getAttribute('src') || '';
|
|
83
|
+
if (src) {
|
|
84
|
+
result.media.push({
|
|
85
|
+
type: 'video',
|
|
86
|
+
url: src
|
|
87
|
+
});
|
|
88
|
+
}
|
|
89
|
+
});
|
|
90
|
+
}
|
|
91
|
+
|
|
92
|
+
// Add images to media
|
|
93
|
+
imageUrls.forEach(url => {
|
|
94
|
+
result.media.push({
|
|
95
|
+
type: 'image',
|
|
96
|
+
url: url
|
|
97
|
+
});
|
|
98
|
+
});
|
|
99
|
+
|
|
100
|
+
return result;
|
|
101
|
+
})()
|
|
102
|
+
`);
|
|
103
|
+
if (!data || !data.media || data.media.length === 0) {
|
|
104
|
+
return [{ index: 0, type: '-', status: 'failed', size: 'No media found' }];
|
|
105
|
+
}
|
|
106
|
+
// Extract cookies for authenticated downloads
|
|
107
|
+
const cookies = await page.evaluate(`(() => document.cookie)()`);
|
|
108
|
+
// Create output directory
|
|
109
|
+
const outputDir = path.join(output, noteId);
|
|
110
|
+
fs.mkdirSync(outputDir, { recursive: true });
|
|
111
|
+
// Download all media files
|
|
112
|
+
const tracker = new DownloadProgressTracker(data.media.length, true);
|
|
113
|
+
const results = [];
|
|
114
|
+
for (let i = 0; i < data.media.length; i++) {
|
|
115
|
+
const media = data.media[i];
|
|
116
|
+
const ext = media.type === 'video' ? 'mp4' : 'jpg';
|
|
117
|
+
const filename = `${noteId}_${i + 1}.${ext}`;
|
|
118
|
+
const destPath = path.join(outputDir, filename);
|
|
119
|
+
const progressBar = tracker.onFileStart(filename, i);
|
|
120
|
+
try {
|
|
121
|
+
const result = await httpDownload(media.url, destPath, {
|
|
122
|
+
cookies: typeof cookies === 'string' ? cookies : '',
|
|
123
|
+
timeout: 60000,
|
|
124
|
+
onProgress: (received, total) => {
|
|
125
|
+
if (progressBar)
|
|
126
|
+
progressBar.update(received, total);
|
|
127
|
+
},
|
|
128
|
+
});
|
|
129
|
+
if (progressBar) {
|
|
130
|
+
progressBar.complete(result.success, result.success ? formatBytes(result.size) : undefined);
|
|
131
|
+
}
|
|
132
|
+
tracker.onFileComplete(result.success);
|
|
133
|
+
results.push({
|
|
134
|
+
index: i + 1,
|
|
135
|
+
type: media.type,
|
|
136
|
+
status: result.success ? 'success' : 'failed',
|
|
137
|
+
size: result.success ? formatBytes(result.size) : (result.error || 'unknown error'),
|
|
138
|
+
});
|
|
139
|
+
}
|
|
140
|
+
catch (err) {
|
|
141
|
+
if (progressBar)
|
|
142
|
+
progressBar.fail(err.message);
|
|
143
|
+
tracker.onFileComplete(false);
|
|
144
|
+
results.push({
|
|
145
|
+
index: i + 1,
|
|
146
|
+
type: media.type,
|
|
147
|
+
status: 'failed',
|
|
148
|
+
size: err.message,
|
|
149
|
+
});
|
|
150
|
+
}
|
|
151
|
+
}
|
|
152
|
+
tracker.finish();
|
|
153
|
+
return results;
|
|
154
|
+
},
|
|
155
|
+
});
|
|
@@ -37,7 +37,7 @@ cli({
|
|
|
37
37
|
const linkEl = el.querySelector('a[href*="/explore/"], a[href*="/search_result/"], a[href*="/note/"]');
|
|
38
38
|
|
|
39
39
|
const href = linkEl?.getAttribute('href') || '';
|
|
40
|
-
const noteId = href.match(/\\/(?:explore|note)\\/([a-
|
|
40
|
+
const noteId = href.match(/\\/(?:explore|note)\\/([a-zA-Z0-9]+)/)?.[1] || '';
|
|
41
41
|
|
|
42
42
|
results.push({
|
|
43
43
|
title: (titleEl?.textContent || '').trim(),
|
|
@@ -0,0 +1,15 @@
|
|
|
1
|
+
export interface XhsUserPageSnapshot {
|
|
2
|
+
noteGroups?: unknown;
|
|
3
|
+
pageData?: unknown;
|
|
4
|
+
}
|
|
5
|
+
export interface XhsUserNoteRow {
|
|
6
|
+
id: string;
|
|
7
|
+
title: string;
|
|
8
|
+
type: string;
|
|
9
|
+
likes: string;
|
|
10
|
+
url: string;
|
|
11
|
+
}
|
|
12
|
+
export declare function normalizeXhsUserId(input: string): string;
|
|
13
|
+
export declare function flattenXhsNoteGroups(noteGroups: unknown): any[];
|
|
14
|
+
export declare function buildXhsNoteUrl(userId: string, noteId: string, xsecToken?: string): string;
|
|
15
|
+
export declare function extractXhsUserNotes(snapshot: XhsUserPageSnapshot, fallbackUserId: string): XhsUserNoteRow[];
|
|
@@ -0,0 +1,67 @@
|
|
|
1
|
+
function toCleanString(value) {
|
|
2
|
+
return typeof value === 'string' ? value.trim() : value == null ? '' : String(value).trim();
|
|
3
|
+
}
|
|
4
|
+
export function normalizeXhsUserId(input) {
|
|
5
|
+
const trimmed = toCleanString(input);
|
|
6
|
+
const withoutQuery = trimmed.replace(/[?#].*$/, '');
|
|
7
|
+
const matched = withoutQuery.match(/\/user\/profile\/([a-zA-Z0-9]+)/);
|
|
8
|
+
if (matched?.[1])
|
|
9
|
+
return matched[1];
|
|
10
|
+
return withoutQuery.replace(/\/+$/, '').split('/').pop() ?? withoutQuery;
|
|
11
|
+
}
|
|
12
|
+
export function flattenXhsNoteGroups(noteGroups) {
|
|
13
|
+
if (!Array.isArray(noteGroups))
|
|
14
|
+
return [];
|
|
15
|
+
const notes = [];
|
|
16
|
+
for (const group of noteGroups) {
|
|
17
|
+
if (!group)
|
|
18
|
+
continue;
|
|
19
|
+
if (Array.isArray(group)) {
|
|
20
|
+
for (const item of group) {
|
|
21
|
+
if (item)
|
|
22
|
+
notes.push(item);
|
|
23
|
+
}
|
|
24
|
+
continue;
|
|
25
|
+
}
|
|
26
|
+
notes.push(group);
|
|
27
|
+
}
|
|
28
|
+
return notes;
|
|
29
|
+
}
|
|
30
|
+
export function buildXhsNoteUrl(userId, noteId, xsecToken) {
|
|
31
|
+
const cleanUserId = toCleanString(userId);
|
|
32
|
+
const cleanNoteId = toCleanString(noteId);
|
|
33
|
+
if (!cleanUserId || !cleanNoteId)
|
|
34
|
+
return '';
|
|
35
|
+
const url = new URL(`https://www.xiaohongshu.com/user/profile/${cleanUserId}/${cleanNoteId}`);
|
|
36
|
+
const cleanToken = toCleanString(xsecToken);
|
|
37
|
+
if (cleanToken) {
|
|
38
|
+
url.searchParams.set('xsec_token', cleanToken);
|
|
39
|
+
url.searchParams.set('xsec_source', 'pc_user');
|
|
40
|
+
}
|
|
41
|
+
return url.toString();
|
|
42
|
+
}
|
|
43
|
+
export function extractXhsUserNotes(snapshot, fallbackUserId) {
|
|
44
|
+
const notes = flattenXhsNoteGroups(snapshot.noteGroups);
|
|
45
|
+
const rows = [];
|
|
46
|
+
const seen = new Set();
|
|
47
|
+
for (const entry of notes) {
|
|
48
|
+
const noteCard = entry?.noteCard ?? entry?.note_card ?? entry;
|
|
49
|
+
if (!noteCard || typeof noteCard !== 'object')
|
|
50
|
+
continue;
|
|
51
|
+
const noteId = toCleanString(noteCard.noteId ?? noteCard.note_id ?? entry?.noteId ?? entry?.note_id ?? entry?.id);
|
|
52
|
+
if (!noteId || seen.has(noteId))
|
|
53
|
+
continue;
|
|
54
|
+
seen.add(noteId);
|
|
55
|
+
const userId = toCleanString(noteCard.user?.userId ?? noteCard.user?.user_id ?? fallbackUserId);
|
|
56
|
+
const xsecToken = toCleanString(entry?.xsecToken ?? entry?.xsec_token ?? noteCard.xsecToken ?? noteCard.xsec_token);
|
|
57
|
+
const likes = toCleanString(noteCard.interactInfo?.likedCount ?? noteCard.interact_info?.liked_count ?? 0) || '0';
|
|
58
|
+
rows.push({
|
|
59
|
+
id: noteId,
|
|
60
|
+
title: toCleanString(noteCard.displayTitle ?? noteCard.display_title ?? noteCard.title),
|
|
61
|
+
type: toCleanString(noteCard.type),
|
|
62
|
+
likes,
|
|
63
|
+
url: buildXhsNoteUrl(userId || fallbackUserId, noteId, xsecToken),
|
|
64
|
+
});
|
|
65
|
+
}
|
|
66
|
+
return rows;
|
|
67
|
+
}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
export {};
|
|
@@ -0,0 +1,81 @@
|
|
|
1
|
+
import { describe, expect, it } from 'vitest';
|
|
2
|
+
import { buildXhsNoteUrl, extractXhsUserNotes, flattenXhsNoteGroups, normalizeXhsUserId, } from './user-helpers.js';
|
|
3
|
+
describe('normalizeXhsUserId', () => {
|
|
4
|
+
it('extracts the profile id from a full Xiaohongshu URL', () => {
|
|
5
|
+
expect(normalizeXhsUserId('https://www.xiaohongshu.com/user/profile/615529370000000002026001?xsec_source=pc_search')).toBe('615529370000000002026001');
|
|
6
|
+
});
|
|
7
|
+
it('keeps a bare profile id unchanged', () => {
|
|
8
|
+
expect(normalizeXhsUserId('615529370000000002026001')).toBe('615529370000000002026001');
|
|
9
|
+
});
|
|
10
|
+
});
|
|
11
|
+
describe('flattenXhsNoteGroups', () => {
|
|
12
|
+
it('flattens grouped note arrays and ignores empty groups', () => {
|
|
13
|
+
expect(flattenXhsNoteGroups([[{ id: 'a' }], [], null, [{ id: 'b' }]])).toEqual([
|
|
14
|
+
{ id: 'a' },
|
|
15
|
+
{ id: 'b' },
|
|
16
|
+
]);
|
|
17
|
+
});
|
|
18
|
+
});
|
|
19
|
+
describe('buildXhsNoteUrl', () => {
|
|
20
|
+
it('includes xsec token when available', () => {
|
|
21
|
+
expect(buildXhsNoteUrl('user123', 'note456', 'token789')).toBe('https://www.xiaohongshu.com/user/profile/user123/note456?xsec_token=token789&xsec_source=pc_user');
|
|
22
|
+
});
|
|
23
|
+
});
|
|
24
|
+
describe('extractXhsUserNotes', () => {
|
|
25
|
+
it('normalizes grouped note cards into CLI rows', () => {
|
|
26
|
+
const rows = extractXhsUserNotes({
|
|
27
|
+
noteGroups: [
|
|
28
|
+
[
|
|
29
|
+
{
|
|
30
|
+
id: 'note-1',
|
|
31
|
+
xsecToken: 'abc',
|
|
32
|
+
noteCard: {
|
|
33
|
+
noteId: 'note-1',
|
|
34
|
+
displayTitle: 'First note',
|
|
35
|
+
type: 'video',
|
|
36
|
+
interactInfo: { likedCount: '4.6万' },
|
|
37
|
+
user: { userId: 'user-1' },
|
|
38
|
+
},
|
|
39
|
+
},
|
|
40
|
+
{
|
|
41
|
+
noteCard: {
|
|
42
|
+
note_id: 'note-2',
|
|
43
|
+
display_title: 'Second note',
|
|
44
|
+
type: 'normal',
|
|
45
|
+
interact_info: { liked_count: 42 },
|
|
46
|
+
},
|
|
47
|
+
},
|
|
48
|
+
],
|
|
49
|
+
[],
|
|
50
|
+
],
|
|
51
|
+
}, 'fallback-user');
|
|
52
|
+
expect(rows).toEqual([
|
|
53
|
+
{
|
|
54
|
+
id: 'note-1',
|
|
55
|
+
title: 'First note',
|
|
56
|
+
type: 'video',
|
|
57
|
+
likes: '4.6万',
|
|
58
|
+
url: 'https://www.xiaohongshu.com/user/profile/user-1/note-1?xsec_token=abc&xsec_source=pc_user',
|
|
59
|
+
},
|
|
60
|
+
{
|
|
61
|
+
id: 'note-2',
|
|
62
|
+
title: 'Second note',
|
|
63
|
+
type: 'normal',
|
|
64
|
+
likes: '42',
|
|
65
|
+
url: 'https://www.xiaohongshu.com/user/profile/fallback-user/note-2',
|
|
66
|
+
},
|
|
67
|
+
]);
|
|
68
|
+
});
|
|
69
|
+
it('deduplicates repeated notes by note id', () => {
|
|
70
|
+
const rows = extractXhsUserNotes({
|
|
71
|
+
noteGroups: [
|
|
72
|
+
[
|
|
73
|
+
{ noteCard: { noteId: 'dup-1', displayTitle: 'keep me' } },
|
|
74
|
+
{ noteCard: { noteId: 'dup-1', displayTitle: 'drop me' } },
|
|
75
|
+
],
|
|
76
|
+
],
|
|
77
|
+
}, 'fallback-user');
|
|
78
|
+
expect(rows).toHaveLength(1);
|
|
79
|
+
expect(rows[0]?.title).toBe('keep me');
|
|
80
|
+
});
|
|
81
|
+
});
|
|
@@ -1,40 +1,57 @@
|
|
|
1
1
|
import { cli, Strategy } from '../../registry.js';
|
|
2
|
+
import { extractXhsUserNotes, normalizeXhsUserId } from './user-helpers.js';
|
|
3
|
+
async function readUserSnapshot(page) {
|
|
4
|
+
return await page.evaluate(`
|
|
5
|
+
(() => {
|
|
6
|
+
const safeClone = (value) => {
|
|
7
|
+
try {
|
|
8
|
+
return JSON.parse(JSON.stringify(value ?? null));
|
|
9
|
+
} catch {
|
|
10
|
+
return null;
|
|
11
|
+
}
|
|
12
|
+
};
|
|
13
|
+
|
|
14
|
+
const userStore = window.__INITIAL_STATE__?.user || {};
|
|
15
|
+
return {
|
|
16
|
+
noteGroups: safeClone(userStore.notes?._value || userStore.notes || []),
|
|
17
|
+
pageData: safeClone(userStore.userPageData?._value || userStore.userPageData || {}),
|
|
18
|
+
};
|
|
19
|
+
})()
|
|
20
|
+
`);
|
|
21
|
+
}
|
|
2
22
|
cli({
|
|
3
23
|
site: 'xiaohongshu',
|
|
4
24
|
name: 'user',
|
|
5
|
-
description: 'Get
|
|
6
|
-
domain: 'xiaohongshu.com',
|
|
7
|
-
strategy: Strategy.
|
|
25
|
+
description: 'Get public notes from a Xiaohongshu user profile',
|
|
26
|
+
domain: 'www.xiaohongshu.com',
|
|
27
|
+
strategy: Strategy.COOKIE,
|
|
8
28
|
browser: true,
|
|
9
29
|
args: [
|
|
10
|
-
{ name: 'id', type: 'string', required: true },
|
|
11
|
-
{ name: 'limit', type: 'int', default: 15 },
|
|
30
|
+
{ name: 'id', type: 'string', required: true, help: 'User id or profile URL' },
|
|
31
|
+
{ name: 'limit', type: 'int', default: 15, help: 'Number of notes to return' },
|
|
12
32
|
],
|
|
13
33
|
columns: ['id', 'title', 'type', 'likes', 'url'],
|
|
14
34
|
func: async (page, kwargs) => {
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
await page.
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
});
|
|
35
|
-
}
|
|
36
|
-
}
|
|
35
|
+
const userId = normalizeXhsUserId(String(kwargs.id));
|
|
36
|
+
const limit = Math.max(1, Number(kwargs.limit ?? 15));
|
|
37
|
+
await page.goto(`https://www.xiaohongshu.com/user/profile/${userId}`);
|
|
38
|
+
await page.wait(3);
|
|
39
|
+
let snapshot = await readUserSnapshot(page);
|
|
40
|
+
let results = extractXhsUserNotes(snapshot ?? {}, userId);
|
|
41
|
+
let previousCount = results.length;
|
|
42
|
+
for (let i = 0; results.length < limit && i < 4; i += 1) {
|
|
43
|
+
await page.autoScroll({ times: 1, delayMs: 1500 });
|
|
44
|
+
await page.wait(1);
|
|
45
|
+
snapshot = await readUserSnapshot(page);
|
|
46
|
+
const nextResults = extractXhsUserNotes(snapshot ?? {}, userId);
|
|
47
|
+
if (nextResults.length <= previousCount)
|
|
48
|
+
break;
|
|
49
|
+
results = nextResults;
|
|
50
|
+
previousCount = nextResults.length;
|
|
51
|
+
}
|
|
52
|
+
if (results.length === 0) {
|
|
53
|
+
throw new Error('No public notes found for this Xiaohongshu user.');
|
|
37
54
|
}
|
|
38
|
-
return results.slice(0,
|
|
39
|
-
}
|
|
55
|
+
return results.slice(0, limit);
|
|
56
|
+
},
|
|
40
57
|
});
|
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Zhihu download — export articles to Markdown format.
|
|
3
|
+
*
|
|
4
|
+
* Usage:
|
|
5
|
+
* opencli zhihu download --url "https://zhuanlan.zhihu.com/p/xxx" --output ./zhihu
|
|
6
|
+
*/
|
|
7
|
+
/**
|
|
8
|
+
* Convert HTML content to Markdown.
|
|
9
|
+
* This is a simplified converter for Zhihu article content.
|
|
10
|
+
*/
|
|
11
|
+
export declare function htmlToMarkdown(html: string): string;
|
|
@@ -0,0 +1,186 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Zhihu download — export articles to Markdown format.
|
|
3
|
+
*
|
|
4
|
+
* Usage:
|
|
5
|
+
* opencli zhihu download --url "https://zhuanlan.zhihu.com/p/xxx" --output ./zhihu
|
|
6
|
+
*/
|
|
7
|
+
import * as fs from 'node:fs';
|
|
8
|
+
import * as path from 'node:path';
|
|
9
|
+
import { cli, Strategy } from '../../registry.js';
|
|
10
|
+
import { sanitizeFilename, httpDownload } from '../../download/index.js';
|
|
11
|
+
import { formatBytes } from '../../download/progress.js';
|
|
12
|
+
/**
|
|
13
|
+
* Convert HTML content to Markdown.
|
|
14
|
+
* This is a simplified converter for Zhihu article content.
|
|
15
|
+
*/
|
|
16
|
+
export function htmlToMarkdown(html) {
|
|
17
|
+
let md = html;
|
|
18
|
+
// Remove script and style tags
|
|
19
|
+
md = md.replace(/<script[^>]*>[\s\S]*?<\/script>/gi, '');
|
|
20
|
+
md = md.replace(/<style[^>]*>[\s\S]*?<\/style>/gi, '');
|
|
21
|
+
// Convert headers
|
|
22
|
+
md = md.replace(/<h1[^>]*>(.*?)<\/h1>/gi, '# $1\n\n');
|
|
23
|
+
md = md.replace(/<h2[^>]*>(.*?)<\/h2>/gi, '## $1\n\n');
|
|
24
|
+
md = md.replace(/<h3[^>]*>(.*?)<\/h3>/gi, '### $1\n\n');
|
|
25
|
+
md = md.replace(/<h4[^>]*>(.*?)<\/h4>/gi, '#### $1\n\n');
|
|
26
|
+
// Convert paragraphs
|
|
27
|
+
md = md.replace(/<p[^>]*>([\s\S]*?)<\/p>/gi, '$1\n\n');
|
|
28
|
+
// Convert links
|
|
29
|
+
md = md.replace(/<a[^>]*href="([^"]*)"[^>]*>(.*?)<\/a>/gi, '[$2]($1)');
|
|
30
|
+
// Convert images
|
|
31
|
+
md = md.replace(/<img[^>]*src="([^"]*)"[^>]*alt="([^"]*)"[^>]*\/?>/gi, '');
|
|
32
|
+
md = md.replace(/<img[^>]*src="([^"]*)"[^>]*\/?>/gi, '');
|
|
33
|
+
// Convert lists
|
|
34
|
+
md = md.replace(/<ul[^>]*>([\s\S]*?)<\/ul>/gi, (match, content) => {
|
|
35
|
+
return content.replace(/<li[^>]*>([\s\S]*?)<\/li>/gi, '- $1\n') + '\n';
|
|
36
|
+
});
|
|
37
|
+
md = md.replace(/<ol[^>]*>([\s\S]*?)<\/ol>/gi, (match, content) => {
|
|
38
|
+
let index = 0;
|
|
39
|
+
return content.replace(/<li[^>]*>([\s\S]*?)<\/li>/gi, (_itemMatch, itemContent) => `${++index}. ${itemContent}\n`) + '\n';
|
|
40
|
+
});
|
|
41
|
+
// Convert bold and italic
|
|
42
|
+
md = md.replace(/<strong[^>]*>(.*?)<\/strong>/gi, '**$1**');
|
|
43
|
+
md = md.replace(/<b[^>]*>(.*?)<\/b>/gi, '**$1**');
|
|
44
|
+
md = md.replace(/<em[^>]*>(.*?)<\/em>/gi, '*$1*');
|
|
45
|
+
md = md.replace(/<i[^>]*>(.*?)<\/i>/gi, '*$1*');
|
|
46
|
+
// Convert code blocks
|
|
47
|
+
md = md.replace(/<pre[^>]*><code[^>]*>([\s\S]*?)<\/code><\/pre>/gi, '```\n$1\n```\n\n');
|
|
48
|
+
md = md.replace(/<code[^>]*>(.*?)<\/code>/gi, '`$1`');
|
|
49
|
+
// Convert blockquotes
|
|
50
|
+
md = md.replace(/<blockquote[^>]*>([\s\S]*?)<\/blockquote>/gi, (match, content) => {
|
|
51
|
+
return content.split('\n').map((line) => `> ${line}`).join('\n') + '\n\n';
|
|
52
|
+
});
|
|
53
|
+
// Convert line breaks
|
|
54
|
+
md = md.replace(/<br\s*\/?>/gi, '\n');
|
|
55
|
+
// Remove remaining HTML tags
|
|
56
|
+
md = md.replace(/<[^>]+>/g, '');
|
|
57
|
+
// Decode HTML entities
|
|
58
|
+
md = md.replace(/ /g, ' ');
|
|
59
|
+
md = md.replace(/</g, '<');
|
|
60
|
+
md = md.replace(/>/g, '>');
|
|
61
|
+
md = md.replace(/&/g, '&');
|
|
62
|
+
md = md.replace(/"/g, '"');
|
|
63
|
+
// Clean up extra whitespace
|
|
64
|
+
md = md.replace(/\n{3,}/g, '\n\n');
|
|
65
|
+
md = md.trim();
|
|
66
|
+
return md;
|
|
67
|
+
}
|
|
68
|
+
cli({
|
|
69
|
+
site: 'zhihu',
|
|
70
|
+
name: 'download',
|
|
71
|
+
description: '导出知乎文章为 Markdown 格式',
|
|
72
|
+
domain: 'zhuanlan.zhihu.com',
|
|
73
|
+
strategy: Strategy.COOKIE,
|
|
74
|
+
args: [
|
|
75
|
+
{ name: 'url', required: true, help: 'Article URL (zhuanlan.zhihu.com/p/xxx)' },
|
|
76
|
+
{ name: 'output', default: './zhihu-articles', help: 'Output directory' },
|
|
77
|
+
{ name: 'download-images', type: 'boolean', default: false, help: 'Download images locally' },
|
|
78
|
+
],
|
|
79
|
+
columns: ['title', 'author', 'status', 'size'],
|
|
80
|
+
func: async (page, kwargs) => {
|
|
81
|
+
const url = kwargs.url;
|
|
82
|
+
const output = kwargs.output;
|
|
83
|
+
const downloadImages = kwargs['download-images'];
|
|
84
|
+
// Navigate to article page
|
|
85
|
+
await page.goto(url);
|
|
86
|
+
await page.wait(3);
|
|
87
|
+
// Extract article content
|
|
88
|
+
const data = await page.evaluate(`
|
|
89
|
+
(() => {
|
|
90
|
+
const result = {
|
|
91
|
+
title: '',
|
|
92
|
+
author: '',
|
|
93
|
+
content: '',
|
|
94
|
+
publishTime: '',
|
|
95
|
+
images: []
|
|
96
|
+
};
|
|
97
|
+
|
|
98
|
+
// Get title
|
|
99
|
+
const titleEl = document.querySelector('.Post-Title, h1.ContentItem-title, .ArticleTitle');
|
|
100
|
+
result.title = titleEl?.textContent?.trim() || 'untitled';
|
|
101
|
+
|
|
102
|
+
// Get author
|
|
103
|
+
const authorEl = document.querySelector('.AuthorInfo-name, .UserLink-link');
|
|
104
|
+
result.author = authorEl?.textContent?.trim() || 'unknown';
|
|
105
|
+
|
|
106
|
+
// Get publish time
|
|
107
|
+
const timeEl = document.querySelector('.ContentItem-time, .Post-Time');
|
|
108
|
+
result.publishTime = timeEl?.textContent?.trim() || '';
|
|
109
|
+
|
|
110
|
+
// Get content HTML
|
|
111
|
+
const contentEl = document.querySelector('.Post-RichTextContainer, .RichText, .ArticleContent');
|
|
112
|
+
if (contentEl) {
|
|
113
|
+
result.content = contentEl.innerHTML;
|
|
114
|
+
|
|
115
|
+
// Extract image URLs
|
|
116
|
+
contentEl.querySelectorAll('img').forEach(img => {
|
|
117
|
+
const src = img.getAttribute('data-original') || img.getAttribute('data-actualsrc') || img.src;
|
|
118
|
+
if (src && !src.includes('data:image')) {
|
|
119
|
+
result.images.push(src);
|
|
120
|
+
}
|
|
121
|
+
});
|
|
122
|
+
}
|
|
123
|
+
|
|
124
|
+
return result;
|
|
125
|
+
})()
|
|
126
|
+
`);
|
|
127
|
+
if (!data || !data.content) {
|
|
128
|
+
return [{
|
|
129
|
+
title: 'Error',
|
|
130
|
+
author: '-',
|
|
131
|
+
status: 'failed',
|
|
132
|
+
size: 'Could not extract article content',
|
|
133
|
+
}];
|
|
134
|
+
}
|
|
135
|
+
// Create output directory
|
|
136
|
+
fs.mkdirSync(output, { recursive: true });
|
|
137
|
+
// Convert HTML to Markdown
|
|
138
|
+
let markdown = htmlToMarkdown(data.content);
|
|
139
|
+
// Create frontmatter
|
|
140
|
+
const frontmatter = [
|
|
141
|
+
'---',
|
|
142
|
+
`title: "${data.title.replace(/"/g, '\\"')}"`,
|
|
143
|
+
`author: "${data.author.replace(/"/g, '\\"')}"`,
|
|
144
|
+
`source: "${url}"`,
|
|
145
|
+
data.publishTime ? `date: "${data.publishTime}"` : '',
|
|
146
|
+
'---',
|
|
147
|
+
'',
|
|
148
|
+
].filter(Boolean).join('\n');
|
|
149
|
+
// Download images if requested
|
|
150
|
+
if (downloadImages && data.images && data.images.length > 0) {
|
|
151
|
+
const imagesDir = path.join(output, 'images');
|
|
152
|
+
fs.mkdirSync(imagesDir, { recursive: true });
|
|
153
|
+
const cookies = await page.evaluate(`(() => document.cookie)()`);
|
|
154
|
+
for (let i = 0; i < data.images.length; i++) {
|
|
155
|
+
const imgUrl = data.images[i];
|
|
156
|
+
const ext = imgUrl.match(/\.(jpg|jpeg|png|gif|webp)/i)?.[1] || 'jpg';
|
|
157
|
+
const imgFilename = `img_${i + 1}.${ext}`;
|
|
158
|
+
const imgPath = path.join(imagesDir, imgFilename);
|
|
159
|
+
try {
|
|
160
|
+
await httpDownload(imgUrl, imgPath, {
|
|
161
|
+
cookies: typeof cookies === 'string' ? cookies : '',
|
|
162
|
+
timeout: 30000,
|
|
163
|
+
});
|
|
164
|
+
// Replace image URL in markdown with local path
|
|
165
|
+
markdown = markdown.replace(new RegExp(imgUrl.replace(/[.*+?^${}()|[\]\\]/g, '\\$&'), 'g'), `./images/${imgFilename}`);
|
|
166
|
+
}
|
|
167
|
+
catch {
|
|
168
|
+
// Keep original URL if download fails
|
|
169
|
+
}
|
|
170
|
+
}
|
|
171
|
+
}
|
|
172
|
+
// Write markdown file
|
|
173
|
+
const safeTitle = sanitizeFilename(data.title, 100);
|
|
174
|
+
const filename = `${safeTitle}.md`;
|
|
175
|
+
const filePath = path.join(output, filename);
|
|
176
|
+
const fullContent = frontmatter + '\n' + markdown;
|
|
177
|
+
fs.writeFileSync(filePath, fullContent, 'utf-8');
|
|
178
|
+
const size = Buffer.byteLength(fullContent, 'utf-8');
|
|
179
|
+
return [{
|
|
180
|
+
title: data.title,
|
|
181
|
+
author: data.author,
|
|
182
|
+
status: 'success',
|
|
183
|
+
size: formatBytes(size),
|
|
184
|
+
}];
|
|
185
|
+
},
|
|
186
|
+
});
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
export {};
|
|
@@ -0,0 +1,10 @@
|
|
|
1
|
+
import { describe, expect, it } from 'vitest';
|
|
2
|
+
import { htmlToMarkdown } from './download.js';
|
|
3
|
+
describe('htmlToMarkdown', () => {
|
|
4
|
+
it('renders ordered lists with the original list item content', () => {
|
|
5
|
+
const html = '<ol><li>First item</li><li>Second item</li></ol>';
|
|
6
|
+
expect(htmlToMarkdown(html)).toContain('1. First item');
|
|
7
|
+
expect(htmlToMarkdown(html)).toContain('2. Second item');
|
|
8
|
+
expect(htmlToMarkdown(html)).not.toContain('$1');
|
|
9
|
+
});
|
|
10
|
+
});
|
package/dist/daemon.d.ts
ADDED
|
@@ -0,0 +1,13 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* opencli micro-daemon — HTTP + WebSocket bridge between CLI and Chrome Extension.
|
|
3
|
+
*
|
|
4
|
+
* Architecture:
|
|
5
|
+
* CLI → HTTP POST /command → daemon → WebSocket → Extension
|
|
6
|
+
* Extension → WebSocket result → daemon → HTTP response → CLI
|
|
7
|
+
*
|
|
8
|
+
* Lifecycle:
|
|
9
|
+
* - Auto-spawned by opencli on first browser command
|
|
10
|
+
* - Auto-exits after 5 minutes of idle
|
|
11
|
+
* - Listens on localhost:19825
|
|
12
|
+
*/
|
|
13
|
+
export {};
|