@jackwener/opencli 1.3.3 → 1.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.github/pull_request_template.md +3 -1
- package/.github/workflows/build-extension.yml +7 -1
- package/.github/workflows/ci.yml +29 -3
- package/.github/workflows/docs.yml +1 -1
- package/.github/workflows/e2e-headed.yml +20 -0
- package/.github/workflows/release.yml +1 -1
- package/.github/workflows/security.yml +0 -3
- package/CHANGELOG.md +55 -0
- package/CONTRIBUTING.md +6 -3
- package/README.md +30 -3
- package/README.zh-CN.md +30 -3
- package/SKILL.md +7 -1
- package/TESTING.md +1 -0
- package/chatwise-opencli.ps1 +82 -0
- package/dist/analysis.d.ts +38 -0
- package/dist/analysis.js +166 -0
- package/dist/browser/cdp.d.ts +0 -4
- package/dist/browser/cdp.js +53 -41
- package/dist/browser/cdp.test.d.ts +1 -0
- package/dist/browser/cdp.test.js +52 -0
- package/dist/browser/dom-snapshot.d.ts +2 -2
- package/dist/browser/dom-snapshot.js +54 -1
- package/dist/browser/dom-snapshot.test.js +36 -0
- package/dist/browser/index.d.ts +2 -2
- package/dist/browser/index.js +1 -1
- package/dist/browser/mcp.d.ts +0 -2
- package/dist/browser/mcp.js +2 -3
- package/dist/browser/page.d.ts +4 -3
- package/dist/browser/page.js +34 -37
- package/dist/browser/stealth.d.ts +0 -2
- package/dist/browser/stealth.js +24 -9
- package/dist/browser.test.js +2 -2
- package/dist/build-manifest.js +15 -9
- package/dist/build-manifest.test.js +12 -0
- package/dist/cascade.js +4 -2
- package/dist/cli-manifest.json +639 -258
- package/dist/cli.js +57 -29
- package/dist/clis/_shared/desktop-commands.d.ts +22 -0
- package/dist/clis/_shared/desktop-commands.js +108 -0
- package/dist/clis/antigravity/serve.js +5 -2
- package/dist/clis/arxiv/search.js +1 -1
- package/dist/clis/bilibili/dynamic.test.d.ts +1 -0
- package/dist/clis/bilibili/dynamic.test.js +68 -0
- package/dist/clis/bilibili/favorite.js +4 -2
- package/dist/clis/bilibili/following.js +3 -2
- package/dist/clis/bilibili/subtitle.js +8 -7
- package/dist/clis/bilibili/utils.js +2 -2
- package/dist/clis/boss/batchgreet.js +1 -1
- package/dist/clis/boss/chatlist.js +1 -1
- package/dist/clis/boss/chatmsg.js +1 -1
- package/dist/clis/boss/detail.js +1 -1
- package/dist/clis/boss/exchange.js +1 -1
- package/dist/clis/boss/greet.js +1 -1
- package/dist/clis/boss/invite.js +1 -1
- package/dist/clis/boss/joblist.js +1 -1
- package/dist/clis/boss/mark.js +4 -3
- package/dist/clis/boss/recommend.js +1 -1
- package/dist/clis/boss/resume.js +1 -1
- package/dist/clis/boss/search.js +1 -1
- package/dist/clis/boss/send.js +5 -4
- package/dist/clis/boss/stats.js +1 -1
- package/dist/clis/chatgpt/ask.js +4 -0
- package/dist/clis/chatgpt/new.js +5 -1
- package/dist/clis/chatgpt/read.js +5 -1
- package/dist/clis/chatgpt/send.js +2 -1
- package/dist/clis/chatgpt/status.js +5 -1
- package/dist/clis/chatwise/ask.js +8 -2
- package/dist/clis/chatwise/export.js +2 -0
- package/dist/clis/chatwise/history.js +2 -0
- package/dist/clis/chatwise/model.js +8 -3
- package/dist/clis/chatwise/new.js +3 -18
- package/dist/clis/chatwise/read.js +2 -0
- package/dist/clis/chatwise/screenshot.js +3 -27
- package/dist/clis/chatwise/send.js +8 -2
- package/dist/clis/chatwise/shared.d.ts +2 -0
- package/dist/clis/chatwise/shared.js +6 -0
- package/dist/clis/chatwise/status.js +3 -22
- package/dist/clis/codex/ask.js +6 -2
- package/dist/clis/codex/dump.js +2 -25
- package/dist/clis/codex/new.js +2 -25
- package/dist/clis/codex/screenshot.js +2 -27
- package/dist/clis/codex/send.js +6 -4
- package/dist/clis/codex/status.js +2 -22
- package/dist/clis/cursor/ask.js +2 -1
- package/dist/clis/cursor/composer.js +2 -1
- package/dist/clis/cursor/dump.js +2 -25
- package/dist/clis/cursor/new.js +2 -18
- package/dist/clis/cursor/read.js +2 -1
- package/dist/clis/cursor/screenshot.js +1 -30
- package/dist/clis/cursor/send.js +2 -1
- package/dist/clis/cursor/status.js +2 -21
- package/dist/clis/dictionary/examples.yaml +25 -0
- package/dist/clis/dictionary/search.yaml +27 -0
- package/dist/clis/dictionary/synonyms.yaml +25 -0
- package/dist/clis/douban/book-hot.js +1 -1
- package/dist/clis/douban/movie-hot.js +1 -1
- package/dist/clis/douban/search.js +1 -1
- package/dist/clis/douban/utils.d.ts +4 -1
- package/dist/clis/douban/utils.js +156 -1
- package/dist/clis/doubao/ask.js +1 -1
- package/dist/clis/doubao/new.js +1 -1
- package/dist/clis/doubao/read.js +1 -1
- package/dist/clis/doubao/send.js +1 -1
- package/dist/clis/doubao/status.js +1 -1
- package/dist/clis/doubao-app/ask.js +1 -1
- package/dist/clis/doubao-app/new.js +1 -1
- package/dist/clis/doubao-app/read.js +1 -1
- package/dist/clis/doubao-app/send.js +1 -1
- package/dist/clis/grok/ask.d.ts +4 -0
- package/dist/clis/grok/ask.js +28 -10
- package/dist/clis/grok/ask.test.js +18 -0
- package/dist/clis/jd/item.d.ts +1 -0
- package/dist/clis/jd/item.js +96 -0
- package/dist/clis/jd/item.test.d.ts +1 -0
- package/dist/clis/jd/item.test.js +28 -0
- package/dist/clis/jike/feed.js +1 -1
- package/dist/clis/jike/search.js +1 -1
- package/dist/clis/linkedin/search.js +5 -4
- package/dist/clis/linkedin/timeline.d.ts +21 -0
- package/dist/clis/linkedin/timeline.js +503 -0
- package/dist/clis/linkedin/timeline.test.d.ts +1 -0
- package/dist/clis/linkedin/timeline.test.js +81 -0
- package/dist/clis/medium/feed.js +1 -1
- package/dist/clis/medium/search.js +1 -1
- package/dist/clis/medium/user.js +1 -1
- package/dist/clis/medium/{shared.js → utils.js} +2 -1
- package/dist/clis/pixiv/detail.yaml +49 -0
- package/dist/clis/pixiv/download.d.ts +7 -0
- package/dist/clis/pixiv/download.js +78 -0
- package/dist/clis/pixiv/download.test.d.ts +1 -0
- package/dist/clis/pixiv/download.test.js +87 -0
- package/dist/clis/pixiv/illusts.d.ts +8 -0
- package/dist/clis/pixiv/illusts.js +65 -0
- package/dist/clis/pixiv/illusts.test.d.ts +1 -0
- package/dist/clis/pixiv/illusts.test.js +99 -0
- package/dist/clis/pixiv/ranking.yaml +53 -0
- package/dist/clis/pixiv/search.d.ts +6 -0
- package/dist/clis/pixiv/search.js +43 -0
- package/dist/clis/pixiv/search.test.d.ts +1 -0
- package/dist/clis/pixiv/search.test.js +83 -0
- package/dist/clis/pixiv/test-utils.d.ts +12 -0
- package/dist/clis/pixiv/test-utils.js +23 -0
- package/dist/clis/pixiv/user.yaml +46 -0
- package/dist/clis/pixiv/utils.d.ts +27 -0
- package/dist/clis/pixiv/utils.js +49 -0
- package/dist/clis/reddit/comment.js +2 -1
- package/dist/clis/reddit/read.js +4 -3
- package/dist/clis/reddit/read.test.d.ts +1 -0
- package/dist/clis/reddit/read.test.js +28 -0
- package/dist/clis/reddit/save.js +2 -1
- package/dist/clis/reddit/saved.js +7 -3
- package/dist/clis/reddit/subscribe.js +2 -1
- package/dist/clis/reddit/upvote.js +2 -1
- package/dist/clis/reddit/upvoted.js +7 -3
- package/dist/clis/sinablog/article.js +1 -1
- package/dist/clis/sinablog/hot.js +1 -1
- package/dist/clis/sinablog/user.js +1 -1
- package/dist/clis/substack/feed.js +1 -1
- package/dist/clis/substack/publication.js +1 -1
- package/dist/clis/substack/search.js +3 -2
- package/dist/clis/substack/{shared.js → utils.js} +3 -2
- package/dist/clis/tiktok/search.yaml +2 -1
- package/dist/clis/twitter/accept.js +2 -1
- package/dist/clis/twitter/article.js +4 -1
- package/dist/clis/twitter/block.js +2 -1
- package/dist/clis/twitter/bookmark.js +2 -1
- package/dist/clis/twitter/bookmarks.js +3 -2
- package/dist/clis/twitter/delete.js +2 -1
- package/dist/clis/twitter/follow.js +2 -1
- package/dist/clis/twitter/followers.js +3 -2
- package/dist/clis/twitter/following.js +3 -2
- package/dist/clis/twitter/hide-reply.js +2 -1
- package/dist/clis/twitter/like.js +2 -1
- package/dist/clis/twitter/notifications.js +2 -1
- package/dist/clis/twitter/post.js +2 -1
- package/dist/clis/twitter/profile.js +5 -2
- package/dist/clis/twitter/reply-dm.js +2 -1
- package/dist/clis/twitter/reply.js +2 -1
- package/dist/clis/twitter/search.js +30 -13
- package/dist/clis/twitter/search.test.d.ts +1 -0
- package/dist/clis/twitter/search.test.js +104 -0
- package/dist/clis/twitter/thread.js +2 -2
- package/dist/clis/twitter/timeline.js +3 -2
- package/dist/clis/twitter/trending.js +3 -2
- package/dist/clis/twitter/unblock.js +2 -1
- package/dist/clis/twitter/unbookmark.js +2 -1
- package/dist/clis/twitter/unfollow.js +2 -1
- package/dist/clis/v2ex/daily.js +3 -2
- package/dist/clis/v2ex/me.js +3 -2
- package/dist/clis/v2ex/notifications.js +4 -4
- package/dist/clis/web/read.d.ts +16 -0
- package/dist/clis/web/read.js +202 -0
- package/dist/clis/xueqiu/danjuan-utils.d.ts +55 -0
- package/dist/clis/xueqiu/danjuan-utils.js +126 -0
- package/dist/clis/xueqiu/danjuan-utils.test.d.ts +1 -0
- package/dist/clis/xueqiu/danjuan-utils.test.js +41 -0
- package/dist/clis/xueqiu/fund-holdings.d.ts +1 -0
- package/dist/clis/xueqiu/fund-holdings.js +28 -0
- package/dist/clis/xueqiu/fund-snapshot.d.ts +1 -0
- package/dist/clis/xueqiu/fund-snapshot.js +25 -0
- package/dist/clis/youtube/transcript.js +5 -4
- package/dist/clis/youtube/video.js +3 -2
- package/dist/daemon.js +7 -3
- package/dist/discovery.js +11 -10
- package/dist/doctor.js +2 -1
- package/dist/download/index.d.ts +4 -12
- package/dist/download/index.js +33 -12
- package/dist/download/index.test.js +79 -2
- package/dist/download/media-download.js +4 -2
- package/dist/engine.test.js +76 -4
- package/dist/execution.d.ts +1 -9
- package/dist/execution.js +56 -46
- package/dist/explore.js +12 -111
- package/dist/external-clis.yaml +0 -8
- package/dist/external.js +7 -5
- package/dist/external.test.js +4 -0
- package/dist/generate.d.ts +0 -9
- package/dist/generate.js +4 -20
- package/dist/hooks.d.ts +46 -0
- package/dist/hooks.js +56 -0
- package/dist/hooks.test.d.ts +4 -0
- package/dist/hooks.test.js +92 -0
- package/dist/interceptor.js +70 -23
- package/dist/main.js +2 -0
- package/dist/output.js +12 -6
- package/dist/pipeline/executor.js +1 -1
- package/dist/pipeline/steps/browser.js +1 -3
- package/dist/pipeline/steps/download.js +42 -26
- package/dist/pipeline/steps/download.test.d.ts +1 -0
- package/dist/pipeline/steps/download.test.js +101 -0
- package/dist/pipeline/steps/fetch.js +40 -22
- package/dist/pipeline/steps/fetch.test.d.ts +1 -0
- package/dist/pipeline/steps/fetch.test.js +123 -0
- package/dist/pipeline/steps/transform.js +2 -6
- package/dist/pipeline/template.js +66 -52
- package/dist/pipeline/template.test.js +28 -0
- package/dist/pipeline/transform.test.js +18 -0
- package/dist/plugin.d.ts +40 -1
- package/dist/plugin.js +214 -17
- package/dist/plugin.test.d.ts +1 -1
- package/dist/plugin.test.js +219 -3
- package/dist/record.js +6 -98
- package/dist/registry-api.d.ts +2 -0
- package/dist/registry-api.js +1 -0
- package/dist/registry.d.ts +5 -2
- package/dist/registry.js +1 -2
- package/dist/runtime.d.ts +0 -1
- package/dist/runtime.js +14 -4
- package/dist/snapshotFormatter.d.ts +7 -14
- package/dist/snapshotFormatter.js +38 -78
- package/dist/utils.d.ts +9 -0
- package/dist/utils.js +29 -0
- package/dist/validate.js +3 -5
- package/dist/yaml-schema.d.ts +26 -0
- package/dist/yaml-schema.js +5 -0
- package/docs/.vitepress/config.mts +3 -0
- package/docs/adapters/browser/dictionary.md +27 -0
- package/docs/adapters/browser/jd.md +27 -0
- package/docs/adapters/browser/linkedin.md +6 -0
- package/docs/adapters/browser/pixiv.md +92 -0
- package/docs/adapters/browser/web.md +30 -0
- package/docs/adapters/browser/xueqiu.md +27 -9
- package/docs/adapters/index.md +3 -1
- package/docs/comparison.md +125 -0
- package/docs/developer/contributing.md +21 -2
- package/docs/developer/testing.md +14 -8
- package/docs/developer/ts-adapter.md +18 -0
- package/docs/developer/yaml-adapter.md +16 -0
- package/docs/guide/plugins.md +10 -0
- package/docs/zh/guide/plugins.md +10 -0
- package/extension/dist/background.js +519 -444
- package/extension/manifest.json +1 -1
- package/extension/package.json +1 -1
- package/extension/src/background.test.ts +46 -1
- package/extension/src/background.ts +108 -33
- package/extension/src/cdp.ts +9 -9
- package/package.json +3 -2
- package/scripts/check-doc-coverage.sh +2 -0
- package/src/analysis.ts +170 -0
- package/src/browser/cdp.test.ts +66 -0
- package/src/browser/cdp.ts +59 -44
- package/src/browser/dom-snapshot.test.ts +42 -0
- package/src/browser/dom-snapshot.ts +56 -3
- package/src/browser/index.ts +2 -2
- package/src/browser/mcp.ts +2 -4
- package/src/browser/page.ts +34 -37
- package/src/browser/stealth.ts +24 -10
- package/src/browser.test.ts +2 -2
- package/src/build-manifest.test.ts +14 -0
- package/src/build-manifest.ts +13 -31
- package/src/cascade.ts +5 -3
- package/src/cli.ts +66 -34
- package/src/clis/_shared/desktop-commands.ts +121 -0
- package/src/clis/antigravity/serve.ts +6 -3
- package/src/clis/arxiv/search.ts +1 -1
- package/src/clis/bilibili/dynamic.test.ts +79 -0
- package/src/clis/bilibili/favorite.ts +5 -2
- package/src/clis/bilibili/following.ts +3 -2
- package/src/clis/bilibili/subtitle.ts +8 -7
- package/src/clis/bilibili/utils.ts +2 -2
- package/src/clis/boss/batchgreet.ts +1 -1
- package/src/clis/boss/chatlist.ts +1 -1
- package/src/clis/boss/chatmsg.ts +1 -1
- package/src/clis/boss/detail.ts +1 -1
- package/src/clis/boss/exchange.ts +1 -1
- package/src/clis/boss/greet.ts +1 -1
- package/src/clis/boss/invite.ts +1 -1
- package/src/clis/boss/joblist.ts +1 -1
- package/src/clis/boss/mark.ts +4 -3
- package/src/clis/boss/recommend.ts +1 -1
- package/src/clis/boss/resume.ts +1 -1
- package/src/clis/boss/search.ts +1 -1
- package/src/clis/boss/send.ts +5 -4
- package/src/clis/boss/stats.ts +1 -1
- package/src/clis/chatgpt/ask.ts +5 -0
- package/src/clis/chatgpt/new.ts +7 -2
- package/src/clis/chatgpt/read.ts +7 -2
- package/src/clis/chatgpt/send.ts +3 -2
- package/src/clis/chatgpt/status.ts +6 -1
- package/src/clis/chatwise/ask.ts +7 -2
- package/src/clis/chatwise/export.ts +2 -0
- package/src/clis/chatwise/history.ts +2 -0
- package/src/clis/chatwise/model.ts +7 -3
- package/src/clis/chatwise/new.ts +3 -20
- package/src/clis/chatwise/read.ts +2 -0
- package/src/clis/chatwise/screenshot.ts +3 -32
- package/src/clis/chatwise/send.ts +7 -2
- package/src/clis/chatwise/shared.ts +8 -0
- package/src/clis/chatwise/status.ts +3 -24
- package/src/clis/codex/ask.ts +5 -2
- package/src/clis/codex/dump.ts +2 -27
- package/src/clis/codex/new.ts +2 -28
- package/src/clis/codex/screenshot.ts +2 -32
- package/src/clis/codex/send.ts +5 -4
- package/src/clis/codex/status.ts +2 -24
- package/src/clis/cursor/ask.ts +2 -1
- package/src/clis/cursor/composer.ts +2 -1
- package/src/clis/cursor/dump.ts +2 -27
- package/src/clis/cursor/new.ts +2 -20
- package/src/clis/cursor/read.ts +2 -1
- package/src/clis/cursor/screenshot.ts +1 -36
- package/src/clis/cursor/send.ts +2 -1
- package/src/clis/cursor/status.ts +2 -22
- package/src/clis/dictionary/examples.yaml +25 -0
- package/src/clis/dictionary/search.yaml +27 -0
- package/src/clis/dictionary/synonyms.yaml +25 -0
- package/src/clis/douban/book-hot.ts +1 -1
- package/src/clis/douban/movie-hot.ts +1 -1
- package/src/clis/douban/search.ts +1 -1
- package/src/clis/douban/utils.ts +165 -1
- package/src/clis/doubao/ask.ts +1 -1
- package/src/clis/doubao/new.ts +1 -1
- package/src/clis/doubao/read.ts +1 -1
- package/src/clis/doubao/send.ts +1 -1
- package/src/clis/doubao/status.ts +1 -1
- package/src/clis/doubao-app/ask.ts +1 -1
- package/src/clis/doubao-app/new.ts +1 -1
- package/src/clis/doubao-app/read.ts +1 -1
- package/src/clis/doubao-app/send.ts +1 -1
- package/src/clis/grok/ask.test.ts +25 -0
- package/src/clis/grok/ask.ts +25 -12
- package/src/clis/jd/item.test.ts +35 -0
- package/src/clis/jd/item.ts +101 -0
- package/src/clis/jike/feed.ts +1 -1
- package/src/clis/jike/search.ts +1 -1
- package/src/clis/linkedin/search.ts +5 -4
- package/src/clis/linkedin/timeline.test.ts +99 -0
- package/src/clis/linkedin/timeline.ts +532 -0
- package/src/clis/medium/feed.ts +1 -1
- package/src/clis/medium/search.ts +1 -1
- package/src/clis/medium/user.ts +1 -1
- package/src/clis/medium/{shared.ts → utils.ts} +2 -1
- package/src/clis/pixiv/detail.yaml +49 -0
- package/src/clis/pixiv/download.test.ts +114 -0
- package/src/clis/pixiv/download.ts +91 -0
- package/src/clis/pixiv/illusts.test.ts +115 -0
- package/src/clis/pixiv/illusts.ts +78 -0
- package/src/clis/pixiv/ranking.yaml +53 -0
- package/src/clis/pixiv/search.test.ts +97 -0
- package/src/clis/pixiv/search.ts +53 -0
- package/src/clis/pixiv/test-utils.ts +29 -0
- package/src/clis/pixiv/user.yaml +46 -0
- package/src/clis/pixiv/utils.ts +62 -0
- package/src/clis/reddit/comment.ts +2 -1
- package/src/clis/reddit/read.test.ts +34 -0
- package/src/clis/reddit/read.ts +4 -3
- package/src/clis/reddit/save.ts +2 -1
- package/src/clis/reddit/saved.ts +6 -2
- package/src/clis/reddit/subscribe.ts +2 -1
- package/src/clis/reddit/upvote.ts +2 -1
- package/src/clis/reddit/upvoted.ts +6 -2
- package/src/clis/sinablog/article.ts +1 -1
- package/src/clis/sinablog/hot.ts +1 -1
- package/src/clis/sinablog/user.ts +1 -1
- package/src/clis/substack/feed.ts +1 -1
- package/src/clis/substack/publication.ts +1 -1
- package/src/clis/substack/search.ts +3 -2
- package/src/clis/substack/{shared.ts → utils.ts} +3 -2
- package/src/clis/tiktok/search.yaml +2 -1
- package/src/clis/twitter/accept.ts +2 -1
- package/src/clis/twitter/article.ts +3 -1
- package/src/clis/twitter/block.ts +2 -1
- package/src/clis/twitter/bookmark.ts +2 -1
- package/src/clis/twitter/bookmarks.ts +3 -2
- package/src/clis/twitter/delete.ts +2 -1
- package/src/clis/twitter/follow.ts +2 -1
- package/src/clis/twitter/followers.ts +3 -2
- package/src/clis/twitter/following.ts +3 -2
- package/src/clis/twitter/hide-reply.ts +2 -1
- package/src/clis/twitter/like.ts +2 -1
- package/src/clis/twitter/notifications.ts +2 -1
- package/src/clis/twitter/post.ts +2 -1
- package/src/clis/twitter/profile.ts +4 -2
- package/src/clis/twitter/reply-dm.ts +2 -1
- package/src/clis/twitter/reply.ts +2 -1
- package/src/clis/twitter/search.test.ts +113 -0
- package/src/clis/twitter/search.ts +38 -14
- package/src/clis/twitter/thread.ts +2 -2
- package/src/clis/twitter/timeline.ts +3 -2
- package/src/clis/twitter/trending.ts +3 -2
- package/src/clis/twitter/unblock.ts +2 -1
- package/src/clis/twitter/unbookmark.ts +2 -1
- package/src/clis/twitter/unfollow.ts +2 -1
- package/src/clis/v2ex/daily.ts +3 -2
- package/src/clis/v2ex/me.ts +3 -2
- package/src/clis/v2ex/notifications.ts +3 -4
- package/src/clis/web/read.ts +210 -0
- package/src/clis/xueqiu/danjuan-utils.test.ts +49 -0
- package/src/clis/xueqiu/danjuan-utils.ts +176 -0
- package/src/clis/xueqiu/fund-holdings.ts +32 -0
- package/src/clis/xueqiu/fund-snapshot.ts +27 -0
- package/src/clis/youtube/transcript.ts +5 -4
- package/src/clis/youtube/video.ts +3 -2
- package/src/daemon.ts +5 -4
- package/src/discovery.ts +12 -34
- package/src/doctor.ts +3 -2
- package/src/download/index.test.ts +93 -2
- package/src/download/index.ts +44 -23
- package/src/download/media-download.ts +5 -3
- package/src/engine.test.ts +84 -3
- package/src/execution.ts +62 -46
- package/src/explore.ts +21 -90
- package/src/external-clis.yaml +0 -8
- package/src/external.test.ts +9 -0
- package/src/external.ts +12 -10
- package/src/generate.ts +4 -41
- package/src/hooks.test.ts +126 -0
- package/src/hooks.ts +90 -0
- package/src/interceptor.ts +73 -23
- package/src/main.ts +2 -0
- package/src/output.ts +14 -6
- package/src/pipeline/executor.ts +1 -1
- package/src/pipeline/steps/browser.ts +1 -3
- package/src/pipeline/steps/download.test.ts +136 -0
- package/src/pipeline/steps/download.ts +47 -34
- package/src/pipeline/steps/fetch.test.ts +179 -0
- package/src/pipeline/steps/fetch.ts +39 -23
- package/src/pipeline/steps/transform.ts +2 -6
- package/src/pipeline/template.test.ts +28 -0
- package/src/pipeline/template.ts +67 -79
- package/src/pipeline/transform.test.ts +20 -0
- package/src/plugin.test.ts +251 -3
- package/src/plugin.ts +265 -21
- package/src/record.ts +12 -84
- package/src/registry-api.ts +2 -0
- package/src/registry.ts +7 -4
- package/src/runtime.ts +14 -4
- package/src/snapshotFormatter.ts +43 -121
- package/src/utils.ts +39 -0
- package/src/validate.ts +3 -5
- package/src/yaml-schema.ts +28 -0
- package/tests/e2e/browser-auth.test.ts +25 -0
- package/tests/e2e/plugin-management.test.ts +137 -0
- package/tests/e2e/public-commands.test.ts +34 -1
- package/vitest.config.ts +19 -1
- package/.github/workflows/pkg-pr-new.yml +0 -30
- package/dist/clis/douban/shared.d.ts +0 -4
- package/dist/clis/douban/shared.js +0 -155
- package/src/clis/douban/shared.ts +0 -165
- /package/dist/clis/boss/{common.d.ts → utils.d.ts} +0 -0
- /package/dist/clis/boss/{common.js → utils.js} +0 -0
- /package/dist/clis/doubao/{common.d.ts → utils.d.ts} +0 -0
- /package/dist/clis/doubao/{common.js → utils.js} +0 -0
- /package/dist/clis/doubao-app/{common.d.ts → utils.d.ts} +0 -0
- /package/dist/clis/doubao-app/{common.js → utils.js} +0 -0
- /package/dist/clis/jike/{shared.d.ts → utils.d.ts} +0 -0
- /package/dist/clis/jike/{shared.js → utils.js} +0 -0
- /package/dist/clis/medium/{shared.d.ts → utils.d.ts} +0 -0
- /package/dist/clis/sinablog/{shared.d.ts → utils.d.ts} +0 -0
- /package/dist/clis/sinablog/{shared.js → utils.js} +0 -0
- /package/dist/clis/substack/{shared.d.ts → utils.d.ts} +0 -0
- /package/src/clis/boss/{common.ts → utils.ts} +0 -0
- /package/src/clis/doubao/{common.ts → utils.ts} +0 -0
- /package/src/clis/doubao-app/{common.ts → utils.ts} +0 -0
- /package/src/clis/jike/{shared.ts → utils.ts} +0 -0
- /package/src/clis/sinablog/{shared.ts → utils.ts} +0 -0
|
@@ -1,4 +1,33 @@
|
|
|
1
|
+
import { CommandExecutionError } from '../../errors.js';
|
|
1
2
|
import { cli, Strategy } from '../../registry.js';
|
|
3
|
+
/**
|
|
4
|
+
* Trigger Twitter search SPA navigation and retry once on transient failures.
|
|
5
|
+
*
|
|
6
|
+
* Twitter/X sometimes keeps the page on /explore for a short period even after
|
|
7
|
+
* pushState + popstate. A second attempt is enough for the intermittent cases
|
|
8
|
+
* reported in issue #353 while keeping the flow narrowly scoped.
|
|
9
|
+
*/
|
|
10
|
+
async function navigateToSearch(page, query) {
|
|
11
|
+
const searchUrl = JSON.stringify(`/search?q=${encodeURIComponent(query)}&f=top`);
|
|
12
|
+
let lastPath = '';
|
|
13
|
+
for (let attempt = 1; attempt <= 2; attempt++) {
|
|
14
|
+
await page.evaluate(`
|
|
15
|
+
(() => {
|
|
16
|
+
window.history.pushState({}, '', ${searchUrl});
|
|
17
|
+
window.dispatchEvent(new PopStateEvent('popstate', { state: {} }));
|
|
18
|
+
})()
|
|
19
|
+
`);
|
|
20
|
+
await page.wait(5);
|
|
21
|
+
lastPath = String(await page.evaluate('() => window.location.pathname') || '');
|
|
22
|
+
if (lastPath.startsWith('/search')) {
|
|
23
|
+
return;
|
|
24
|
+
}
|
|
25
|
+
if (attempt < 2) {
|
|
26
|
+
await page.wait(1);
|
|
27
|
+
}
|
|
28
|
+
}
|
|
29
|
+
throw new CommandExecutionError(`SPA navigation to /search failed. Final path: ${lastPath || '(empty)'}. Twitter may have changed its routing.`);
|
|
30
|
+
}
|
|
2
31
|
cli({
|
|
3
32
|
site: 'twitter',
|
|
4
33
|
name: 'search',
|
|
@@ -25,19 +54,7 @@ cli({
|
|
|
25
54
|
// a full page reload, so the interceptor stays alive.
|
|
26
55
|
// Note: the previous approach (nativeSetter + Enter keydown on the
|
|
27
56
|
// search input) does not reliably trigger Twitter's form submission.
|
|
28
|
-
|
|
29
|
-
await page.evaluate(`
|
|
30
|
-
(() => {
|
|
31
|
-
window.history.pushState({}, '', ${searchUrl});
|
|
32
|
-
window.dispatchEvent(new PopStateEvent('popstate', { state: {} }));
|
|
33
|
-
})()
|
|
34
|
-
`);
|
|
35
|
-
await page.wait(5);
|
|
36
|
-
// Verify SPA navigation succeeded
|
|
37
|
-
const currentPath = await page.evaluate('() => window.location.pathname');
|
|
38
|
-
if (!currentPath?.startsWith('/search')) {
|
|
39
|
-
throw new Error('SPA navigation to /search failed. Twitter may have changed its routing.');
|
|
40
|
-
}
|
|
57
|
+
await navigateToSearch(page, query);
|
|
41
58
|
// 4. Scroll to trigger additional pagination
|
|
42
59
|
await page.autoScroll({ times: 3, delayMs: 2000 });
|
|
43
60
|
// 6. Retrieve captured data
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
import './search.js';
|
|
@@ -0,0 +1,104 @@
|
|
|
1
|
+
import { describe, expect, it, vi } from 'vitest';
|
|
2
|
+
import { getRegistry } from '../../registry.js';
|
|
3
|
+
import './search.js';
|
|
4
|
+
describe('twitter search command', () => {
|
|
5
|
+
it('retries transient SPA navigation failures before giving up', async () => {
|
|
6
|
+
const command = getRegistry().get('twitter/search');
|
|
7
|
+
expect(command?.func).toBeTypeOf('function');
|
|
8
|
+
const evaluate = vi.fn()
|
|
9
|
+
.mockResolvedValueOnce(undefined)
|
|
10
|
+
.mockResolvedValueOnce('/explore')
|
|
11
|
+
.mockResolvedValueOnce(undefined)
|
|
12
|
+
.mockResolvedValueOnce('/search');
|
|
13
|
+
const page = {
|
|
14
|
+
goto: vi.fn().mockResolvedValue(undefined),
|
|
15
|
+
wait: vi.fn().mockResolvedValue(undefined),
|
|
16
|
+
installInterceptor: vi.fn().mockResolvedValue(undefined),
|
|
17
|
+
evaluate,
|
|
18
|
+
autoScroll: vi.fn().mockResolvedValue(undefined),
|
|
19
|
+
getInterceptedRequests: vi.fn().mockResolvedValue([
|
|
20
|
+
{
|
|
21
|
+
data: {
|
|
22
|
+
search_by_raw_query: {
|
|
23
|
+
search_timeline: {
|
|
24
|
+
timeline: {
|
|
25
|
+
instructions: [
|
|
26
|
+
{
|
|
27
|
+
type: 'TimelineAddEntries',
|
|
28
|
+
entries: [
|
|
29
|
+
{
|
|
30
|
+
entryId: 'tweet-1',
|
|
31
|
+
content: {
|
|
32
|
+
itemContent: {
|
|
33
|
+
tweet_results: {
|
|
34
|
+
result: {
|
|
35
|
+
rest_id: '1',
|
|
36
|
+
legacy: {
|
|
37
|
+
full_text: 'hello world',
|
|
38
|
+
favorite_count: 7,
|
|
39
|
+
},
|
|
40
|
+
core: {
|
|
41
|
+
user_results: {
|
|
42
|
+
result: {
|
|
43
|
+
core: {
|
|
44
|
+
screen_name: 'alice',
|
|
45
|
+
},
|
|
46
|
+
},
|
|
47
|
+
},
|
|
48
|
+
},
|
|
49
|
+
views: {
|
|
50
|
+
count: '12',
|
|
51
|
+
},
|
|
52
|
+
},
|
|
53
|
+
},
|
|
54
|
+
},
|
|
55
|
+
},
|
|
56
|
+
},
|
|
57
|
+
],
|
|
58
|
+
},
|
|
59
|
+
],
|
|
60
|
+
},
|
|
61
|
+
},
|
|
62
|
+
},
|
|
63
|
+
},
|
|
64
|
+
},
|
|
65
|
+
]),
|
|
66
|
+
};
|
|
67
|
+
const result = await command.func(page, { query: 'from:alice', limit: 5 });
|
|
68
|
+
expect(result).toEqual([
|
|
69
|
+
{
|
|
70
|
+
id: '1',
|
|
71
|
+
author: 'alice',
|
|
72
|
+
text: 'hello world',
|
|
73
|
+
likes: 7,
|
|
74
|
+
views: '12',
|
|
75
|
+
url: 'https://x.com/i/status/1',
|
|
76
|
+
},
|
|
77
|
+
]);
|
|
78
|
+
expect(page.installInterceptor).toHaveBeenCalledWith('SearchTimeline');
|
|
79
|
+
expect(evaluate).toHaveBeenCalledTimes(4);
|
|
80
|
+
});
|
|
81
|
+
it('throws with the final path after both attempts fail', async () => {
|
|
82
|
+
const command = getRegistry().get('twitter/search');
|
|
83
|
+
expect(command?.func).toBeTypeOf('function');
|
|
84
|
+
const evaluate = vi.fn()
|
|
85
|
+
.mockResolvedValueOnce(undefined)
|
|
86
|
+
.mockResolvedValueOnce('/explore')
|
|
87
|
+
.mockResolvedValueOnce(undefined)
|
|
88
|
+
.mockResolvedValueOnce('/login');
|
|
89
|
+
const page = {
|
|
90
|
+
goto: vi.fn().mockResolvedValue(undefined),
|
|
91
|
+
wait: vi.fn().mockResolvedValue(undefined),
|
|
92
|
+
installInterceptor: vi.fn().mockResolvedValue(undefined),
|
|
93
|
+
evaluate,
|
|
94
|
+
autoScroll: vi.fn().mockResolvedValue(undefined),
|
|
95
|
+
getInterceptedRequests: vi.fn(),
|
|
96
|
+
};
|
|
97
|
+
await expect(command.func(page, { query: 'from:alice', limit: 5 }))
|
|
98
|
+
.rejects
|
|
99
|
+
.toThrow('Final path: /login');
|
|
100
|
+
expect(page.autoScroll).not.toHaveBeenCalled();
|
|
101
|
+
expect(page.getInterceptedRequests).not.toHaveBeenCalled();
|
|
102
|
+
expect(evaluate).toHaveBeenCalledTimes(4);
|
|
103
|
+
});
|
|
104
|
+
});
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import { cli, Strategy } from '../../registry.js';
|
|
2
|
-
import { AuthRequiredError } from '../../errors.js';
|
|
2
|
+
import { AuthRequiredError, CommandExecutionError } from '../../errors.js';
|
|
3
3
|
// ── Twitter GraphQL constants ──────────────────────────────────────────
|
|
4
4
|
const BEARER_TOKEN = 'AAAAAAAAAAAAAAAAAAAAANRILgAAAAAAnNwIzUejRCOuH5E6I8xnZz4puTs%3D1Zv7ttfk8LF81IUq16cHjhLTvJu4FA33AGWWjCpTnA';
|
|
5
5
|
const TWEET_DETAIL_QUERY_ID = 'nBS-WpgA6ZG0CyNHD517JQ';
|
|
@@ -136,7 +136,7 @@ cli({
|
|
|
136
136
|
}`);
|
|
137
137
|
if (data?.error) {
|
|
138
138
|
if (allTweets.length === 0)
|
|
139
|
-
throw new
|
|
139
|
+
throw new CommandExecutionError(`HTTP ${data.error}: Tweet not found or queryId expired`);
|
|
140
140
|
break;
|
|
141
141
|
}
|
|
142
142
|
// TypeScript-side: type-safe parsing + cursor extraction
|
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import { AuthRequiredError, CommandExecutionError } from '../../errors.js';
|
|
1
2
|
import { cli, Strategy } from '../../registry.js';
|
|
2
3
|
// ── Twitter GraphQL constants ──────────────────────────────────────────
|
|
3
4
|
const BEARER_TOKEN = 'AAAAAAAAAAAAAAAAAAAAANRILgAAAAAAnNwIzUejRCOuH5E6I8xnZz4puTs%3D1Zv7ttfk8LF81IUq16cHjhLTvJu4FA33AGWWjCpTnA';
|
|
@@ -159,7 +160,7 @@ cli({
|
|
|
159
160
|
return document.cookie.split(';').map(c=>c.trim()).find(c=>c.startsWith('ct0='))?.split('=')[1] || null;
|
|
160
161
|
}`);
|
|
161
162
|
if (!ct0)
|
|
162
|
-
throw new
|
|
163
|
+
throw new AuthRequiredError('x.com', 'Not logged into x.com (no ct0 cookie)');
|
|
163
164
|
// Dynamically resolve queryId for the selected endpoint
|
|
164
165
|
const resolved = await page.evaluate(`async () => {
|
|
165
166
|
try {
|
|
@@ -195,7 +196,7 @@ cli({
|
|
|
195
196
|
}`);
|
|
196
197
|
if (data?.error) {
|
|
197
198
|
if (allTweets.length === 0)
|
|
198
|
-
throw new
|
|
199
|
+
throw new CommandExecutionError(`HTTP ${data.error}: Failed to fetch timeline. queryId may have expired.`);
|
|
199
200
|
break;
|
|
200
201
|
}
|
|
201
202
|
const { tweets, nextCursor } = parseHomeTimeline(data, seen);
|
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
import { cli, Strategy } from '../../registry.js';
|
|
2
|
+
import { AuthRequiredError, EmptyResultError } from '../../errors.js';
|
|
2
3
|
// ── Twitter GraphQL constants ──────────────────────────────────────────
|
|
3
4
|
const BEARER_TOKEN = 'AAAAAAAAAAAAAAAAAAAAANRILgAAAAAAnNwIzUejRCOuH5E6I8xnZz4puTs%3D1Zv7ttfk8LF81IUq16cHjhLTvJu4FA33AGWWjCpTnA';
|
|
4
5
|
// ── CLI definition ────────────────────────────────────────────────────
|
|
@@ -23,7 +24,7 @@ cli({
|
|
|
23
24
|
return document.cookie.split(';').map(c=>c.trim()).find(c=>c.startsWith('ct0='))?.split('=')[1] || null;
|
|
24
25
|
})()`);
|
|
25
26
|
if (!ct0)
|
|
26
|
-
throw new
|
|
27
|
+
throw new AuthRequiredError('x.com', 'Not logged into x.com (no ct0 cookie)');
|
|
27
28
|
// Try legacy guide.json API first (faster than DOM scraping)
|
|
28
29
|
let trends = [];
|
|
29
30
|
const apiData = await page.evaluate(`(async () => {
|
|
@@ -84,7 +85,7 @@ cli({
|
|
|
84
85
|
}
|
|
85
86
|
}
|
|
86
87
|
if (trends.length === 0) {
|
|
87
|
-
throw new
|
|
88
|
+
throw new EmptyResultError('twitter trending', 'API may have changed or login may be required.');
|
|
88
89
|
}
|
|
89
90
|
return trends.slice(0, limit);
|
|
90
91
|
},
|
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import { CommandExecutionError } from '../../errors.js';
|
|
1
2
|
import { cli, Strategy } from '../../registry.js';
|
|
2
3
|
cli({
|
|
3
4
|
site: 'twitter',
|
|
@@ -12,7 +13,7 @@ cli({
|
|
|
12
13
|
columns: ['status', 'message'],
|
|
13
14
|
func: async (page, kwargs) => {
|
|
14
15
|
if (!page)
|
|
15
|
-
throw new
|
|
16
|
+
throw new CommandExecutionError('Browser session required for twitter unblock');
|
|
16
17
|
const username = kwargs.username.replace(/^@/, '');
|
|
17
18
|
await page.goto(`https://x.com/${username}`);
|
|
18
19
|
await page.wait(5);
|
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import { CommandExecutionError } from '../../errors.js';
|
|
1
2
|
import { cli, Strategy } from '../../registry.js';
|
|
2
3
|
cli({
|
|
3
4
|
site: 'twitter',
|
|
@@ -12,7 +13,7 @@ cli({
|
|
|
12
13
|
columns: ['status', 'message'],
|
|
13
14
|
func: async (page, kwargs) => {
|
|
14
15
|
if (!page)
|
|
15
|
-
throw new
|
|
16
|
+
throw new CommandExecutionError('Browser session required for twitter unbookmark');
|
|
16
17
|
await page.goto(kwargs.url);
|
|
17
18
|
await page.wait(5);
|
|
18
19
|
const result = await page.evaluate(`(async () => {
|
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
import { cli, Strategy } from '../../registry.js';
|
|
2
|
+
import { CommandExecutionError } from '../../errors.js';
|
|
2
3
|
cli({
|
|
3
4
|
site: 'twitter',
|
|
4
5
|
name: 'unfollow',
|
|
@@ -12,7 +13,7 @@ cli({
|
|
|
12
13
|
columns: ['status', 'message'],
|
|
13
14
|
func: async (page, kwargs) => {
|
|
14
15
|
if (!page)
|
|
15
|
-
throw new
|
|
16
|
+
throw new CommandExecutionError('Browser session required for twitter unfollow');
|
|
16
17
|
const username = kwargs.username.replace(/^@/, '');
|
|
17
18
|
await page.goto(`https://x.com/${username}`);
|
|
18
19
|
await page.wait(5);
|
package/dist/clis/v2ex/daily.js
CHANGED
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
/**
|
|
2
2
|
* V2EX Daily Check-in adapter.
|
|
3
3
|
*/
|
|
4
|
+
import { CommandExecutionError } from '../../errors.js';
|
|
4
5
|
import { cli, Strategy } from '../../registry.js';
|
|
5
6
|
cli({
|
|
6
7
|
site: 'v2ex',
|
|
@@ -13,7 +14,7 @@ cli({
|
|
|
13
14
|
columns: ['status', 'message'],
|
|
14
15
|
func: async (page) => {
|
|
15
16
|
if (!page)
|
|
16
|
-
throw new
|
|
17
|
+
throw new CommandExecutionError('Browser page required');
|
|
17
18
|
if (process.env.OPENCLI_VERBOSE) {
|
|
18
19
|
console.error('[opencli:v2ex] Navigating to /mission/daily');
|
|
19
20
|
}
|
|
@@ -56,7 +57,7 @@ cli({
|
|
|
56
57
|
console.error(`[opencli:v2ex:debug] Page Title: ${checkResult.debug_title}`);
|
|
57
58
|
console.error(`[opencli:v2ex:debug] Page Body: ${checkResult.debug_body}`);
|
|
58
59
|
}
|
|
59
|
-
throw new
|
|
60
|
+
throw new CommandExecutionError(checkResult.error);
|
|
60
61
|
}
|
|
61
62
|
if (checkResult.claimed) {
|
|
62
63
|
return [{ status: '✅ 已签到', message: checkResult.message }];
|
package/dist/clis/v2ex/me.js
CHANGED
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
/**
|
|
2
2
|
* V2EX Me (Profile/Balance) adapter.
|
|
3
3
|
*/
|
|
4
|
+
import { CommandExecutionError } from '../../errors.js';
|
|
4
5
|
import { cli, Strategy } from '../../registry.js';
|
|
5
6
|
cli({
|
|
6
7
|
site: 'v2ex',
|
|
@@ -13,7 +14,7 @@ cli({
|
|
|
13
14
|
columns: ['username', 'balance', 'unread_notifications', 'daily_reward_ready'],
|
|
14
15
|
func: async (page) => {
|
|
15
16
|
if (!page)
|
|
16
|
-
throw new
|
|
17
|
+
throw new CommandExecutionError('Browser page required');
|
|
17
18
|
if (process.env.OPENCLI_VERBOSE) {
|
|
18
19
|
console.error('[opencli:v2ex] Navigating to /');
|
|
19
20
|
}
|
|
@@ -91,7 +92,7 @@ cli({
|
|
|
91
92
|
console.error(`[opencli:v2ex:debug] Page Title: ${data.debug_title}`);
|
|
92
93
|
console.error(`[opencli:v2ex:debug] Page Body: ${data.debug_body}`);
|
|
93
94
|
}
|
|
94
|
-
throw new
|
|
95
|
+
throw new CommandExecutionError(data.error);
|
|
95
96
|
}
|
|
96
97
|
return [data];
|
|
97
98
|
},
|
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
/**
|
|
2
2
|
* V2EX Notifications adapter.
|
|
3
3
|
*/
|
|
4
|
+
import { CommandExecutionError } from '../../errors.js';
|
|
4
5
|
import { cli, Strategy } from '../../registry.js';
|
|
5
6
|
cli({
|
|
6
7
|
site: 'v2ex',
|
|
@@ -15,7 +16,7 @@ cli({
|
|
|
15
16
|
columns: ['type', 'content', 'time'],
|
|
16
17
|
func: async (page, kwargs) => {
|
|
17
18
|
if (!page)
|
|
18
|
-
throw new
|
|
19
|
+
throw new CommandExecutionError('Browser page required');
|
|
19
20
|
if (process.env.OPENCLI_VERBOSE) {
|
|
20
21
|
console.error('[opencli:v2ex] Navigating to /notifications');
|
|
21
22
|
}
|
|
@@ -62,9 +63,8 @@ cli({
|
|
|
62
63
|
});
|
|
63
64
|
}
|
|
64
65
|
`);
|
|
65
|
-
if (!Array.isArray(data))
|
|
66
|
-
throw new
|
|
67
|
-
}
|
|
66
|
+
if (!Array.isArray(data))
|
|
67
|
+
throw new CommandExecutionError('Failed to parse notifications data');
|
|
68
68
|
const limit = kwargs.limit || 20;
|
|
69
69
|
return data.slice(0, limit);
|
|
70
70
|
},
|
|
@@ -0,0 +1,16 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Generic web page reader — fetch any URL and export as Markdown.
|
|
3
|
+
*
|
|
4
|
+
* Uses browser-side DOM heuristics to extract the main content:
|
|
5
|
+
* 1. <article> element
|
|
6
|
+
* 2. [role="main"] element
|
|
7
|
+
* 3. <main> element
|
|
8
|
+
* 4. Largest text-dense block as fallback
|
|
9
|
+
*
|
|
10
|
+
* Pipes through the shared article-download pipeline (Turndown + image download).
|
|
11
|
+
*
|
|
12
|
+
* Usage:
|
|
13
|
+
* opencli web read --url "https://www.anthropic.com/research/..." --output ./articles
|
|
14
|
+
* opencli web read --url "https://..." --download-images false
|
|
15
|
+
*/
|
|
16
|
+
export {};
|
|
@@ -0,0 +1,202 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Generic web page reader — fetch any URL and export as Markdown.
|
|
3
|
+
*
|
|
4
|
+
* Uses browser-side DOM heuristics to extract the main content:
|
|
5
|
+
* 1. <article> element
|
|
6
|
+
* 2. [role="main"] element
|
|
7
|
+
* 3. <main> element
|
|
8
|
+
* 4. Largest text-dense block as fallback
|
|
9
|
+
*
|
|
10
|
+
* Pipes through the shared article-download pipeline (Turndown + image download).
|
|
11
|
+
*
|
|
12
|
+
* Usage:
|
|
13
|
+
* opencli web read --url "https://www.anthropic.com/research/..." --output ./articles
|
|
14
|
+
* opencli web read --url "https://..." --download-images false
|
|
15
|
+
*/
|
|
16
|
+
import { cli, Strategy } from '../../registry.js';
|
|
17
|
+
import { downloadArticle } from '../../download/article-download.js';
|
|
18
|
+
cli({
|
|
19
|
+
site: 'web',
|
|
20
|
+
name: 'read',
|
|
21
|
+
description: 'Fetch any web page and export as Markdown',
|
|
22
|
+
strategy: Strategy.COOKIE,
|
|
23
|
+
navigateBefore: false, // we handle navigation ourselves
|
|
24
|
+
args: [
|
|
25
|
+
{ name: 'url', required: true, help: 'Any web page URL' },
|
|
26
|
+
{ name: 'output', default: './web-articles', help: 'Output directory' },
|
|
27
|
+
{ name: 'download-images', type: 'boolean', default: true, help: 'Download images locally' },
|
|
28
|
+
{ name: 'wait', type: 'int', default: 3, help: 'Seconds to wait after page load' },
|
|
29
|
+
],
|
|
30
|
+
columns: ['title', 'author', 'publish_time', 'status', 'size'],
|
|
31
|
+
func: async (page, kwargs) => {
|
|
32
|
+
const url = kwargs.url;
|
|
33
|
+
const waitSeconds = kwargs.wait ?? 3;
|
|
34
|
+
// Navigate to the target URL
|
|
35
|
+
await page.goto(url);
|
|
36
|
+
await page.wait(waitSeconds);
|
|
37
|
+
// Extract article content using browser-side heuristics
|
|
38
|
+
const data = await page.evaluate(`
|
|
39
|
+
(() => {
|
|
40
|
+
const result = {
|
|
41
|
+
title: '',
|
|
42
|
+
author: '',
|
|
43
|
+
publishTime: '',
|
|
44
|
+
contentHtml: '',
|
|
45
|
+
imageUrls: []
|
|
46
|
+
};
|
|
47
|
+
|
|
48
|
+
// --- Title extraction ---
|
|
49
|
+
// Priority: og:title > <title> > first <h1>
|
|
50
|
+
const ogTitle = document.querySelector('meta[property="og:title"]');
|
|
51
|
+
if (ogTitle) {
|
|
52
|
+
result.title = ogTitle.getAttribute('content')?.trim() || '';
|
|
53
|
+
}
|
|
54
|
+
if (!result.title) {
|
|
55
|
+
result.title = document.title?.trim() || '';
|
|
56
|
+
}
|
|
57
|
+
if (!result.title) {
|
|
58
|
+
const h1 = document.querySelector('h1');
|
|
59
|
+
result.title = h1?.textContent?.trim() || 'untitled';
|
|
60
|
+
}
|
|
61
|
+
// Strip site suffix (e.g. " | Anthropic", " - Blog")
|
|
62
|
+
result.title = result.title.replace(/\\s*[|\\-–—]\\s*[^|\\-–—]{1,30}$/, '').trim();
|
|
63
|
+
|
|
64
|
+
// --- Author extraction ---
|
|
65
|
+
const authorMeta = document.querySelector(
|
|
66
|
+
'meta[name="author"], meta[property="article:author"], meta[name="twitter:creator"]'
|
|
67
|
+
);
|
|
68
|
+
result.author = authorMeta?.getAttribute('content')?.trim() || '';
|
|
69
|
+
|
|
70
|
+
// --- Publish time extraction ---
|
|
71
|
+
const timeMeta = document.querySelector(
|
|
72
|
+
'meta[property="article:published_time"], meta[name="date"], meta[name="publishdate"], time[datetime]'
|
|
73
|
+
);
|
|
74
|
+
if (timeMeta) {
|
|
75
|
+
result.publishTime = timeMeta.getAttribute('content')
|
|
76
|
+
|| timeMeta.getAttribute('datetime')
|
|
77
|
+
|| timeMeta.textContent?.trim()
|
|
78
|
+
|| '';
|
|
79
|
+
}
|
|
80
|
+
|
|
81
|
+
// --- Content extraction ---
|
|
82
|
+
// Strategy: try semantic elements first, then fall back to largest text block
|
|
83
|
+
let contentEl = null;
|
|
84
|
+
|
|
85
|
+
// 1. <article>
|
|
86
|
+
const articles = document.querySelectorAll('article');
|
|
87
|
+
if (articles.length === 1) {
|
|
88
|
+
contentEl = articles[0];
|
|
89
|
+
} else if (articles.length > 1) {
|
|
90
|
+
// Pick the largest article by text length
|
|
91
|
+
let maxLen = 0;
|
|
92
|
+
articles.forEach(a => {
|
|
93
|
+
const len = a.textContent?.length || 0;
|
|
94
|
+
if (len > maxLen) { maxLen = len; contentEl = a; }
|
|
95
|
+
});
|
|
96
|
+
}
|
|
97
|
+
|
|
98
|
+
// 2. [role="main"]
|
|
99
|
+
if (!contentEl) {
|
|
100
|
+
contentEl = document.querySelector('[role="main"]');
|
|
101
|
+
}
|
|
102
|
+
|
|
103
|
+
// 3. <main>
|
|
104
|
+
if (!contentEl) {
|
|
105
|
+
contentEl = document.querySelector('main');
|
|
106
|
+
}
|
|
107
|
+
|
|
108
|
+
// 4. Largest text-dense block fallback
|
|
109
|
+
if (!contentEl) {
|
|
110
|
+
const candidates = document.querySelectorAll(
|
|
111
|
+
'div[class*="content"], div[class*="article"], div[class*="post"], ' +
|
|
112
|
+
'div[class*="entry"], div[class*="body"], div[id*="content"], ' +
|
|
113
|
+
'div[id*="article"], div[id*="post"], section'
|
|
114
|
+
);
|
|
115
|
+
let maxLen = 0;
|
|
116
|
+
candidates.forEach(c => {
|
|
117
|
+
const len = c.textContent?.length || 0;
|
|
118
|
+
if (len > maxLen) { maxLen = len; contentEl = c; }
|
|
119
|
+
});
|
|
120
|
+
}
|
|
121
|
+
|
|
122
|
+
// 5. Last resort: document.body
|
|
123
|
+
if (!contentEl || (contentEl.textContent?.length || 0) < 200) {
|
|
124
|
+
contentEl = document.body;
|
|
125
|
+
}
|
|
126
|
+
|
|
127
|
+
// Clean up noise elements before extraction
|
|
128
|
+
const clone = contentEl.cloneNode(true);
|
|
129
|
+
const noise = 'nav, header, footer, aside, .sidebar, .nav, .menu, .footer, ' +
|
|
130
|
+
'.header, .comments, .comment, .ad, .ads, .advertisement, .social-share, ' +
|
|
131
|
+
'.related-posts, .newsletter, .cookie-banner, script, style, noscript, iframe';
|
|
132
|
+
clone.querySelectorAll(noise).forEach(el => el.remove());
|
|
133
|
+
|
|
134
|
+
// Deduplicate: some sites (e.g. Anthropic) render each paragraph twice
|
|
135
|
+
// (a visible version + a line-broken animation version with missing spaces).
|
|
136
|
+
// Compare by stripping ALL whitespace so "Hello world" matches "Helloworld".
|
|
137
|
+
const stripWS = (s) => (s || '').replace(/\\s+/g, '');
|
|
138
|
+
const dedup = (parent) => {
|
|
139
|
+
const children = Array.from(parent.children || []);
|
|
140
|
+
for (let i = children.length - 1; i >= 1; i--) {
|
|
141
|
+
const curRaw = children[i].textContent || '';
|
|
142
|
+
const prevRaw = children[i - 1].textContent || '';
|
|
143
|
+
const cur = stripWS(curRaw);
|
|
144
|
+
const prev = stripWS(prevRaw);
|
|
145
|
+
if (cur.length < 20 || prev.length < 20) continue;
|
|
146
|
+
// Exact match after whitespace strip, or >90% overlap
|
|
147
|
+
if (cur === prev) {
|
|
148
|
+
// Keep the one with more proper spacing (more spaces = better formatted)
|
|
149
|
+
const curSpaces = (curRaw.match(/ /g) || []).length;
|
|
150
|
+
const prevSpaces = (prevRaw.match(/ /g) || []).length;
|
|
151
|
+
if (curSpaces >= prevSpaces) children[i - 1].remove();
|
|
152
|
+
else children[i].remove();
|
|
153
|
+
} else if (prev.includes(cur) && cur.length / prev.length > 0.8) {
|
|
154
|
+
children[i].remove();
|
|
155
|
+
} else if (cur.includes(prev) && prev.length / cur.length > 0.8) {
|
|
156
|
+
children[i - 1].remove();
|
|
157
|
+
}
|
|
158
|
+
}
|
|
159
|
+
};
|
|
160
|
+
dedup(clone);
|
|
161
|
+
clone.querySelectorAll('section, div').forEach(el => {
|
|
162
|
+
if (el.children && el.children.length > 2) dedup(el);
|
|
163
|
+
});
|
|
164
|
+
|
|
165
|
+
result.contentHtml = clone.innerHTML;
|
|
166
|
+
|
|
167
|
+
// --- Image extraction ---
|
|
168
|
+
const seen = new Set();
|
|
169
|
+
clone.querySelectorAll('img').forEach(img => {
|
|
170
|
+
const src = img.getAttribute('data-src')
|
|
171
|
+
|| img.getAttribute('data-original')
|
|
172
|
+
|| img.getAttribute('src');
|
|
173
|
+
if (src && !src.startsWith('data:') && !seen.has(src)) {
|
|
174
|
+
seen.add(src);
|
|
175
|
+
result.imageUrls.push(src);
|
|
176
|
+
}
|
|
177
|
+
});
|
|
178
|
+
|
|
179
|
+
return result;
|
|
180
|
+
})()
|
|
181
|
+
`);
|
|
182
|
+
// Determine Referer from URL for image downloads
|
|
183
|
+
let referer = '';
|
|
184
|
+
try {
|
|
185
|
+
const parsed = new URL(url);
|
|
186
|
+
referer = parsed.origin + '/';
|
|
187
|
+
}
|
|
188
|
+
catch { /* ignore */ }
|
|
189
|
+
return downloadArticle({
|
|
190
|
+
title: data?.title || 'untitled',
|
|
191
|
+
author: data?.author,
|
|
192
|
+
publishTime: data?.publishTime,
|
|
193
|
+
sourceUrl: url,
|
|
194
|
+
contentHtml: data?.contentHtml || '',
|
|
195
|
+
imageUrls: data?.imageUrls,
|
|
196
|
+
}, {
|
|
197
|
+
output: kwargs.output,
|
|
198
|
+
downloadImages: kwargs['download-images'],
|
|
199
|
+
imageHeaders: referer ? { Referer: referer } : undefined,
|
|
200
|
+
});
|
|
201
|
+
},
|
|
202
|
+
});
|
|
@@ -0,0 +1,55 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Shared helpers for Danjuan (蛋卷基金) adapters.
|
|
3
|
+
*
|
|
4
|
+
* Core design: a single page.evaluate call fetches the gain overview AND
|
|
5
|
+
* all per-account holdings in parallel (Promise.all), minimising Node↔Browser
|
|
6
|
+
* round-trips to exactly one.
|
|
7
|
+
*/
|
|
8
|
+
import type { IPage } from '../../types.js';
|
|
9
|
+
export declare const DANJUAN_DOMAIN = "danjuanfunds.com";
|
|
10
|
+
export declare const DANJUAN_ASSET_PAGE = "https://danjuanfunds.com/my-money";
|
|
11
|
+
export interface DanjuanAccount {
|
|
12
|
+
accountId: string;
|
|
13
|
+
accountName: string;
|
|
14
|
+
accountType: string;
|
|
15
|
+
accountCode: string;
|
|
16
|
+
marketValue: number | null;
|
|
17
|
+
dailyGain: number | null;
|
|
18
|
+
mainFlag: boolean;
|
|
19
|
+
}
|
|
20
|
+
export interface DanjuanHolding {
|
|
21
|
+
accountId: string;
|
|
22
|
+
accountName: string;
|
|
23
|
+
accountType: string;
|
|
24
|
+
fdCode: string;
|
|
25
|
+
fdName: string;
|
|
26
|
+
category: string;
|
|
27
|
+
marketValue: number | null;
|
|
28
|
+
volume: number | null;
|
|
29
|
+
usableRemainShare: number | null;
|
|
30
|
+
dailyGain: number | null;
|
|
31
|
+
holdGain: number | null;
|
|
32
|
+
holdGainRate: number | null;
|
|
33
|
+
totalGain: number | null;
|
|
34
|
+
nav: number | null;
|
|
35
|
+
marketPercent: number | null;
|
|
36
|
+
}
|
|
37
|
+
export interface DanjuanSnapshot {
|
|
38
|
+
asOf: string | null;
|
|
39
|
+
totalAssetAmount: number | null;
|
|
40
|
+
totalAssetDailyGain: number | null;
|
|
41
|
+
totalAssetHoldGain: number | null;
|
|
42
|
+
totalAssetTotalGain: number | null;
|
|
43
|
+
totalFundMarketValue: number | null;
|
|
44
|
+
accounts: DanjuanAccount[];
|
|
45
|
+
holdings: DanjuanHolding[];
|
|
46
|
+
}
|
|
47
|
+
/**
|
|
48
|
+
* Fetch the complete Danjuan fund picture in ONE browser round-trip.
|
|
49
|
+
*
|
|
50
|
+
* Inside the browser context we:
|
|
51
|
+
* 1. Fetch the gain/assets overview (contains account list)
|
|
52
|
+
* 2. Promise.all → fetch every account's holdings in parallel
|
|
53
|
+
* 3. Return the combined result to Node
|
|
54
|
+
*/
|
|
55
|
+
export declare function fetchDanjuanAll(page: IPage): Promise<DanjuanSnapshot>;
|