@jackwener/opencli 1.3.3 → 1.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.github/pull_request_template.md +3 -1
- package/.github/workflows/build-extension.yml +7 -1
- package/.github/workflows/ci.yml +29 -3
- package/.github/workflows/docs.yml +1 -1
- package/.github/workflows/e2e-headed.yml +20 -0
- package/.github/workflows/release.yml +1 -1
- package/.github/workflows/security.yml +0 -3
- package/CHANGELOG.md +55 -0
- package/CONTRIBUTING.md +6 -3
- package/README.md +30 -3
- package/README.zh-CN.md +30 -3
- package/SKILL.md +7 -1
- package/TESTING.md +1 -0
- package/chatwise-opencli.ps1 +82 -0
- package/dist/analysis.d.ts +38 -0
- package/dist/analysis.js +166 -0
- package/dist/browser/cdp.d.ts +0 -4
- package/dist/browser/cdp.js +53 -41
- package/dist/browser/cdp.test.d.ts +1 -0
- package/dist/browser/cdp.test.js +52 -0
- package/dist/browser/dom-snapshot.d.ts +2 -2
- package/dist/browser/dom-snapshot.js +54 -1
- package/dist/browser/dom-snapshot.test.js +36 -0
- package/dist/browser/index.d.ts +2 -2
- package/dist/browser/index.js +1 -1
- package/dist/browser/mcp.d.ts +0 -2
- package/dist/browser/mcp.js +2 -3
- package/dist/browser/page.d.ts +4 -3
- package/dist/browser/page.js +34 -37
- package/dist/browser/stealth.d.ts +0 -2
- package/dist/browser/stealth.js +24 -9
- package/dist/browser.test.js +2 -2
- package/dist/build-manifest.js +15 -9
- package/dist/build-manifest.test.js +12 -0
- package/dist/cascade.js +4 -2
- package/dist/cli-manifest.json +639 -258
- package/dist/cli.js +57 -29
- package/dist/clis/_shared/desktop-commands.d.ts +22 -0
- package/dist/clis/_shared/desktop-commands.js +108 -0
- package/dist/clis/antigravity/serve.js +5 -2
- package/dist/clis/arxiv/search.js +1 -1
- package/dist/clis/bilibili/dynamic.test.d.ts +1 -0
- package/dist/clis/bilibili/dynamic.test.js +68 -0
- package/dist/clis/bilibili/favorite.js +4 -2
- package/dist/clis/bilibili/following.js +3 -2
- package/dist/clis/bilibili/subtitle.js +8 -7
- package/dist/clis/bilibili/utils.js +2 -2
- package/dist/clis/boss/batchgreet.js +1 -1
- package/dist/clis/boss/chatlist.js +1 -1
- package/dist/clis/boss/chatmsg.js +1 -1
- package/dist/clis/boss/detail.js +1 -1
- package/dist/clis/boss/exchange.js +1 -1
- package/dist/clis/boss/greet.js +1 -1
- package/dist/clis/boss/invite.js +1 -1
- package/dist/clis/boss/joblist.js +1 -1
- package/dist/clis/boss/mark.js +4 -3
- package/dist/clis/boss/recommend.js +1 -1
- package/dist/clis/boss/resume.js +1 -1
- package/dist/clis/boss/search.js +1 -1
- package/dist/clis/boss/send.js +5 -4
- package/dist/clis/boss/stats.js +1 -1
- package/dist/clis/chatgpt/ask.js +4 -0
- package/dist/clis/chatgpt/new.js +5 -1
- package/dist/clis/chatgpt/read.js +5 -1
- package/dist/clis/chatgpt/send.js +2 -1
- package/dist/clis/chatgpt/status.js +5 -1
- package/dist/clis/chatwise/ask.js +8 -2
- package/dist/clis/chatwise/export.js +2 -0
- package/dist/clis/chatwise/history.js +2 -0
- package/dist/clis/chatwise/model.js +8 -3
- package/dist/clis/chatwise/new.js +3 -18
- package/dist/clis/chatwise/read.js +2 -0
- package/dist/clis/chatwise/screenshot.js +3 -27
- package/dist/clis/chatwise/send.js +8 -2
- package/dist/clis/chatwise/shared.d.ts +2 -0
- package/dist/clis/chatwise/shared.js +6 -0
- package/dist/clis/chatwise/status.js +3 -22
- package/dist/clis/codex/ask.js +6 -2
- package/dist/clis/codex/dump.js +2 -25
- package/dist/clis/codex/new.js +2 -25
- package/dist/clis/codex/screenshot.js +2 -27
- package/dist/clis/codex/send.js +6 -4
- package/dist/clis/codex/status.js +2 -22
- package/dist/clis/cursor/ask.js +2 -1
- package/dist/clis/cursor/composer.js +2 -1
- package/dist/clis/cursor/dump.js +2 -25
- package/dist/clis/cursor/new.js +2 -18
- package/dist/clis/cursor/read.js +2 -1
- package/dist/clis/cursor/screenshot.js +1 -30
- package/dist/clis/cursor/send.js +2 -1
- package/dist/clis/cursor/status.js +2 -21
- package/dist/clis/dictionary/examples.yaml +25 -0
- package/dist/clis/dictionary/search.yaml +27 -0
- package/dist/clis/dictionary/synonyms.yaml +25 -0
- package/dist/clis/douban/book-hot.js +1 -1
- package/dist/clis/douban/movie-hot.js +1 -1
- package/dist/clis/douban/search.js +1 -1
- package/dist/clis/douban/utils.d.ts +4 -1
- package/dist/clis/douban/utils.js +156 -1
- package/dist/clis/doubao/ask.js +1 -1
- package/dist/clis/doubao/new.js +1 -1
- package/dist/clis/doubao/read.js +1 -1
- package/dist/clis/doubao/send.js +1 -1
- package/dist/clis/doubao/status.js +1 -1
- package/dist/clis/doubao-app/ask.js +1 -1
- package/dist/clis/doubao-app/new.js +1 -1
- package/dist/clis/doubao-app/read.js +1 -1
- package/dist/clis/doubao-app/send.js +1 -1
- package/dist/clis/grok/ask.d.ts +4 -0
- package/dist/clis/grok/ask.js +28 -10
- package/dist/clis/grok/ask.test.js +18 -0
- package/dist/clis/jd/item.d.ts +1 -0
- package/dist/clis/jd/item.js +96 -0
- package/dist/clis/jd/item.test.d.ts +1 -0
- package/dist/clis/jd/item.test.js +28 -0
- package/dist/clis/jike/feed.js +1 -1
- package/dist/clis/jike/search.js +1 -1
- package/dist/clis/linkedin/search.js +5 -4
- package/dist/clis/linkedin/timeline.d.ts +21 -0
- package/dist/clis/linkedin/timeline.js +503 -0
- package/dist/clis/linkedin/timeline.test.d.ts +1 -0
- package/dist/clis/linkedin/timeline.test.js +81 -0
- package/dist/clis/medium/feed.js +1 -1
- package/dist/clis/medium/search.js +1 -1
- package/dist/clis/medium/user.js +1 -1
- package/dist/clis/medium/{shared.js → utils.js} +2 -1
- package/dist/clis/pixiv/detail.yaml +49 -0
- package/dist/clis/pixiv/download.d.ts +7 -0
- package/dist/clis/pixiv/download.js +78 -0
- package/dist/clis/pixiv/download.test.d.ts +1 -0
- package/dist/clis/pixiv/download.test.js +87 -0
- package/dist/clis/pixiv/illusts.d.ts +8 -0
- package/dist/clis/pixiv/illusts.js +65 -0
- package/dist/clis/pixiv/illusts.test.d.ts +1 -0
- package/dist/clis/pixiv/illusts.test.js +99 -0
- package/dist/clis/pixiv/ranking.yaml +53 -0
- package/dist/clis/pixiv/search.d.ts +6 -0
- package/dist/clis/pixiv/search.js +43 -0
- package/dist/clis/pixiv/search.test.d.ts +1 -0
- package/dist/clis/pixiv/search.test.js +83 -0
- package/dist/clis/pixiv/test-utils.d.ts +12 -0
- package/dist/clis/pixiv/test-utils.js +23 -0
- package/dist/clis/pixiv/user.yaml +46 -0
- package/dist/clis/pixiv/utils.d.ts +27 -0
- package/dist/clis/pixiv/utils.js +49 -0
- package/dist/clis/reddit/comment.js +2 -1
- package/dist/clis/reddit/read.js +4 -3
- package/dist/clis/reddit/read.test.d.ts +1 -0
- package/dist/clis/reddit/read.test.js +28 -0
- package/dist/clis/reddit/save.js +2 -1
- package/dist/clis/reddit/saved.js +7 -3
- package/dist/clis/reddit/subscribe.js +2 -1
- package/dist/clis/reddit/upvote.js +2 -1
- package/dist/clis/reddit/upvoted.js +7 -3
- package/dist/clis/sinablog/article.js +1 -1
- package/dist/clis/sinablog/hot.js +1 -1
- package/dist/clis/sinablog/user.js +1 -1
- package/dist/clis/substack/feed.js +1 -1
- package/dist/clis/substack/publication.js +1 -1
- package/dist/clis/substack/search.js +3 -2
- package/dist/clis/substack/{shared.js → utils.js} +3 -2
- package/dist/clis/tiktok/search.yaml +2 -1
- package/dist/clis/twitter/accept.js +2 -1
- package/dist/clis/twitter/article.js +4 -1
- package/dist/clis/twitter/block.js +2 -1
- package/dist/clis/twitter/bookmark.js +2 -1
- package/dist/clis/twitter/bookmarks.js +3 -2
- package/dist/clis/twitter/delete.js +2 -1
- package/dist/clis/twitter/follow.js +2 -1
- package/dist/clis/twitter/followers.js +3 -2
- package/dist/clis/twitter/following.js +3 -2
- package/dist/clis/twitter/hide-reply.js +2 -1
- package/dist/clis/twitter/like.js +2 -1
- package/dist/clis/twitter/notifications.js +2 -1
- package/dist/clis/twitter/post.js +2 -1
- package/dist/clis/twitter/profile.js +5 -2
- package/dist/clis/twitter/reply-dm.js +2 -1
- package/dist/clis/twitter/reply.js +2 -1
- package/dist/clis/twitter/search.js +30 -13
- package/dist/clis/twitter/search.test.d.ts +1 -0
- package/dist/clis/twitter/search.test.js +104 -0
- package/dist/clis/twitter/thread.js +2 -2
- package/dist/clis/twitter/timeline.js +3 -2
- package/dist/clis/twitter/trending.js +3 -2
- package/dist/clis/twitter/unblock.js +2 -1
- package/dist/clis/twitter/unbookmark.js +2 -1
- package/dist/clis/twitter/unfollow.js +2 -1
- package/dist/clis/v2ex/daily.js +3 -2
- package/dist/clis/v2ex/me.js +3 -2
- package/dist/clis/v2ex/notifications.js +4 -4
- package/dist/clis/web/read.d.ts +16 -0
- package/dist/clis/web/read.js +202 -0
- package/dist/clis/xueqiu/danjuan-utils.d.ts +55 -0
- package/dist/clis/xueqiu/danjuan-utils.js +126 -0
- package/dist/clis/xueqiu/danjuan-utils.test.d.ts +1 -0
- package/dist/clis/xueqiu/danjuan-utils.test.js +41 -0
- package/dist/clis/xueqiu/fund-holdings.d.ts +1 -0
- package/dist/clis/xueqiu/fund-holdings.js +28 -0
- package/dist/clis/xueqiu/fund-snapshot.d.ts +1 -0
- package/dist/clis/xueqiu/fund-snapshot.js +25 -0
- package/dist/clis/youtube/transcript.js +5 -4
- package/dist/clis/youtube/video.js +3 -2
- package/dist/daemon.js +7 -3
- package/dist/discovery.js +11 -10
- package/dist/doctor.js +2 -1
- package/dist/download/index.d.ts +4 -12
- package/dist/download/index.js +33 -12
- package/dist/download/index.test.js +79 -2
- package/dist/download/media-download.js +4 -2
- package/dist/engine.test.js +76 -4
- package/dist/execution.d.ts +1 -9
- package/dist/execution.js +56 -46
- package/dist/explore.js +12 -111
- package/dist/external-clis.yaml +0 -8
- package/dist/external.js +7 -5
- package/dist/external.test.js +4 -0
- package/dist/generate.d.ts +0 -9
- package/dist/generate.js +4 -20
- package/dist/hooks.d.ts +46 -0
- package/dist/hooks.js +56 -0
- package/dist/hooks.test.d.ts +4 -0
- package/dist/hooks.test.js +92 -0
- package/dist/interceptor.js +70 -23
- package/dist/main.js +2 -0
- package/dist/output.js +12 -6
- package/dist/pipeline/executor.js +1 -1
- package/dist/pipeline/steps/browser.js +1 -3
- package/dist/pipeline/steps/download.js +42 -26
- package/dist/pipeline/steps/download.test.d.ts +1 -0
- package/dist/pipeline/steps/download.test.js +101 -0
- package/dist/pipeline/steps/fetch.js +40 -22
- package/dist/pipeline/steps/fetch.test.d.ts +1 -0
- package/dist/pipeline/steps/fetch.test.js +123 -0
- package/dist/pipeline/steps/transform.js +2 -6
- package/dist/pipeline/template.js +66 -52
- package/dist/pipeline/template.test.js +28 -0
- package/dist/pipeline/transform.test.js +18 -0
- package/dist/plugin.d.ts +40 -1
- package/dist/plugin.js +214 -17
- package/dist/plugin.test.d.ts +1 -1
- package/dist/plugin.test.js +219 -3
- package/dist/record.js +6 -98
- package/dist/registry-api.d.ts +2 -0
- package/dist/registry-api.js +1 -0
- package/dist/registry.d.ts +5 -2
- package/dist/registry.js +1 -2
- package/dist/runtime.d.ts +0 -1
- package/dist/runtime.js +14 -4
- package/dist/snapshotFormatter.d.ts +7 -14
- package/dist/snapshotFormatter.js +38 -78
- package/dist/utils.d.ts +9 -0
- package/dist/utils.js +29 -0
- package/dist/validate.js +3 -5
- package/dist/yaml-schema.d.ts +26 -0
- package/dist/yaml-schema.js +5 -0
- package/docs/.vitepress/config.mts +3 -0
- package/docs/adapters/browser/dictionary.md +27 -0
- package/docs/adapters/browser/jd.md +27 -0
- package/docs/adapters/browser/linkedin.md +6 -0
- package/docs/adapters/browser/pixiv.md +92 -0
- package/docs/adapters/browser/web.md +30 -0
- package/docs/adapters/browser/xueqiu.md +27 -9
- package/docs/adapters/index.md +3 -1
- package/docs/comparison.md +125 -0
- package/docs/developer/contributing.md +21 -2
- package/docs/developer/testing.md +14 -8
- package/docs/developer/ts-adapter.md +18 -0
- package/docs/developer/yaml-adapter.md +16 -0
- package/docs/guide/plugins.md +10 -0
- package/docs/zh/guide/plugins.md +10 -0
- package/extension/dist/background.js +519 -444
- package/extension/manifest.json +1 -1
- package/extension/package.json +1 -1
- package/extension/src/background.test.ts +46 -1
- package/extension/src/background.ts +108 -33
- package/extension/src/cdp.ts +9 -9
- package/package.json +3 -2
- package/scripts/check-doc-coverage.sh +2 -0
- package/src/analysis.ts +170 -0
- package/src/browser/cdp.test.ts +66 -0
- package/src/browser/cdp.ts +59 -44
- package/src/browser/dom-snapshot.test.ts +42 -0
- package/src/browser/dom-snapshot.ts +56 -3
- package/src/browser/index.ts +2 -2
- package/src/browser/mcp.ts +2 -4
- package/src/browser/page.ts +34 -37
- package/src/browser/stealth.ts +24 -10
- package/src/browser.test.ts +2 -2
- package/src/build-manifest.test.ts +14 -0
- package/src/build-manifest.ts +13 -31
- package/src/cascade.ts +5 -3
- package/src/cli.ts +66 -34
- package/src/clis/_shared/desktop-commands.ts +121 -0
- package/src/clis/antigravity/serve.ts +6 -3
- package/src/clis/arxiv/search.ts +1 -1
- package/src/clis/bilibili/dynamic.test.ts +79 -0
- package/src/clis/bilibili/favorite.ts +5 -2
- package/src/clis/bilibili/following.ts +3 -2
- package/src/clis/bilibili/subtitle.ts +8 -7
- package/src/clis/bilibili/utils.ts +2 -2
- package/src/clis/boss/batchgreet.ts +1 -1
- package/src/clis/boss/chatlist.ts +1 -1
- package/src/clis/boss/chatmsg.ts +1 -1
- package/src/clis/boss/detail.ts +1 -1
- package/src/clis/boss/exchange.ts +1 -1
- package/src/clis/boss/greet.ts +1 -1
- package/src/clis/boss/invite.ts +1 -1
- package/src/clis/boss/joblist.ts +1 -1
- package/src/clis/boss/mark.ts +4 -3
- package/src/clis/boss/recommend.ts +1 -1
- package/src/clis/boss/resume.ts +1 -1
- package/src/clis/boss/search.ts +1 -1
- package/src/clis/boss/send.ts +5 -4
- package/src/clis/boss/stats.ts +1 -1
- package/src/clis/chatgpt/ask.ts +5 -0
- package/src/clis/chatgpt/new.ts +7 -2
- package/src/clis/chatgpt/read.ts +7 -2
- package/src/clis/chatgpt/send.ts +3 -2
- package/src/clis/chatgpt/status.ts +6 -1
- package/src/clis/chatwise/ask.ts +7 -2
- package/src/clis/chatwise/export.ts +2 -0
- package/src/clis/chatwise/history.ts +2 -0
- package/src/clis/chatwise/model.ts +7 -3
- package/src/clis/chatwise/new.ts +3 -20
- package/src/clis/chatwise/read.ts +2 -0
- package/src/clis/chatwise/screenshot.ts +3 -32
- package/src/clis/chatwise/send.ts +7 -2
- package/src/clis/chatwise/shared.ts +8 -0
- package/src/clis/chatwise/status.ts +3 -24
- package/src/clis/codex/ask.ts +5 -2
- package/src/clis/codex/dump.ts +2 -27
- package/src/clis/codex/new.ts +2 -28
- package/src/clis/codex/screenshot.ts +2 -32
- package/src/clis/codex/send.ts +5 -4
- package/src/clis/codex/status.ts +2 -24
- package/src/clis/cursor/ask.ts +2 -1
- package/src/clis/cursor/composer.ts +2 -1
- package/src/clis/cursor/dump.ts +2 -27
- package/src/clis/cursor/new.ts +2 -20
- package/src/clis/cursor/read.ts +2 -1
- package/src/clis/cursor/screenshot.ts +1 -36
- package/src/clis/cursor/send.ts +2 -1
- package/src/clis/cursor/status.ts +2 -22
- package/src/clis/dictionary/examples.yaml +25 -0
- package/src/clis/dictionary/search.yaml +27 -0
- package/src/clis/dictionary/synonyms.yaml +25 -0
- package/src/clis/douban/book-hot.ts +1 -1
- package/src/clis/douban/movie-hot.ts +1 -1
- package/src/clis/douban/search.ts +1 -1
- package/src/clis/douban/utils.ts +165 -1
- package/src/clis/doubao/ask.ts +1 -1
- package/src/clis/doubao/new.ts +1 -1
- package/src/clis/doubao/read.ts +1 -1
- package/src/clis/doubao/send.ts +1 -1
- package/src/clis/doubao/status.ts +1 -1
- package/src/clis/doubao-app/ask.ts +1 -1
- package/src/clis/doubao-app/new.ts +1 -1
- package/src/clis/doubao-app/read.ts +1 -1
- package/src/clis/doubao-app/send.ts +1 -1
- package/src/clis/grok/ask.test.ts +25 -0
- package/src/clis/grok/ask.ts +25 -12
- package/src/clis/jd/item.test.ts +35 -0
- package/src/clis/jd/item.ts +101 -0
- package/src/clis/jike/feed.ts +1 -1
- package/src/clis/jike/search.ts +1 -1
- package/src/clis/linkedin/search.ts +5 -4
- package/src/clis/linkedin/timeline.test.ts +99 -0
- package/src/clis/linkedin/timeline.ts +532 -0
- package/src/clis/medium/feed.ts +1 -1
- package/src/clis/medium/search.ts +1 -1
- package/src/clis/medium/user.ts +1 -1
- package/src/clis/medium/{shared.ts → utils.ts} +2 -1
- package/src/clis/pixiv/detail.yaml +49 -0
- package/src/clis/pixiv/download.test.ts +114 -0
- package/src/clis/pixiv/download.ts +91 -0
- package/src/clis/pixiv/illusts.test.ts +115 -0
- package/src/clis/pixiv/illusts.ts +78 -0
- package/src/clis/pixiv/ranking.yaml +53 -0
- package/src/clis/pixiv/search.test.ts +97 -0
- package/src/clis/pixiv/search.ts +53 -0
- package/src/clis/pixiv/test-utils.ts +29 -0
- package/src/clis/pixiv/user.yaml +46 -0
- package/src/clis/pixiv/utils.ts +62 -0
- package/src/clis/reddit/comment.ts +2 -1
- package/src/clis/reddit/read.test.ts +34 -0
- package/src/clis/reddit/read.ts +4 -3
- package/src/clis/reddit/save.ts +2 -1
- package/src/clis/reddit/saved.ts +6 -2
- package/src/clis/reddit/subscribe.ts +2 -1
- package/src/clis/reddit/upvote.ts +2 -1
- package/src/clis/reddit/upvoted.ts +6 -2
- package/src/clis/sinablog/article.ts +1 -1
- package/src/clis/sinablog/hot.ts +1 -1
- package/src/clis/sinablog/user.ts +1 -1
- package/src/clis/substack/feed.ts +1 -1
- package/src/clis/substack/publication.ts +1 -1
- package/src/clis/substack/search.ts +3 -2
- package/src/clis/substack/{shared.ts → utils.ts} +3 -2
- package/src/clis/tiktok/search.yaml +2 -1
- package/src/clis/twitter/accept.ts +2 -1
- package/src/clis/twitter/article.ts +3 -1
- package/src/clis/twitter/block.ts +2 -1
- package/src/clis/twitter/bookmark.ts +2 -1
- package/src/clis/twitter/bookmarks.ts +3 -2
- package/src/clis/twitter/delete.ts +2 -1
- package/src/clis/twitter/follow.ts +2 -1
- package/src/clis/twitter/followers.ts +3 -2
- package/src/clis/twitter/following.ts +3 -2
- package/src/clis/twitter/hide-reply.ts +2 -1
- package/src/clis/twitter/like.ts +2 -1
- package/src/clis/twitter/notifications.ts +2 -1
- package/src/clis/twitter/post.ts +2 -1
- package/src/clis/twitter/profile.ts +4 -2
- package/src/clis/twitter/reply-dm.ts +2 -1
- package/src/clis/twitter/reply.ts +2 -1
- package/src/clis/twitter/search.test.ts +113 -0
- package/src/clis/twitter/search.ts +38 -14
- package/src/clis/twitter/thread.ts +2 -2
- package/src/clis/twitter/timeline.ts +3 -2
- package/src/clis/twitter/trending.ts +3 -2
- package/src/clis/twitter/unblock.ts +2 -1
- package/src/clis/twitter/unbookmark.ts +2 -1
- package/src/clis/twitter/unfollow.ts +2 -1
- package/src/clis/v2ex/daily.ts +3 -2
- package/src/clis/v2ex/me.ts +3 -2
- package/src/clis/v2ex/notifications.ts +3 -4
- package/src/clis/web/read.ts +210 -0
- package/src/clis/xueqiu/danjuan-utils.test.ts +49 -0
- package/src/clis/xueqiu/danjuan-utils.ts +176 -0
- package/src/clis/xueqiu/fund-holdings.ts +32 -0
- package/src/clis/xueqiu/fund-snapshot.ts +27 -0
- package/src/clis/youtube/transcript.ts +5 -4
- package/src/clis/youtube/video.ts +3 -2
- package/src/daemon.ts +5 -4
- package/src/discovery.ts +12 -34
- package/src/doctor.ts +3 -2
- package/src/download/index.test.ts +93 -2
- package/src/download/index.ts +44 -23
- package/src/download/media-download.ts +5 -3
- package/src/engine.test.ts +84 -3
- package/src/execution.ts +62 -46
- package/src/explore.ts +21 -90
- package/src/external-clis.yaml +0 -8
- package/src/external.test.ts +9 -0
- package/src/external.ts +12 -10
- package/src/generate.ts +4 -41
- package/src/hooks.test.ts +126 -0
- package/src/hooks.ts +90 -0
- package/src/interceptor.ts +73 -23
- package/src/main.ts +2 -0
- package/src/output.ts +14 -6
- package/src/pipeline/executor.ts +1 -1
- package/src/pipeline/steps/browser.ts +1 -3
- package/src/pipeline/steps/download.test.ts +136 -0
- package/src/pipeline/steps/download.ts +47 -34
- package/src/pipeline/steps/fetch.test.ts +179 -0
- package/src/pipeline/steps/fetch.ts +39 -23
- package/src/pipeline/steps/transform.ts +2 -6
- package/src/pipeline/template.test.ts +28 -0
- package/src/pipeline/template.ts +67 -79
- package/src/pipeline/transform.test.ts +20 -0
- package/src/plugin.test.ts +251 -3
- package/src/plugin.ts +265 -21
- package/src/record.ts +12 -84
- package/src/registry-api.ts +2 -0
- package/src/registry.ts +7 -4
- package/src/runtime.ts +14 -4
- package/src/snapshotFormatter.ts +43 -121
- package/src/utils.ts +39 -0
- package/src/validate.ts +3 -5
- package/src/yaml-schema.ts +28 -0
- package/tests/e2e/browser-auth.test.ts +25 -0
- package/tests/e2e/plugin-management.test.ts +137 -0
- package/tests/e2e/public-commands.test.ts +34 -1
- package/vitest.config.ts +19 -1
- package/.github/workflows/pkg-pr-new.yml +0 -30
- package/dist/clis/douban/shared.d.ts +0 -4
- package/dist/clis/douban/shared.js +0 -155
- package/src/clis/douban/shared.ts +0 -165
- /package/dist/clis/boss/{common.d.ts → utils.d.ts} +0 -0
- /package/dist/clis/boss/{common.js → utils.js} +0 -0
- /package/dist/clis/doubao/{common.d.ts → utils.d.ts} +0 -0
- /package/dist/clis/doubao/{common.js → utils.js} +0 -0
- /package/dist/clis/doubao-app/{common.d.ts → utils.d.ts} +0 -0
- /package/dist/clis/doubao-app/{common.js → utils.js} +0 -0
- /package/dist/clis/jike/{shared.d.ts → utils.d.ts} +0 -0
- /package/dist/clis/jike/{shared.js → utils.js} +0 -0
- /package/dist/clis/medium/{shared.d.ts → utils.d.ts} +0 -0
- /package/dist/clis/sinablog/{shared.d.ts → utils.d.ts} +0 -0
- /package/dist/clis/sinablog/{shared.js → utils.js} +0 -0
- /package/dist/clis/substack/{shared.d.ts → utils.d.ts} +0 -0
- /package/src/clis/boss/{common.ts → utils.ts} +0 -0
- /package/src/clis/doubao/{common.ts → utils.ts} +0 -0
- /package/src/clis/doubao-app/{common.ts → utils.ts} +0 -0
- /package/src/clis/jike/{shared.ts → utils.ts} +0 -0
- /package/src/clis/sinablog/{shared.ts → utils.ts} +0 -0
|
@@ -0,0 +1,136 @@
|
|
|
1
|
+
import { beforeEach, describe, expect, it, vi } from 'vitest';
|
|
2
|
+
import * as os from 'node:os';
|
|
3
|
+
import * as path from 'node:path';
|
|
4
|
+
import type { IPage } from '../../types.js';
|
|
5
|
+
|
|
6
|
+
const { mockHttpDownload, mockYtdlpDownload, mockExportCookiesToNetscape } = vi.hoisted(() => ({
|
|
7
|
+
mockHttpDownload: vi.fn(),
|
|
8
|
+
mockYtdlpDownload: vi.fn(),
|
|
9
|
+
mockExportCookiesToNetscape: vi.fn(),
|
|
10
|
+
}));
|
|
11
|
+
|
|
12
|
+
vi.mock('../../download/index.js', async () => {
|
|
13
|
+
const actual = await vi.importActual<typeof import('../../download/index.js')>('../../download/index.js');
|
|
14
|
+
return {
|
|
15
|
+
...actual,
|
|
16
|
+
httpDownload: mockHttpDownload,
|
|
17
|
+
ytdlpDownload: mockYtdlpDownload,
|
|
18
|
+
exportCookiesToNetscape: mockExportCookiesToNetscape,
|
|
19
|
+
};
|
|
20
|
+
});
|
|
21
|
+
|
|
22
|
+
import { stepDownload } from './download.js';
|
|
23
|
+
|
|
24
|
+
function createMockPage(getCookies: IPage['getCookies']): IPage {
|
|
25
|
+
return {
|
|
26
|
+
goto: vi.fn(),
|
|
27
|
+
evaluate: vi.fn().mockResolvedValue(null),
|
|
28
|
+
getCookies,
|
|
29
|
+
snapshot: vi.fn().mockResolvedValue(''),
|
|
30
|
+
click: vi.fn(),
|
|
31
|
+
typeText: vi.fn(),
|
|
32
|
+
pressKey: vi.fn(),
|
|
33
|
+
scrollTo: vi.fn(),
|
|
34
|
+
getFormState: vi.fn().mockResolvedValue({}),
|
|
35
|
+
wait: vi.fn(),
|
|
36
|
+
tabs: vi.fn().mockResolvedValue([]),
|
|
37
|
+
closeTab: vi.fn(),
|
|
38
|
+
newTab: vi.fn(),
|
|
39
|
+
selectTab: vi.fn(),
|
|
40
|
+
networkRequests: vi.fn().mockResolvedValue([]),
|
|
41
|
+
consoleMessages: vi.fn().mockResolvedValue([]),
|
|
42
|
+
scroll: vi.fn(),
|
|
43
|
+
autoScroll: vi.fn(),
|
|
44
|
+
installInterceptor: vi.fn(),
|
|
45
|
+
getInterceptedRequests: vi.fn().mockResolvedValue([]),
|
|
46
|
+
screenshot: vi.fn().mockResolvedValue(''),
|
|
47
|
+
};
|
|
48
|
+
}
|
|
49
|
+
|
|
50
|
+
describe('stepDownload', () => {
|
|
51
|
+
beforeEach(() => {
|
|
52
|
+
mockHttpDownload.mockReset();
|
|
53
|
+
mockHttpDownload.mockResolvedValue({ success: true, size: 2 });
|
|
54
|
+
mockYtdlpDownload.mockReset();
|
|
55
|
+
mockYtdlpDownload.mockResolvedValue({ success: true, size: 2 });
|
|
56
|
+
mockExportCookiesToNetscape.mockReset();
|
|
57
|
+
});
|
|
58
|
+
|
|
59
|
+
it('scopes browser cookies to each direct-download target domain', async () => {
|
|
60
|
+
const page = createMockPage(vi.fn().mockImplementation(async (opts?: { domain?: string }) => {
|
|
61
|
+
const domain = opts?.domain ?? 'unknown';
|
|
62
|
+
return [{ name: 'sid', value: domain, domain }];
|
|
63
|
+
}));
|
|
64
|
+
|
|
65
|
+
await stepDownload(
|
|
66
|
+
page,
|
|
67
|
+
{
|
|
68
|
+
url: '${{ item.url }}',
|
|
69
|
+
dir: path.join(os.tmpdir(), 'opencli-download-test'),
|
|
70
|
+
filename: '${{ index }}.txt',
|
|
71
|
+
progress: false,
|
|
72
|
+
concurrency: 1,
|
|
73
|
+
},
|
|
74
|
+
[
|
|
75
|
+
{ url: 'https://a.example/file-1.txt' },
|
|
76
|
+
{ url: 'https://b.example/file-2.txt' },
|
|
77
|
+
],
|
|
78
|
+
{},
|
|
79
|
+
);
|
|
80
|
+
|
|
81
|
+
expect(mockHttpDownload).toHaveBeenNthCalledWith(
|
|
82
|
+
1,
|
|
83
|
+
'https://a.example/file-1.txt',
|
|
84
|
+
path.join(os.tmpdir(), 'opencli-download-test', '0.txt'),
|
|
85
|
+
expect.objectContaining({ cookies: 'sid=a.example' }),
|
|
86
|
+
);
|
|
87
|
+
expect(mockHttpDownload).toHaveBeenNthCalledWith(
|
|
88
|
+
2,
|
|
89
|
+
'https://b.example/file-2.txt',
|
|
90
|
+
path.join(os.tmpdir(), 'opencli-download-test', '1.txt'),
|
|
91
|
+
expect.objectContaining({ cookies: 'sid=b.example' }),
|
|
92
|
+
);
|
|
93
|
+
});
|
|
94
|
+
|
|
95
|
+
it('builds yt-dlp cookies from all target domains instead of only the first item', async () => {
|
|
96
|
+
const getCookies = vi.fn().mockImplementation(async (opts?: { domain?: string }) => {
|
|
97
|
+
const domain = opts?.domain ?? 'unknown';
|
|
98
|
+
return [{
|
|
99
|
+
name: `sid-${domain}`,
|
|
100
|
+
value: domain,
|
|
101
|
+
domain,
|
|
102
|
+
path: '/',
|
|
103
|
+
secure: false,
|
|
104
|
+
httpOnly: false,
|
|
105
|
+
}];
|
|
106
|
+
});
|
|
107
|
+
const page = createMockPage(getCookies);
|
|
108
|
+
|
|
109
|
+
await stepDownload(
|
|
110
|
+
page,
|
|
111
|
+
{
|
|
112
|
+
url: '${{ item.url }}',
|
|
113
|
+
dir: '/tmp/opencli-download-test',
|
|
114
|
+
filename: '${{ index }}.mp4',
|
|
115
|
+
progress: false,
|
|
116
|
+
concurrency: 1,
|
|
117
|
+
},
|
|
118
|
+
[
|
|
119
|
+
{ url: 'https://www.youtube.com/watch?v=one' },
|
|
120
|
+
{ url: 'https://www.bilibili.com/video/BV1xx411c7mD' },
|
|
121
|
+
],
|
|
122
|
+
{},
|
|
123
|
+
);
|
|
124
|
+
|
|
125
|
+
expect(getCookies).toHaveBeenCalledWith({ domain: 'www.youtube.com' });
|
|
126
|
+
expect(getCookies).toHaveBeenCalledWith({ domain: 'www.bilibili.com' });
|
|
127
|
+
expect(mockExportCookiesToNetscape).toHaveBeenCalledWith(
|
|
128
|
+
expect.arrayContaining([
|
|
129
|
+
expect.objectContaining({ name: 'sid-www.youtube.com', domain: 'www.youtube.com' }),
|
|
130
|
+
expect.objectContaining({ name: 'sid-www.bilibili.com', domain: 'www.bilibili.com' }),
|
|
131
|
+
]),
|
|
132
|
+
expect.any(String),
|
|
133
|
+
);
|
|
134
|
+
expect(mockYtdlpDownload).toHaveBeenCalledTimes(2);
|
|
135
|
+
});
|
|
136
|
+
});
|
|
@@ -13,6 +13,7 @@ import * as path from 'node:path';
|
|
|
13
13
|
import * as os from 'node:os';
|
|
14
14
|
import type { IPage } from '../../types.js';
|
|
15
15
|
import { render } from '../template.js';
|
|
16
|
+
import { getErrorMessage } from '../../errors.js';
|
|
16
17
|
import {
|
|
17
18
|
httpDownload,
|
|
18
19
|
ytdlpDownload,
|
|
@@ -26,6 +27,7 @@ import {
|
|
|
26
27
|
formatCookieHeader,
|
|
27
28
|
} from '../../download/index.js';
|
|
28
29
|
import { DownloadProgressTracker, formatBytes } from '../../download/progress.js';
|
|
30
|
+
import { mapConcurrent } from '../../utils.js';
|
|
29
31
|
|
|
30
32
|
export interface DownloadResult {
|
|
31
33
|
status: 'success' | 'skipped' | 'failed';
|
|
@@ -35,35 +37,14 @@ export interface DownloadResult {
|
|
|
35
37
|
duration?: number;
|
|
36
38
|
}
|
|
37
39
|
|
|
38
|
-
/**
|
|
39
|
-
* Simple async concurrency limiter for downloads.
|
|
40
|
-
*/
|
|
41
|
-
async function mapConcurrent<T, R>(
|
|
42
|
-
items: T[],
|
|
43
|
-
limit: number,
|
|
44
|
-
fn: (item: T, index: number) => Promise<R>,
|
|
45
|
-
): Promise<R[]> {
|
|
46
|
-
const results: R[] = new Array(items.length);
|
|
47
|
-
let index = 0;
|
|
48
|
-
|
|
49
|
-
async function worker() {
|
|
50
|
-
while (index < items.length) {
|
|
51
|
-
const i = index++;
|
|
52
|
-
results[i] = await fn(items[i], i);
|
|
53
|
-
}
|
|
54
|
-
}
|
|
55
40
|
|
|
56
|
-
const workers = Array.from({ length: Math.min(limit, items.length) }, () => worker());
|
|
57
|
-
await Promise.all(workers);
|
|
58
|
-
return results;
|
|
59
|
-
}
|
|
60
41
|
|
|
61
42
|
/**
|
|
62
43
|
* Extract cookies from browser page.
|
|
63
44
|
*/
|
|
64
|
-
async function extractBrowserCookies(page: IPage, domain
|
|
45
|
+
async function extractBrowserCookies(page: IPage, domain: string): Promise<string> {
|
|
65
46
|
try {
|
|
66
|
-
const cookies = await page.getCookies(
|
|
47
|
+
const cookies = await page.getCookies({ domain });
|
|
67
48
|
return formatCookieHeader(cookies);
|
|
68
49
|
} catch {
|
|
69
50
|
return '';
|
|
@@ -94,6 +75,16 @@ async function extractCookiesArray(
|
|
|
94
75
|
}
|
|
95
76
|
}
|
|
96
77
|
|
|
78
|
+
function dedupeCookies(
|
|
79
|
+
cookies: Array<{ name: string; value: string; domain: string; path: string; secure: boolean; httpOnly: boolean }>,
|
|
80
|
+
): Array<{ name: string; value: string; domain: string; path: string; secure: boolean; httpOnly: boolean }> {
|
|
81
|
+
const deduped = new Map<string, { name: string; value: string; domain: string; path: string; secure: boolean; httpOnly: boolean }>();
|
|
82
|
+
for (const cookie of cookies) {
|
|
83
|
+
deduped.set(`${cookie.domain}\t${cookie.path}\t${cookie.name}`, cookie);
|
|
84
|
+
}
|
|
85
|
+
return [...deduped.values()];
|
|
86
|
+
}
|
|
87
|
+
|
|
97
88
|
/**
|
|
98
89
|
* Download step handler for YAML pipelines.
|
|
99
90
|
*
|
|
@@ -143,23 +134,29 @@ export async function stepDownload(
|
|
|
143
134
|
// Create progress tracker
|
|
144
135
|
const tracker = new DownloadProgressTracker(items.length, showProgress);
|
|
145
136
|
|
|
146
|
-
//
|
|
147
|
-
|
|
137
|
+
// Cache cookie lookups per domain so mixed-domain batches stay isolated without repeated browser calls.
|
|
138
|
+
const cookieHeaderCache = new Map<string, Promise<string>>();
|
|
148
139
|
let cookiesFile: string | undefined;
|
|
149
140
|
|
|
150
141
|
if (page) {
|
|
151
|
-
cookies = await extractBrowserCookies(page);
|
|
152
|
-
|
|
153
142
|
// For yt-dlp, we need to export cookies to Netscape format
|
|
154
143
|
if (useYtdlp || items.some((item, index) => {
|
|
155
144
|
const url = String(render(urlTemplate, { args, data, item, index }));
|
|
156
145
|
return requiresYtdlp(url);
|
|
157
146
|
})) {
|
|
158
147
|
try {
|
|
159
|
-
|
|
160
|
-
|
|
161
|
-
|
|
162
|
-
|
|
148
|
+
const ytdlpDomains = [...new Set(items.flatMap((item, index) => {
|
|
149
|
+
const url = String(render(urlTemplate, { args, data, item, index }));
|
|
150
|
+
if (!useYtdlp && !requiresYtdlp(url)) return [];
|
|
151
|
+
try {
|
|
152
|
+
return [new URL(url).hostname];
|
|
153
|
+
} catch {
|
|
154
|
+
return [];
|
|
155
|
+
}
|
|
156
|
+
}))];
|
|
157
|
+
const cookiesArray = dedupeCookies(
|
|
158
|
+
(await Promise.all(ytdlpDomains.map((domain) => extractCookiesArray(page, domain)))).flat(),
|
|
159
|
+
);
|
|
163
160
|
|
|
164
161
|
if (cookiesArray.length > 0) {
|
|
165
162
|
const tempDir = getTempDir();
|
|
@@ -254,6 +251,21 @@ export async function stepDownload(
|
|
|
254
251
|
}
|
|
255
252
|
} else {
|
|
256
253
|
// Direct HTTP download
|
|
254
|
+
let cookies = '';
|
|
255
|
+
if (page) {
|
|
256
|
+
try {
|
|
257
|
+
const targetDomain = new URL(url).hostname;
|
|
258
|
+
let cookiePromise = cookieHeaderCache.get(targetDomain);
|
|
259
|
+
if (!cookiePromise) {
|
|
260
|
+
cookiePromise = extractBrowserCookies(page, targetDomain);
|
|
261
|
+
cookieHeaderCache.set(targetDomain, cookiePromise);
|
|
262
|
+
}
|
|
263
|
+
cookies = await cookiePromise;
|
|
264
|
+
} catch {
|
|
265
|
+
cookies = '';
|
|
266
|
+
}
|
|
267
|
+
}
|
|
268
|
+
|
|
257
269
|
result = await httpDownload(url, destPath, {
|
|
258
270
|
cookies,
|
|
259
271
|
timeout,
|
|
@@ -268,10 +280,11 @@ export async function stepDownload(
|
|
|
268
280
|
progressBar.complete(result.success, result.success ? formatBytes(result.size) : undefined);
|
|
269
281
|
}
|
|
270
282
|
}
|
|
271
|
-
} catch (err
|
|
272
|
-
|
|
283
|
+
} catch (err) {
|
|
284
|
+
const msg = getErrorMessage(err);
|
|
285
|
+
result = { success: false, size: 0, error: msg };
|
|
273
286
|
if (progressBar) {
|
|
274
|
-
progressBar.fail(
|
|
287
|
+
progressBar.fail(msg);
|
|
275
288
|
}
|
|
276
289
|
}
|
|
277
290
|
|
|
@@ -0,0 +1,179 @@
|
|
|
1
|
+
import { afterEach, describe, expect, it, vi } from 'vitest';
|
|
2
|
+
import { CliError } from '../../errors.js';
|
|
3
|
+
import type { IPage } from '../../types.js';
|
|
4
|
+
import { stepFetch } from './fetch.js';
|
|
5
|
+
|
|
6
|
+
afterEach(() => {
|
|
7
|
+
vi.restoreAllMocks();
|
|
8
|
+
vi.unstubAllGlobals();
|
|
9
|
+
});
|
|
10
|
+
|
|
11
|
+
describe('stepFetch', () => {
|
|
12
|
+
// W1 + W4: non-browser single fetch throws CliError with FETCH_ERROR code and full message
|
|
13
|
+
it('throws CliError with FETCH_ERROR code on non-ok responses without a browser session', async () => {
|
|
14
|
+
const jsonMock = vi.fn().mockResolvedValue({ error: 'rate limited' });
|
|
15
|
+
const fetchMock = vi.fn().mockResolvedValue({
|
|
16
|
+
ok: false,
|
|
17
|
+
status: 429,
|
|
18
|
+
statusText: 'Too Many Requests',
|
|
19
|
+
json: jsonMock,
|
|
20
|
+
});
|
|
21
|
+
vi.stubGlobal('fetch', fetchMock);
|
|
22
|
+
|
|
23
|
+
const err = await stepFetch(null, { url: 'https://api.example.com/items' }, null, {}).catch((e: unknown) => e);
|
|
24
|
+
expect(err).toBeInstanceOf(CliError);
|
|
25
|
+
expect((err as CliError).code).toBe('FETCH_ERROR');
|
|
26
|
+
expect((err as CliError).message).toBe('HTTP 429 Too Many Requests from https://api.example.com/items');
|
|
27
|
+
expect(jsonMock).not.toHaveBeenCalled();
|
|
28
|
+
});
|
|
29
|
+
|
|
30
|
+
// W1 + W3: browser single fetch returns error status from evaluate, outer code throws CliError
|
|
31
|
+
it('throws CliError with FETCH_ERROR code on non-ok responses inside the browser session', async () => {
|
|
32
|
+
const jsonMock = vi.fn().mockResolvedValue({ error: 'auth required' });
|
|
33
|
+
const fetchMock = vi.fn().mockResolvedValue({
|
|
34
|
+
ok: false,
|
|
35
|
+
status: 401,
|
|
36
|
+
statusText: 'Unauthorized',
|
|
37
|
+
json: jsonMock,
|
|
38
|
+
});
|
|
39
|
+
vi.stubGlobal('fetch', fetchMock);
|
|
40
|
+
|
|
41
|
+
// Simulate real CDP behavior: evaluate returns a value, errors are thrown outside
|
|
42
|
+
const page = {
|
|
43
|
+
evaluate: vi.fn(async (js: string) => Function(`return (${js})`)()()),
|
|
44
|
+
} as unknown as IPage;
|
|
45
|
+
|
|
46
|
+
const err = await stepFetch(page, { url: 'https://api.example.com/items' }, null, {}).catch((e: unknown) => e);
|
|
47
|
+
expect(err).toBeInstanceOf(CliError);
|
|
48
|
+
expect((err as CliError).code).toBe('FETCH_ERROR');
|
|
49
|
+
expect((err as CliError).message).toBe('HTTP 401 Unauthorized from https://api.example.com/items');
|
|
50
|
+
expect(jsonMock).not.toHaveBeenCalled();
|
|
51
|
+
});
|
|
52
|
+
|
|
53
|
+
it('returns per-item HTTP errors for batch fetches without a browser session', async () => {
|
|
54
|
+
const jsonMock = vi.fn().mockResolvedValue({ error: 'upstream unavailable' });
|
|
55
|
+
const fetchMock = vi.fn().mockResolvedValue({
|
|
56
|
+
ok: false,
|
|
57
|
+
status: 503,
|
|
58
|
+
statusText: 'Service Unavailable',
|
|
59
|
+
json: jsonMock,
|
|
60
|
+
});
|
|
61
|
+
vi.stubGlobal('fetch', fetchMock);
|
|
62
|
+
|
|
63
|
+
await expect(stepFetch(
|
|
64
|
+
null,
|
|
65
|
+
{ url: 'https://api.example.com/items/${{ item.id }}' },
|
|
66
|
+
[{ id: 1 }],
|
|
67
|
+
{},
|
|
68
|
+
)).resolves.toEqual([
|
|
69
|
+
{ error: 'HTTP 503 Service Unavailable from https://api.example.com/items/1' },
|
|
70
|
+
]);
|
|
71
|
+
expect(jsonMock).not.toHaveBeenCalled();
|
|
72
|
+
});
|
|
73
|
+
|
|
74
|
+
it('returns per-item HTTP errors for batch browser fetches', async () => {
|
|
75
|
+
const jsonMock = vi.fn().mockResolvedValue({ error: 'upstream unavailable' });
|
|
76
|
+
const fetchMock = vi.fn().mockResolvedValue({
|
|
77
|
+
ok: false,
|
|
78
|
+
status: 503,
|
|
79
|
+
statusText: 'Service Unavailable',
|
|
80
|
+
json: jsonMock,
|
|
81
|
+
});
|
|
82
|
+
vi.stubGlobal('fetch', fetchMock);
|
|
83
|
+
|
|
84
|
+
const page = {
|
|
85
|
+
evaluate: vi.fn(async (js: string) => Function(`return (${js})`)()()),
|
|
86
|
+
} as unknown as IPage;
|
|
87
|
+
|
|
88
|
+
await expect(stepFetch(
|
|
89
|
+
page,
|
|
90
|
+
{ url: 'https://api.example.com/items/${{ item.id }}' },
|
|
91
|
+
[{ id: 1 }],
|
|
92
|
+
{},
|
|
93
|
+
)).resolves.toEqual([
|
|
94
|
+
{ error: 'HTTP 503 Service Unavailable from https://api.example.com/items/1' },
|
|
95
|
+
]);
|
|
96
|
+
expect(jsonMock).not.toHaveBeenCalled();
|
|
97
|
+
});
|
|
98
|
+
|
|
99
|
+
it('stringifies non-Error batch browser failures consistently', async () => {
|
|
100
|
+
vi.stubGlobal('fetch', vi.fn().mockRejectedValue('socket hang up'));
|
|
101
|
+
|
|
102
|
+
const page = {
|
|
103
|
+
evaluate: vi.fn(async (js: string) => Function(`return (${js})`)()()),
|
|
104
|
+
} as unknown as IPage;
|
|
105
|
+
|
|
106
|
+
await expect(stepFetch(
|
|
107
|
+
page,
|
|
108
|
+
{ url: 'https://api.example.com/items/${{ item.id }}' },
|
|
109
|
+
[{ id: 1 }],
|
|
110
|
+
{},
|
|
111
|
+
)).resolves.toEqual([
|
|
112
|
+
{ error: 'socket hang up' },
|
|
113
|
+
]);
|
|
114
|
+
});
|
|
115
|
+
|
|
116
|
+
it('stringifies non-Error batch non-browser failures consistently', async () => {
|
|
117
|
+
vi.stubGlobal('fetch', vi.fn().mockRejectedValue('socket hang up'));
|
|
118
|
+
|
|
119
|
+
await expect(stepFetch(
|
|
120
|
+
null,
|
|
121
|
+
{ url: 'https://api.example.com/items/${{ item.id }}' },
|
|
122
|
+
[{ id: 1 }],
|
|
123
|
+
{},
|
|
124
|
+
)).resolves.toEqual([
|
|
125
|
+
{ error: 'socket hang up' },
|
|
126
|
+
]);
|
|
127
|
+
});
|
|
128
|
+
|
|
129
|
+
// W2: batch item failures emit a warning log
|
|
130
|
+
it('logs a warning for each failed batch item in non-browser mode', async () => {
|
|
131
|
+
const { log } = await import('../../logger.js');
|
|
132
|
+
const warnSpy = vi.spyOn(log, 'warn');
|
|
133
|
+
|
|
134
|
+
vi.stubGlobal('fetch', vi.fn().mockResolvedValue({
|
|
135
|
+
ok: false,
|
|
136
|
+
status: 503,
|
|
137
|
+
statusText: 'Service Unavailable',
|
|
138
|
+
json: vi.fn(),
|
|
139
|
+
}));
|
|
140
|
+
|
|
141
|
+
await stepFetch(
|
|
142
|
+
null,
|
|
143
|
+
{ url: 'https://api.example.com/items/${{ item.id }}' },
|
|
144
|
+
[{ id: 1 }, { id: 2 }],
|
|
145
|
+
{},
|
|
146
|
+
);
|
|
147
|
+
|
|
148
|
+
expect(warnSpy).toHaveBeenCalledTimes(2);
|
|
149
|
+
expect(warnSpy).toHaveBeenCalledWith(expect.stringContaining('https://api.example.com/items/1'));
|
|
150
|
+
expect(warnSpy).toHaveBeenCalledWith(expect.stringContaining('https://api.example.com/items/2'));
|
|
151
|
+
});
|
|
152
|
+
|
|
153
|
+
it('logs a warning for each failed batch item in browser mode', async () => {
|
|
154
|
+
const { log } = await import('../../logger.js');
|
|
155
|
+
const warnSpy = vi.spyOn(log, 'warn');
|
|
156
|
+
|
|
157
|
+
vi.stubGlobal('fetch', vi.fn().mockResolvedValue({
|
|
158
|
+
ok: false,
|
|
159
|
+
status: 502,
|
|
160
|
+
statusText: 'Bad Gateway',
|
|
161
|
+
json: vi.fn(),
|
|
162
|
+
}));
|
|
163
|
+
|
|
164
|
+
const page = {
|
|
165
|
+
evaluate: vi.fn(async (js: string) => Function(`return (${js})`)()()),
|
|
166
|
+
} as unknown as IPage;
|
|
167
|
+
|
|
168
|
+
await stepFetch(
|
|
169
|
+
page,
|
|
170
|
+
{ url: 'https://api.example.com/items/${{ item.id }}' },
|
|
171
|
+
[{ id: 1 }, { id: 2 }],
|
|
172
|
+
{},
|
|
173
|
+
);
|
|
174
|
+
|
|
175
|
+
expect(warnSpy).toHaveBeenCalledTimes(2);
|
|
176
|
+
expect(warnSpy).toHaveBeenCalledWith(expect.stringContaining('https://api.example.com/items/1'));
|
|
177
|
+
expect(warnSpy).toHaveBeenCalledWith(expect.stringContaining('https://api.example.com/items/2'));
|
|
178
|
+
});
|
|
179
|
+
});
|
|
@@ -2,29 +2,14 @@
|
|
|
2
2
|
* Pipeline step: fetch — HTTP API requests.
|
|
3
3
|
*/
|
|
4
4
|
|
|
5
|
+
import { CliError, getErrorMessage } from '../../errors.js';
|
|
6
|
+
import { log } from '../../logger.js';
|
|
5
7
|
import type { IPage } from '../../types.js';
|
|
6
8
|
import { render } from '../template.js';
|
|
7
9
|
|
|
8
|
-
|
|
9
|
-
return typeof value === 'object' && value !== null && !Array.isArray(value);
|
|
10
|
-
}
|
|
11
|
-
|
|
12
|
-
/** Simple async concurrency limiter */
|
|
13
|
-
async function mapConcurrent<T, R>(items: T[], limit: number, fn: (item: T, index: number) => Promise<R>): Promise<R[]> {
|
|
14
|
-
const results: R[] = new Array(items.length);
|
|
15
|
-
let index = 0;
|
|
10
|
+
import { isRecord, mapConcurrent } from '../../utils.js';
|
|
16
11
|
|
|
17
|
-
async function worker() {
|
|
18
|
-
while (index < items.length) {
|
|
19
|
-
const i = index++;
|
|
20
|
-
results[i] = await fn(items[i], i);
|
|
21
|
-
}
|
|
22
|
-
}
|
|
23
12
|
|
|
24
|
-
const workers = Array.from({ length: Math.min(limit, items.length) }, () => worker());
|
|
25
|
-
await Promise.all(workers);
|
|
26
|
-
return results;
|
|
27
|
-
}
|
|
28
13
|
|
|
29
14
|
/** Single URL fetch helper */
|
|
30
15
|
async function fetchSingle(
|
|
@@ -45,20 +30,33 @@ async function fetchSingle(
|
|
|
45
30
|
|
|
46
31
|
if (page === null) {
|
|
47
32
|
const resp = await fetch(finalUrl, { method: method.toUpperCase(), headers: renderedHeaders });
|
|
33
|
+
if (!resp.ok) {
|
|
34
|
+
throw new CliError('FETCH_ERROR', `HTTP ${resp.status} ${resp.statusText} from ${finalUrl}`);
|
|
35
|
+
}
|
|
48
36
|
return resp.json();
|
|
49
37
|
}
|
|
50
38
|
|
|
51
39
|
const headersJs = JSON.stringify(renderedHeaders);
|
|
52
40
|
const urlJs = JSON.stringify(finalUrl);
|
|
53
41
|
const methodJs = JSON.stringify(method.toUpperCase());
|
|
54
|
-
|
|
42
|
+
// Return error status instead of throwing inside evaluate to avoid CDP wrapper
|
|
43
|
+
// rewriting the message (CDP prepends "Evaluate error: " to thrown errors).
|
|
44
|
+
const result = await page.evaluate(`
|
|
55
45
|
async () => {
|
|
56
46
|
const resp = await fetch(${urlJs}, {
|
|
57
47
|
method: ${methodJs}, headers: ${headersJs}, credentials: "include"
|
|
58
48
|
});
|
|
49
|
+
if (!resp.ok) {
|
|
50
|
+
return { __httpError: resp.status, statusText: resp.statusText };
|
|
51
|
+
}
|
|
59
52
|
return await resp.json();
|
|
60
53
|
}
|
|
61
54
|
`);
|
|
55
|
+
if (result && typeof result === 'object' && '__httpError' in result) {
|
|
56
|
+
const { __httpError: status, statusText } = result as { __httpError: number; statusText: string };
|
|
57
|
+
throw new CliError('FETCH_ERROR', `HTTP ${status} ${statusText} from ${finalUrl}`);
|
|
58
|
+
}
|
|
59
|
+
return result;
|
|
62
60
|
}
|
|
63
61
|
|
|
64
62
|
/**
|
|
@@ -72,10 +70,11 @@ async function fetchBatchInBrowser(
|
|
|
72
70
|
): Promise<unknown[]> {
|
|
73
71
|
const headersJs = JSON.stringify(headers);
|
|
74
72
|
const urlsJs = JSON.stringify(urls);
|
|
73
|
+
const methodJs = JSON.stringify(method);
|
|
75
74
|
return (await page.evaluate(`
|
|
76
75
|
async () => {
|
|
77
76
|
const urls = ${urlsJs};
|
|
78
|
-
const method =
|
|
77
|
+
const method = ${methodJs};
|
|
79
78
|
const headers = ${headersJs};
|
|
80
79
|
const concurrency = ${concurrency};
|
|
81
80
|
|
|
@@ -87,9 +86,13 @@ async function fetchBatchInBrowser(
|
|
|
87
86
|
const i = idx++;
|
|
88
87
|
try {
|
|
89
88
|
const resp = await fetch(urls[i], { method, headers, credentials: "include" });
|
|
89
|
+
if (!resp.ok) {
|
|
90
|
+
throw new Error('HTTP ' + resp.status + ' ' + resp.statusText + ' from ' + urls[i]);
|
|
91
|
+
}
|
|
90
92
|
results[i] = await resp.json();
|
|
91
93
|
} catch (e) {
|
|
92
|
-
results[i] = { error: e.message };
|
|
94
|
+
results[i] = { error: e instanceof Error ? e.message : String(e) };
|
|
95
|
+
// Note: getErrorMessage() is a Node.js utility — can't use it inside evaluate()
|
|
93
96
|
}
|
|
94
97
|
}
|
|
95
98
|
}
|
|
@@ -130,13 +133,26 @@ export async function stepFetch(page: IPage | null, params: unknown, data: unkno
|
|
|
130
133
|
|
|
131
134
|
// BATCH IPC: if browser is available, batch all fetches into a single evaluate() call
|
|
132
135
|
if (page !== null) {
|
|
133
|
-
|
|
136
|
+
const results = await fetchBatchInBrowser(page, urls, method.toUpperCase(), renderedHeaders, concurrency);
|
|
137
|
+
for (let i = 0; i < results.length; i++) {
|
|
138
|
+
const r = results[i];
|
|
139
|
+
if (r && typeof r === 'object' && 'error' in r) {
|
|
140
|
+
log.warn(`Batch fetch failed for ${urls[i]}: ${(r as { error: string }).error}`);
|
|
141
|
+
}
|
|
142
|
+
}
|
|
143
|
+
return results;
|
|
134
144
|
}
|
|
135
145
|
|
|
136
146
|
// Non-browser: use concurrent pool (already optimized)
|
|
137
147
|
return mapConcurrent(data, concurrency, async (item, index) => {
|
|
138
148
|
const itemUrl = String(render(urlTemplate, { args, data, item, index }));
|
|
139
|
-
|
|
149
|
+
try {
|
|
150
|
+
return await fetchSingle(null, itemUrl, method, queryParams, headers, args, data);
|
|
151
|
+
} catch (error) {
|
|
152
|
+
const message = getErrorMessage(error);
|
|
153
|
+
log.warn(`Batch fetch failed for ${itemUrl}: ${message}`);
|
|
154
|
+
return { error: message };
|
|
155
|
+
}
|
|
140
156
|
});
|
|
141
157
|
}
|
|
142
158
|
const url = render(urlOrObj, { args, data });
|
|
@@ -5,9 +5,7 @@
|
|
|
5
5
|
import type { IPage } from '../../types.js';
|
|
6
6
|
import { render, evalExpr } from '../template.js';
|
|
7
7
|
|
|
8
|
-
|
|
9
|
-
return typeof value === 'object' && value !== null && !Array.isArray(value);
|
|
10
|
-
}
|
|
8
|
+
import { isRecord } from '../../utils.js';
|
|
11
9
|
|
|
12
10
|
export async function stepSelect(_page: IPage | null, params: unknown, data: unknown, args: Record<string, unknown>): Promise<unknown> {
|
|
13
11
|
const pathStr = String(render(params, { args, data }));
|
|
@@ -62,9 +60,7 @@ export async function stepSort(_page: IPage | null, params: unknown, data: unkno
|
|
|
62
60
|
return [...data].sort((a, b) => {
|
|
63
61
|
const left = isRecord(a) ? a[key] : undefined;
|
|
64
62
|
const right = isRecord(b) ? b[key] : undefined;
|
|
65
|
-
const
|
|
66
|
-
const vb = right ?? '';
|
|
67
|
-
const cmp = va < vb ? -1 : va > vb ? 1 : 0;
|
|
63
|
+
const cmp = String(left ?? '').localeCompare(String(right ?? ''), undefined, { numeric: true });
|
|
68
64
|
return reverse ? -cmp : cmp;
|
|
69
65
|
});
|
|
70
66
|
}
|
|
@@ -54,6 +54,31 @@ describe('evalExpr', () => {
|
|
|
54
54
|
it('evaluates || with truthy left', () => {
|
|
55
55
|
expect(evalExpr("item.name || 'N/A'", { item: { name: 'Alice' } })).toBe('Alice');
|
|
56
56
|
});
|
|
57
|
+
it('evaluates chained || fallback (issue #303)', () => {
|
|
58
|
+
// When first two are falsy, should evaluate through to the string literal
|
|
59
|
+
expect(evalExpr("item.a || item.b || 'default'", { item: {} })).toBe('default');
|
|
60
|
+
});
|
|
61
|
+
it('evaluates chained || with middle value truthy', () => {
|
|
62
|
+
expect(evalExpr("item.a || item.b || 'default'", { item: { b: 'middle' } })).toBe('middle');
|
|
63
|
+
});
|
|
64
|
+
it('evaluates chained || with first value truthy', () => {
|
|
65
|
+
expect(evalExpr("item.a || item.b || 'default'", { item: { a: 'first', b: 'middle' } })).toBe('first');
|
|
66
|
+
});
|
|
67
|
+
it('evaluates || with 0 as falsy left (JS semantics)', () => {
|
|
68
|
+
expect(evalExpr("item.count || 'N/A'", { item: { count: 0 } })).toBe('N/A');
|
|
69
|
+
});
|
|
70
|
+
it('evaluates || with empty string as falsy left', () => {
|
|
71
|
+
expect(evalExpr("item.name || 'unknown'", { item: { name: '' } })).toBe('unknown');
|
|
72
|
+
});
|
|
73
|
+
it('evaluates || with numeric fallback returning number type', () => {
|
|
74
|
+
expect(evalExpr('item.a || 42', { item: {} })).toBe(42);
|
|
75
|
+
});
|
|
76
|
+
it('evaluates 4-way chained ||', () => {
|
|
77
|
+
expect(evalExpr("item.a || item.b || item.c || 'last'", { item: { c: 'third' } })).toBe('third');
|
|
78
|
+
});
|
|
79
|
+
it('handles || combined with pipe filter', () => {
|
|
80
|
+
expect(evalExpr("item.a || item.b | upper", { item: { b: 'hello' } })).toBe('HELLO');
|
|
81
|
+
});
|
|
57
82
|
it('resolves simple path', () => {
|
|
58
83
|
expect(evalExpr('item.title', { item: { title: 'Test' } })).toBe('Test');
|
|
59
84
|
});
|
|
@@ -66,6 +91,9 @@ describe('evalExpr', () => {
|
|
|
66
91
|
it('evaluates method calls on values', () => {
|
|
67
92
|
expect(evalExpr("args.username.startsWith('@') ? args.username : '@' + args.username", { args: { username: 'alice' } })).toBe('@alice');
|
|
68
93
|
});
|
|
94
|
+
it('rejects constructor-based sandbox escapes', () => {
|
|
95
|
+
expect(evalExpr("args['cons' + 'tructor']['constructor']('return process')()", { args: {} })).toBeUndefined();
|
|
96
|
+
});
|
|
69
97
|
it('applies join filter', () => {
|
|
70
98
|
expect(evalExpr('item.tags | join(,)', { item: { tags: ['a', 'b', 'c'] } })).toBe('a,b,c');
|
|
71
99
|
});
|