@jackwener/opencli 1.3.3 → 1.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.github/pull_request_template.md +3 -1
- package/.github/workflows/build-extension.yml +7 -1
- package/.github/workflows/ci.yml +29 -3
- package/.github/workflows/docs.yml +1 -1
- package/.github/workflows/e2e-headed.yml +20 -0
- package/.github/workflows/release.yml +1 -1
- package/.github/workflows/security.yml +0 -3
- package/CHANGELOG.md +55 -0
- package/CONTRIBUTING.md +6 -3
- package/README.md +30 -3
- package/README.zh-CN.md +30 -3
- package/SKILL.md +7 -1
- package/TESTING.md +1 -0
- package/chatwise-opencli.ps1 +82 -0
- package/dist/analysis.d.ts +38 -0
- package/dist/analysis.js +166 -0
- package/dist/browser/cdp.d.ts +0 -4
- package/dist/browser/cdp.js +53 -41
- package/dist/browser/cdp.test.d.ts +1 -0
- package/dist/browser/cdp.test.js +52 -0
- package/dist/browser/dom-snapshot.d.ts +2 -2
- package/dist/browser/dom-snapshot.js +54 -1
- package/dist/browser/dom-snapshot.test.js +36 -0
- package/dist/browser/index.d.ts +2 -2
- package/dist/browser/index.js +1 -1
- package/dist/browser/mcp.d.ts +0 -2
- package/dist/browser/mcp.js +2 -3
- package/dist/browser/page.d.ts +4 -3
- package/dist/browser/page.js +34 -37
- package/dist/browser/stealth.d.ts +0 -2
- package/dist/browser/stealth.js +24 -9
- package/dist/browser.test.js +2 -2
- package/dist/build-manifest.js +15 -9
- package/dist/build-manifest.test.js +12 -0
- package/dist/cascade.js +4 -2
- package/dist/cli-manifest.json +639 -258
- package/dist/cli.js +57 -29
- package/dist/clis/_shared/desktop-commands.d.ts +22 -0
- package/dist/clis/_shared/desktop-commands.js +108 -0
- package/dist/clis/antigravity/serve.js +5 -2
- package/dist/clis/arxiv/search.js +1 -1
- package/dist/clis/bilibili/dynamic.test.d.ts +1 -0
- package/dist/clis/bilibili/dynamic.test.js +68 -0
- package/dist/clis/bilibili/favorite.js +4 -2
- package/dist/clis/bilibili/following.js +3 -2
- package/dist/clis/bilibili/subtitle.js +8 -7
- package/dist/clis/bilibili/utils.js +2 -2
- package/dist/clis/boss/batchgreet.js +1 -1
- package/dist/clis/boss/chatlist.js +1 -1
- package/dist/clis/boss/chatmsg.js +1 -1
- package/dist/clis/boss/detail.js +1 -1
- package/dist/clis/boss/exchange.js +1 -1
- package/dist/clis/boss/greet.js +1 -1
- package/dist/clis/boss/invite.js +1 -1
- package/dist/clis/boss/joblist.js +1 -1
- package/dist/clis/boss/mark.js +4 -3
- package/dist/clis/boss/recommend.js +1 -1
- package/dist/clis/boss/resume.js +1 -1
- package/dist/clis/boss/search.js +1 -1
- package/dist/clis/boss/send.js +5 -4
- package/dist/clis/boss/stats.js +1 -1
- package/dist/clis/chatgpt/ask.js +4 -0
- package/dist/clis/chatgpt/new.js +5 -1
- package/dist/clis/chatgpt/read.js +5 -1
- package/dist/clis/chatgpt/send.js +2 -1
- package/dist/clis/chatgpt/status.js +5 -1
- package/dist/clis/chatwise/ask.js +8 -2
- package/dist/clis/chatwise/export.js +2 -0
- package/dist/clis/chatwise/history.js +2 -0
- package/dist/clis/chatwise/model.js +8 -3
- package/dist/clis/chatwise/new.js +3 -18
- package/dist/clis/chatwise/read.js +2 -0
- package/dist/clis/chatwise/screenshot.js +3 -27
- package/dist/clis/chatwise/send.js +8 -2
- package/dist/clis/chatwise/shared.d.ts +2 -0
- package/dist/clis/chatwise/shared.js +6 -0
- package/dist/clis/chatwise/status.js +3 -22
- package/dist/clis/codex/ask.js +6 -2
- package/dist/clis/codex/dump.js +2 -25
- package/dist/clis/codex/new.js +2 -25
- package/dist/clis/codex/screenshot.js +2 -27
- package/dist/clis/codex/send.js +6 -4
- package/dist/clis/codex/status.js +2 -22
- package/dist/clis/cursor/ask.js +2 -1
- package/dist/clis/cursor/composer.js +2 -1
- package/dist/clis/cursor/dump.js +2 -25
- package/dist/clis/cursor/new.js +2 -18
- package/dist/clis/cursor/read.js +2 -1
- package/dist/clis/cursor/screenshot.js +1 -30
- package/dist/clis/cursor/send.js +2 -1
- package/dist/clis/cursor/status.js +2 -21
- package/dist/clis/dictionary/examples.yaml +25 -0
- package/dist/clis/dictionary/search.yaml +27 -0
- package/dist/clis/dictionary/synonyms.yaml +25 -0
- package/dist/clis/douban/book-hot.js +1 -1
- package/dist/clis/douban/movie-hot.js +1 -1
- package/dist/clis/douban/search.js +1 -1
- package/dist/clis/douban/utils.d.ts +4 -1
- package/dist/clis/douban/utils.js +156 -1
- package/dist/clis/doubao/ask.js +1 -1
- package/dist/clis/doubao/new.js +1 -1
- package/dist/clis/doubao/read.js +1 -1
- package/dist/clis/doubao/send.js +1 -1
- package/dist/clis/doubao/status.js +1 -1
- package/dist/clis/doubao-app/ask.js +1 -1
- package/dist/clis/doubao-app/new.js +1 -1
- package/dist/clis/doubao-app/read.js +1 -1
- package/dist/clis/doubao-app/send.js +1 -1
- package/dist/clis/grok/ask.d.ts +4 -0
- package/dist/clis/grok/ask.js +28 -10
- package/dist/clis/grok/ask.test.js +18 -0
- package/dist/clis/jd/item.d.ts +1 -0
- package/dist/clis/jd/item.js +96 -0
- package/dist/clis/jd/item.test.d.ts +1 -0
- package/dist/clis/jd/item.test.js +28 -0
- package/dist/clis/jike/feed.js +1 -1
- package/dist/clis/jike/search.js +1 -1
- package/dist/clis/linkedin/search.js +5 -4
- package/dist/clis/linkedin/timeline.d.ts +21 -0
- package/dist/clis/linkedin/timeline.js +503 -0
- package/dist/clis/linkedin/timeline.test.d.ts +1 -0
- package/dist/clis/linkedin/timeline.test.js +81 -0
- package/dist/clis/medium/feed.js +1 -1
- package/dist/clis/medium/search.js +1 -1
- package/dist/clis/medium/user.js +1 -1
- package/dist/clis/medium/{shared.js → utils.js} +2 -1
- package/dist/clis/pixiv/detail.yaml +49 -0
- package/dist/clis/pixiv/download.d.ts +7 -0
- package/dist/clis/pixiv/download.js +78 -0
- package/dist/clis/pixiv/download.test.d.ts +1 -0
- package/dist/clis/pixiv/download.test.js +87 -0
- package/dist/clis/pixiv/illusts.d.ts +8 -0
- package/dist/clis/pixiv/illusts.js +65 -0
- package/dist/clis/pixiv/illusts.test.d.ts +1 -0
- package/dist/clis/pixiv/illusts.test.js +99 -0
- package/dist/clis/pixiv/ranking.yaml +53 -0
- package/dist/clis/pixiv/search.d.ts +6 -0
- package/dist/clis/pixiv/search.js +43 -0
- package/dist/clis/pixiv/search.test.d.ts +1 -0
- package/dist/clis/pixiv/search.test.js +83 -0
- package/dist/clis/pixiv/test-utils.d.ts +12 -0
- package/dist/clis/pixiv/test-utils.js +23 -0
- package/dist/clis/pixiv/user.yaml +46 -0
- package/dist/clis/pixiv/utils.d.ts +27 -0
- package/dist/clis/pixiv/utils.js +49 -0
- package/dist/clis/reddit/comment.js +2 -1
- package/dist/clis/reddit/read.js +4 -3
- package/dist/clis/reddit/read.test.d.ts +1 -0
- package/dist/clis/reddit/read.test.js +28 -0
- package/dist/clis/reddit/save.js +2 -1
- package/dist/clis/reddit/saved.js +7 -3
- package/dist/clis/reddit/subscribe.js +2 -1
- package/dist/clis/reddit/upvote.js +2 -1
- package/dist/clis/reddit/upvoted.js +7 -3
- package/dist/clis/sinablog/article.js +1 -1
- package/dist/clis/sinablog/hot.js +1 -1
- package/dist/clis/sinablog/user.js +1 -1
- package/dist/clis/substack/feed.js +1 -1
- package/dist/clis/substack/publication.js +1 -1
- package/dist/clis/substack/search.js +3 -2
- package/dist/clis/substack/{shared.js → utils.js} +3 -2
- package/dist/clis/tiktok/search.yaml +2 -1
- package/dist/clis/twitter/accept.js +2 -1
- package/dist/clis/twitter/article.js +4 -1
- package/dist/clis/twitter/block.js +2 -1
- package/dist/clis/twitter/bookmark.js +2 -1
- package/dist/clis/twitter/bookmarks.js +3 -2
- package/dist/clis/twitter/delete.js +2 -1
- package/dist/clis/twitter/follow.js +2 -1
- package/dist/clis/twitter/followers.js +3 -2
- package/dist/clis/twitter/following.js +3 -2
- package/dist/clis/twitter/hide-reply.js +2 -1
- package/dist/clis/twitter/like.js +2 -1
- package/dist/clis/twitter/notifications.js +2 -1
- package/dist/clis/twitter/post.js +2 -1
- package/dist/clis/twitter/profile.js +5 -2
- package/dist/clis/twitter/reply-dm.js +2 -1
- package/dist/clis/twitter/reply.js +2 -1
- package/dist/clis/twitter/search.js +30 -13
- package/dist/clis/twitter/search.test.d.ts +1 -0
- package/dist/clis/twitter/search.test.js +104 -0
- package/dist/clis/twitter/thread.js +2 -2
- package/dist/clis/twitter/timeline.js +3 -2
- package/dist/clis/twitter/trending.js +3 -2
- package/dist/clis/twitter/unblock.js +2 -1
- package/dist/clis/twitter/unbookmark.js +2 -1
- package/dist/clis/twitter/unfollow.js +2 -1
- package/dist/clis/v2ex/daily.js +3 -2
- package/dist/clis/v2ex/me.js +3 -2
- package/dist/clis/v2ex/notifications.js +4 -4
- package/dist/clis/web/read.d.ts +16 -0
- package/dist/clis/web/read.js +202 -0
- package/dist/clis/xueqiu/danjuan-utils.d.ts +55 -0
- package/dist/clis/xueqiu/danjuan-utils.js +126 -0
- package/dist/clis/xueqiu/danjuan-utils.test.d.ts +1 -0
- package/dist/clis/xueqiu/danjuan-utils.test.js +41 -0
- package/dist/clis/xueqiu/fund-holdings.d.ts +1 -0
- package/dist/clis/xueqiu/fund-holdings.js +28 -0
- package/dist/clis/xueqiu/fund-snapshot.d.ts +1 -0
- package/dist/clis/xueqiu/fund-snapshot.js +25 -0
- package/dist/clis/youtube/transcript.js +5 -4
- package/dist/clis/youtube/video.js +3 -2
- package/dist/daemon.js +7 -3
- package/dist/discovery.js +11 -10
- package/dist/doctor.js +2 -1
- package/dist/download/index.d.ts +4 -12
- package/dist/download/index.js +33 -12
- package/dist/download/index.test.js +79 -2
- package/dist/download/media-download.js +4 -2
- package/dist/engine.test.js +76 -4
- package/dist/execution.d.ts +1 -9
- package/dist/execution.js +56 -46
- package/dist/explore.js +12 -111
- package/dist/external-clis.yaml +0 -8
- package/dist/external.js +7 -5
- package/dist/external.test.js +4 -0
- package/dist/generate.d.ts +0 -9
- package/dist/generate.js +4 -20
- package/dist/hooks.d.ts +46 -0
- package/dist/hooks.js +56 -0
- package/dist/hooks.test.d.ts +4 -0
- package/dist/hooks.test.js +92 -0
- package/dist/interceptor.js +70 -23
- package/dist/main.js +2 -0
- package/dist/output.js +12 -6
- package/dist/pipeline/executor.js +1 -1
- package/dist/pipeline/steps/browser.js +1 -3
- package/dist/pipeline/steps/download.js +42 -26
- package/dist/pipeline/steps/download.test.d.ts +1 -0
- package/dist/pipeline/steps/download.test.js +101 -0
- package/dist/pipeline/steps/fetch.js +40 -22
- package/dist/pipeline/steps/fetch.test.d.ts +1 -0
- package/dist/pipeline/steps/fetch.test.js +123 -0
- package/dist/pipeline/steps/transform.js +2 -6
- package/dist/pipeline/template.js +66 -52
- package/dist/pipeline/template.test.js +28 -0
- package/dist/pipeline/transform.test.js +18 -0
- package/dist/plugin.d.ts +40 -1
- package/dist/plugin.js +214 -17
- package/dist/plugin.test.d.ts +1 -1
- package/dist/plugin.test.js +219 -3
- package/dist/record.js +6 -98
- package/dist/registry-api.d.ts +2 -0
- package/dist/registry-api.js +1 -0
- package/dist/registry.d.ts +5 -2
- package/dist/registry.js +1 -2
- package/dist/runtime.d.ts +0 -1
- package/dist/runtime.js +14 -4
- package/dist/snapshotFormatter.d.ts +7 -14
- package/dist/snapshotFormatter.js +38 -78
- package/dist/utils.d.ts +9 -0
- package/dist/utils.js +29 -0
- package/dist/validate.js +3 -5
- package/dist/yaml-schema.d.ts +26 -0
- package/dist/yaml-schema.js +5 -0
- package/docs/.vitepress/config.mts +3 -0
- package/docs/adapters/browser/dictionary.md +27 -0
- package/docs/adapters/browser/jd.md +27 -0
- package/docs/adapters/browser/linkedin.md +6 -0
- package/docs/adapters/browser/pixiv.md +92 -0
- package/docs/adapters/browser/web.md +30 -0
- package/docs/adapters/browser/xueqiu.md +27 -9
- package/docs/adapters/index.md +3 -1
- package/docs/comparison.md +125 -0
- package/docs/developer/contributing.md +21 -2
- package/docs/developer/testing.md +14 -8
- package/docs/developer/ts-adapter.md +18 -0
- package/docs/developer/yaml-adapter.md +16 -0
- package/docs/guide/plugins.md +10 -0
- package/docs/zh/guide/plugins.md +10 -0
- package/extension/dist/background.js +519 -444
- package/extension/manifest.json +1 -1
- package/extension/package.json +1 -1
- package/extension/src/background.test.ts +46 -1
- package/extension/src/background.ts +108 -33
- package/extension/src/cdp.ts +9 -9
- package/package.json +3 -2
- package/scripts/check-doc-coverage.sh +2 -0
- package/src/analysis.ts +170 -0
- package/src/browser/cdp.test.ts +66 -0
- package/src/browser/cdp.ts +59 -44
- package/src/browser/dom-snapshot.test.ts +42 -0
- package/src/browser/dom-snapshot.ts +56 -3
- package/src/browser/index.ts +2 -2
- package/src/browser/mcp.ts +2 -4
- package/src/browser/page.ts +34 -37
- package/src/browser/stealth.ts +24 -10
- package/src/browser.test.ts +2 -2
- package/src/build-manifest.test.ts +14 -0
- package/src/build-manifest.ts +13 -31
- package/src/cascade.ts +5 -3
- package/src/cli.ts +66 -34
- package/src/clis/_shared/desktop-commands.ts +121 -0
- package/src/clis/antigravity/serve.ts +6 -3
- package/src/clis/arxiv/search.ts +1 -1
- package/src/clis/bilibili/dynamic.test.ts +79 -0
- package/src/clis/bilibili/favorite.ts +5 -2
- package/src/clis/bilibili/following.ts +3 -2
- package/src/clis/bilibili/subtitle.ts +8 -7
- package/src/clis/bilibili/utils.ts +2 -2
- package/src/clis/boss/batchgreet.ts +1 -1
- package/src/clis/boss/chatlist.ts +1 -1
- package/src/clis/boss/chatmsg.ts +1 -1
- package/src/clis/boss/detail.ts +1 -1
- package/src/clis/boss/exchange.ts +1 -1
- package/src/clis/boss/greet.ts +1 -1
- package/src/clis/boss/invite.ts +1 -1
- package/src/clis/boss/joblist.ts +1 -1
- package/src/clis/boss/mark.ts +4 -3
- package/src/clis/boss/recommend.ts +1 -1
- package/src/clis/boss/resume.ts +1 -1
- package/src/clis/boss/search.ts +1 -1
- package/src/clis/boss/send.ts +5 -4
- package/src/clis/boss/stats.ts +1 -1
- package/src/clis/chatgpt/ask.ts +5 -0
- package/src/clis/chatgpt/new.ts +7 -2
- package/src/clis/chatgpt/read.ts +7 -2
- package/src/clis/chatgpt/send.ts +3 -2
- package/src/clis/chatgpt/status.ts +6 -1
- package/src/clis/chatwise/ask.ts +7 -2
- package/src/clis/chatwise/export.ts +2 -0
- package/src/clis/chatwise/history.ts +2 -0
- package/src/clis/chatwise/model.ts +7 -3
- package/src/clis/chatwise/new.ts +3 -20
- package/src/clis/chatwise/read.ts +2 -0
- package/src/clis/chatwise/screenshot.ts +3 -32
- package/src/clis/chatwise/send.ts +7 -2
- package/src/clis/chatwise/shared.ts +8 -0
- package/src/clis/chatwise/status.ts +3 -24
- package/src/clis/codex/ask.ts +5 -2
- package/src/clis/codex/dump.ts +2 -27
- package/src/clis/codex/new.ts +2 -28
- package/src/clis/codex/screenshot.ts +2 -32
- package/src/clis/codex/send.ts +5 -4
- package/src/clis/codex/status.ts +2 -24
- package/src/clis/cursor/ask.ts +2 -1
- package/src/clis/cursor/composer.ts +2 -1
- package/src/clis/cursor/dump.ts +2 -27
- package/src/clis/cursor/new.ts +2 -20
- package/src/clis/cursor/read.ts +2 -1
- package/src/clis/cursor/screenshot.ts +1 -36
- package/src/clis/cursor/send.ts +2 -1
- package/src/clis/cursor/status.ts +2 -22
- package/src/clis/dictionary/examples.yaml +25 -0
- package/src/clis/dictionary/search.yaml +27 -0
- package/src/clis/dictionary/synonyms.yaml +25 -0
- package/src/clis/douban/book-hot.ts +1 -1
- package/src/clis/douban/movie-hot.ts +1 -1
- package/src/clis/douban/search.ts +1 -1
- package/src/clis/douban/utils.ts +165 -1
- package/src/clis/doubao/ask.ts +1 -1
- package/src/clis/doubao/new.ts +1 -1
- package/src/clis/doubao/read.ts +1 -1
- package/src/clis/doubao/send.ts +1 -1
- package/src/clis/doubao/status.ts +1 -1
- package/src/clis/doubao-app/ask.ts +1 -1
- package/src/clis/doubao-app/new.ts +1 -1
- package/src/clis/doubao-app/read.ts +1 -1
- package/src/clis/doubao-app/send.ts +1 -1
- package/src/clis/grok/ask.test.ts +25 -0
- package/src/clis/grok/ask.ts +25 -12
- package/src/clis/jd/item.test.ts +35 -0
- package/src/clis/jd/item.ts +101 -0
- package/src/clis/jike/feed.ts +1 -1
- package/src/clis/jike/search.ts +1 -1
- package/src/clis/linkedin/search.ts +5 -4
- package/src/clis/linkedin/timeline.test.ts +99 -0
- package/src/clis/linkedin/timeline.ts +532 -0
- package/src/clis/medium/feed.ts +1 -1
- package/src/clis/medium/search.ts +1 -1
- package/src/clis/medium/user.ts +1 -1
- package/src/clis/medium/{shared.ts → utils.ts} +2 -1
- package/src/clis/pixiv/detail.yaml +49 -0
- package/src/clis/pixiv/download.test.ts +114 -0
- package/src/clis/pixiv/download.ts +91 -0
- package/src/clis/pixiv/illusts.test.ts +115 -0
- package/src/clis/pixiv/illusts.ts +78 -0
- package/src/clis/pixiv/ranking.yaml +53 -0
- package/src/clis/pixiv/search.test.ts +97 -0
- package/src/clis/pixiv/search.ts +53 -0
- package/src/clis/pixiv/test-utils.ts +29 -0
- package/src/clis/pixiv/user.yaml +46 -0
- package/src/clis/pixiv/utils.ts +62 -0
- package/src/clis/reddit/comment.ts +2 -1
- package/src/clis/reddit/read.test.ts +34 -0
- package/src/clis/reddit/read.ts +4 -3
- package/src/clis/reddit/save.ts +2 -1
- package/src/clis/reddit/saved.ts +6 -2
- package/src/clis/reddit/subscribe.ts +2 -1
- package/src/clis/reddit/upvote.ts +2 -1
- package/src/clis/reddit/upvoted.ts +6 -2
- package/src/clis/sinablog/article.ts +1 -1
- package/src/clis/sinablog/hot.ts +1 -1
- package/src/clis/sinablog/user.ts +1 -1
- package/src/clis/substack/feed.ts +1 -1
- package/src/clis/substack/publication.ts +1 -1
- package/src/clis/substack/search.ts +3 -2
- package/src/clis/substack/{shared.ts → utils.ts} +3 -2
- package/src/clis/tiktok/search.yaml +2 -1
- package/src/clis/twitter/accept.ts +2 -1
- package/src/clis/twitter/article.ts +3 -1
- package/src/clis/twitter/block.ts +2 -1
- package/src/clis/twitter/bookmark.ts +2 -1
- package/src/clis/twitter/bookmarks.ts +3 -2
- package/src/clis/twitter/delete.ts +2 -1
- package/src/clis/twitter/follow.ts +2 -1
- package/src/clis/twitter/followers.ts +3 -2
- package/src/clis/twitter/following.ts +3 -2
- package/src/clis/twitter/hide-reply.ts +2 -1
- package/src/clis/twitter/like.ts +2 -1
- package/src/clis/twitter/notifications.ts +2 -1
- package/src/clis/twitter/post.ts +2 -1
- package/src/clis/twitter/profile.ts +4 -2
- package/src/clis/twitter/reply-dm.ts +2 -1
- package/src/clis/twitter/reply.ts +2 -1
- package/src/clis/twitter/search.test.ts +113 -0
- package/src/clis/twitter/search.ts +38 -14
- package/src/clis/twitter/thread.ts +2 -2
- package/src/clis/twitter/timeline.ts +3 -2
- package/src/clis/twitter/trending.ts +3 -2
- package/src/clis/twitter/unblock.ts +2 -1
- package/src/clis/twitter/unbookmark.ts +2 -1
- package/src/clis/twitter/unfollow.ts +2 -1
- package/src/clis/v2ex/daily.ts +3 -2
- package/src/clis/v2ex/me.ts +3 -2
- package/src/clis/v2ex/notifications.ts +3 -4
- package/src/clis/web/read.ts +210 -0
- package/src/clis/xueqiu/danjuan-utils.test.ts +49 -0
- package/src/clis/xueqiu/danjuan-utils.ts +176 -0
- package/src/clis/xueqiu/fund-holdings.ts +32 -0
- package/src/clis/xueqiu/fund-snapshot.ts +27 -0
- package/src/clis/youtube/transcript.ts +5 -4
- package/src/clis/youtube/video.ts +3 -2
- package/src/daemon.ts +5 -4
- package/src/discovery.ts +12 -34
- package/src/doctor.ts +3 -2
- package/src/download/index.test.ts +93 -2
- package/src/download/index.ts +44 -23
- package/src/download/media-download.ts +5 -3
- package/src/engine.test.ts +84 -3
- package/src/execution.ts +62 -46
- package/src/explore.ts +21 -90
- package/src/external-clis.yaml +0 -8
- package/src/external.test.ts +9 -0
- package/src/external.ts +12 -10
- package/src/generate.ts +4 -41
- package/src/hooks.test.ts +126 -0
- package/src/hooks.ts +90 -0
- package/src/interceptor.ts +73 -23
- package/src/main.ts +2 -0
- package/src/output.ts +14 -6
- package/src/pipeline/executor.ts +1 -1
- package/src/pipeline/steps/browser.ts +1 -3
- package/src/pipeline/steps/download.test.ts +136 -0
- package/src/pipeline/steps/download.ts +47 -34
- package/src/pipeline/steps/fetch.test.ts +179 -0
- package/src/pipeline/steps/fetch.ts +39 -23
- package/src/pipeline/steps/transform.ts +2 -6
- package/src/pipeline/template.test.ts +28 -0
- package/src/pipeline/template.ts +67 -79
- package/src/pipeline/transform.test.ts +20 -0
- package/src/plugin.test.ts +251 -3
- package/src/plugin.ts +265 -21
- package/src/record.ts +12 -84
- package/src/registry-api.ts +2 -0
- package/src/registry.ts +7 -4
- package/src/runtime.ts +14 -4
- package/src/snapshotFormatter.ts +43 -121
- package/src/utils.ts +39 -0
- package/src/validate.ts +3 -5
- package/src/yaml-schema.ts +28 -0
- package/tests/e2e/browser-auth.test.ts +25 -0
- package/tests/e2e/plugin-management.test.ts +137 -0
- package/tests/e2e/public-commands.test.ts +34 -1
- package/vitest.config.ts +19 -1
- package/.github/workflows/pkg-pr-new.yml +0 -30
- package/dist/clis/douban/shared.d.ts +0 -4
- package/dist/clis/douban/shared.js +0 -155
- package/src/clis/douban/shared.ts +0 -165
- /package/dist/clis/boss/{common.d.ts → utils.d.ts} +0 -0
- /package/dist/clis/boss/{common.js → utils.js} +0 -0
- /package/dist/clis/doubao/{common.d.ts → utils.d.ts} +0 -0
- /package/dist/clis/doubao/{common.js → utils.js} +0 -0
- /package/dist/clis/doubao-app/{common.d.ts → utils.d.ts} +0 -0
- /package/dist/clis/doubao-app/{common.js → utils.js} +0 -0
- /package/dist/clis/jike/{shared.d.ts → utils.d.ts} +0 -0
- /package/dist/clis/jike/{shared.js → utils.js} +0 -0
- /package/dist/clis/medium/{shared.d.ts → utils.d.ts} +0 -0
- /package/dist/clis/sinablog/{shared.d.ts → utils.d.ts} +0 -0
- /package/dist/clis/sinablog/{shared.js → utils.js} +0 -0
- /package/dist/clis/substack/{shared.d.ts → utils.d.ts} +0 -0
- /package/src/clis/boss/{common.ts → utils.ts} +0 -0
- /package/src/clis/doubao/{common.ts → utils.ts} +0 -0
- /package/src/clis/doubao-app/{common.ts → utils.ts} +0 -0
- /package/src/clis/jike/{shared.ts → utils.ts} +0 -0
- /package/src/clis/sinablog/{shared.ts → utils.ts} +0 -0
|
@@ -0,0 +1,101 @@
|
|
|
1
|
+
import { beforeEach, describe, expect, it, vi } from 'vitest';
|
|
2
|
+
import * as os from 'node:os';
|
|
3
|
+
import * as path from 'node:path';
|
|
4
|
+
const { mockHttpDownload, mockYtdlpDownload, mockExportCookiesToNetscape } = vi.hoisted(() => ({
|
|
5
|
+
mockHttpDownload: vi.fn(),
|
|
6
|
+
mockYtdlpDownload: vi.fn(),
|
|
7
|
+
mockExportCookiesToNetscape: vi.fn(),
|
|
8
|
+
}));
|
|
9
|
+
vi.mock('../../download/index.js', async () => {
|
|
10
|
+
const actual = await vi.importActual('../../download/index.js');
|
|
11
|
+
return {
|
|
12
|
+
...actual,
|
|
13
|
+
httpDownload: mockHttpDownload,
|
|
14
|
+
ytdlpDownload: mockYtdlpDownload,
|
|
15
|
+
exportCookiesToNetscape: mockExportCookiesToNetscape,
|
|
16
|
+
};
|
|
17
|
+
});
|
|
18
|
+
import { stepDownload } from './download.js';
|
|
19
|
+
function createMockPage(getCookies) {
|
|
20
|
+
return {
|
|
21
|
+
goto: vi.fn(),
|
|
22
|
+
evaluate: vi.fn().mockResolvedValue(null),
|
|
23
|
+
getCookies,
|
|
24
|
+
snapshot: vi.fn().mockResolvedValue(''),
|
|
25
|
+
click: vi.fn(),
|
|
26
|
+
typeText: vi.fn(),
|
|
27
|
+
pressKey: vi.fn(),
|
|
28
|
+
scrollTo: vi.fn(),
|
|
29
|
+
getFormState: vi.fn().mockResolvedValue({}),
|
|
30
|
+
wait: vi.fn(),
|
|
31
|
+
tabs: vi.fn().mockResolvedValue([]),
|
|
32
|
+
closeTab: vi.fn(),
|
|
33
|
+
newTab: vi.fn(),
|
|
34
|
+
selectTab: vi.fn(),
|
|
35
|
+
networkRequests: vi.fn().mockResolvedValue([]),
|
|
36
|
+
consoleMessages: vi.fn().mockResolvedValue([]),
|
|
37
|
+
scroll: vi.fn(),
|
|
38
|
+
autoScroll: vi.fn(),
|
|
39
|
+
installInterceptor: vi.fn(),
|
|
40
|
+
getInterceptedRequests: vi.fn().mockResolvedValue([]),
|
|
41
|
+
screenshot: vi.fn().mockResolvedValue(''),
|
|
42
|
+
};
|
|
43
|
+
}
|
|
44
|
+
describe('stepDownload', () => {
|
|
45
|
+
beforeEach(() => {
|
|
46
|
+
mockHttpDownload.mockReset();
|
|
47
|
+
mockHttpDownload.mockResolvedValue({ success: true, size: 2 });
|
|
48
|
+
mockYtdlpDownload.mockReset();
|
|
49
|
+
mockYtdlpDownload.mockResolvedValue({ success: true, size: 2 });
|
|
50
|
+
mockExportCookiesToNetscape.mockReset();
|
|
51
|
+
});
|
|
52
|
+
it('scopes browser cookies to each direct-download target domain', async () => {
|
|
53
|
+
const page = createMockPage(vi.fn().mockImplementation(async (opts) => {
|
|
54
|
+
const domain = opts?.domain ?? 'unknown';
|
|
55
|
+
return [{ name: 'sid', value: domain, domain }];
|
|
56
|
+
}));
|
|
57
|
+
await stepDownload(page, {
|
|
58
|
+
url: '${{ item.url }}',
|
|
59
|
+
dir: path.join(os.tmpdir(), 'opencli-download-test'),
|
|
60
|
+
filename: '${{ index }}.txt',
|
|
61
|
+
progress: false,
|
|
62
|
+
concurrency: 1,
|
|
63
|
+
}, [
|
|
64
|
+
{ url: 'https://a.example/file-1.txt' },
|
|
65
|
+
{ url: 'https://b.example/file-2.txt' },
|
|
66
|
+
], {});
|
|
67
|
+
expect(mockHttpDownload).toHaveBeenNthCalledWith(1, 'https://a.example/file-1.txt', path.join(os.tmpdir(), 'opencli-download-test', '0.txt'), expect.objectContaining({ cookies: 'sid=a.example' }));
|
|
68
|
+
expect(mockHttpDownload).toHaveBeenNthCalledWith(2, 'https://b.example/file-2.txt', path.join(os.tmpdir(), 'opencli-download-test', '1.txt'), expect.objectContaining({ cookies: 'sid=b.example' }));
|
|
69
|
+
});
|
|
70
|
+
it('builds yt-dlp cookies from all target domains instead of only the first item', async () => {
|
|
71
|
+
const getCookies = vi.fn().mockImplementation(async (opts) => {
|
|
72
|
+
const domain = opts?.domain ?? 'unknown';
|
|
73
|
+
return [{
|
|
74
|
+
name: `sid-${domain}`,
|
|
75
|
+
value: domain,
|
|
76
|
+
domain,
|
|
77
|
+
path: '/',
|
|
78
|
+
secure: false,
|
|
79
|
+
httpOnly: false,
|
|
80
|
+
}];
|
|
81
|
+
});
|
|
82
|
+
const page = createMockPage(getCookies);
|
|
83
|
+
await stepDownload(page, {
|
|
84
|
+
url: '${{ item.url }}',
|
|
85
|
+
dir: '/tmp/opencli-download-test',
|
|
86
|
+
filename: '${{ index }}.mp4',
|
|
87
|
+
progress: false,
|
|
88
|
+
concurrency: 1,
|
|
89
|
+
}, [
|
|
90
|
+
{ url: 'https://www.youtube.com/watch?v=one' },
|
|
91
|
+
{ url: 'https://www.bilibili.com/video/BV1xx411c7mD' },
|
|
92
|
+
], {});
|
|
93
|
+
expect(getCookies).toHaveBeenCalledWith({ domain: 'www.youtube.com' });
|
|
94
|
+
expect(getCookies).toHaveBeenCalledWith({ domain: 'www.bilibili.com' });
|
|
95
|
+
expect(mockExportCookiesToNetscape).toHaveBeenCalledWith(expect.arrayContaining([
|
|
96
|
+
expect.objectContaining({ name: 'sid-www.youtube.com', domain: 'www.youtube.com' }),
|
|
97
|
+
expect.objectContaining({ name: 'sid-www.bilibili.com', domain: 'www.bilibili.com' }),
|
|
98
|
+
]), expect.any(String));
|
|
99
|
+
expect(mockYtdlpDownload).toHaveBeenCalledTimes(2);
|
|
100
|
+
});
|
|
101
|
+
});
|
|
@@ -1,24 +1,10 @@
|
|
|
1
1
|
/**
|
|
2
2
|
* Pipeline step: fetch — HTTP API requests.
|
|
3
3
|
*/
|
|
4
|
+
import { CliError, getErrorMessage } from '../../errors.js';
|
|
5
|
+
import { log } from '../../logger.js';
|
|
4
6
|
import { render } from '../template.js';
|
|
5
|
-
|
|
6
|
-
return typeof value === 'object' && value !== null && !Array.isArray(value);
|
|
7
|
-
}
|
|
8
|
-
/** Simple async concurrency limiter */
|
|
9
|
-
async function mapConcurrent(items, limit, fn) {
|
|
10
|
-
const results = new Array(items.length);
|
|
11
|
-
let index = 0;
|
|
12
|
-
async function worker() {
|
|
13
|
-
while (index < items.length) {
|
|
14
|
-
const i = index++;
|
|
15
|
-
results[i] = await fn(items[i], i);
|
|
16
|
-
}
|
|
17
|
-
}
|
|
18
|
-
const workers = Array.from({ length: Math.min(limit, items.length) }, () => worker());
|
|
19
|
-
await Promise.all(workers);
|
|
20
|
-
return results;
|
|
21
|
-
}
|
|
7
|
+
import { isRecord, mapConcurrent } from '../../utils.js';
|
|
22
8
|
/** Single URL fetch helper */
|
|
23
9
|
async function fetchSingle(page, url, method, queryParams, headers, args, data) {
|
|
24
10
|
const renderedParams = {};
|
|
@@ -34,19 +20,32 @@ async function fetchSingle(page, url, method, queryParams, headers, args, data)
|
|
|
34
20
|
}
|
|
35
21
|
if (page === null) {
|
|
36
22
|
const resp = await fetch(finalUrl, { method: method.toUpperCase(), headers: renderedHeaders });
|
|
23
|
+
if (!resp.ok) {
|
|
24
|
+
throw new CliError('FETCH_ERROR', `HTTP ${resp.status} ${resp.statusText} from ${finalUrl}`);
|
|
25
|
+
}
|
|
37
26
|
return resp.json();
|
|
38
27
|
}
|
|
39
28
|
const headersJs = JSON.stringify(renderedHeaders);
|
|
40
29
|
const urlJs = JSON.stringify(finalUrl);
|
|
41
30
|
const methodJs = JSON.stringify(method.toUpperCase());
|
|
42
|
-
|
|
31
|
+
// Return error status instead of throwing inside evaluate to avoid CDP wrapper
|
|
32
|
+
// rewriting the message (CDP prepends "Evaluate error: " to thrown errors).
|
|
33
|
+
const result = await page.evaluate(`
|
|
43
34
|
async () => {
|
|
44
35
|
const resp = await fetch(${urlJs}, {
|
|
45
36
|
method: ${methodJs}, headers: ${headersJs}, credentials: "include"
|
|
46
37
|
});
|
|
38
|
+
if (!resp.ok) {
|
|
39
|
+
return { __httpError: resp.status, statusText: resp.statusText };
|
|
40
|
+
}
|
|
47
41
|
return await resp.json();
|
|
48
42
|
}
|
|
49
43
|
`);
|
|
44
|
+
if (result && typeof result === 'object' && '__httpError' in result) {
|
|
45
|
+
const { __httpError: status, statusText } = result;
|
|
46
|
+
throw new CliError('FETCH_ERROR', `HTTP ${status} ${statusText} from ${finalUrl}`);
|
|
47
|
+
}
|
|
48
|
+
return result;
|
|
50
49
|
}
|
|
51
50
|
/**
|
|
52
51
|
* Batch fetch: send all URLs into the browser as a single evaluate() call.
|
|
@@ -56,10 +55,11 @@ async function fetchSingle(page, url, method, queryParams, headers, args, data)
|
|
|
56
55
|
async function fetchBatchInBrowser(page, urls, method, headers, concurrency) {
|
|
57
56
|
const headersJs = JSON.stringify(headers);
|
|
58
57
|
const urlsJs = JSON.stringify(urls);
|
|
58
|
+
const methodJs = JSON.stringify(method);
|
|
59
59
|
return (await page.evaluate(`
|
|
60
60
|
async () => {
|
|
61
61
|
const urls = ${urlsJs};
|
|
62
|
-
const method =
|
|
62
|
+
const method = ${methodJs};
|
|
63
63
|
const headers = ${headersJs};
|
|
64
64
|
const concurrency = ${concurrency};
|
|
65
65
|
|
|
@@ -71,9 +71,13 @@ async function fetchBatchInBrowser(page, urls, method, headers, concurrency) {
|
|
|
71
71
|
const i = idx++;
|
|
72
72
|
try {
|
|
73
73
|
const resp = await fetch(urls[i], { method, headers, credentials: "include" });
|
|
74
|
+
if (!resp.ok) {
|
|
75
|
+
throw new Error('HTTP ' + resp.status + ' ' + resp.statusText + ' from ' + urls[i]);
|
|
76
|
+
}
|
|
74
77
|
results[i] = await resp.json();
|
|
75
78
|
} catch (e) {
|
|
76
|
-
results[i] = { error: e.message };
|
|
79
|
+
results[i] = { error: e instanceof Error ? e.message : String(e) };
|
|
80
|
+
// Note: getErrorMessage() is a Node.js utility — can't use it inside evaluate()
|
|
77
81
|
}
|
|
78
82
|
}
|
|
79
83
|
}
|
|
@@ -111,12 +115,26 @@ export async function stepFetch(page, params, data, args) {
|
|
|
111
115
|
});
|
|
112
116
|
// BATCH IPC: if browser is available, batch all fetches into a single evaluate() call
|
|
113
117
|
if (page !== null) {
|
|
114
|
-
|
|
118
|
+
const results = await fetchBatchInBrowser(page, urls, method.toUpperCase(), renderedHeaders, concurrency);
|
|
119
|
+
for (let i = 0; i < results.length; i++) {
|
|
120
|
+
const r = results[i];
|
|
121
|
+
if (r && typeof r === 'object' && 'error' in r) {
|
|
122
|
+
log.warn(`Batch fetch failed for ${urls[i]}: ${r.error}`);
|
|
123
|
+
}
|
|
124
|
+
}
|
|
125
|
+
return results;
|
|
115
126
|
}
|
|
116
127
|
// Non-browser: use concurrent pool (already optimized)
|
|
117
128
|
return mapConcurrent(data, concurrency, async (item, index) => {
|
|
118
129
|
const itemUrl = String(render(urlTemplate, { args, data, item, index }));
|
|
119
|
-
|
|
130
|
+
try {
|
|
131
|
+
return await fetchSingle(null, itemUrl, method, queryParams, headers, args, data);
|
|
132
|
+
}
|
|
133
|
+
catch (error) {
|
|
134
|
+
const message = getErrorMessage(error);
|
|
135
|
+
log.warn(`Batch fetch failed for ${itemUrl}: ${message}`);
|
|
136
|
+
return { error: message };
|
|
137
|
+
}
|
|
120
138
|
});
|
|
121
139
|
}
|
|
122
140
|
const url = render(urlOrObj, { args, data });
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
export {};
|
|
@@ -0,0 +1,123 @@
|
|
|
1
|
+
import { afterEach, describe, expect, it, vi } from 'vitest';
|
|
2
|
+
import { CliError } from '../../errors.js';
|
|
3
|
+
import { stepFetch } from './fetch.js';
|
|
4
|
+
afterEach(() => {
|
|
5
|
+
vi.restoreAllMocks();
|
|
6
|
+
vi.unstubAllGlobals();
|
|
7
|
+
});
|
|
8
|
+
describe('stepFetch', () => {
|
|
9
|
+
// W1 + W4: non-browser single fetch throws CliError with FETCH_ERROR code and full message
|
|
10
|
+
it('throws CliError with FETCH_ERROR code on non-ok responses without a browser session', async () => {
|
|
11
|
+
const jsonMock = vi.fn().mockResolvedValue({ error: 'rate limited' });
|
|
12
|
+
const fetchMock = vi.fn().mockResolvedValue({
|
|
13
|
+
ok: false,
|
|
14
|
+
status: 429,
|
|
15
|
+
statusText: 'Too Many Requests',
|
|
16
|
+
json: jsonMock,
|
|
17
|
+
});
|
|
18
|
+
vi.stubGlobal('fetch', fetchMock);
|
|
19
|
+
const err = await stepFetch(null, { url: 'https://api.example.com/items' }, null, {}).catch((e) => e);
|
|
20
|
+
expect(err).toBeInstanceOf(CliError);
|
|
21
|
+
expect(err.code).toBe('FETCH_ERROR');
|
|
22
|
+
expect(err.message).toBe('HTTP 429 Too Many Requests from https://api.example.com/items');
|
|
23
|
+
expect(jsonMock).not.toHaveBeenCalled();
|
|
24
|
+
});
|
|
25
|
+
// W1 + W3: browser single fetch returns error status from evaluate, outer code throws CliError
|
|
26
|
+
it('throws CliError with FETCH_ERROR code on non-ok responses inside the browser session', async () => {
|
|
27
|
+
const jsonMock = vi.fn().mockResolvedValue({ error: 'auth required' });
|
|
28
|
+
const fetchMock = vi.fn().mockResolvedValue({
|
|
29
|
+
ok: false,
|
|
30
|
+
status: 401,
|
|
31
|
+
statusText: 'Unauthorized',
|
|
32
|
+
json: jsonMock,
|
|
33
|
+
});
|
|
34
|
+
vi.stubGlobal('fetch', fetchMock);
|
|
35
|
+
// Simulate real CDP behavior: evaluate returns a value, errors are thrown outside
|
|
36
|
+
const page = {
|
|
37
|
+
evaluate: vi.fn(async (js) => Function(`return (${js})`)()()),
|
|
38
|
+
};
|
|
39
|
+
const err = await stepFetch(page, { url: 'https://api.example.com/items' }, null, {}).catch((e) => e);
|
|
40
|
+
expect(err).toBeInstanceOf(CliError);
|
|
41
|
+
expect(err.code).toBe('FETCH_ERROR');
|
|
42
|
+
expect(err.message).toBe('HTTP 401 Unauthorized from https://api.example.com/items');
|
|
43
|
+
expect(jsonMock).not.toHaveBeenCalled();
|
|
44
|
+
});
|
|
45
|
+
it('returns per-item HTTP errors for batch fetches without a browser session', async () => {
|
|
46
|
+
const jsonMock = vi.fn().mockResolvedValue({ error: 'upstream unavailable' });
|
|
47
|
+
const fetchMock = vi.fn().mockResolvedValue({
|
|
48
|
+
ok: false,
|
|
49
|
+
status: 503,
|
|
50
|
+
statusText: 'Service Unavailable',
|
|
51
|
+
json: jsonMock,
|
|
52
|
+
});
|
|
53
|
+
vi.stubGlobal('fetch', fetchMock);
|
|
54
|
+
await expect(stepFetch(null, { url: 'https://api.example.com/items/${{ item.id }}' }, [{ id: 1 }], {})).resolves.toEqual([
|
|
55
|
+
{ error: 'HTTP 503 Service Unavailable from https://api.example.com/items/1' },
|
|
56
|
+
]);
|
|
57
|
+
expect(jsonMock).not.toHaveBeenCalled();
|
|
58
|
+
});
|
|
59
|
+
it('returns per-item HTTP errors for batch browser fetches', async () => {
|
|
60
|
+
const jsonMock = vi.fn().mockResolvedValue({ error: 'upstream unavailable' });
|
|
61
|
+
const fetchMock = vi.fn().mockResolvedValue({
|
|
62
|
+
ok: false,
|
|
63
|
+
status: 503,
|
|
64
|
+
statusText: 'Service Unavailable',
|
|
65
|
+
json: jsonMock,
|
|
66
|
+
});
|
|
67
|
+
vi.stubGlobal('fetch', fetchMock);
|
|
68
|
+
const page = {
|
|
69
|
+
evaluate: vi.fn(async (js) => Function(`return (${js})`)()()),
|
|
70
|
+
};
|
|
71
|
+
await expect(stepFetch(page, { url: 'https://api.example.com/items/${{ item.id }}' }, [{ id: 1 }], {})).resolves.toEqual([
|
|
72
|
+
{ error: 'HTTP 503 Service Unavailable from https://api.example.com/items/1' },
|
|
73
|
+
]);
|
|
74
|
+
expect(jsonMock).not.toHaveBeenCalled();
|
|
75
|
+
});
|
|
76
|
+
it('stringifies non-Error batch browser failures consistently', async () => {
|
|
77
|
+
vi.stubGlobal('fetch', vi.fn().mockRejectedValue('socket hang up'));
|
|
78
|
+
const page = {
|
|
79
|
+
evaluate: vi.fn(async (js) => Function(`return (${js})`)()()),
|
|
80
|
+
};
|
|
81
|
+
await expect(stepFetch(page, { url: 'https://api.example.com/items/${{ item.id }}' }, [{ id: 1 }], {})).resolves.toEqual([
|
|
82
|
+
{ error: 'socket hang up' },
|
|
83
|
+
]);
|
|
84
|
+
});
|
|
85
|
+
it('stringifies non-Error batch non-browser failures consistently', async () => {
|
|
86
|
+
vi.stubGlobal('fetch', vi.fn().mockRejectedValue('socket hang up'));
|
|
87
|
+
await expect(stepFetch(null, { url: 'https://api.example.com/items/${{ item.id }}' }, [{ id: 1 }], {})).resolves.toEqual([
|
|
88
|
+
{ error: 'socket hang up' },
|
|
89
|
+
]);
|
|
90
|
+
});
|
|
91
|
+
// W2: batch item failures emit a warning log
|
|
92
|
+
it('logs a warning for each failed batch item in non-browser mode', async () => {
|
|
93
|
+
const { log } = await import('../../logger.js');
|
|
94
|
+
const warnSpy = vi.spyOn(log, 'warn');
|
|
95
|
+
vi.stubGlobal('fetch', vi.fn().mockResolvedValue({
|
|
96
|
+
ok: false,
|
|
97
|
+
status: 503,
|
|
98
|
+
statusText: 'Service Unavailable',
|
|
99
|
+
json: vi.fn(),
|
|
100
|
+
}));
|
|
101
|
+
await stepFetch(null, { url: 'https://api.example.com/items/${{ item.id }}' }, [{ id: 1 }, { id: 2 }], {});
|
|
102
|
+
expect(warnSpy).toHaveBeenCalledTimes(2);
|
|
103
|
+
expect(warnSpy).toHaveBeenCalledWith(expect.stringContaining('https://api.example.com/items/1'));
|
|
104
|
+
expect(warnSpy).toHaveBeenCalledWith(expect.stringContaining('https://api.example.com/items/2'));
|
|
105
|
+
});
|
|
106
|
+
it('logs a warning for each failed batch item in browser mode', async () => {
|
|
107
|
+
const { log } = await import('../../logger.js');
|
|
108
|
+
const warnSpy = vi.spyOn(log, 'warn');
|
|
109
|
+
vi.stubGlobal('fetch', vi.fn().mockResolvedValue({
|
|
110
|
+
ok: false,
|
|
111
|
+
status: 502,
|
|
112
|
+
statusText: 'Bad Gateway',
|
|
113
|
+
json: vi.fn(),
|
|
114
|
+
}));
|
|
115
|
+
const page = {
|
|
116
|
+
evaluate: vi.fn(async (js) => Function(`return (${js})`)()()),
|
|
117
|
+
};
|
|
118
|
+
await stepFetch(page, { url: 'https://api.example.com/items/${{ item.id }}' }, [{ id: 1 }, { id: 2 }], {});
|
|
119
|
+
expect(warnSpy).toHaveBeenCalledTimes(2);
|
|
120
|
+
expect(warnSpy).toHaveBeenCalledWith(expect.stringContaining('https://api.example.com/items/1'));
|
|
121
|
+
expect(warnSpy).toHaveBeenCalledWith(expect.stringContaining('https://api.example.com/items/2'));
|
|
122
|
+
});
|
|
123
|
+
});
|
|
@@ -2,9 +2,7 @@
|
|
|
2
2
|
* Pipeline steps: data transforms — select, map, filter, sort, limit.
|
|
3
3
|
*/
|
|
4
4
|
import { render, evalExpr } from '../template.js';
|
|
5
|
-
|
|
6
|
-
return typeof value === 'object' && value !== null && !Array.isArray(value);
|
|
7
|
-
}
|
|
5
|
+
import { isRecord } from '../../utils.js';
|
|
8
6
|
export async function stepSelect(_page, params, data, args) {
|
|
9
7
|
const pathStr = String(render(params, { args, data }));
|
|
10
8
|
if (data && typeof data === 'object') {
|
|
@@ -61,9 +59,7 @@ export async function stepSort(_page, params, data, _args) {
|
|
|
61
59
|
return [...data].sort((a, b) => {
|
|
62
60
|
const left = isRecord(a) ? a[key] : undefined;
|
|
63
61
|
const right = isRecord(b) ? b[key] : undefined;
|
|
64
|
-
const
|
|
65
|
-
const vb = right ?? '';
|
|
66
|
-
const cmp = va < vb ? -1 : va > vb ? 1 : 0;
|
|
62
|
+
const cmp = String(left ?? '').localeCompare(String(right ?? ''), undefined, { numeric: true });
|
|
67
63
|
return reverse ? -cmp : cmp;
|
|
68
64
|
});
|
|
69
65
|
}
|
|
@@ -1,9 +1,8 @@
|
|
|
1
1
|
/**
|
|
2
2
|
* Pipeline template engine: ${{ ... }} expression rendering.
|
|
3
3
|
*/
|
|
4
|
-
|
|
5
|
-
|
|
6
|
-
}
|
|
4
|
+
import vm from 'node:vm';
|
|
5
|
+
import { isRecord } from '../utils.js';
|
|
7
6
|
export function render(template, ctx) {
|
|
8
7
|
if (typeof template !== 'string')
|
|
9
8
|
return template;
|
|
@@ -29,46 +28,28 @@ export function evalExpr(expr, ctx) {
|
|
|
29
28
|
const data = ctx.data;
|
|
30
29
|
const index = ctx.index ?? 0;
|
|
31
30
|
// ── Pipe filters: expr | filter1(arg) | filter2 ──
|
|
32
|
-
//
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
let
|
|
37
|
-
|
|
38
|
-
result = applyFilter(segments[i], result);
|
|
31
|
+
// Split on single | (not ||) so "item.a || item.b | upper" works correctly.
|
|
32
|
+
const pipeSegments = expr.split(/(?<!\|)\|(?!\|)/).map(s => s.trim());
|
|
33
|
+
if (pipeSegments.length > 1) {
|
|
34
|
+
let result = evalExpr(pipeSegments[0], ctx);
|
|
35
|
+
for (let i = 1; i < pipeSegments.length; i++) {
|
|
36
|
+
result = applyFilter(pipeSegments[i], result);
|
|
39
37
|
}
|
|
40
38
|
return result;
|
|
41
39
|
}
|
|
42
|
-
//
|
|
43
|
-
const
|
|
44
|
-
if (
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
if (!isNaN(numVal)) {
|
|
51
|
-
switch (op) {
|
|
52
|
-
case '+': return numVal + num;
|
|
53
|
-
case '-': return numVal - num;
|
|
54
|
-
case '*': return numVal * num;
|
|
55
|
-
case '/': return num !== 0 ? numVal / num : 0;
|
|
56
|
-
}
|
|
57
|
-
}
|
|
58
|
-
}
|
|
59
|
-
}
|
|
60
|
-
// JS-like fallback expression: item.tweetCount || 'N/A'
|
|
61
|
-
const orMatch = expr.match(/^(.+?)\s*\|\|\s*(.+)$/);
|
|
62
|
-
if (orMatch) {
|
|
63
|
-
const left = evalExpr(orMatch[1].trim(), ctx);
|
|
64
|
-
if (left)
|
|
65
|
-
return left;
|
|
66
|
-
const right = orMatch[2].trim();
|
|
67
|
-
return right.replace(/^['"]|['"]$/g, '');
|
|
68
|
-
}
|
|
40
|
+
// Fast path: quoted string literal — skip VM overhead
|
|
41
|
+
const strLit = expr.match(/^(['"])(.*)\1$/);
|
|
42
|
+
if (strLit)
|
|
43
|
+
return strLit[2];
|
|
44
|
+
// Fast path: numeric literal
|
|
45
|
+
if (/^\d+(\.\d+)?$/.test(expr))
|
|
46
|
+
return Number(expr);
|
|
47
|
+
// Try resolving as a simple dotted path (item.foo.bar, args.limit, index)
|
|
69
48
|
const resolved = resolvePath(expr, { args, item, data, index });
|
|
70
49
|
if (resolved !== null && resolved !== undefined)
|
|
71
50
|
return resolved;
|
|
51
|
+
// Fallback: evaluate as JS in a sandboxed VM.
|
|
52
|
+
// Handles ||, ??, arithmetic, ternary, method calls, etc. natively.
|
|
72
53
|
return evalJsExpr(expr, { args, item, data, index });
|
|
73
54
|
}
|
|
74
55
|
/**
|
|
@@ -87,7 +68,7 @@ function applyFilter(filterExpr, value) {
|
|
|
87
68
|
case 'default': {
|
|
88
69
|
if (value === null || value === undefined || value === '') {
|
|
89
70
|
const intVal = parseInt(filterArg, 10);
|
|
90
|
-
if (!isNaN(intVal) && String(intVal) === filterArg.trim())
|
|
71
|
+
if (!Number.isNaN(intVal) && String(intVal) === filterArg.trim())
|
|
91
72
|
return intVal;
|
|
92
73
|
return filterArg;
|
|
93
74
|
}
|
|
@@ -103,7 +84,7 @@ function applyFilter(filterExpr, value) {
|
|
|
103
84
|
return typeof value === 'string' ? value.trim() : value;
|
|
104
85
|
case 'truncate': {
|
|
105
86
|
const n = parseInt(filterArg, 10) || 50;
|
|
106
|
-
return typeof value === 'string' && value.length > n ? value.slice(0, n)
|
|
87
|
+
return typeof value === 'string' && value.length > n ? `${value.slice(0, n)}...` : value;
|
|
107
88
|
}
|
|
108
89
|
case 'replace': {
|
|
109
90
|
if (typeof value !== 'string')
|
|
@@ -132,6 +113,7 @@ function applyFilter(filterExpr, value) {
|
|
|
132
113
|
case 'sanitize':
|
|
133
114
|
// Remove invalid filename characters
|
|
134
115
|
return typeof value === 'string'
|
|
116
|
+
// biome-ignore lint/suspicious/noControlCharactersInRegex: intentional - strips C0 control chars from filenames
|
|
135
117
|
? value.replace(/[<>:"/\\|?*\x00-\x1f]/g, '_')
|
|
136
118
|
: value;
|
|
137
119
|
case 'ext': {
|
|
@@ -196,26 +178,58 @@ export function resolvePath(pathStr, ctx) {
|
|
|
196
178
|
}
|
|
197
179
|
/**
|
|
198
180
|
* Evaluate arbitrary JS expressions as a last-resort fallback.
|
|
199
|
-
*
|
|
200
|
-
|
|
201
|
-
|
|
202
|
-
|
|
203
|
-
*
|
|
204
|
-
*
|
|
205
|
-
* If opencli ever loads untrusted third-party adapters, this MUST be replaced
|
|
206
|
-
* with a proper sandboxed evaluator.
|
|
181
|
+
* Runs inside a `node:vm` sandbox with dynamic code generation disabled.
|
|
182
|
+
*/
|
|
183
|
+
const FORBIDDEN_EXPR_PATTERNS = /\b(constructor|__proto__|prototype|globalThis|process|require|import|eval)\b/;
|
|
184
|
+
/**
|
|
185
|
+
* Deep-copy plain data to sever prototype chains, preventing sandbox escape
|
|
186
|
+
* via `args.constructor.constructor('return process')()` etc.
|
|
207
187
|
*/
|
|
188
|
+
function sanitizeContext(obj) {
|
|
189
|
+
if (obj === null || obj === undefined)
|
|
190
|
+
return obj;
|
|
191
|
+
if (typeof obj !== 'object' && typeof obj !== 'function')
|
|
192
|
+
return obj;
|
|
193
|
+
try {
|
|
194
|
+
return JSON.parse(JSON.stringify(obj));
|
|
195
|
+
}
|
|
196
|
+
catch {
|
|
197
|
+
return {};
|
|
198
|
+
}
|
|
199
|
+
}
|
|
208
200
|
function evalJsExpr(expr, ctx) {
|
|
209
201
|
// Guard against absurdly long expressions that could indicate injection.
|
|
210
202
|
if (expr.length > 2000)
|
|
211
203
|
return undefined;
|
|
212
|
-
|
|
213
|
-
|
|
214
|
-
|
|
204
|
+
// Block obvious sandbox escape attempts.
|
|
205
|
+
if (FORBIDDEN_EXPR_PATTERNS.test(expr))
|
|
206
|
+
return undefined;
|
|
207
|
+
const args = sanitizeContext(ctx.args ?? {});
|
|
208
|
+
const item = sanitizeContext(ctx.item ?? {});
|
|
209
|
+
const data = sanitizeContext(ctx.data);
|
|
215
210
|
const index = ctx.index ?? 0;
|
|
216
211
|
try {
|
|
217
|
-
|
|
218
|
-
|
|
212
|
+
return vm.runInNewContext(`(${expr})`, {
|
|
213
|
+
args,
|
|
214
|
+
item,
|
|
215
|
+
data,
|
|
216
|
+
index,
|
|
217
|
+
encodeURIComponent,
|
|
218
|
+
decodeURIComponent,
|
|
219
|
+
JSON,
|
|
220
|
+
Math,
|
|
221
|
+
Number,
|
|
222
|
+
String,
|
|
223
|
+
Boolean,
|
|
224
|
+
Array,
|
|
225
|
+
Date,
|
|
226
|
+
}, {
|
|
227
|
+
timeout: 50,
|
|
228
|
+
contextCodeGeneration: {
|
|
229
|
+
strings: false,
|
|
230
|
+
wasm: false,
|
|
231
|
+
},
|
|
232
|
+
});
|
|
219
233
|
}
|
|
220
234
|
catch {
|
|
221
235
|
return undefined;
|
|
@@ -51,6 +51,31 @@ describe('evalExpr', () => {
|
|
|
51
51
|
it('evaluates || with truthy left', () => {
|
|
52
52
|
expect(evalExpr("item.name || 'N/A'", { item: { name: 'Alice' } })).toBe('Alice');
|
|
53
53
|
});
|
|
54
|
+
it('evaluates chained || fallback (issue #303)', () => {
|
|
55
|
+
// When first two are falsy, should evaluate through to the string literal
|
|
56
|
+
expect(evalExpr("item.a || item.b || 'default'", { item: {} })).toBe('default');
|
|
57
|
+
});
|
|
58
|
+
it('evaluates chained || with middle value truthy', () => {
|
|
59
|
+
expect(evalExpr("item.a || item.b || 'default'", { item: { b: 'middle' } })).toBe('middle');
|
|
60
|
+
});
|
|
61
|
+
it('evaluates chained || with first value truthy', () => {
|
|
62
|
+
expect(evalExpr("item.a || item.b || 'default'", { item: { a: 'first', b: 'middle' } })).toBe('first');
|
|
63
|
+
});
|
|
64
|
+
it('evaluates || with 0 as falsy left (JS semantics)', () => {
|
|
65
|
+
expect(evalExpr("item.count || 'N/A'", { item: { count: 0 } })).toBe('N/A');
|
|
66
|
+
});
|
|
67
|
+
it('evaluates || with empty string as falsy left', () => {
|
|
68
|
+
expect(evalExpr("item.name || 'unknown'", { item: { name: '' } })).toBe('unknown');
|
|
69
|
+
});
|
|
70
|
+
it('evaluates || with numeric fallback returning number type', () => {
|
|
71
|
+
expect(evalExpr('item.a || 42', { item: {} })).toBe(42);
|
|
72
|
+
});
|
|
73
|
+
it('evaluates 4-way chained ||', () => {
|
|
74
|
+
expect(evalExpr("item.a || item.b || item.c || 'last'", { item: { c: 'third' } })).toBe('third');
|
|
75
|
+
});
|
|
76
|
+
it('handles || combined with pipe filter', () => {
|
|
77
|
+
expect(evalExpr("item.a || item.b | upper", { item: { b: 'hello' } })).toBe('HELLO');
|
|
78
|
+
});
|
|
54
79
|
it('resolves simple path', () => {
|
|
55
80
|
expect(evalExpr('item.title', { item: { title: 'Test' } })).toBe('Test');
|
|
56
81
|
});
|
|
@@ -63,6 +88,9 @@ describe('evalExpr', () => {
|
|
|
63
88
|
it('evaluates method calls on values', () => {
|
|
64
89
|
expect(evalExpr("args.username.startsWith('@') ? args.username : '@' + args.username", { args: { username: 'alice' } })).toBe('@alice');
|
|
65
90
|
});
|
|
91
|
+
it('rejects constructor-based sandbox escapes', () => {
|
|
92
|
+
expect(evalExpr("args['cons' + 'tructor']['constructor']('return process')()", { args: {} })).toBeUndefined();
|
|
93
|
+
});
|
|
66
94
|
it('applies join filter', () => {
|
|
67
95
|
expect(evalExpr('item.tags | join(,)', { item: { tags: ['a', 'b', 'c'] } })).toBe('a,b,c');
|
|
68
96
|
});
|
|
@@ -85,6 +85,24 @@ describe('stepSort', () => {
|
|
|
85
85
|
await stepSort(null, 'score', SAMPLE_DATA, {});
|
|
86
86
|
expect(SAMPLE_DATA).toEqual(original);
|
|
87
87
|
});
|
|
88
|
+
it('sorts string-encoded numbers naturally by default', async () => {
|
|
89
|
+
const data = [
|
|
90
|
+
{ name: 'A', volume: '99' },
|
|
91
|
+
{ name: 'B', volume: '1000' },
|
|
92
|
+
{ name: 'C', volume: '250' },
|
|
93
|
+
];
|
|
94
|
+
const result = await stepSort(null, { by: 'volume', order: 'desc' }, data, {});
|
|
95
|
+
expect(result.map((r) => r.name)).toEqual(['B', 'C', 'A']);
|
|
96
|
+
});
|
|
97
|
+
it('handles missing fields gracefully', async () => {
|
|
98
|
+
const data = [
|
|
99
|
+
{ name: 'A', value: '10' },
|
|
100
|
+
{ name: 'B' },
|
|
101
|
+
{ name: 'C', value: '5' },
|
|
102
|
+
];
|
|
103
|
+
const result = await stepSort(null, { by: 'value', order: 'asc' }, data, {});
|
|
104
|
+
expect(result.map((r) => r.name)).toEqual(['B', 'C', 'A']);
|
|
105
|
+
});
|
|
88
106
|
});
|
|
89
107
|
describe('stepLimit', () => {
|
|
90
108
|
it('limits array to N items', async () => {
|