@jackwener/opencli 1.3.3 → 1.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.github/pull_request_template.md +3 -1
- package/.github/workflows/build-extension.yml +7 -1
- package/.github/workflows/ci.yml +29 -3
- package/.github/workflows/docs.yml +1 -1
- package/.github/workflows/e2e-headed.yml +20 -0
- package/.github/workflows/release.yml +1 -1
- package/.github/workflows/security.yml +0 -3
- package/CHANGELOG.md +55 -0
- package/CONTRIBUTING.md +6 -3
- package/README.md +30 -3
- package/README.zh-CN.md +30 -3
- package/SKILL.md +7 -1
- package/TESTING.md +1 -0
- package/chatwise-opencli.ps1 +82 -0
- package/dist/analysis.d.ts +38 -0
- package/dist/analysis.js +166 -0
- package/dist/browser/cdp.d.ts +0 -4
- package/dist/browser/cdp.js +53 -41
- package/dist/browser/cdp.test.d.ts +1 -0
- package/dist/browser/cdp.test.js +52 -0
- package/dist/browser/dom-snapshot.d.ts +2 -2
- package/dist/browser/dom-snapshot.js +54 -1
- package/dist/browser/dom-snapshot.test.js +36 -0
- package/dist/browser/index.d.ts +2 -2
- package/dist/browser/index.js +1 -1
- package/dist/browser/mcp.d.ts +0 -2
- package/dist/browser/mcp.js +2 -3
- package/dist/browser/page.d.ts +4 -3
- package/dist/browser/page.js +34 -37
- package/dist/browser/stealth.d.ts +0 -2
- package/dist/browser/stealth.js +24 -9
- package/dist/browser.test.js +2 -2
- package/dist/build-manifest.js +15 -9
- package/dist/build-manifest.test.js +12 -0
- package/dist/cascade.js +4 -2
- package/dist/cli-manifest.json +639 -258
- package/dist/cli.js +57 -29
- package/dist/clis/_shared/desktop-commands.d.ts +22 -0
- package/dist/clis/_shared/desktop-commands.js +108 -0
- package/dist/clis/antigravity/serve.js +5 -2
- package/dist/clis/arxiv/search.js +1 -1
- package/dist/clis/bilibili/dynamic.test.d.ts +1 -0
- package/dist/clis/bilibili/dynamic.test.js +68 -0
- package/dist/clis/bilibili/favorite.js +4 -2
- package/dist/clis/bilibili/following.js +3 -2
- package/dist/clis/bilibili/subtitle.js +8 -7
- package/dist/clis/bilibili/utils.js +2 -2
- package/dist/clis/boss/batchgreet.js +1 -1
- package/dist/clis/boss/chatlist.js +1 -1
- package/dist/clis/boss/chatmsg.js +1 -1
- package/dist/clis/boss/detail.js +1 -1
- package/dist/clis/boss/exchange.js +1 -1
- package/dist/clis/boss/greet.js +1 -1
- package/dist/clis/boss/invite.js +1 -1
- package/dist/clis/boss/joblist.js +1 -1
- package/dist/clis/boss/mark.js +4 -3
- package/dist/clis/boss/recommend.js +1 -1
- package/dist/clis/boss/resume.js +1 -1
- package/dist/clis/boss/search.js +1 -1
- package/dist/clis/boss/send.js +5 -4
- package/dist/clis/boss/stats.js +1 -1
- package/dist/clis/chatgpt/ask.js +4 -0
- package/dist/clis/chatgpt/new.js +5 -1
- package/dist/clis/chatgpt/read.js +5 -1
- package/dist/clis/chatgpt/send.js +2 -1
- package/dist/clis/chatgpt/status.js +5 -1
- package/dist/clis/chatwise/ask.js +8 -2
- package/dist/clis/chatwise/export.js +2 -0
- package/dist/clis/chatwise/history.js +2 -0
- package/dist/clis/chatwise/model.js +8 -3
- package/dist/clis/chatwise/new.js +3 -18
- package/dist/clis/chatwise/read.js +2 -0
- package/dist/clis/chatwise/screenshot.js +3 -27
- package/dist/clis/chatwise/send.js +8 -2
- package/dist/clis/chatwise/shared.d.ts +2 -0
- package/dist/clis/chatwise/shared.js +6 -0
- package/dist/clis/chatwise/status.js +3 -22
- package/dist/clis/codex/ask.js +6 -2
- package/dist/clis/codex/dump.js +2 -25
- package/dist/clis/codex/new.js +2 -25
- package/dist/clis/codex/screenshot.js +2 -27
- package/dist/clis/codex/send.js +6 -4
- package/dist/clis/codex/status.js +2 -22
- package/dist/clis/cursor/ask.js +2 -1
- package/dist/clis/cursor/composer.js +2 -1
- package/dist/clis/cursor/dump.js +2 -25
- package/dist/clis/cursor/new.js +2 -18
- package/dist/clis/cursor/read.js +2 -1
- package/dist/clis/cursor/screenshot.js +1 -30
- package/dist/clis/cursor/send.js +2 -1
- package/dist/clis/cursor/status.js +2 -21
- package/dist/clis/dictionary/examples.yaml +25 -0
- package/dist/clis/dictionary/search.yaml +27 -0
- package/dist/clis/dictionary/synonyms.yaml +25 -0
- package/dist/clis/douban/book-hot.js +1 -1
- package/dist/clis/douban/movie-hot.js +1 -1
- package/dist/clis/douban/search.js +1 -1
- package/dist/clis/douban/utils.d.ts +4 -1
- package/dist/clis/douban/utils.js +156 -1
- package/dist/clis/doubao/ask.js +1 -1
- package/dist/clis/doubao/new.js +1 -1
- package/dist/clis/doubao/read.js +1 -1
- package/dist/clis/doubao/send.js +1 -1
- package/dist/clis/doubao/status.js +1 -1
- package/dist/clis/doubao-app/ask.js +1 -1
- package/dist/clis/doubao-app/new.js +1 -1
- package/dist/clis/doubao-app/read.js +1 -1
- package/dist/clis/doubao-app/send.js +1 -1
- package/dist/clis/grok/ask.d.ts +4 -0
- package/dist/clis/grok/ask.js +28 -10
- package/dist/clis/grok/ask.test.js +18 -0
- package/dist/clis/jd/item.d.ts +1 -0
- package/dist/clis/jd/item.js +96 -0
- package/dist/clis/jd/item.test.d.ts +1 -0
- package/dist/clis/jd/item.test.js +28 -0
- package/dist/clis/jike/feed.js +1 -1
- package/dist/clis/jike/search.js +1 -1
- package/dist/clis/linkedin/search.js +5 -4
- package/dist/clis/linkedin/timeline.d.ts +21 -0
- package/dist/clis/linkedin/timeline.js +503 -0
- package/dist/clis/linkedin/timeline.test.d.ts +1 -0
- package/dist/clis/linkedin/timeline.test.js +81 -0
- package/dist/clis/medium/feed.js +1 -1
- package/dist/clis/medium/search.js +1 -1
- package/dist/clis/medium/user.js +1 -1
- package/dist/clis/medium/{shared.js → utils.js} +2 -1
- package/dist/clis/pixiv/detail.yaml +49 -0
- package/dist/clis/pixiv/download.d.ts +7 -0
- package/dist/clis/pixiv/download.js +78 -0
- package/dist/clis/pixiv/download.test.d.ts +1 -0
- package/dist/clis/pixiv/download.test.js +87 -0
- package/dist/clis/pixiv/illusts.d.ts +8 -0
- package/dist/clis/pixiv/illusts.js +65 -0
- package/dist/clis/pixiv/illusts.test.d.ts +1 -0
- package/dist/clis/pixiv/illusts.test.js +99 -0
- package/dist/clis/pixiv/ranking.yaml +53 -0
- package/dist/clis/pixiv/search.d.ts +6 -0
- package/dist/clis/pixiv/search.js +43 -0
- package/dist/clis/pixiv/search.test.d.ts +1 -0
- package/dist/clis/pixiv/search.test.js +83 -0
- package/dist/clis/pixiv/test-utils.d.ts +12 -0
- package/dist/clis/pixiv/test-utils.js +23 -0
- package/dist/clis/pixiv/user.yaml +46 -0
- package/dist/clis/pixiv/utils.d.ts +27 -0
- package/dist/clis/pixiv/utils.js +49 -0
- package/dist/clis/reddit/comment.js +2 -1
- package/dist/clis/reddit/read.js +4 -3
- package/dist/clis/reddit/read.test.d.ts +1 -0
- package/dist/clis/reddit/read.test.js +28 -0
- package/dist/clis/reddit/save.js +2 -1
- package/dist/clis/reddit/saved.js +7 -3
- package/dist/clis/reddit/subscribe.js +2 -1
- package/dist/clis/reddit/upvote.js +2 -1
- package/dist/clis/reddit/upvoted.js +7 -3
- package/dist/clis/sinablog/article.js +1 -1
- package/dist/clis/sinablog/hot.js +1 -1
- package/dist/clis/sinablog/user.js +1 -1
- package/dist/clis/substack/feed.js +1 -1
- package/dist/clis/substack/publication.js +1 -1
- package/dist/clis/substack/search.js +3 -2
- package/dist/clis/substack/{shared.js → utils.js} +3 -2
- package/dist/clis/tiktok/search.yaml +2 -1
- package/dist/clis/twitter/accept.js +2 -1
- package/dist/clis/twitter/article.js +4 -1
- package/dist/clis/twitter/block.js +2 -1
- package/dist/clis/twitter/bookmark.js +2 -1
- package/dist/clis/twitter/bookmarks.js +3 -2
- package/dist/clis/twitter/delete.js +2 -1
- package/dist/clis/twitter/follow.js +2 -1
- package/dist/clis/twitter/followers.js +3 -2
- package/dist/clis/twitter/following.js +3 -2
- package/dist/clis/twitter/hide-reply.js +2 -1
- package/dist/clis/twitter/like.js +2 -1
- package/dist/clis/twitter/notifications.js +2 -1
- package/dist/clis/twitter/post.js +2 -1
- package/dist/clis/twitter/profile.js +5 -2
- package/dist/clis/twitter/reply-dm.js +2 -1
- package/dist/clis/twitter/reply.js +2 -1
- package/dist/clis/twitter/search.js +30 -13
- package/dist/clis/twitter/search.test.d.ts +1 -0
- package/dist/clis/twitter/search.test.js +104 -0
- package/dist/clis/twitter/thread.js +2 -2
- package/dist/clis/twitter/timeline.js +3 -2
- package/dist/clis/twitter/trending.js +3 -2
- package/dist/clis/twitter/unblock.js +2 -1
- package/dist/clis/twitter/unbookmark.js +2 -1
- package/dist/clis/twitter/unfollow.js +2 -1
- package/dist/clis/v2ex/daily.js +3 -2
- package/dist/clis/v2ex/me.js +3 -2
- package/dist/clis/v2ex/notifications.js +4 -4
- package/dist/clis/web/read.d.ts +16 -0
- package/dist/clis/web/read.js +202 -0
- package/dist/clis/xueqiu/danjuan-utils.d.ts +55 -0
- package/dist/clis/xueqiu/danjuan-utils.js +126 -0
- package/dist/clis/xueqiu/danjuan-utils.test.d.ts +1 -0
- package/dist/clis/xueqiu/danjuan-utils.test.js +41 -0
- package/dist/clis/xueqiu/fund-holdings.d.ts +1 -0
- package/dist/clis/xueqiu/fund-holdings.js +28 -0
- package/dist/clis/xueqiu/fund-snapshot.d.ts +1 -0
- package/dist/clis/xueqiu/fund-snapshot.js +25 -0
- package/dist/clis/youtube/transcript.js +5 -4
- package/dist/clis/youtube/video.js +3 -2
- package/dist/daemon.js +7 -3
- package/dist/discovery.js +11 -10
- package/dist/doctor.js +2 -1
- package/dist/download/index.d.ts +4 -12
- package/dist/download/index.js +33 -12
- package/dist/download/index.test.js +79 -2
- package/dist/download/media-download.js +4 -2
- package/dist/engine.test.js +76 -4
- package/dist/execution.d.ts +1 -9
- package/dist/execution.js +56 -46
- package/dist/explore.js +12 -111
- package/dist/external-clis.yaml +0 -8
- package/dist/external.js +7 -5
- package/dist/external.test.js +4 -0
- package/dist/generate.d.ts +0 -9
- package/dist/generate.js +4 -20
- package/dist/hooks.d.ts +46 -0
- package/dist/hooks.js +56 -0
- package/dist/hooks.test.d.ts +4 -0
- package/dist/hooks.test.js +92 -0
- package/dist/interceptor.js +70 -23
- package/dist/main.js +2 -0
- package/dist/output.js +12 -6
- package/dist/pipeline/executor.js +1 -1
- package/dist/pipeline/steps/browser.js +1 -3
- package/dist/pipeline/steps/download.js +42 -26
- package/dist/pipeline/steps/download.test.d.ts +1 -0
- package/dist/pipeline/steps/download.test.js +101 -0
- package/dist/pipeline/steps/fetch.js +40 -22
- package/dist/pipeline/steps/fetch.test.d.ts +1 -0
- package/dist/pipeline/steps/fetch.test.js +123 -0
- package/dist/pipeline/steps/transform.js +2 -6
- package/dist/pipeline/template.js +66 -52
- package/dist/pipeline/template.test.js +28 -0
- package/dist/pipeline/transform.test.js +18 -0
- package/dist/plugin.d.ts +40 -1
- package/dist/plugin.js +214 -17
- package/dist/plugin.test.d.ts +1 -1
- package/dist/plugin.test.js +219 -3
- package/dist/record.js +6 -98
- package/dist/registry-api.d.ts +2 -0
- package/dist/registry-api.js +1 -0
- package/dist/registry.d.ts +5 -2
- package/dist/registry.js +1 -2
- package/dist/runtime.d.ts +0 -1
- package/dist/runtime.js +14 -4
- package/dist/snapshotFormatter.d.ts +7 -14
- package/dist/snapshotFormatter.js +38 -78
- package/dist/utils.d.ts +9 -0
- package/dist/utils.js +29 -0
- package/dist/validate.js +3 -5
- package/dist/yaml-schema.d.ts +26 -0
- package/dist/yaml-schema.js +5 -0
- package/docs/.vitepress/config.mts +3 -0
- package/docs/adapters/browser/dictionary.md +27 -0
- package/docs/adapters/browser/jd.md +27 -0
- package/docs/adapters/browser/linkedin.md +6 -0
- package/docs/adapters/browser/pixiv.md +92 -0
- package/docs/adapters/browser/web.md +30 -0
- package/docs/adapters/browser/xueqiu.md +27 -9
- package/docs/adapters/index.md +3 -1
- package/docs/comparison.md +125 -0
- package/docs/developer/contributing.md +21 -2
- package/docs/developer/testing.md +14 -8
- package/docs/developer/ts-adapter.md +18 -0
- package/docs/developer/yaml-adapter.md +16 -0
- package/docs/guide/plugins.md +10 -0
- package/docs/zh/guide/plugins.md +10 -0
- package/extension/dist/background.js +519 -444
- package/extension/manifest.json +1 -1
- package/extension/package.json +1 -1
- package/extension/src/background.test.ts +46 -1
- package/extension/src/background.ts +108 -33
- package/extension/src/cdp.ts +9 -9
- package/package.json +3 -2
- package/scripts/check-doc-coverage.sh +2 -0
- package/src/analysis.ts +170 -0
- package/src/browser/cdp.test.ts +66 -0
- package/src/browser/cdp.ts +59 -44
- package/src/browser/dom-snapshot.test.ts +42 -0
- package/src/browser/dom-snapshot.ts +56 -3
- package/src/browser/index.ts +2 -2
- package/src/browser/mcp.ts +2 -4
- package/src/browser/page.ts +34 -37
- package/src/browser/stealth.ts +24 -10
- package/src/browser.test.ts +2 -2
- package/src/build-manifest.test.ts +14 -0
- package/src/build-manifest.ts +13 -31
- package/src/cascade.ts +5 -3
- package/src/cli.ts +66 -34
- package/src/clis/_shared/desktop-commands.ts +121 -0
- package/src/clis/antigravity/serve.ts +6 -3
- package/src/clis/arxiv/search.ts +1 -1
- package/src/clis/bilibili/dynamic.test.ts +79 -0
- package/src/clis/bilibili/favorite.ts +5 -2
- package/src/clis/bilibili/following.ts +3 -2
- package/src/clis/bilibili/subtitle.ts +8 -7
- package/src/clis/bilibili/utils.ts +2 -2
- package/src/clis/boss/batchgreet.ts +1 -1
- package/src/clis/boss/chatlist.ts +1 -1
- package/src/clis/boss/chatmsg.ts +1 -1
- package/src/clis/boss/detail.ts +1 -1
- package/src/clis/boss/exchange.ts +1 -1
- package/src/clis/boss/greet.ts +1 -1
- package/src/clis/boss/invite.ts +1 -1
- package/src/clis/boss/joblist.ts +1 -1
- package/src/clis/boss/mark.ts +4 -3
- package/src/clis/boss/recommend.ts +1 -1
- package/src/clis/boss/resume.ts +1 -1
- package/src/clis/boss/search.ts +1 -1
- package/src/clis/boss/send.ts +5 -4
- package/src/clis/boss/stats.ts +1 -1
- package/src/clis/chatgpt/ask.ts +5 -0
- package/src/clis/chatgpt/new.ts +7 -2
- package/src/clis/chatgpt/read.ts +7 -2
- package/src/clis/chatgpt/send.ts +3 -2
- package/src/clis/chatgpt/status.ts +6 -1
- package/src/clis/chatwise/ask.ts +7 -2
- package/src/clis/chatwise/export.ts +2 -0
- package/src/clis/chatwise/history.ts +2 -0
- package/src/clis/chatwise/model.ts +7 -3
- package/src/clis/chatwise/new.ts +3 -20
- package/src/clis/chatwise/read.ts +2 -0
- package/src/clis/chatwise/screenshot.ts +3 -32
- package/src/clis/chatwise/send.ts +7 -2
- package/src/clis/chatwise/shared.ts +8 -0
- package/src/clis/chatwise/status.ts +3 -24
- package/src/clis/codex/ask.ts +5 -2
- package/src/clis/codex/dump.ts +2 -27
- package/src/clis/codex/new.ts +2 -28
- package/src/clis/codex/screenshot.ts +2 -32
- package/src/clis/codex/send.ts +5 -4
- package/src/clis/codex/status.ts +2 -24
- package/src/clis/cursor/ask.ts +2 -1
- package/src/clis/cursor/composer.ts +2 -1
- package/src/clis/cursor/dump.ts +2 -27
- package/src/clis/cursor/new.ts +2 -20
- package/src/clis/cursor/read.ts +2 -1
- package/src/clis/cursor/screenshot.ts +1 -36
- package/src/clis/cursor/send.ts +2 -1
- package/src/clis/cursor/status.ts +2 -22
- package/src/clis/dictionary/examples.yaml +25 -0
- package/src/clis/dictionary/search.yaml +27 -0
- package/src/clis/dictionary/synonyms.yaml +25 -0
- package/src/clis/douban/book-hot.ts +1 -1
- package/src/clis/douban/movie-hot.ts +1 -1
- package/src/clis/douban/search.ts +1 -1
- package/src/clis/douban/utils.ts +165 -1
- package/src/clis/doubao/ask.ts +1 -1
- package/src/clis/doubao/new.ts +1 -1
- package/src/clis/doubao/read.ts +1 -1
- package/src/clis/doubao/send.ts +1 -1
- package/src/clis/doubao/status.ts +1 -1
- package/src/clis/doubao-app/ask.ts +1 -1
- package/src/clis/doubao-app/new.ts +1 -1
- package/src/clis/doubao-app/read.ts +1 -1
- package/src/clis/doubao-app/send.ts +1 -1
- package/src/clis/grok/ask.test.ts +25 -0
- package/src/clis/grok/ask.ts +25 -12
- package/src/clis/jd/item.test.ts +35 -0
- package/src/clis/jd/item.ts +101 -0
- package/src/clis/jike/feed.ts +1 -1
- package/src/clis/jike/search.ts +1 -1
- package/src/clis/linkedin/search.ts +5 -4
- package/src/clis/linkedin/timeline.test.ts +99 -0
- package/src/clis/linkedin/timeline.ts +532 -0
- package/src/clis/medium/feed.ts +1 -1
- package/src/clis/medium/search.ts +1 -1
- package/src/clis/medium/user.ts +1 -1
- package/src/clis/medium/{shared.ts → utils.ts} +2 -1
- package/src/clis/pixiv/detail.yaml +49 -0
- package/src/clis/pixiv/download.test.ts +114 -0
- package/src/clis/pixiv/download.ts +91 -0
- package/src/clis/pixiv/illusts.test.ts +115 -0
- package/src/clis/pixiv/illusts.ts +78 -0
- package/src/clis/pixiv/ranking.yaml +53 -0
- package/src/clis/pixiv/search.test.ts +97 -0
- package/src/clis/pixiv/search.ts +53 -0
- package/src/clis/pixiv/test-utils.ts +29 -0
- package/src/clis/pixiv/user.yaml +46 -0
- package/src/clis/pixiv/utils.ts +62 -0
- package/src/clis/reddit/comment.ts +2 -1
- package/src/clis/reddit/read.test.ts +34 -0
- package/src/clis/reddit/read.ts +4 -3
- package/src/clis/reddit/save.ts +2 -1
- package/src/clis/reddit/saved.ts +6 -2
- package/src/clis/reddit/subscribe.ts +2 -1
- package/src/clis/reddit/upvote.ts +2 -1
- package/src/clis/reddit/upvoted.ts +6 -2
- package/src/clis/sinablog/article.ts +1 -1
- package/src/clis/sinablog/hot.ts +1 -1
- package/src/clis/sinablog/user.ts +1 -1
- package/src/clis/substack/feed.ts +1 -1
- package/src/clis/substack/publication.ts +1 -1
- package/src/clis/substack/search.ts +3 -2
- package/src/clis/substack/{shared.ts → utils.ts} +3 -2
- package/src/clis/tiktok/search.yaml +2 -1
- package/src/clis/twitter/accept.ts +2 -1
- package/src/clis/twitter/article.ts +3 -1
- package/src/clis/twitter/block.ts +2 -1
- package/src/clis/twitter/bookmark.ts +2 -1
- package/src/clis/twitter/bookmarks.ts +3 -2
- package/src/clis/twitter/delete.ts +2 -1
- package/src/clis/twitter/follow.ts +2 -1
- package/src/clis/twitter/followers.ts +3 -2
- package/src/clis/twitter/following.ts +3 -2
- package/src/clis/twitter/hide-reply.ts +2 -1
- package/src/clis/twitter/like.ts +2 -1
- package/src/clis/twitter/notifications.ts +2 -1
- package/src/clis/twitter/post.ts +2 -1
- package/src/clis/twitter/profile.ts +4 -2
- package/src/clis/twitter/reply-dm.ts +2 -1
- package/src/clis/twitter/reply.ts +2 -1
- package/src/clis/twitter/search.test.ts +113 -0
- package/src/clis/twitter/search.ts +38 -14
- package/src/clis/twitter/thread.ts +2 -2
- package/src/clis/twitter/timeline.ts +3 -2
- package/src/clis/twitter/trending.ts +3 -2
- package/src/clis/twitter/unblock.ts +2 -1
- package/src/clis/twitter/unbookmark.ts +2 -1
- package/src/clis/twitter/unfollow.ts +2 -1
- package/src/clis/v2ex/daily.ts +3 -2
- package/src/clis/v2ex/me.ts +3 -2
- package/src/clis/v2ex/notifications.ts +3 -4
- package/src/clis/web/read.ts +210 -0
- package/src/clis/xueqiu/danjuan-utils.test.ts +49 -0
- package/src/clis/xueqiu/danjuan-utils.ts +176 -0
- package/src/clis/xueqiu/fund-holdings.ts +32 -0
- package/src/clis/xueqiu/fund-snapshot.ts +27 -0
- package/src/clis/youtube/transcript.ts +5 -4
- package/src/clis/youtube/video.ts +3 -2
- package/src/daemon.ts +5 -4
- package/src/discovery.ts +12 -34
- package/src/doctor.ts +3 -2
- package/src/download/index.test.ts +93 -2
- package/src/download/index.ts +44 -23
- package/src/download/media-download.ts +5 -3
- package/src/engine.test.ts +84 -3
- package/src/execution.ts +62 -46
- package/src/explore.ts +21 -90
- package/src/external-clis.yaml +0 -8
- package/src/external.test.ts +9 -0
- package/src/external.ts +12 -10
- package/src/generate.ts +4 -41
- package/src/hooks.test.ts +126 -0
- package/src/hooks.ts +90 -0
- package/src/interceptor.ts +73 -23
- package/src/main.ts +2 -0
- package/src/output.ts +14 -6
- package/src/pipeline/executor.ts +1 -1
- package/src/pipeline/steps/browser.ts +1 -3
- package/src/pipeline/steps/download.test.ts +136 -0
- package/src/pipeline/steps/download.ts +47 -34
- package/src/pipeline/steps/fetch.test.ts +179 -0
- package/src/pipeline/steps/fetch.ts +39 -23
- package/src/pipeline/steps/transform.ts +2 -6
- package/src/pipeline/template.test.ts +28 -0
- package/src/pipeline/template.ts +67 -79
- package/src/pipeline/transform.test.ts +20 -0
- package/src/plugin.test.ts +251 -3
- package/src/plugin.ts +265 -21
- package/src/record.ts +12 -84
- package/src/registry-api.ts +2 -0
- package/src/registry.ts +7 -4
- package/src/runtime.ts +14 -4
- package/src/snapshotFormatter.ts +43 -121
- package/src/utils.ts +39 -0
- package/src/validate.ts +3 -5
- package/src/yaml-schema.ts +28 -0
- package/tests/e2e/browser-auth.test.ts +25 -0
- package/tests/e2e/plugin-management.test.ts +137 -0
- package/tests/e2e/public-commands.test.ts +34 -1
- package/vitest.config.ts +19 -1
- package/.github/workflows/pkg-pr-new.yml +0 -30
- package/dist/clis/douban/shared.d.ts +0 -4
- package/dist/clis/douban/shared.js +0 -155
- package/src/clis/douban/shared.ts +0 -165
- /package/dist/clis/boss/{common.d.ts → utils.d.ts} +0 -0
- /package/dist/clis/boss/{common.js → utils.js} +0 -0
- /package/dist/clis/doubao/{common.d.ts → utils.d.ts} +0 -0
- /package/dist/clis/doubao/{common.js → utils.js} +0 -0
- /package/dist/clis/doubao-app/{common.d.ts → utils.d.ts} +0 -0
- /package/dist/clis/doubao-app/{common.js → utils.js} +0 -0
- /package/dist/clis/jike/{shared.d.ts → utils.d.ts} +0 -0
- /package/dist/clis/jike/{shared.js → utils.js} +0 -0
- /package/dist/clis/medium/{shared.d.ts → utils.d.ts} +0 -0
- /package/dist/clis/sinablog/{shared.d.ts → utils.d.ts} +0 -0
- /package/dist/clis/sinablog/{shared.js → utils.js} +0 -0
- /package/dist/clis/substack/{shared.d.ts → utils.d.ts} +0 -0
- /package/src/clis/boss/{common.ts → utils.ts} +0 -0
- /package/src/clis/doubao/{common.ts → utils.ts} +0 -0
- /package/src/clis/doubao-app/{common.ts → utils.ts} +0 -0
- /package/src/clis/jike/{shared.ts → utils.ts} +0 -0
- /package/src/clis/sinablog/{shared.ts → utils.ts} +0 -0
|
@@ -9,6 +9,7 @@
|
|
|
9
9
|
|
|
10
10
|
import * as fs from 'node:fs';
|
|
11
11
|
import * as path from 'node:path';
|
|
12
|
+
import { getErrorMessage } from '../errors.js';
|
|
12
13
|
import {
|
|
13
14
|
httpDownload,
|
|
14
15
|
ytdlpDownload,
|
|
@@ -153,15 +154,16 @@ export async function downloadMedia(
|
|
|
153
154
|
status: result.success ? 'success' : 'failed',
|
|
154
155
|
size: result.success ? formatBytes(result.size) : (result.error || 'unknown error'),
|
|
155
156
|
});
|
|
156
|
-
} catch (err
|
|
157
|
-
|
|
157
|
+
} catch (err) {
|
|
158
|
+
const msg = getErrorMessage(err);
|
|
159
|
+
if (progressBar) progressBar.fail(msg);
|
|
158
160
|
tracker.onFileComplete(false);
|
|
159
161
|
|
|
160
162
|
results.push({
|
|
161
163
|
index: i + 1,
|
|
162
164
|
type: media.type,
|
|
163
165
|
status: 'failed',
|
|
164
|
-
size:
|
|
166
|
+
size: msg,
|
|
165
167
|
});
|
|
166
168
|
}
|
|
167
169
|
}
|
package/src/engine.test.ts
CHANGED
|
@@ -2,17 +2,20 @@ import { describe, it, expect, vi, beforeEach, afterEach } from 'vitest';
|
|
|
2
2
|
import { discoverClis, discoverPlugins, PLUGINS_DIR } from './discovery.js';
|
|
3
3
|
import { executeCommand } from './execution.js';
|
|
4
4
|
import { getRegistry, cli, Strategy } from './registry.js';
|
|
5
|
+
import { clearAllHooks, onAfterExecute } from './hooks.js';
|
|
5
6
|
import * as fs from 'node:fs';
|
|
7
|
+
import * as os from 'node:os';
|
|
6
8
|
import * as path from 'node:path';
|
|
9
|
+
import { pathToFileURL } from 'node:url';
|
|
7
10
|
|
|
8
11
|
describe('discoverClis', () => {
|
|
9
12
|
it('handles non-existent directories gracefully', async () => {
|
|
10
13
|
// Should not throw for missing directories
|
|
11
|
-
await expect(discoverClis('
|
|
14
|
+
await expect(discoverClis(path.join(os.tmpdir(), 'nonexistent-opencli-test-dir'))).resolves.not.toThrow();
|
|
12
15
|
});
|
|
13
16
|
|
|
14
17
|
it('imports only CLI command modules during filesystem discovery', async () => {
|
|
15
|
-
const tempRoot = await fs.promises.mkdtemp(path.join(
|
|
18
|
+
const tempRoot = await fs.promises.mkdtemp(path.join(os.tmpdir(), 'opencli-discovery-'));
|
|
16
19
|
const siteDir = path.join(tempRoot, 'temp-site');
|
|
17
20
|
const helperPath = path.join(siteDir, 'helper.ts');
|
|
18
21
|
const commandPath = path.join(siteDir, 'hello.ts');
|
|
@@ -24,7 +27,7 @@ globalThis.__opencli_helper_loaded__ = true;
|
|
|
24
27
|
export const helper = true;
|
|
25
28
|
`);
|
|
26
29
|
await fs.promises.writeFile(commandPath, `
|
|
27
|
-
import { cli, Strategy } from '${path.join(process.cwd(), 'src', 'registry.ts')}';
|
|
30
|
+
import { cli, Strategy } from '${pathToFileURL(path.join(process.cwd(), 'src', 'registry.ts')).href}';
|
|
28
31
|
cli({
|
|
29
32
|
site: 'temp-site',
|
|
30
33
|
name: 'hello',
|
|
@@ -45,6 +48,36 @@ cli({
|
|
|
45
48
|
await fs.promises.rm(tempRoot, { recursive: true, force: true });
|
|
46
49
|
}
|
|
47
50
|
});
|
|
51
|
+
|
|
52
|
+
it('falls back to filesystem discovery when the manifest is invalid', async () => {
|
|
53
|
+
const tempBuildRoot = await fs.promises.mkdtemp(path.join(os.tmpdir(), 'opencli-manifest-fallback-'));
|
|
54
|
+
const distDir = path.join(tempBuildRoot, 'dist');
|
|
55
|
+
const siteDir = path.join(distDir, 'fallback-site');
|
|
56
|
+
const commandPath = path.join(siteDir, 'hello.ts');
|
|
57
|
+
const manifestPath = path.join(tempBuildRoot, 'cli-manifest.json');
|
|
58
|
+
|
|
59
|
+
try {
|
|
60
|
+
await fs.promises.mkdir(siteDir, { recursive: true });
|
|
61
|
+
await fs.promises.writeFile(manifestPath, '{ invalid json');
|
|
62
|
+
await fs.promises.writeFile(commandPath, `
|
|
63
|
+
import { cli, Strategy } from '${pathToFileURL(path.join(process.cwd(), 'src', 'registry.ts')).href}';
|
|
64
|
+
cli({
|
|
65
|
+
site: 'fallback-site',
|
|
66
|
+
name: 'hello',
|
|
67
|
+
description: 'hello command',
|
|
68
|
+
strategy: Strategy.PUBLIC,
|
|
69
|
+
browser: false,
|
|
70
|
+
func: async () => [{ ok: true }],
|
|
71
|
+
});
|
|
72
|
+
`);
|
|
73
|
+
|
|
74
|
+
await discoverClis(distDir);
|
|
75
|
+
|
|
76
|
+
expect(getRegistry().get('fallback-site/hello')).toBeDefined();
|
|
77
|
+
} finally {
|
|
78
|
+
await fs.promises.rm(tempBuildRoot, { recursive: true, force: true });
|
|
79
|
+
}
|
|
80
|
+
});
|
|
48
81
|
});
|
|
49
82
|
|
|
50
83
|
describe('discoverPlugins', () => {
|
|
@@ -88,6 +121,11 @@ columns: [message]
|
|
|
88
121
|
});
|
|
89
122
|
|
|
90
123
|
describe('executeCommand', () => {
|
|
124
|
+
beforeEach(() => {
|
|
125
|
+
clearAllHooks();
|
|
126
|
+
vi.unstubAllEnvs();
|
|
127
|
+
});
|
|
128
|
+
|
|
91
129
|
it('accepts kebab-case option names after Commander camelCases them', async () => {
|
|
92
130
|
const cmd = cli({
|
|
93
131
|
site: 'test-engine',
|
|
@@ -165,4 +203,47 @@ describe('executeCommand', () => {
|
|
|
165
203
|
await executeCommand(cmd, {}, true);
|
|
166
204
|
expect(receivedDebug).toBe(true);
|
|
167
205
|
});
|
|
206
|
+
|
|
207
|
+
it('fires onAfterExecute even when command execution throws', async () => {
|
|
208
|
+
const seen: Array<{ error?: unknown; finishedAt?: number }> = [];
|
|
209
|
+
onAfterExecute((ctx) => {
|
|
210
|
+
seen.push({ error: ctx.error, finishedAt: ctx.finishedAt });
|
|
211
|
+
});
|
|
212
|
+
|
|
213
|
+
const cmd = cli({
|
|
214
|
+
site: 'test-engine',
|
|
215
|
+
name: 'failing-test',
|
|
216
|
+
description: 'failing command',
|
|
217
|
+
browser: false,
|
|
218
|
+
strategy: Strategy.PUBLIC,
|
|
219
|
+
func: async () => {
|
|
220
|
+
throw new Error('boom');
|
|
221
|
+
},
|
|
222
|
+
});
|
|
223
|
+
|
|
224
|
+
await expect(executeCommand(cmd, {})).rejects.toThrow('boom');
|
|
225
|
+
expect(seen).toHaveLength(1);
|
|
226
|
+
expect(seen[0].error).toBeInstanceOf(Error);
|
|
227
|
+
expect((seen[0].error as Error).message).toBe('boom');
|
|
228
|
+
expect(typeof seen[0].finishedAt).toBe('number');
|
|
229
|
+
});
|
|
230
|
+
|
|
231
|
+
it('fails fast for chatwise commands when OPENCLI_CDP_ENDPOINT is missing', async () => {
|
|
232
|
+
const cmd = cli({
|
|
233
|
+
site: 'chatwise',
|
|
234
|
+
name: 'status',
|
|
235
|
+
description: 'chatwise status',
|
|
236
|
+
browser: true,
|
|
237
|
+
strategy: Strategy.PUBLIC,
|
|
238
|
+
requiredEnv: [
|
|
239
|
+
{
|
|
240
|
+
name: 'OPENCLI_CDP_ENDPOINT',
|
|
241
|
+
help: 'Set OPENCLI_CDP_ENDPOINT before running chatwise commands.',
|
|
242
|
+
},
|
|
243
|
+
],
|
|
244
|
+
func: async () => [{ ok: true }],
|
|
245
|
+
});
|
|
246
|
+
|
|
247
|
+
await expect(executeCommand(cmd, {})).rejects.toThrow('requires environment variable OPENCLI_CDP_ENDPOINT');
|
|
248
|
+
});
|
|
168
249
|
});
|
package/src/execution.ts
CHANGED
|
@@ -7,6 +7,7 @@
|
|
|
7
7
|
* 3. Domain pre-navigation for cookie/header strategies
|
|
8
8
|
* 4. Timeout enforcement
|
|
9
9
|
* 5. Lazy-loading of TS modules from manifest
|
|
10
|
+
* 6. Lifecycle hooks (onBeforeExecute / onAfterExecute)
|
|
10
11
|
*/
|
|
11
12
|
|
|
12
13
|
import { type CliCommand, type InternalCliCommand, type Arg, Strategy, getRegistry, fullName } from './registry.js';
|
|
@@ -16,22 +17,17 @@ import { executePipeline } from './pipeline/index.js';
|
|
|
16
17
|
import { AdapterLoadError, ArgumentError, CommandExecutionError, getErrorMessage } from './errors.js';
|
|
17
18
|
import { shouldUseBrowserSession } from './capabilityRouting.js';
|
|
18
19
|
import { getBrowserFactory, browserSession, runWithTimeout, DEFAULT_BROWSER_COMMAND_TIMEOUT } from './runtime.js';
|
|
20
|
+
import { emitHook, type HookContext } from './hooks.js';
|
|
19
21
|
|
|
20
|
-
/** Set of TS module paths that have been loaded */
|
|
21
22
|
const _loadedModules = new Set<string>();
|
|
22
23
|
type CommandArgs = Record<string, unknown>;
|
|
23
24
|
|
|
24
|
-
|
|
25
|
-
/**
|
|
26
|
-
* Validates and coerces arguments based on the command's Arg definitions.
|
|
27
|
-
*/
|
|
28
25
|
export function coerceAndValidateArgs(cmdArgs: Arg[], kwargs: CommandArgs): CommandArgs {
|
|
29
26
|
const result: CommandArgs = { ...kwargs };
|
|
30
27
|
|
|
31
28
|
for (const argDef of cmdArgs) {
|
|
32
29
|
const val = result[argDef.name];
|
|
33
|
-
|
|
34
|
-
// 1. Check required
|
|
30
|
+
|
|
35
31
|
if (argDef.required && (val === undefined || val === null || val === '')) {
|
|
36
32
|
throw new ArgumentError(
|
|
37
33
|
`Argument "${argDef.name}" is required.`,
|
|
@@ -40,7 +36,6 @@ export function coerceAndValidateArgs(cmdArgs: Arg[], kwargs: CommandArgs): Comm
|
|
|
40
36
|
}
|
|
41
37
|
|
|
42
38
|
if (val !== undefined && val !== null) {
|
|
43
|
-
// 2. Type coercion
|
|
44
39
|
if (argDef.type === 'int' || argDef.type === 'number') {
|
|
45
40
|
const num = Number(val);
|
|
46
41
|
if (Number.isNaN(num)) {
|
|
@@ -58,7 +53,6 @@ export function coerceAndValidateArgs(cmdArgs: Arg[], kwargs: CommandArgs): Comm
|
|
|
58
53
|
}
|
|
59
54
|
}
|
|
60
55
|
|
|
61
|
-
// 3. Choices validation
|
|
62
56
|
const coercedVal = result[argDef.name];
|
|
63
57
|
if (argDef.choices && argDef.choices.length > 0) {
|
|
64
58
|
if (!argDef.choices.map(String).includes(String(coercedVal))) {
|
|
@@ -72,16 +66,12 @@ export function coerceAndValidateArgs(cmdArgs: Arg[], kwargs: CommandArgs): Comm
|
|
|
72
66
|
return result;
|
|
73
67
|
}
|
|
74
68
|
|
|
75
|
-
/**
|
|
76
|
-
* Run a command's func or pipeline against a page.
|
|
77
|
-
*/
|
|
78
69
|
async function runCommand(
|
|
79
70
|
cmd: CliCommand,
|
|
80
71
|
page: IPage | null,
|
|
81
72
|
kwargs: CommandArgs,
|
|
82
73
|
debug: boolean,
|
|
83
74
|
): Promise<unknown> {
|
|
84
|
-
// Lazy-load TS module on first execution (manifest fast-path)
|
|
85
75
|
const internal = cmd as InternalCliCommand;
|
|
86
76
|
if (internal._lazy && internal._modulePath) {
|
|
87
77
|
const modulePath = internal._modulePath;
|
|
@@ -96,13 +86,18 @@ async function runCommand(
|
|
|
96
86
|
);
|
|
97
87
|
}
|
|
98
88
|
}
|
|
99
|
-
|
|
89
|
+
|
|
100
90
|
const updated = getRegistry().get(fullName(cmd));
|
|
101
|
-
if (updated?.func)
|
|
91
|
+
if (updated?.func) {
|
|
92
|
+
if (!page && updated.browser !== false) {
|
|
93
|
+
throw new CommandExecutionError(`Command ${fullName(cmd)} requires a browser session but none was provided`);
|
|
94
|
+
}
|
|
95
|
+
return updated.func(page as IPage, kwargs, debug);
|
|
96
|
+
}
|
|
102
97
|
if (updated?.pipeline) return executePipeline(page, updated.pipeline, { args: kwargs, debug });
|
|
103
98
|
}
|
|
104
99
|
|
|
105
|
-
if (cmd.func) return cmd.func(page
|
|
100
|
+
if (cmd.func) return cmd.func(page as IPage, kwargs, debug);
|
|
106
101
|
if (cmd.pipeline) return executePipeline(page, cmd.pipeline, { args: kwargs, debug });
|
|
107
102
|
throw new CommandExecutionError(
|
|
108
103
|
`Command ${fullName(cmd)} has no func or pipeline`,
|
|
@@ -110,30 +105,29 @@ async function runCommand(
|
|
|
110
105
|
);
|
|
111
106
|
}
|
|
112
107
|
|
|
113
|
-
/**
|
|
114
|
-
* Resolve the pre-navigation URL for a command, or null to skip.
|
|
115
|
-
*
|
|
116
|
-
* COOKIE/HEADER strategies need the browser on the target domain so
|
|
117
|
-
* `fetch(url, { credentials: 'include' })` carries cookies.
|
|
118
|
-
* Adapters that handle their own navigation set `navigateBefore: false`.
|
|
119
|
-
*/
|
|
120
108
|
function resolvePreNav(cmd: CliCommand): string | null {
|
|
121
109
|
if (cmd.navigateBefore === false) return null;
|
|
122
110
|
if (typeof cmd.navigateBefore === 'string') return cmd.navigateBefore;
|
|
123
111
|
|
|
124
|
-
// Default: pre-navigate for COOKIE/HEADER strategies with a domain
|
|
125
112
|
if ((cmd.strategy === Strategy.COOKIE || cmd.strategy === Strategy.HEADER) && cmd.domain) {
|
|
126
113
|
return `https://${cmd.domain}`;
|
|
127
114
|
}
|
|
128
115
|
return null;
|
|
129
116
|
}
|
|
130
117
|
|
|
131
|
-
|
|
132
|
-
|
|
133
|
-
|
|
134
|
-
|
|
135
|
-
|
|
136
|
-
|
|
118
|
+
function ensureRequiredEnv(cmd: CliCommand): void {
|
|
119
|
+
const missing = (cmd.requiredEnv ?? []).find(({ name }) => {
|
|
120
|
+
const value = process.env[name];
|
|
121
|
+
return value === undefined || value === null || value === '';
|
|
122
|
+
});
|
|
123
|
+
if (!missing) return;
|
|
124
|
+
|
|
125
|
+
throw new CommandExecutionError(
|
|
126
|
+
`Command ${fullName(cmd)} requires environment variable ${missing.name}.`,
|
|
127
|
+
missing.help ?? `Set ${missing.name} before running ${fullName(cmd)}.`,
|
|
128
|
+
);
|
|
129
|
+
}
|
|
130
|
+
|
|
137
131
|
export async function executeCommand(
|
|
138
132
|
cmd: CliCommand,
|
|
139
133
|
rawKwargs: CommandArgs,
|
|
@@ -147,22 +141,44 @@ export async function executeCommand(
|
|
|
147
141
|
throw new ArgumentError(getErrorMessage(err));
|
|
148
142
|
}
|
|
149
143
|
|
|
150
|
-
|
|
151
|
-
|
|
152
|
-
|
|
153
|
-
|
|
154
|
-
|
|
155
|
-
|
|
156
|
-
|
|
157
|
-
|
|
158
|
-
|
|
159
|
-
|
|
160
|
-
|
|
161
|
-
|
|
162
|
-
|
|
163
|
-
|
|
144
|
+
const hookCtx: HookContext = {
|
|
145
|
+
command: fullName(cmd),
|
|
146
|
+
args: kwargs,
|
|
147
|
+
startedAt: Date.now(),
|
|
148
|
+
};
|
|
149
|
+
await emitHook('onBeforeExecute', hookCtx);
|
|
150
|
+
|
|
151
|
+
let result: unknown;
|
|
152
|
+
try {
|
|
153
|
+
if (shouldUseBrowserSession(cmd)) {
|
|
154
|
+
ensureRequiredEnv(cmd);
|
|
155
|
+
const BrowserFactory = getBrowserFactory();
|
|
156
|
+
result = await browserSession(BrowserFactory, async (page) => {
|
|
157
|
+
const preNavUrl = resolvePreNav(cmd);
|
|
158
|
+
if (preNavUrl) {
|
|
159
|
+
try {
|
|
160
|
+
await page.goto(preNavUrl);
|
|
161
|
+
await page.wait(2);
|
|
162
|
+
} catch (err) {
|
|
163
|
+
if (debug) console.error(`[pre-nav] Failed to navigate to ${preNavUrl}: ${err instanceof Error ? err.message : err}`);
|
|
164
|
+
}
|
|
165
|
+
}
|
|
166
|
+
return runWithTimeout(runCommand(cmd, page, kwargs, debug), {
|
|
167
|
+
timeout: cmd.timeoutSeconds ?? DEFAULT_BROWSER_COMMAND_TIMEOUT,
|
|
168
|
+
label: fullName(cmd),
|
|
169
|
+
});
|
|
170
|
+
}, { workspace: `site:${cmd.site}` });
|
|
171
|
+
} else {
|
|
172
|
+
result = await runCommand(cmd, null, kwargs, debug);
|
|
173
|
+
}
|
|
174
|
+
} catch (err) {
|
|
175
|
+
hookCtx.error = err;
|
|
176
|
+
hookCtx.finishedAt = Date.now();
|
|
177
|
+
await emitHook('onAfterExecute', hookCtx);
|
|
178
|
+
throw err;
|
|
164
179
|
}
|
|
165
180
|
|
|
166
|
-
|
|
167
|
-
|
|
181
|
+
hookCtx.finishedAt = Date.now();
|
|
182
|
+
await emitHook('onAfterExecute', hookCtx, result);
|
|
183
|
+
return result;
|
|
168
184
|
}
|
package/src/explore.ts
CHANGED
|
@@ -10,12 +10,22 @@ import * as fs from 'node:fs';
|
|
|
10
10
|
import * as path from 'node:path';
|
|
11
11
|
import { DEFAULT_BROWSER_EXPLORE_TIMEOUT, browserSession, runWithTimeout } from './runtime.js';
|
|
12
12
|
import type { IBrowserFactory } from './runtime.js';
|
|
13
|
-
import {
|
|
13
|
+
import { LIMIT_PARAMS } from './constants.js';
|
|
14
14
|
import { detectFramework } from './scripts/framework.js';
|
|
15
15
|
import { discoverStores } from './scripts/store.js';
|
|
16
16
|
import { interactFuzz } from './scripts/interact.js';
|
|
17
17
|
import type { IPage } from './types.js';
|
|
18
18
|
import { log } from './logger.js';
|
|
19
|
+
import {
|
|
20
|
+
urlToPattern,
|
|
21
|
+
findArrayPath,
|
|
22
|
+
flattenFields,
|
|
23
|
+
detectFieldRoles,
|
|
24
|
+
inferCapabilityName,
|
|
25
|
+
inferStrategy,
|
|
26
|
+
detectAuthFromHeaders,
|
|
27
|
+
classifyQueryParams,
|
|
28
|
+
} from './analysis.js';
|
|
19
29
|
|
|
20
30
|
// ── Site name detection ────────────────────────────────────────────────────
|
|
21
31
|
|
|
@@ -48,10 +58,6 @@ export function slugify(value: string): string {
|
|
|
48
58
|
return value.trim().toLowerCase().replace(/[^a-zA-Z0-9]+/g, '-').replace(/^-|-$/g, '') || 'site';
|
|
49
59
|
}
|
|
50
60
|
|
|
51
|
-
// ── Field & capability inference ───────────────────────────────────────────
|
|
52
|
-
|
|
53
|
-
// (constants now imported from constants.ts)
|
|
54
|
-
|
|
55
61
|
// ── Network analysis ───────────────────────────────────────────────────────
|
|
56
62
|
|
|
57
63
|
interface NetworkEntry {
|
|
@@ -160,68 +166,16 @@ function parseNetworkRequests(raw: unknown): NetworkEntry[] {
|
|
|
160
166
|
return [];
|
|
161
167
|
}
|
|
162
168
|
|
|
163
|
-
function urlToPattern(url: string): string {
|
|
164
|
-
try {
|
|
165
|
-
const p = new URL(url);
|
|
166
|
-
const pathNorm = p.pathname.replace(/\/\d+/g, '/{id}').replace(/\/[0-9a-fA-F]{8,}/g, '/{hex}').replace(/\/BV[a-zA-Z0-9]{10}/g, '/{bvid}');
|
|
167
|
-
const params: string[] = [];
|
|
168
|
-
p.searchParams.forEach((_v, k) => { if (!VOLATILE_PARAMS.has(k)) params.push(k); });
|
|
169
|
-
return `${p.host}${pathNorm}${params.length ? '?' + params.sort().map(k => `${k}={}`).join('&') : ''}`;
|
|
170
|
-
} catch { return url; }
|
|
171
|
-
}
|
|
172
|
-
|
|
173
|
-
function detectAuthIndicators(headers?: Record<string, string>): string[] {
|
|
174
|
-
if (!headers) return [];
|
|
175
|
-
const indicators: string[] = [];
|
|
176
|
-
const keys = Object.keys(headers).map(k => k.toLowerCase());
|
|
177
|
-
if (keys.some(k => k === 'authorization')) indicators.push('bearer');
|
|
178
|
-
if (keys.some(k => k.startsWith('x-csrf') || k.startsWith('x-xsrf'))) indicators.push('csrf');
|
|
179
|
-
if (keys.some(k => k.startsWith('x-s') || k === 'x-t' || k === 'x-s-common')) indicators.push('signature');
|
|
180
|
-
return indicators;
|
|
181
|
-
}
|
|
182
|
-
|
|
183
169
|
function analyzeResponseBody(body: unknown): AnalyzedEndpoint['responseAnalysis'] {
|
|
184
170
|
if (!body || typeof body !== 'object') return null;
|
|
185
|
-
const
|
|
186
|
-
|
|
187
|
-
function findArrays(obj: unknown, path: string, depth: number) {
|
|
188
|
-
if (depth > 4) return;
|
|
189
|
-
if (Array.isArray(obj) && obj.length >= 2 && obj.some(item => item && typeof item === 'object' && !Array.isArray(item))) {
|
|
190
|
-
candidates.push({ path, items: obj });
|
|
191
|
-
}
|
|
192
|
-
if (obj && typeof obj === 'object' && !Array.isArray(obj)) {
|
|
193
|
-
for (const [key, val] of Object.entries(obj)) findArrays(val, path ? `${path}.${key}` : key, depth + 1);
|
|
194
|
-
}
|
|
195
|
-
}
|
|
196
|
-
findArrays(body, '', 0);
|
|
197
|
-
if (!candidates.length) return null;
|
|
171
|
+
const result = findArrayPath(body);
|
|
172
|
+
if (!result) return null;
|
|
198
173
|
|
|
199
|
-
|
|
200
|
-
const best = candidates[0];
|
|
201
|
-
const sample = best.items[0];
|
|
174
|
+
const sample = result.items[0];
|
|
202
175
|
const sampleFields = sample && typeof sample === 'object' ? flattenFields(sample, '', 2) : [];
|
|
176
|
+
const detectedFields = detectFieldRoles(sampleFields);
|
|
203
177
|
|
|
204
|
-
|
|
205
|
-
for (const [role, aliases] of Object.entries(FIELD_ROLES)) {
|
|
206
|
-
for (const f of sampleFields) {
|
|
207
|
-
if (aliases.includes(f.split('.').pop()?.toLowerCase() ?? '')) { detectedFields[role] = f; break; }
|
|
208
|
-
}
|
|
209
|
-
}
|
|
210
|
-
|
|
211
|
-
return { itemPath: best.path || null, itemCount: best.items.length, detectedFields, sampleFields };
|
|
212
|
-
}
|
|
213
|
-
|
|
214
|
-
function flattenFields(obj: unknown, prefix: string, maxDepth: number): string[] {
|
|
215
|
-
if (maxDepth <= 0 || !obj || typeof obj !== 'object') return [];
|
|
216
|
-
const names: string[] = [];
|
|
217
|
-
const record = obj as Record<string, unknown>;
|
|
218
|
-
for (const key of Object.keys(record)) {
|
|
219
|
-
const full = prefix ? `${prefix}.${key}` : key;
|
|
220
|
-
names.push(full);
|
|
221
|
-
const val = record[key];
|
|
222
|
-
if (val && typeof val === 'object' && !Array.isArray(val)) names.push(...flattenFields(val, full, maxDepth - 1));
|
|
223
|
-
}
|
|
224
|
-
return names;
|
|
178
|
+
return { itemPath: result.path || null, itemCount: result.items.length, detectedFields, sampleFields };
|
|
225
179
|
}
|
|
226
180
|
|
|
227
181
|
function isBooleanRecord(value: unknown): value is Record<string, boolean> {
|
|
@@ -243,28 +197,6 @@ function scoreEndpoint(ep: { contentType: string; responseAnalysis: AnalyzedEndp
|
|
|
243
197
|
return s;
|
|
244
198
|
}
|
|
245
199
|
|
|
246
|
-
function inferCapabilityName(url: string, goal?: string): string {
|
|
247
|
-
if (goal) return goal;
|
|
248
|
-
const u = url.toLowerCase();
|
|
249
|
-
if (u.includes('hot') || u.includes('popular') || u.includes('ranking') || u.includes('trending')) return 'hot';
|
|
250
|
-
if (u.includes('search')) return 'search';
|
|
251
|
-
if (u.includes('feed') || u.includes('timeline') || u.includes('dynamic')) return 'feed';
|
|
252
|
-
if (u.includes('comment') || u.includes('reply')) return 'comments';
|
|
253
|
-
if (u.includes('history')) return 'history';
|
|
254
|
-
if (u.includes('profile') || u.includes('userinfo') || u.includes('/me')) return 'me';
|
|
255
|
-
if (u.includes('favorite') || u.includes('collect') || u.includes('bookmark')) return 'favorite';
|
|
256
|
-
try {
|
|
257
|
-
const segs = new URL(url).pathname.split('/').filter(s => s && !s.match(/^\d+$/) && !s.match(/^[0-9a-f]{8,}$/i));
|
|
258
|
-
if (segs.length) return segs[segs.length - 1].replace(/[^a-z0-9]/gi, '_').toLowerCase();
|
|
259
|
-
} catch {}
|
|
260
|
-
return 'data';
|
|
261
|
-
}
|
|
262
|
-
|
|
263
|
-
function inferStrategy(authIndicators: string[]): string {
|
|
264
|
-
if (authIndicators.includes('signature')) return 'intercept';
|
|
265
|
-
if (authIndicators.includes('bearer') || authIndicators.includes('csrf')) return 'header';
|
|
266
|
-
return 'cookie';
|
|
267
|
-
}
|
|
268
200
|
|
|
269
201
|
// ── Framework detection ────────────────────────────────────────────────────
|
|
270
202
|
|
|
@@ -300,15 +232,14 @@ function analyzeEndpoints(networkEntries: NetworkEntry[]): { analyzed: AnalyzedE
|
|
|
300
232
|
const key = `${entry.method}:${pattern}`;
|
|
301
233
|
if (seen.has(key)) continue;
|
|
302
234
|
|
|
303
|
-
const
|
|
304
|
-
try { new URL(entry.url).searchParams.forEach((_v, k) => { if (!VOLATILE_PARAMS.has(k)) qp.push(k); }); } catch {}
|
|
235
|
+
const { params: qp, hasSearch, hasPagination, hasLimit } = classifyQueryParams(entry.url);
|
|
305
236
|
|
|
306
237
|
const ep: AnalyzedEndpoint = {
|
|
307
238
|
pattern, method: entry.method, url: entry.url, status: entry.status, contentType: ct,
|
|
308
|
-
queryParams: qp, hasSearchParam:
|
|
309
|
-
hasPaginationParam:
|
|
310
|
-
hasLimitParam: qp.some(p => LIMIT_PARAMS.has(p)),
|
|
311
|
-
authIndicators:
|
|
239
|
+
queryParams: qp, hasSearchParam: hasSearch,
|
|
240
|
+
hasPaginationParam: hasPagination,
|
|
241
|
+
hasLimitParam: hasLimit || qp.some(p => LIMIT_PARAMS.has(p)),
|
|
242
|
+
authIndicators: detectAuthFromHeaders(entry.requestHeaders),
|
|
312
243
|
responseAnalysis: entry.responseBody ? analyzeResponseBody(entry.responseBody) : null,
|
|
313
244
|
score: 0,
|
|
314
245
|
};
|
package/src/external-clis.yaml
CHANGED
|
@@ -22,14 +22,6 @@
|
|
|
22
22
|
install:
|
|
23
23
|
default: "npm install -g @readwiseio/readwise-cli"
|
|
24
24
|
|
|
25
|
-
- name: kubectl
|
|
26
|
-
binary: kubectl
|
|
27
|
-
description: "Kubernetes command-line tool"
|
|
28
|
-
homepage: "https://kubernetes.io/docs/reference/kubectl/"
|
|
29
|
-
tags: [kubernetes, k8s, devops]
|
|
30
|
-
install:
|
|
31
|
-
mac: "brew install kubectl"
|
|
32
|
-
|
|
33
25
|
- name: docker
|
|
34
26
|
binary: docker
|
|
35
27
|
description: "Docker command-line interface"
|
package/src/external.test.ts
CHANGED
|
@@ -33,6 +33,15 @@ describe('parseCommand', () => {
|
|
|
33
33
|
'Install command contains unsafe shell operators',
|
|
34
34
|
);
|
|
35
35
|
});
|
|
36
|
+
|
|
37
|
+
it('rejects command substitution and multiline input', () => {
|
|
38
|
+
expect(() => parseCommand('brew install $(whoami)')).toThrow(
|
|
39
|
+
'Install command contains unsafe shell operators',
|
|
40
|
+
);
|
|
41
|
+
expect(() => parseCommand('brew install gh\nrm -rf /')).toThrow(
|
|
42
|
+
'Install command contains unsafe shell operators',
|
|
43
|
+
);
|
|
44
|
+
});
|
|
36
45
|
});
|
|
37
46
|
|
|
38
47
|
describe('installExternalCli', () => {
|
package/src/external.ts
CHANGED
|
@@ -6,6 +6,7 @@ import { spawnSync, execFileSync } from 'node:child_process';
|
|
|
6
6
|
import yaml from 'js-yaml';
|
|
7
7
|
import chalk from 'chalk';
|
|
8
8
|
import { log } from './logger.js';
|
|
9
|
+
import { getErrorMessage } from './errors.js';
|
|
9
10
|
|
|
10
11
|
const __dirname = path.dirname(fileURLToPath(import.meta.url));
|
|
11
12
|
|
|
@@ -41,8 +42,8 @@ export function loadExternalClis(): ExternalCliConfig[] {
|
|
|
41
42
|
const parsed = (yaml.load(raw) || []) as ExternalCliConfig[];
|
|
42
43
|
for (const item of parsed) configs.set(item.name, item);
|
|
43
44
|
}
|
|
44
|
-
} catch (err
|
|
45
|
-
log.warn(`Failed to parse built-in external-clis.yaml: ${err
|
|
45
|
+
} catch (err) {
|
|
46
|
+
log.warn(`Failed to parse built-in external-clis.yaml: ${getErrorMessage(err)}`);
|
|
46
47
|
}
|
|
47
48
|
|
|
48
49
|
// 2. Load user custom
|
|
@@ -55,8 +56,8 @@ export function loadExternalClis(): ExternalCliConfig[] {
|
|
|
55
56
|
configs.set(item.name, item); // Overwrite built-in if duplicated
|
|
56
57
|
}
|
|
57
58
|
}
|
|
58
|
-
} catch (err
|
|
59
|
-
log.warn(`Failed to parse user external-clis.yaml: ${err
|
|
59
|
+
} catch (err) {
|
|
60
|
+
log.warn(`Failed to parse user external-clis.yaml: ${getErrorMessage(err)}`);
|
|
60
61
|
}
|
|
61
62
|
|
|
62
63
|
return Array.from(configs.values()).sort((a, b) => a.name.localeCompare(b.name));
|
|
@@ -94,7 +95,7 @@ export function getInstallCmd(installConfig?: ExternalCliInstall): string | null
|
|
|
94
95
|
* Object with `binary` and `args` fields, or throws on unsafe input.
|
|
95
96
|
*/
|
|
96
97
|
export function parseCommand(cmd: string): { binary: string; args: string[] } {
|
|
97
|
-
const shellOperators = /&&|\|\|?|;|[
|
|
98
|
+
const shellOperators = /&&|\|\|?|;|[><`$#\n\r]|\$\(/;
|
|
98
99
|
if (shellOperators.test(cmd)) {
|
|
99
100
|
throw new Error(
|
|
100
101
|
`Install command contains unsafe shell operators and cannot be executed securely: "${cmd}". ` +
|
|
@@ -118,8 +119,9 @@ export function parseCommand(cmd: string): { binary: string; args: string[] } {
|
|
|
118
119
|
return { binary, args };
|
|
119
120
|
}
|
|
120
121
|
|
|
121
|
-
function shouldRetryWithCmdShim(binary: string, err:
|
|
122
|
-
|
|
122
|
+
function shouldRetryWithCmdShim(binary: string, err: unknown): boolean {
|
|
123
|
+
const code = err instanceof Error ? (err as NodeJS.ErrnoException).code : undefined;
|
|
124
|
+
return os.platform() === 'win32' && !path.extname(binary) && code === 'ENOENT';
|
|
123
125
|
}
|
|
124
126
|
|
|
125
127
|
function runInstallCommand(cmd: string): void {
|
|
@@ -127,7 +129,7 @@ function runInstallCommand(cmd: string): void {
|
|
|
127
129
|
|
|
128
130
|
try {
|
|
129
131
|
execFileSync(binary, args, { stdio: 'inherit' });
|
|
130
|
-
} catch (err
|
|
132
|
+
} catch (err) {
|
|
131
133
|
if (shouldRetryWithCmdShim(binary, err)) {
|
|
132
134
|
execFileSync(`${binary}.cmd`, args, { stdio: 'inherit' });
|
|
133
135
|
return;
|
|
@@ -156,8 +158,8 @@ export function installExternalCli(cli: ExternalCliConfig): boolean {
|
|
|
156
158
|
runInstallCommand(cmd);
|
|
157
159
|
console.log(chalk.green(`✅ Installed '${cli.name}' successfully.\n`));
|
|
158
160
|
return true;
|
|
159
|
-
} catch (err
|
|
160
|
-
console.error(chalk.red(`❌ Failed to install '${cli.name}': ${err
|
|
161
|
+
} catch (err) {
|
|
162
|
+
console.error(chalk.red(`❌ Failed to install '${cli.name}': ${getErrorMessage(err)}`));
|
|
161
163
|
return false;
|
|
162
164
|
}
|
|
163
165
|
}
|