@jackwener/opencli 1.3.3 → 1.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.github/pull_request_template.md +3 -1
- package/.github/workflows/build-extension.yml +7 -1
- package/.github/workflows/ci.yml +29 -3
- package/.github/workflows/docs.yml +1 -1
- package/.github/workflows/e2e-headed.yml +20 -0
- package/.github/workflows/release.yml +1 -1
- package/.github/workflows/security.yml +0 -3
- package/CHANGELOG.md +55 -0
- package/CONTRIBUTING.md +6 -3
- package/README.md +30 -3
- package/README.zh-CN.md +30 -3
- package/SKILL.md +7 -1
- package/TESTING.md +1 -0
- package/chatwise-opencli.ps1 +82 -0
- package/dist/analysis.d.ts +38 -0
- package/dist/analysis.js +166 -0
- package/dist/browser/cdp.d.ts +0 -4
- package/dist/browser/cdp.js +53 -41
- package/dist/browser/cdp.test.d.ts +1 -0
- package/dist/browser/cdp.test.js +52 -0
- package/dist/browser/dom-snapshot.d.ts +2 -2
- package/dist/browser/dom-snapshot.js +54 -1
- package/dist/browser/dom-snapshot.test.js +36 -0
- package/dist/browser/index.d.ts +2 -2
- package/dist/browser/index.js +1 -1
- package/dist/browser/mcp.d.ts +0 -2
- package/dist/browser/mcp.js +2 -3
- package/dist/browser/page.d.ts +4 -3
- package/dist/browser/page.js +34 -37
- package/dist/browser/stealth.d.ts +0 -2
- package/dist/browser/stealth.js +24 -9
- package/dist/browser.test.js +2 -2
- package/dist/build-manifest.js +15 -9
- package/dist/build-manifest.test.js +12 -0
- package/dist/cascade.js +4 -2
- package/dist/cli-manifest.json +639 -258
- package/dist/cli.js +57 -29
- package/dist/clis/_shared/desktop-commands.d.ts +22 -0
- package/dist/clis/_shared/desktop-commands.js +108 -0
- package/dist/clis/antigravity/serve.js +5 -2
- package/dist/clis/arxiv/search.js +1 -1
- package/dist/clis/bilibili/dynamic.test.d.ts +1 -0
- package/dist/clis/bilibili/dynamic.test.js +68 -0
- package/dist/clis/bilibili/favorite.js +4 -2
- package/dist/clis/bilibili/following.js +3 -2
- package/dist/clis/bilibili/subtitle.js +8 -7
- package/dist/clis/bilibili/utils.js +2 -2
- package/dist/clis/boss/batchgreet.js +1 -1
- package/dist/clis/boss/chatlist.js +1 -1
- package/dist/clis/boss/chatmsg.js +1 -1
- package/dist/clis/boss/detail.js +1 -1
- package/dist/clis/boss/exchange.js +1 -1
- package/dist/clis/boss/greet.js +1 -1
- package/dist/clis/boss/invite.js +1 -1
- package/dist/clis/boss/joblist.js +1 -1
- package/dist/clis/boss/mark.js +4 -3
- package/dist/clis/boss/recommend.js +1 -1
- package/dist/clis/boss/resume.js +1 -1
- package/dist/clis/boss/search.js +1 -1
- package/dist/clis/boss/send.js +5 -4
- package/dist/clis/boss/stats.js +1 -1
- package/dist/clis/chatgpt/ask.js +4 -0
- package/dist/clis/chatgpt/new.js +5 -1
- package/dist/clis/chatgpt/read.js +5 -1
- package/dist/clis/chatgpt/send.js +2 -1
- package/dist/clis/chatgpt/status.js +5 -1
- package/dist/clis/chatwise/ask.js +8 -2
- package/dist/clis/chatwise/export.js +2 -0
- package/dist/clis/chatwise/history.js +2 -0
- package/dist/clis/chatwise/model.js +8 -3
- package/dist/clis/chatwise/new.js +3 -18
- package/dist/clis/chatwise/read.js +2 -0
- package/dist/clis/chatwise/screenshot.js +3 -27
- package/dist/clis/chatwise/send.js +8 -2
- package/dist/clis/chatwise/shared.d.ts +2 -0
- package/dist/clis/chatwise/shared.js +6 -0
- package/dist/clis/chatwise/status.js +3 -22
- package/dist/clis/codex/ask.js +6 -2
- package/dist/clis/codex/dump.js +2 -25
- package/dist/clis/codex/new.js +2 -25
- package/dist/clis/codex/screenshot.js +2 -27
- package/dist/clis/codex/send.js +6 -4
- package/dist/clis/codex/status.js +2 -22
- package/dist/clis/cursor/ask.js +2 -1
- package/dist/clis/cursor/composer.js +2 -1
- package/dist/clis/cursor/dump.js +2 -25
- package/dist/clis/cursor/new.js +2 -18
- package/dist/clis/cursor/read.js +2 -1
- package/dist/clis/cursor/screenshot.js +1 -30
- package/dist/clis/cursor/send.js +2 -1
- package/dist/clis/cursor/status.js +2 -21
- package/dist/clis/dictionary/examples.yaml +25 -0
- package/dist/clis/dictionary/search.yaml +27 -0
- package/dist/clis/dictionary/synonyms.yaml +25 -0
- package/dist/clis/douban/book-hot.js +1 -1
- package/dist/clis/douban/movie-hot.js +1 -1
- package/dist/clis/douban/search.js +1 -1
- package/dist/clis/douban/utils.d.ts +4 -1
- package/dist/clis/douban/utils.js +156 -1
- package/dist/clis/doubao/ask.js +1 -1
- package/dist/clis/doubao/new.js +1 -1
- package/dist/clis/doubao/read.js +1 -1
- package/dist/clis/doubao/send.js +1 -1
- package/dist/clis/doubao/status.js +1 -1
- package/dist/clis/doubao-app/ask.js +1 -1
- package/dist/clis/doubao-app/new.js +1 -1
- package/dist/clis/doubao-app/read.js +1 -1
- package/dist/clis/doubao-app/send.js +1 -1
- package/dist/clis/grok/ask.d.ts +4 -0
- package/dist/clis/grok/ask.js +28 -10
- package/dist/clis/grok/ask.test.js +18 -0
- package/dist/clis/jd/item.d.ts +1 -0
- package/dist/clis/jd/item.js +96 -0
- package/dist/clis/jd/item.test.d.ts +1 -0
- package/dist/clis/jd/item.test.js +28 -0
- package/dist/clis/jike/feed.js +1 -1
- package/dist/clis/jike/search.js +1 -1
- package/dist/clis/linkedin/search.js +5 -4
- package/dist/clis/linkedin/timeline.d.ts +21 -0
- package/dist/clis/linkedin/timeline.js +503 -0
- package/dist/clis/linkedin/timeline.test.d.ts +1 -0
- package/dist/clis/linkedin/timeline.test.js +81 -0
- package/dist/clis/medium/feed.js +1 -1
- package/dist/clis/medium/search.js +1 -1
- package/dist/clis/medium/user.js +1 -1
- package/dist/clis/medium/{shared.js → utils.js} +2 -1
- package/dist/clis/pixiv/detail.yaml +49 -0
- package/dist/clis/pixiv/download.d.ts +7 -0
- package/dist/clis/pixiv/download.js +78 -0
- package/dist/clis/pixiv/download.test.d.ts +1 -0
- package/dist/clis/pixiv/download.test.js +87 -0
- package/dist/clis/pixiv/illusts.d.ts +8 -0
- package/dist/clis/pixiv/illusts.js +65 -0
- package/dist/clis/pixiv/illusts.test.d.ts +1 -0
- package/dist/clis/pixiv/illusts.test.js +99 -0
- package/dist/clis/pixiv/ranking.yaml +53 -0
- package/dist/clis/pixiv/search.d.ts +6 -0
- package/dist/clis/pixiv/search.js +43 -0
- package/dist/clis/pixiv/search.test.d.ts +1 -0
- package/dist/clis/pixiv/search.test.js +83 -0
- package/dist/clis/pixiv/test-utils.d.ts +12 -0
- package/dist/clis/pixiv/test-utils.js +23 -0
- package/dist/clis/pixiv/user.yaml +46 -0
- package/dist/clis/pixiv/utils.d.ts +27 -0
- package/dist/clis/pixiv/utils.js +49 -0
- package/dist/clis/reddit/comment.js +2 -1
- package/dist/clis/reddit/read.js +4 -3
- package/dist/clis/reddit/read.test.d.ts +1 -0
- package/dist/clis/reddit/read.test.js +28 -0
- package/dist/clis/reddit/save.js +2 -1
- package/dist/clis/reddit/saved.js +7 -3
- package/dist/clis/reddit/subscribe.js +2 -1
- package/dist/clis/reddit/upvote.js +2 -1
- package/dist/clis/reddit/upvoted.js +7 -3
- package/dist/clis/sinablog/article.js +1 -1
- package/dist/clis/sinablog/hot.js +1 -1
- package/dist/clis/sinablog/user.js +1 -1
- package/dist/clis/substack/feed.js +1 -1
- package/dist/clis/substack/publication.js +1 -1
- package/dist/clis/substack/search.js +3 -2
- package/dist/clis/substack/{shared.js → utils.js} +3 -2
- package/dist/clis/tiktok/search.yaml +2 -1
- package/dist/clis/twitter/accept.js +2 -1
- package/dist/clis/twitter/article.js +4 -1
- package/dist/clis/twitter/block.js +2 -1
- package/dist/clis/twitter/bookmark.js +2 -1
- package/dist/clis/twitter/bookmarks.js +3 -2
- package/dist/clis/twitter/delete.js +2 -1
- package/dist/clis/twitter/follow.js +2 -1
- package/dist/clis/twitter/followers.js +3 -2
- package/dist/clis/twitter/following.js +3 -2
- package/dist/clis/twitter/hide-reply.js +2 -1
- package/dist/clis/twitter/like.js +2 -1
- package/dist/clis/twitter/notifications.js +2 -1
- package/dist/clis/twitter/post.js +2 -1
- package/dist/clis/twitter/profile.js +5 -2
- package/dist/clis/twitter/reply-dm.js +2 -1
- package/dist/clis/twitter/reply.js +2 -1
- package/dist/clis/twitter/search.js +30 -13
- package/dist/clis/twitter/search.test.d.ts +1 -0
- package/dist/clis/twitter/search.test.js +104 -0
- package/dist/clis/twitter/thread.js +2 -2
- package/dist/clis/twitter/timeline.js +3 -2
- package/dist/clis/twitter/trending.js +3 -2
- package/dist/clis/twitter/unblock.js +2 -1
- package/dist/clis/twitter/unbookmark.js +2 -1
- package/dist/clis/twitter/unfollow.js +2 -1
- package/dist/clis/v2ex/daily.js +3 -2
- package/dist/clis/v2ex/me.js +3 -2
- package/dist/clis/v2ex/notifications.js +4 -4
- package/dist/clis/web/read.d.ts +16 -0
- package/dist/clis/web/read.js +202 -0
- package/dist/clis/xueqiu/danjuan-utils.d.ts +55 -0
- package/dist/clis/xueqiu/danjuan-utils.js +126 -0
- package/dist/clis/xueqiu/danjuan-utils.test.d.ts +1 -0
- package/dist/clis/xueqiu/danjuan-utils.test.js +41 -0
- package/dist/clis/xueqiu/fund-holdings.d.ts +1 -0
- package/dist/clis/xueqiu/fund-holdings.js +28 -0
- package/dist/clis/xueqiu/fund-snapshot.d.ts +1 -0
- package/dist/clis/xueqiu/fund-snapshot.js +25 -0
- package/dist/clis/youtube/transcript.js +5 -4
- package/dist/clis/youtube/video.js +3 -2
- package/dist/daemon.js +7 -3
- package/dist/discovery.js +11 -10
- package/dist/doctor.js +2 -1
- package/dist/download/index.d.ts +4 -12
- package/dist/download/index.js +33 -12
- package/dist/download/index.test.js +79 -2
- package/dist/download/media-download.js +4 -2
- package/dist/engine.test.js +76 -4
- package/dist/execution.d.ts +1 -9
- package/dist/execution.js +56 -46
- package/dist/explore.js +12 -111
- package/dist/external-clis.yaml +0 -8
- package/dist/external.js +7 -5
- package/dist/external.test.js +4 -0
- package/dist/generate.d.ts +0 -9
- package/dist/generate.js +4 -20
- package/dist/hooks.d.ts +46 -0
- package/dist/hooks.js +56 -0
- package/dist/hooks.test.d.ts +4 -0
- package/dist/hooks.test.js +92 -0
- package/dist/interceptor.js +70 -23
- package/dist/main.js +2 -0
- package/dist/output.js +12 -6
- package/dist/pipeline/executor.js +1 -1
- package/dist/pipeline/steps/browser.js +1 -3
- package/dist/pipeline/steps/download.js +42 -26
- package/dist/pipeline/steps/download.test.d.ts +1 -0
- package/dist/pipeline/steps/download.test.js +101 -0
- package/dist/pipeline/steps/fetch.js +40 -22
- package/dist/pipeline/steps/fetch.test.d.ts +1 -0
- package/dist/pipeline/steps/fetch.test.js +123 -0
- package/dist/pipeline/steps/transform.js +2 -6
- package/dist/pipeline/template.js +66 -52
- package/dist/pipeline/template.test.js +28 -0
- package/dist/pipeline/transform.test.js +18 -0
- package/dist/plugin.d.ts +40 -1
- package/dist/plugin.js +214 -17
- package/dist/plugin.test.d.ts +1 -1
- package/dist/plugin.test.js +219 -3
- package/dist/record.js +6 -98
- package/dist/registry-api.d.ts +2 -0
- package/dist/registry-api.js +1 -0
- package/dist/registry.d.ts +5 -2
- package/dist/registry.js +1 -2
- package/dist/runtime.d.ts +0 -1
- package/dist/runtime.js +14 -4
- package/dist/snapshotFormatter.d.ts +7 -14
- package/dist/snapshotFormatter.js +38 -78
- package/dist/utils.d.ts +9 -0
- package/dist/utils.js +29 -0
- package/dist/validate.js +3 -5
- package/dist/yaml-schema.d.ts +26 -0
- package/dist/yaml-schema.js +5 -0
- package/docs/.vitepress/config.mts +3 -0
- package/docs/adapters/browser/dictionary.md +27 -0
- package/docs/adapters/browser/jd.md +27 -0
- package/docs/adapters/browser/linkedin.md +6 -0
- package/docs/adapters/browser/pixiv.md +92 -0
- package/docs/adapters/browser/web.md +30 -0
- package/docs/adapters/browser/xueqiu.md +27 -9
- package/docs/adapters/index.md +3 -1
- package/docs/comparison.md +125 -0
- package/docs/developer/contributing.md +21 -2
- package/docs/developer/testing.md +14 -8
- package/docs/developer/ts-adapter.md +18 -0
- package/docs/developer/yaml-adapter.md +16 -0
- package/docs/guide/plugins.md +10 -0
- package/docs/zh/guide/plugins.md +10 -0
- package/extension/dist/background.js +519 -444
- package/extension/manifest.json +1 -1
- package/extension/package.json +1 -1
- package/extension/src/background.test.ts +46 -1
- package/extension/src/background.ts +108 -33
- package/extension/src/cdp.ts +9 -9
- package/package.json +3 -2
- package/scripts/check-doc-coverage.sh +2 -0
- package/src/analysis.ts +170 -0
- package/src/browser/cdp.test.ts +66 -0
- package/src/browser/cdp.ts +59 -44
- package/src/browser/dom-snapshot.test.ts +42 -0
- package/src/browser/dom-snapshot.ts +56 -3
- package/src/browser/index.ts +2 -2
- package/src/browser/mcp.ts +2 -4
- package/src/browser/page.ts +34 -37
- package/src/browser/stealth.ts +24 -10
- package/src/browser.test.ts +2 -2
- package/src/build-manifest.test.ts +14 -0
- package/src/build-manifest.ts +13 -31
- package/src/cascade.ts +5 -3
- package/src/cli.ts +66 -34
- package/src/clis/_shared/desktop-commands.ts +121 -0
- package/src/clis/antigravity/serve.ts +6 -3
- package/src/clis/arxiv/search.ts +1 -1
- package/src/clis/bilibili/dynamic.test.ts +79 -0
- package/src/clis/bilibili/favorite.ts +5 -2
- package/src/clis/bilibili/following.ts +3 -2
- package/src/clis/bilibili/subtitle.ts +8 -7
- package/src/clis/bilibili/utils.ts +2 -2
- package/src/clis/boss/batchgreet.ts +1 -1
- package/src/clis/boss/chatlist.ts +1 -1
- package/src/clis/boss/chatmsg.ts +1 -1
- package/src/clis/boss/detail.ts +1 -1
- package/src/clis/boss/exchange.ts +1 -1
- package/src/clis/boss/greet.ts +1 -1
- package/src/clis/boss/invite.ts +1 -1
- package/src/clis/boss/joblist.ts +1 -1
- package/src/clis/boss/mark.ts +4 -3
- package/src/clis/boss/recommend.ts +1 -1
- package/src/clis/boss/resume.ts +1 -1
- package/src/clis/boss/search.ts +1 -1
- package/src/clis/boss/send.ts +5 -4
- package/src/clis/boss/stats.ts +1 -1
- package/src/clis/chatgpt/ask.ts +5 -0
- package/src/clis/chatgpt/new.ts +7 -2
- package/src/clis/chatgpt/read.ts +7 -2
- package/src/clis/chatgpt/send.ts +3 -2
- package/src/clis/chatgpt/status.ts +6 -1
- package/src/clis/chatwise/ask.ts +7 -2
- package/src/clis/chatwise/export.ts +2 -0
- package/src/clis/chatwise/history.ts +2 -0
- package/src/clis/chatwise/model.ts +7 -3
- package/src/clis/chatwise/new.ts +3 -20
- package/src/clis/chatwise/read.ts +2 -0
- package/src/clis/chatwise/screenshot.ts +3 -32
- package/src/clis/chatwise/send.ts +7 -2
- package/src/clis/chatwise/shared.ts +8 -0
- package/src/clis/chatwise/status.ts +3 -24
- package/src/clis/codex/ask.ts +5 -2
- package/src/clis/codex/dump.ts +2 -27
- package/src/clis/codex/new.ts +2 -28
- package/src/clis/codex/screenshot.ts +2 -32
- package/src/clis/codex/send.ts +5 -4
- package/src/clis/codex/status.ts +2 -24
- package/src/clis/cursor/ask.ts +2 -1
- package/src/clis/cursor/composer.ts +2 -1
- package/src/clis/cursor/dump.ts +2 -27
- package/src/clis/cursor/new.ts +2 -20
- package/src/clis/cursor/read.ts +2 -1
- package/src/clis/cursor/screenshot.ts +1 -36
- package/src/clis/cursor/send.ts +2 -1
- package/src/clis/cursor/status.ts +2 -22
- package/src/clis/dictionary/examples.yaml +25 -0
- package/src/clis/dictionary/search.yaml +27 -0
- package/src/clis/dictionary/synonyms.yaml +25 -0
- package/src/clis/douban/book-hot.ts +1 -1
- package/src/clis/douban/movie-hot.ts +1 -1
- package/src/clis/douban/search.ts +1 -1
- package/src/clis/douban/utils.ts +165 -1
- package/src/clis/doubao/ask.ts +1 -1
- package/src/clis/doubao/new.ts +1 -1
- package/src/clis/doubao/read.ts +1 -1
- package/src/clis/doubao/send.ts +1 -1
- package/src/clis/doubao/status.ts +1 -1
- package/src/clis/doubao-app/ask.ts +1 -1
- package/src/clis/doubao-app/new.ts +1 -1
- package/src/clis/doubao-app/read.ts +1 -1
- package/src/clis/doubao-app/send.ts +1 -1
- package/src/clis/grok/ask.test.ts +25 -0
- package/src/clis/grok/ask.ts +25 -12
- package/src/clis/jd/item.test.ts +35 -0
- package/src/clis/jd/item.ts +101 -0
- package/src/clis/jike/feed.ts +1 -1
- package/src/clis/jike/search.ts +1 -1
- package/src/clis/linkedin/search.ts +5 -4
- package/src/clis/linkedin/timeline.test.ts +99 -0
- package/src/clis/linkedin/timeline.ts +532 -0
- package/src/clis/medium/feed.ts +1 -1
- package/src/clis/medium/search.ts +1 -1
- package/src/clis/medium/user.ts +1 -1
- package/src/clis/medium/{shared.ts → utils.ts} +2 -1
- package/src/clis/pixiv/detail.yaml +49 -0
- package/src/clis/pixiv/download.test.ts +114 -0
- package/src/clis/pixiv/download.ts +91 -0
- package/src/clis/pixiv/illusts.test.ts +115 -0
- package/src/clis/pixiv/illusts.ts +78 -0
- package/src/clis/pixiv/ranking.yaml +53 -0
- package/src/clis/pixiv/search.test.ts +97 -0
- package/src/clis/pixiv/search.ts +53 -0
- package/src/clis/pixiv/test-utils.ts +29 -0
- package/src/clis/pixiv/user.yaml +46 -0
- package/src/clis/pixiv/utils.ts +62 -0
- package/src/clis/reddit/comment.ts +2 -1
- package/src/clis/reddit/read.test.ts +34 -0
- package/src/clis/reddit/read.ts +4 -3
- package/src/clis/reddit/save.ts +2 -1
- package/src/clis/reddit/saved.ts +6 -2
- package/src/clis/reddit/subscribe.ts +2 -1
- package/src/clis/reddit/upvote.ts +2 -1
- package/src/clis/reddit/upvoted.ts +6 -2
- package/src/clis/sinablog/article.ts +1 -1
- package/src/clis/sinablog/hot.ts +1 -1
- package/src/clis/sinablog/user.ts +1 -1
- package/src/clis/substack/feed.ts +1 -1
- package/src/clis/substack/publication.ts +1 -1
- package/src/clis/substack/search.ts +3 -2
- package/src/clis/substack/{shared.ts → utils.ts} +3 -2
- package/src/clis/tiktok/search.yaml +2 -1
- package/src/clis/twitter/accept.ts +2 -1
- package/src/clis/twitter/article.ts +3 -1
- package/src/clis/twitter/block.ts +2 -1
- package/src/clis/twitter/bookmark.ts +2 -1
- package/src/clis/twitter/bookmarks.ts +3 -2
- package/src/clis/twitter/delete.ts +2 -1
- package/src/clis/twitter/follow.ts +2 -1
- package/src/clis/twitter/followers.ts +3 -2
- package/src/clis/twitter/following.ts +3 -2
- package/src/clis/twitter/hide-reply.ts +2 -1
- package/src/clis/twitter/like.ts +2 -1
- package/src/clis/twitter/notifications.ts +2 -1
- package/src/clis/twitter/post.ts +2 -1
- package/src/clis/twitter/profile.ts +4 -2
- package/src/clis/twitter/reply-dm.ts +2 -1
- package/src/clis/twitter/reply.ts +2 -1
- package/src/clis/twitter/search.test.ts +113 -0
- package/src/clis/twitter/search.ts +38 -14
- package/src/clis/twitter/thread.ts +2 -2
- package/src/clis/twitter/timeline.ts +3 -2
- package/src/clis/twitter/trending.ts +3 -2
- package/src/clis/twitter/unblock.ts +2 -1
- package/src/clis/twitter/unbookmark.ts +2 -1
- package/src/clis/twitter/unfollow.ts +2 -1
- package/src/clis/v2ex/daily.ts +3 -2
- package/src/clis/v2ex/me.ts +3 -2
- package/src/clis/v2ex/notifications.ts +3 -4
- package/src/clis/web/read.ts +210 -0
- package/src/clis/xueqiu/danjuan-utils.test.ts +49 -0
- package/src/clis/xueqiu/danjuan-utils.ts +176 -0
- package/src/clis/xueqiu/fund-holdings.ts +32 -0
- package/src/clis/xueqiu/fund-snapshot.ts +27 -0
- package/src/clis/youtube/transcript.ts +5 -4
- package/src/clis/youtube/video.ts +3 -2
- package/src/daemon.ts +5 -4
- package/src/discovery.ts +12 -34
- package/src/doctor.ts +3 -2
- package/src/download/index.test.ts +93 -2
- package/src/download/index.ts +44 -23
- package/src/download/media-download.ts +5 -3
- package/src/engine.test.ts +84 -3
- package/src/execution.ts +62 -46
- package/src/explore.ts +21 -90
- package/src/external-clis.yaml +0 -8
- package/src/external.test.ts +9 -0
- package/src/external.ts +12 -10
- package/src/generate.ts +4 -41
- package/src/hooks.test.ts +126 -0
- package/src/hooks.ts +90 -0
- package/src/interceptor.ts +73 -23
- package/src/main.ts +2 -0
- package/src/output.ts +14 -6
- package/src/pipeline/executor.ts +1 -1
- package/src/pipeline/steps/browser.ts +1 -3
- package/src/pipeline/steps/download.test.ts +136 -0
- package/src/pipeline/steps/download.ts +47 -34
- package/src/pipeline/steps/fetch.test.ts +179 -0
- package/src/pipeline/steps/fetch.ts +39 -23
- package/src/pipeline/steps/transform.ts +2 -6
- package/src/pipeline/template.test.ts +28 -0
- package/src/pipeline/template.ts +67 -79
- package/src/pipeline/transform.test.ts +20 -0
- package/src/plugin.test.ts +251 -3
- package/src/plugin.ts +265 -21
- package/src/record.ts +12 -84
- package/src/registry-api.ts +2 -0
- package/src/registry.ts +7 -4
- package/src/runtime.ts +14 -4
- package/src/snapshotFormatter.ts +43 -121
- package/src/utils.ts +39 -0
- package/src/validate.ts +3 -5
- package/src/yaml-schema.ts +28 -0
- package/tests/e2e/browser-auth.test.ts +25 -0
- package/tests/e2e/plugin-management.test.ts +137 -0
- package/tests/e2e/public-commands.test.ts +34 -1
- package/vitest.config.ts +19 -1
- package/.github/workflows/pkg-pr-new.yml +0 -30
- package/dist/clis/douban/shared.d.ts +0 -4
- package/dist/clis/douban/shared.js +0 -155
- package/src/clis/douban/shared.ts +0 -165
- /package/dist/clis/boss/{common.d.ts → utils.d.ts} +0 -0
- /package/dist/clis/boss/{common.js → utils.js} +0 -0
- /package/dist/clis/doubao/{common.d.ts → utils.d.ts} +0 -0
- /package/dist/clis/doubao/{common.js → utils.js} +0 -0
- /package/dist/clis/doubao-app/{common.d.ts → utils.d.ts} +0 -0
- /package/dist/clis/doubao-app/{common.js → utils.js} +0 -0
- /package/dist/clis/jike/{shared.d.ts → utils.d.ts} +0 -0
- /package/dist/clis/jike/{shared.js → utils.js} +0 -0
- /package/dist/clis/medium/{shared.d.ts → utils.d.ts} +0 -0
- /package/dist/clis/sinablog/{shared.d.ts → utils.d.ts} +0 -0
- /package/dist/clis/sinablog/{shared.js → utils.js} +0 -0
- /package/dist/clis/substack/{shared.d.ts → utils.d.ts} +0 -0
- /package/src/clis/boss/{common.ts → utils.ts} +0 -0
- /package/src/clis/doubao/{common.ts → utils.ts} +0 -0
- /package/src/clis/doubao-app/{common.ts → utils.ts} +0 -0
- /package/src/clis/jike/{shared.ts → utils.ts} +0 -0
- /package/src/clis/sinablog/{shared.ts → utils.ts} +0 -0
package/dist/analysis.js
ADDED
|
@@ -0,0 +1,166 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Shared API analysis helpers used by both explore.ts and record.ts.
|
|
3
|
+
*
|
|
4
|
+
* Extracts common logic for:
|
|
5
|
+
* - URL pattern normalization
|
|
6
|
+
* - Array path discovery in JSON responses
|
|
7
|
+
* - Field role detection
|
|
8
|
+
* - Auth indicator inference
|
|
9
|
+
* - Capability name inference
|
|
10
|
+
* - Strategy inference
|
|
11
|
+
*/
|
|
12
|
+
import { VOLATILE_PARAMS, SEARCH_PARAMS, PAGINATION_PARAMS, LIMIT_PARAMS, FIELD_ROLES, } from './constants.js';
|
|
13
|
+
// ── URL pattern normalization ───────────────────────────────────────────────
|
|
14
|
+
/** Normalize a full URL into a pattern (replace IDs, strip volatile params). */
|
|
15
|
+
export function urlToPattern(url) {
|
|
16
|
+
try {
|
|
17
|
+
const p = new URL(url);
|
|
18
|
+
const pathNorm = p.pathname
|
|
19
|
+
.replace(/\/\d+/g, '/{id}')
|
|
20
|
+
.replace(/\/[0-9a-fA-F]{8,}/g, '/{hex}')
|
|
21
|
+
.replace(/\/BV[a-zA-Z0-9]{10}/g, '/{bvid}');
|
|
22
|
+
const params = [];
|
|
23
|
+
p.searchParams.forEach((_v, k) => { if (!VOLATILE_PARAMS.has(k))
|
|
24
|
+
params.push(k); });
|
|
25
|
+
return `${p.host}${pathNorm}${params.length ? '?' + params.sort().map(k => `${k}={}`).join('&') : ''}`;
|
|
26
|
+
}
|
|
27
|
+
catch {
|
|
28
|
+
return url;
|
|
29
|
+
}
|
|
30
|
+
}
|
|
31
|
+
/** Find the best (largest) array of objects in a JSON response body. */
|
|
32
|
+
export function findArrayPath(obj, depth = 0) {
|
|
33
|
+
if (depth > 5 || !obj || typeof obj !== 'object')
|
|
34
|
+
return null;
|
|
35
|
+
if (Array.isArray(obj)) {
|
|
36
|
+
if (obj.length >= 2 && obj.some(i => i && typeof i === 'object' && !Array.isArray(i))) {
|
|
37
|
+
return { path: '', items: obj };
|
|
38
|
+
}
|
|
39
|
+
return null;
|
|
40
|
+
}
|
|
41
|
+
let best = null;
|
|
42
|
+
for (const [key, val] of Object.entries(obj)) {
|
|
43
|
+
const found = findArrayPath(val, depth + 1);
|
|
44
|
+
if (found) {
|
|
45
|
+
const fullPath = found.path ? `${key}.${found.path}` : key;
|
|
46
|
+
const candidate = { path: fullPath, items: found.items };
|
|
47
|
+
if (!best || candidate.items.length > best.items.length)
|
|
48
|
+
best = candidate;
|
|
49
|
+
}
|
|
50
|
+
}
|
|
51
|
+
return best;
|
|
52
|
+
}
|
|
53
|
+
// ── Field flattening & role detection ───────────────────────────────────────
|
|
54
|
+
/** Flatten nested object keys up to maxDepth. */
|
|
55
|
+
export function flattenFields(obj, prefix, maxDepth) {
|
|
56
|
+
if (maxDepth <= 0 || !obj || typeof obj !== 'object')
|
|
57
|
+
return [];
|
|
58
|
+
const names = [];
|
|
59
|
+
const record = obj;
|
|
60
|
+
for (const key of Object.keys(record)) {
|
|
61
|
+
const full = prefix ? `${prefix}.${key}` : key;
|
|
62
|
+
names.push(full);
|
|
63
|
+
const val = record[key];
|
|
64
|
+
if (val && typeof val === 'object' && !Array.isArray(val))
|
|
65
|
+
names.push(...flattenFields(val, full, maxDepth - 1));
|
|
66
|
+
}
|
|
67
|
+
return names;
|
|
68
|
+
}
|
|
69
|
+
/** Detect semantic field roles (title, url, author, etc.) from sample fields. */
|
|
70
|
+
export function detectFieldRoles(sampleFields) {
|
|
71
|
+
const detectedFields = {};
|
|
72
|
+
for (const [role, aliases] of Object.entries(FIELD_ROLES)) {
|
|
73
|
+
for (const f of sampleFields) {
|
|
74
|
+
if (aliases.includes(f.split('.').pop()?.toLowerCase() ?? '')) {
|
|
75
|
+
detectedFields[role] = f;
|
|
76
|
+
break;
|
|
77
|
+
}
|
|
78
|
+
}
|
|
79
|
+
}
|
|
80
|
+
return detectedFields;
|
|
81
|
+
}
|
|
82
|
+
// ── Capability name inference ───────────────────────────────────────────────
|
|
83
|
+
/** Infer a CLI capability name from a URL. */
|
|
84
|
+
export function inferCapabilityName(url, goal) {
|
|
85
|
+
if (goal)
|
|
86
|
+
return goal;
|
|
87
|
+
const u = url.toLowerCase();
|
|
88
|
+
if (u.includes('hot') || u.includes('popular') || u.includes('ranking') || u.includes('trending'))
|
|
89
|
+
return 'hot';
|
|
90
|
+
if (u.includes('search'))
|
|
91
|
+
return 'search';
|
|
92
|
+
if (u.includes('feed') || u.includes('timeline') || u.includes('dynamic'))
|
|
93
|
+
return 'feed';
|
|
94
|
+
if (u.includes('comment') || u.includes('reply'))
|
|
95
|
+
return 'comments';
|
|
96
|
+
if (u.includes('history'))
|
|
97
|
+
return 'history';
|
|
98
|
+
if (u.includes('profile') || u.includes('userinfo') || u.includes('/me'))
|
|
99
|
+
return 'me';
|
|
100
|
+
if (u.includes('favorite') || u.includes('collect') || u.includes('bookmark'))
|
|
101
|
+
return 'favorite';
|
|
102
|
+
try {
|
|
103
|
+
const segs = new URL(url).pathname
|
|
104
|
+
.split('/')
|
|
105
|
+
.filter(s => s && !s.match(/^\d+$/) && !s.match(/^[0-9a-f]{8,}$/i) && !s.match(/^v\d+$/));
|
|
106
|
+
if (segs.length)
|
|
107
|
+
return segs[segs.length - 1].replace(/[^a-z0-9]/gi, '_').toLowerCase();
|
|
108
|
+
}
|
|
109
|
+
catch { }
|
|
110
|
+
return 'data';
|
|
111
|
+
}
|
|
112
|
+
// ── Strategy inference ──────────────────────────────────────────────────────
|
|
113
|
+
/** Infer auth strategy from detected indicators. */
|
|
114
|
+
export function inferStrategy(authIndicators) {
|
|
115
|
+
if (authIndicators.includes('signature'))
|
|
116
|
+
return 'intercept';
|
|
117
|
+
if (authIndicators.includes('bearer') || authIndicators.includes('csrf'))
|
|
118
|
+
return 'header';
|
|
119
|
+
return 'cookie';
|
|
120
|
+
}
|
|
121
|
+
// ── Auth indicator detection ────────────────────────────────────────────────
|
|
122
|
+
/** Detect auth indicators from HTTP headers. */
|
|
123
|
+
export function detectAuthFromHeaders(headers) {
|
|
124
|
+
if (!headers)
|
|
125
|
+
return [];
|
|
126
|
+
const indicators = [];
|
|
127
|
+
const keys = Object.keys(headers).map(k => k.toLowerCase());
|
|
128
|
+
if (keys.some(k => k === 'authorization'))
|
|
129
|
+
indicators.push('bearer');
|
|
130
|
+
if (keys.some(k => k.startsWith('x-csrf') || k.startsWith('x-xsrf')))
|
|
131
|
+
indicators.push('csrf');
|
|
132
|
+
if (keys.some(k => k.startsWith('x-s') || k === 'x-t' || k === 'x-s-common'))
|
|
133
|
+
indicators.push('signature');
|
|
134
|
+
return indicators;
|
|
135
|
+
}
|
|
136
|
+
/** Detect auth indicators from URL and response body (heuristic). */
|
|
137
|
+
export function detectAuthFromContent(url, body) {
|
|
138
|
+
const indicators = [];
|
|
139
|
+
if (body && typeof body === 'object') {
|
|
140
|
+
const keys = Object.keys(body).map(k => k.toLowerCase());
|
|
141
|
+
if (keys.some(k => k.includes('sign') || k === 'w_rid' || k.includes('token'))) {
|
|
142
|
+
indicators.push('signature');
|
|
143
|
+
}
|
|
144
|
+
}
|
|
145
|
+
if (url.includes('/wbi/') || url.includes('w_rid='))
|
|
146
|
+
indicators.push('signature');
|
|
147
|
+
if (url.includes('bearer') || url.includes('access_token'))
|
|
148
|
+
indicators.push('bearer');
|
|
149
|
+
return indicators;
|
|
150
|
+
}
|
|
151
|
+
// ── Query param classification ──────────────────────────────────────────────
|
|
152
|
+
/** Extract non-volatile query params and classify them. */
|
|
153
|
+
export function classifyQueryParams(url) {
|
|
154
|
+
const params = [];
|
|
155
|
+
try {
|
|
156
|
+
new URL(url).searchParams.forEach((_v, k) => { if (!VOLATILE_PARAMS.has(k))
|
|
157
|
+
params.push(k); });
|
|
158
|
+
}
|
|
159
|
+
catch { }
|
|
160
|
+
return {
|
|
161
|
+
params,
|
|
162
|
+
hasSearch: params.some(p => SEARCH_PARAMS.has(p)),
|
|
163
|
+
hasPagination: params.some(p => PAGINATION_PARAMS.has(p)),
|
|
164
|
+
hasLimit: params.some(p => LIMIT_PARAMS.has(p)),
|
|
165
|
+
};
|
|
166
|
+
}
|
package/dist/browser/cdp.d.ts
CHANGED
|
@@ -24,13 +24,9 @@ export declare class CDPBridge {
|
|
|
24
24
|
workspace?: string;
|
|
25
25
|
}): Promise<IPage>;
|
|
26
26
|
close(): Promise<void>;
|
|
27
|
-
/** Send a CDP command with timeout guard (P0 fix #4) */
|
|
28
27
|
send(method: string, params?: Record<string, unknown>, timeoutMs?: number): Promise<unknown>;
|
|
29
|
-
/** Listen for a CDP event */
|
|
30
28
|
on(event: string, handler: (params: unknown) => void): void;
|
|
31
|
-
/** Remove a CDP event listener */
|
|
32
29
|
off(event: string, handler: (params: unknown) => void): void;
|
|
33
|
-
/** Wait for a CDP event to fire (one-shot) */
|
|
34
30
|
waitForEvent(event: string, timeoutMs?: number): Promise<unknown>;
|
|
35
31
|
}
|
|
36
32
|
declare function selectCDPTarget(targets: CDPTarget[]): CDPTarget | undefined;
|
package/dist/browser/cdp.js
CHANGED
|
@@ -8,27 +8,28 @@
|
|
|
8
8
|
* - Shared DOM helper methods extracted to reduce duplication with Page (P1 #5)
|
|
9
9
|
*/
|
|
10
10
|
import { WebSocket } from 'ws';
|
|
11
|
+
import { request as httpRequest } from 'node:http';
|
|
12
|
+
import { request as httpsRequest } from 'node:https';
|
|
11
13
|
import { wrapForEval } from './utils.js';
|
|
12
14
|
import { generateSnapshotJs, scrollToRefJs, getFormStateJs } from './dom-snapshot.js';
|
|
13
15
|
import { generateStealthJs } from './stealth.js';
|
|
14
16
|
import { clickJs, typeTextJs, pressKeyJs, waitForTextJs, scrollJs, autoScrollJs, networkRequestsJs, waitForDomStableJs, } from './dom-helpers.js';
|
|
15
|
-
|
|
17
|
+
import { isRecord, saveBase64ToFile } from '../utils.js';
|
|
18
|
+
const CDP_SEND_TIMEOUT = 30_000;
|
|
16
19
|
export class CDPBridge {
|
|
17
20
|
_ws = null;
|
|
18
21
|
_idCounter = 0;
|
|
19
22
|
_pending = new Map();
|
|
20
23
|
_eventListeners = new Map();
|
|
21
24
|
async connect(opts) {
|
|
25
|
+
if (this._ws)
|
|
26
|
+
throw new Error('CDPBridge is already connected. Call close() before reconnecting.');
|
|
22
27
|
const endpoint = process.env.OPENCLI_CDP_ENDPOINT;
|
|
23
28
|
if (!endpoint)
|
|
24
29
|
throw new Error('OPENCLI_CDP_ENDPOINT is not set');
|
|
25
|
-
// If it's a direct ws:// URL, use it. Otherwise, fetch the /json endpoint to find a page.
|
|
26
30
|
let wsUrl = endpoint;
|
|
27
31
|
if (endpoint.startsWith('http')) {
|
|
28
|
-
const
|
|
29
|
-
if (!res.ok)
|
|
30
|
-
throw new Error(`Failed to fetch CDP targets: ${res.statusText}`);
|
|
31
|
-
const targets = await res.json();
|
|
32
|
+
const targets = await fetchJsonDirect(`${endpoint.replace(/\/$/, '')}/json`);
|
|
32
33
|
const target = selectCDPTarget(targets);
|
|
33
34
|
if (!target || !target.webSocketDebuggerUrl) {
|
|
34
35
|
throw new Error('No inspectable targets found at CDP endpoint');
|
|
@@ -37,19 +38,16 @@ export class CDPBridge {
|
|
|
37
38
|
}
|
|
38
39
|
return new Promise((resolve, reject) => {
|
|
39
40
|
const ws = new WebSocket(wsUrl);
|
|
40
|
-
const timeoutMs = (opts?.timeout ?? 10) * 1000;
|
|
41
|
+
const timeoutMs = (opts?.timeout ?? 10) * 1000;
|
|
41
42
|
const timeout = setTimeout(() => reject(new Error('CDP connect timeout')), timeoutMs);
|
|
42
43
|
ws.on('open', async () => {
|
|
43
44
|
clearTimeout(timeout);
|
|
44
45
|
this._ws = ws;
|
|
45
|
-
// Register stealth script to run before any page JS on every navigation.
|
|
46
46
|
try {
|
|
47
47
|
await this.send('Page.enable');
|
|
48
48
|
await this.send('Page.addScriptToEvaluateOnNewDocument', { source: generateStealthJs() });
|
|
49
49
|
}
|
|
50
|
-
catch {
|
|
51
|
-
// Non-fatal: stealth is best-effort
|
|
52
|
-
}
|
|
50
|
+
catch { }
|
|
53
51
|
resolve(new CDPPage(this));
|
|
54
52
|
});
|
|
55
53
|
ws.on('error', (err) => {
|
|
@@ -59,7 +57,6 @@ export class CDPBridge {
|
|
|
59
57
|
ws.on('message', (data) => {
|
|
60
58
|
try {
|
|
61
59
|
const msg = JSON.parse(data.toString());
|
|
62
|
-
// Handle command responses
|
|
63
60
|
if (msg.id && this._pending.has(msg.id)) {
|
|
64
61
|
const entry = this._pending.get(msg.id);
|
|
65
62
|
clearTimeout(entry.timer);
|
|
@@ -71,7 +68,6 @@ export class CDPBridge {
|
|
|
71
68
|
entry.resolve(msg.result);
|
|
72
69
|
}
|
|
73
70
|
}
|
|
74
|
-
// Handle CDP events
|
|
75
71
|
if (msg.method) {
|
|
76
72
|
const listeners = this._eventListeners.get(msg.method);
|
|
77
73
|
if (listeners) {
|
|
@@ -80,9 +76,7 @@ export class CDPBridge {
|
|
|
80
76
|
}
|
|
81
77
|
}
|
|
82
78
|
}
|
|
83
|
-
catch {
|
|
84
|
-
// ignore parsing errors
|
|
85
|
-
}
|
|
79
|
+
catch { }
|
|
86
80
|
});
|
|
87
81
|
});
|
|
88
82
|
}
|
|
@@ -98,7 +92,6 @@ export class CDPBridge {
|
|
|
98
92
|
this._pending.clear();
|
|
99
93
|
this._eventListeners.clear();
|
|
100
94
|
}
|
|
101
|
-
/** Send a CDP command with timeout guard (P0 fix #4) */
|
|
102
95
|
async send(method, params = {}, timeoutMs = CDP_SEND_TIMEOUT) {
|
|
103
96
|
if (!this._ws || this._ws.readyState !== WebSocket.OPEN) {
|
|
104
97
|
throw new Error('CDP connection is not open');
|
|
@@ -113,7 +106,6 @@ export class CDPBridge {
|
|
|
113
106
|
this._ws.send(JSON.stringify({ id, method, params }));
|
|
114
107
|
});
|
|
115
108
|
}
|
|
116
|
-
/** Listen for a CDP event */
|
|
117
109
|
on(event, handler) {
|
|
118
110
|
let set = this._eventListeners.get(event);
|
|
119
111
|
if (!set) {
|
|
@@ -122,11 +114,9 @@ export class CDPBridge {
|
|
|
122
114
|
}
|
|
123
115
|
set.add(handler);
|
|
124
116
|
}
|
|
125
|
-
/** Remove a CDP event listener */
|
|
126
117
|
off(event, handler) {
|
|
127
118
|
this._eventListeners.get(event)?.delete(handler);
|
|
128
119
|
}
|
|
129
|
-
/** Wait for a CDP event to fire (one-shot) */
|
|
130
120
|
waitForEvent(event, timeoutMs = 15_000) {
|
|
131
121
|
return new Promise((resolve, reject) => {
|
|
132
122
|
const timer = setTimeout(() => {
|
|
@@ -144,18 +134,18 @@ export class CDPBridge {
|
|
|
144
134
|
}
|
|
145
135
|
class CDPPage {
|
|
146
136
|
bridge;
|
|
137
|
+
_pageEnabled = false;
|
|
147
138
|
constructor(bridge) {
|
|
148
139
|
this.bridge = bridge;
|
|
149
140
|
}
|
|
150
|
-
/** Navigate with proper load event waiting (P1 fix #3) */
|
|
151
141
|
async goto(url, options) {
|
|
152
|
-
|
|
153
|
-
|
|
154
|
-
.
|
|
142
|
+
if (!this._pageEnabled) {
|
|
143
|
+
await this.bridge.send('Page.enable');
|
|
144
|
+
this._pageEnabled = true;
|
|
145
|
+
}
|
|
146
|
+
const loadPromise = this.bridge.waitForEvent('Page.loadEventFired', 30_000).catch(() => { });
|
|
155
147
|
await this.bridge.send('Page.navigate', { url });
|
|
156
148
|
await loadPromise;
|
|
157
|
-
// Smart settle: use DOM stability detection instead of fixed sleep.
|
|
158
|
-
// settleMs is now a timeout cap (default 1000ms), not a fixed wait.
|
|
159
149
|
if (options?.waitUntil !== 'none') {
|
|
160
150
|
const maxMs = options?.settleMs ?? 1000;
|
|
161
151
|
await this.evaluate(waitForDomStableJs(maxMs, Math.min(500, maxMs)));
|
|
@@ -166,7 +156,7 @@ class CDPPage {
|
|
|
166
156
|
const result = await this.bridge.send('Runtime.evaluate', {
|
|
167
157
|
expression,
|
|
168
158
|
returnByValue: true,
|
|
169
|
-
awaitPromise: true
|
|
159
|
+
awaitPromise: true,
|
|
170
160
|
});
|
|
171
161
|
if (result.exceptionDetails) {
|
|
172
162
|
throw new Error('Evaluate error: ' + (result.exceptionDetails.exception?.description || 'Unknown exception'));
|
|
@@ -178,7 +168,7 @@ class CDPPage {
|
|
|
178
168
|
const cookies = isRecord(result) && Array.isArray(result.cookies) ? result.cookies : [];
|
|
179
169
|
const domain = opts.domain;
|
|
180
170
|
return domain
|
|
181
|
-
? cookies.filter((cookie) => isCookie(cookie) && cookie.domain
|
|
171
|
+
? cookies.filter((cookie) => isCookie(cookie) && matchesCookieDomain(cookie.domain, domain))
|
|
182
172
|
: cookies;
|
|
183
173
|
}
|
|
184
174
|
async snapshot(opts = {}) {
|
|
@@ -192,7 +182,6 @@ class CDPPage {
|
|
|
192
182
|
});
|
|
193
183
|
return this.evaluate(snapshotJs);
|
|
194
184
|
}
|
|
195
|
-
// ── Shared DOM operations (P1 fix #5 — using dom-helpers.ts) ──
|
|
196
185
|
async click(ref) {
|
|
197
186
|
await this.evaluate(clickJs(ref));
|
|
198
187
|
}
|
|
@@ -210,12 +199,12 @@ class CDPPage {
|
|
|
210
199
|
}
|
|
211
200
|
async wait(options) {
|
|
212
201
|
if (typeof options === 'number') {
|
|
213
|
-
await new Promise(resolve => setTimeout(resolve, options * 1000));
|
|
202
|
+
await new Promise((resolve) => setTimeout(resolve, options * 1000));
|
|
214
203
|
return;
|
|
215
204
|
}
|
|
216
205
|
if (typeof options.time === 'number') {
|
|
217
206
|
const waitTime = options.time;
|
|
218
|
-
await new Promise(resolve => setTimeout(resolve, waitTime * 1000));
|
|
207
|
+
await new Promise((resolve) => setTimeout(resolve, waitTime * 1000));
|
|
219
208
|
return;
|
|
220
209
|
}
|
|
221
210
|
if (options.text) {
|
|
@@ -223,7 +212,6 @@ class CDPPage {
|
|
|
223
212
|
await this.evaluate(waitForTextJs(options.text, timeout));
|
|
224
213
|
}
|
|
225
214
|
}
|
|
226
|
-
// ── Implemented methods (P1 fix #2) ──
|
|
227
215
|
async scroll(direction = 'down', amount = 500) {
|
|
228
216
|
await this.evaluate(scrollJs(direction, amount));
|
|
229
217
|
}
|
|
@@ -240,11 +228,7 @@ class CDPPage {
|
|
|
240
228
|
});
|
|
241
229
|
const base64 = isRecord(result) && typeof result.data === 'string' ? result.data : '';
|
|
242
230
|
if (options.path) {
|
|
243
|
-
|
|
244
|
-
const path = await import('node:path');
|
|
245
|
-
const dir = path.dirname(options.path);
|
|
246
|
-
await fs.promises.mkdir(dir, { recursive: true });
|
|
247
|
-
await fs.promises.writeFile(options.path, Buffer.from(base64, 'base64'));
|
|
231
|
+
await saveBase64ToFile(base64, options.path);
|
|
248
232
|
}
|
|
249
233
|
return base64;
|
|
250
234
|
}
|
|
@@ -280,16 +264,18 @@ class CDPPage {
|
|
|
280
264
|
return Array.isArray(result) ? result : [];
|
|
281
265
|
}
|
|
282
266
|
}
|
|
283
|
-
function isRecord(value) {
|
|
284
|
-
return typeof value === 'object' && value !== null && !Array.isArray(value);
|
|
285
|
-
}
|
|
286
267
|
function isCookie(value) {
|
|
287
268
|
return isRecord(value)
|
|
288
269
|
&& typeof value.name === 'string'
|
|
289
270
|
&& typeof value.value === 'string'
|
|
290
271
|
&& typeof value.domain === 'string';
|
|
291
272
|
}
|
|
292
|
-
|
|
273
|
+
function matchesCookieDomain(cookieDomain, targetDomain) {
|
|
274
|
+
const normalizedCookieDomain = cookieDomain.replace(/^\./, '').toLowerCase();
|
|
275
|
+
const normalizedTargetDomain = targetDomain.replace(/^\./, '').toLowerCase();
|
|
276
|
+
return normalizedTargetDomain === normalizedCookieDomain
|
|
277
|
+
|| normalizedTargetDomain.endsWith(`.${normalizedCookieDomain}`);
|
|
278
|
+
}
|
|
293
279
|
function selectCDPTarget(targets) {
|
|
294
280
|
const preferredPattern = compilePreferredPattern(process.env.OPENCLI_CDP_TARGET);
|
|
295
281
|
const ranked = targets
|
|
@@ -375,3 +361,29 @@ export const __test__ = {
|
|
|
375
361
|
selectCDPTarget,
|
|
376
362
|
scoreCDPTarget,
|
|
377
363
|
};
|
|
364
|
+
function fetchJsonDirect(url) {
|
|
365
|
+
return new Promise((resolve, reject) => {
|
|
366
|
+
const parsed = new URL(url);
|
|
367
|
+
const request = (parsed.protocol === 'https:' ? httpsRequest : httpRequest)(parsed, (res) => {
|
|
368
|
+
const statusCode = res.statusCode ?? 0;
|
|
369
|
+
if (statusCode < 200 || statusCode >= 300) {
|
|
370
|
+
res.resume();
|
|
371
|
+
reject(new Error(`Failed to fetch CDP targets: HTTP ${statusCode}`));
|
|
372
|
+
return;
|
|
373
|
+
}
|
|
374
|
+
const chunks = [];
|
|
375
|
+
res.on('data', (chunk) => chunks.push(Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk)));
|
|
376
|
+
res.on('end', () => {
|
|
377
|
+
try {
|
|
378
|
+
resolve(JSON.parse(Buffer.concat(chunks).toString('utf8')));
|
|
379
|
+
}
|
|
380
|
+
catch (error) {
|
|
381
|
+
reject(error instanceof Error ? error : new Error(String(error)));
|
|
382
|
+
}
|
|
383
|
+
});
|
|
384
|
+
});
|
|
385
|
+
request.on('error', reject);
|
|
386
|
+
request.setTimeout(10_000, () => request.destroy(new Error('Timed out fetching CDP targets')));
|
|
387
|
+
request.end();
|
|
388
|
+
});
|
|
389
|
+
}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
export {};
|
|
@@ -0,0 +1,52 @@
|
|
|
1
|
+
import { beforeEach, describe, expect, it, vi } from 'vitest';
|
|
2
|
+
const { MockWebSocket } = vi.hoisted(() => {
|
|
3
|
+
class MockWebSocket {
|
|
4
|
+
static OPEN = 1;
|
|
5
|
+
readyState = 1;
|
|
6
|
+
handlers = new Map();
|
|
7
|
+
constructor(_url) {
|
|
8
|
+
queueMicrotask(() => this.emit('open'));
|
|
9
|
+
}
|
|
10
|
+
on(event, handler) {
|
|
11
|
+
const handlers = this.handlers.get(event) ?? [];
|
|
12
|
+
handlers.push(handler);
|
|
13
|
+
this.handlers.set(event, handlers);
|
|
14
|
+
}
|
|
15
|
+
send(_message) { }
|
|
16
|
+
close() {
|
|
17
|
+
this.readyState = 3;
|
|
18
|
+
}
|
|
19
|
+
emit(event, ...args) {
|
|
20
|
+
for (const handler of this.handlers.get(event) ?? []) {
|
|
21
|
+
handler(...args);
|
|
22
|
+
}
|
|
23
|
+
}
|
|
24
|
+
}
|
|
25
|
+
return { MockWebSocket };
|
|
26
|
+
});
|
|
27
|
+
vi.mock('ws', () => ({
|
|
28
|
+
WebSocket: MockWebSocket,
|
|
29
|
+
}));
|
|
30
|
+
import { CDPBridge } from './cdp.js';
|
|
31
|
+
describe('CDPBridge cookies', () => {
|
|
32
|
+
beforeEach(() => {
|
|
33
|
+
vi.unstubAllEnvs();
|
|
34
|
+
});
|
|
35
|
+
it('filters cookies by actual domain match instead of substring match', async () => {
|
|
36
|
+
vi.stubEnv('OPENCLI_CDP_ENDPOINT', 'ws://127.0.0.1:9222/devtools/page/1');
|
|
37
|
+
const bridge = new CDPBridge();
|
|
38
|
+
vi.spyOn(bridge, 'send').mockResolvedValue({
|
|
39
|
+
cookies: [
|
|
40
|
+
{ name: 'good', value: '1', domain: '.example.com' },
|
|
41
|
+
{ name: 'exact', value: '2', domain: 'example.com' },
|
|
42
|
+
{ name: 'bad', value: '3', domain: 'notexample.com' },
|
|
43
|
+
],
|
|
44
|
+
});
|
|
45
|
+
const page = await bridge.connect();
|
|
46
|
+
const cookies = await page.getCookies({ domain: 'example.com' });
|
|
47
|
+
expect(cookies).toEqual([
|
|
48
|
+
{ name: 'good', value: '1', domain: '.example.com' },
|
|
49
|
+
{ name: 'exact', value: '2', domain: 'example.com' },
|
|
50
|
+
]);
|
|
51
|
+
});
|
|
52
|
+
});
|
|
@@ -23,7 +23,7 @@
|
|
|
23
23
|
* - scrollToRefJs(ref) — scroll to a data-opencli-ref element
|
|
24
24
|
* - getFormStateJs() — extract all form fields as structured JSON
|
|
25
25
|
*/
|
|
26
|
-
export interface
|
|
26
|
+
export interface DomSnapshotOptions {
|
|
27
27
|
/** Extra pixels beyond viewport to include (default 800) */
|
|
28
28
|
viewportExpand?: number;
|
|
29
29
|
/** Maximum DOM depth to traverse (default 50) */
|
|
@@ -83,4 +83,4 @@ export declare function getFormStateJs(): string;
|
|
|
83
83
|
* - `|iframe|` — iframe content
|
|
84
84
|
* - `|table|` — markdown table rendering
|
|
85
85
|
*/
|
|
86
|
-
export declare function generateSnapshotJs(opts?:
|
|
86
|
+
export declare function generateSnapshotJs(opts?: DomSnapshotOptions): string;
|
|
@@ -230,6 +230,13 @@ export function generateSnapshotJs(opts = {}) {
|
|
|
230
230
|
|
|
231
231
|
const AD_SELECTOR_RE = /\\b(ad[_-]?(?:banner|container|wrapper|slot|unit|block|frame|leaderboard|sidebar)|google[_-]?ad|sponsored|adsbygoogle|banner[_-]?ad)\\b/i;
|
|
232
232
|
|
|
233
|
+
// Search element indicators for heuristic detection
|
|
234
|
+
const SEARCH_INDICATORS = new Set([
|
|
235
|
+
'search', 'magnify', 'glass', 'lookup', 'find', 'query',
|
|
236
|
+
'search-icon', 'search-btn', 'search-button', 'searchbox',
|
|
237
|
+
'fa-search', 'icon-search', 'btn-search',
|
|
238
|
+
]);
|
|
239
|
+
|
|
233
240
|
// ── Viewport & Layout Helpers ──────────────────────────────────────
|
|
234
241
|
|
|
235
242
|
const vw = window.innerWidth;
|
|
@@ -298,19 +305,65 @@ export function generateSnapshotJs(opts = {}) {
|
|
|
298
305
|
|
|
299
306
|
// ── Interactivity Detection ────────────────────────────────────────
|
|
300
307
|
|
|
308
|
+
// Check if element contains a form control within limited depth (handles label/span wrappers)
|
|
309
|
+
function hasFormControlDescendant(el, maxDepth = 2) {
|
|
310
|
+
if (maxDepth <= 0) return false;
|
|
311
|
+
for (const child of el.children || []) {
|
|
312
|
+
const tag = child.tagName?.toLowerCase();
|
|
313
|
+
if (tag === 'input' || tag === 'select' || tag === 'textarea') return true;
|
|
314
|
+
if (hasFormControlDescendant(child, maxDepth - 1)) return true;
|
|
315
|
+
}
|
|
316
|
+
return false;
|
|
317
|
+
}
|
|
318
|
+
|
|
301
319
|
function isInteractive(el) {
|
|
302
320
|
const tag = el.tagName.toLowerCase();
|
|
303
321
|
if (INTERACTIVE_TAGS.has(tag)) {
|
|
304
|
-
|
|
322
|
+
// Skip labels that proxy via "for" to avoid double-activating external inputs
|
|
323
|
+
if (tag === 'label') {
|
|
324
|
+
if (el.hasAttribute('for')) return false;
|
|
325
|
+
// Detect labels that wrap form controls up to two levels deep (label > span > input)
|
|
326
|
+
if (hasFormControlDescendant(el, 2)) return true;
|
|
327
|
+
}
|
|
305
328
|
if (el.disabled && (tag === 'button' || tag === 'input')) return false;
|
|
306
329
|
return true;
|
|
307
330
|
}
|
|
331
|
+
// Span wrappers for UI components - check if they contain form controls
|
|
332
|
+
if (tag === 'span') {
|
|
333
|
+
if (hasFormControlDescendant(el, 2)) return true;
|
|
334
|
+
}
|
|
308
335
|
const role = el.getAttribute('role');
|
|
309
336
|
if (role && INTERACTIVE_ROLES.has(role)) return true;
|
|
310
337
|
if (el.hasAttribute('onclick') || el.hasAttribute('onmousedown') || el.hasAttribute('ontouchstart')) return true;
|
|
311
338
|
if (el.hasAttribute('tabindex') && el.getAttribute('tabindex') !== '-1') return true;
|
|
312
339
|
try { if (window.getComputedStyle(el).cursor === 'pointer') return true; } catch {}
|
|
313
340
|
if (el.isContentEditable && el.getAttribute('contenteditable') !== 'false') return true;
|
|
341
|
+
// Search element heuristic detection
|
|
342
|
+
if (isSearchElement(el)) return true;
|
|
343
|
+
return false;
|
|
344
|
+
}
|
|
345
|
+
|
|
346
|
+
function isSearchElement(el) {
|
|
347
|
+
// Check class names for search indicators
|
|
348
|
+
const className = el.className?.toLowerCase() || '';
|
|
349
|
+
const classes = className.split(/\\s+/).filter(Boolean);
|
|
350
|
+
for (const cls of classes) {
|
|
351
|
+
const cleaned = cls.replace(/[^a-z0-9-]/g, '');
|
|
352
|
+
if (SEARCH_INDICATORS.has(cleaned)) return true;
|
|
353
|
+
}
|
|
354
|
+
// Check id for search indicators
|
|
355
|
+
const id = el.id?.toLowerCase() || '';
|
|
356
|
+
const cleanedId = id.replace(/[^a-z0-9-]/g, '');
|
|
357
|
+
if (SEARCH_INDICATORS.has(cleanedId)) return true;
|
|
358
|
+
// Check data-* attributes for search functionality
|
|
359
|
+
for (const attr of el.attributes || []) {
|
|
360
|
+
if (attr.name.startsWith('data-')) {
|
|
361
|
+
const value = attr.value.toLowerCase();
|
|
362
|
+
for (const kw of SEARCH_INDICATORS) {
|
|
363
|
+
if (value.includes(kw)) return true;
|
|
364
|
+
}
|
|
365
|
+
}
|
|
366
|
+
}
|
|
314
367
|
return false;
|
|
315
368
|
}
|
|
316
369
|
|
|
@@ -210,3 +210,39 @@ describe('getFormStateJs', () => {
|
|
|
210
210
|
expect(js).toContain('data-opencli-ref');
|
|
211
211
|
});
|
|
212
212
|
});
|
|
213
|
+
describe('Search Element Detection', () => {
|
|
214
|
+
it('includes SEARCH_INDICATORS set', () => {
|
|
215
|
+
const js = generateSnapshotJs();
|
|
216
|
+
expect(js).toContain('SEARCH_INDICATORS');
|
|
217
|
+
expect(js).toContain('search');
|
|
218
|
+
expect(js).toContain('magnify');
|
|
219
|
+
expect(js).toContain('glass');
|
|
220
|
+
});
|
|
221
|
+
it('includes hasFormControlDescendant function', () => {
|
|
222
|
+
const js = generateSnapshotJs();
|
|
223
|
+
expect(js).toContain('hasFormControlDescendant');
|
|
224
|
+
expect(js).toContain('input');
|
|
225
|
+
expect(js).toContain('select');
|
|
226
|
+
expect(js).toContain('textarea');
|
|
227
|
+
});
|
|
228
|
+
it('includes isSearchElement function', () => {
|
|
229
|
+
const js = generateSnapshotJs();
|
|
230
|
+
expect(js).toContain('isSearchElement');
|
|
231
|
+
expect(js).toContain('className');
|
|
232
|
+
expect(js).toContain('data-');
|
|
233
|
+
});
|
|
234
|
+
it('checks label wrapper detection in isInteractive', () => {
|
|
235
|
+
const js = generateSnapshotJs();
|
|
236
|
+
// Label elements without "for" attribute should check for form control descendants
|
|
237
|
+
expect(js).toContain('hasFormControlDescendant(el, 2)');
|
|
238
|
+
});
|
|
239
|
+
it('checks span wrapper detection in isInteractive', () => {
|
|
240
|
+
const js = generateSnapshotJs();
|
|
241
|
+
// Span elements should check for form control descendants
|
|
242
|
+
expect(js).toContain("tag === 'span'");
|
|
243
|
+
});
|
|
244
|
+
it('integrates search element detection into isInteractive', () => {
|
|
245
|
+
const js = generateSnapshotJs();
|
|
246
|
+
expect(js).toContain('isSearchElement(el)');
|
|
247
|
+
});
|
|
248
|
+
});
|
package/dist/browser/index.d.ts
CHANGED
|
@@ -5,12 +5,12 @@
|
|
|
5
5
|
* External code should import from './browser/index.js' (or './browser.js' via Node resolution).
|
|
6
6
|
*/
|
|
7
7
|
export { Page } from './page.js';
|
|
8
|
-
export { BrowserBridge
|
|
8
|
+
export { BrowserBridge } from './mcp.js';
|
|
9
9
|
export { CDPBridge } from './cdp.js';
|
|
10
10
|
export { isDaemonRunning } from './daemon-client.js';
|
|
11
11
|
export { generateSnapshotJs, scrollToRefJs, getFormStateJs } from './dom-snapshot.js';
|
|
12
12
|
export { generateStealthJs } from './stealth.js';
|
|
13
|
-
export type {
|
|
13
|
+
export type { DomSnapshotOptions } from './dom-snapshot.js';
|
|
14
14
|
import { extractTabEntries, diffTabIndexes, appendLimited } from './tabs.js';
|
|
15
15
|
import { withTimeoutMs } from '../runtime.js';
|
|
16
16
|
export declare const __test__: {
|
package/dist/browser/index.js
CHANGED
|
@@ -5,7 +5,7 @@
|
|
|
5
5
|
* External code should import from './browser/index.js' (or './browser.js' via Node resolution).
|
|
6
6
|
*/
|
|
7
7
|
export { Page } from './page.js';
|
|
8
|
-
export { BrowserBridge
|
|
8
|
+
export { BrowserBridge } from './mcp.js';
|
|
9
9
|
export { CDPBridge } from './cdp.js';
|
|
10
10
|
export { isDaemonRunning } from './daemon-client.js';
|
|
11
11
|
export { generateSnapshotJs, scrollToRefJs, getFormStateJs } from './dom-snapshot.js';
|
package/dist/browser/mcp.d.ts
CHANGED
package/dist/browser/mcp.js
CHANGED
|
@@ -7,6 +7,7 @@ import * as path from 'node:path';
|
|
|
7
7
|
import * as fs from 'node:fs';
|
|
8
8
|
import { Page } from './page.js';
|
|
9
9
|
import { isDaemonRunning, isExtensionConnected } from './daemon-client.js';
|
|
10
|
+
import { DEFAULT_DAEMON_PORT } from '../constants.js';
|
|
10
11
|
const DAEMON_SPAWN_TIMEOUT = 10000; // 10s to wait for daemon + extension
|
|
11
12
|
/**
|
|
12
13
|
* Browser factory: manages daemon lifecycle and provides IPage instances.
|
|
@@ -95,8 +96,6 @@ export class BrowserBridge {
|
|
|
95
96
|
}
|
|
96
97
|
throw new Error('Failed to start opencli daemon. Try running manually:\n' +
|
|
97
98
|
` node ${daemonPath}\n` +
|
|
98
|
-
|
|
99
|
+
`Make sure port ${DEFAULT_DAEMON_PORT} is available.`);
|
|
99
100
|
}
|
|
100
101
|
}
|
|
101
|
-
/** @deprecated Use BrowserBridge instead */
|
|
102
|
-
export const PlaywrightMCP = BrowserBridge;
|