@sovovs/bycli 1.8.0 → 2.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/cli-manifest.json +0 -61
- package/dist/src/adapter-shadow.js +2 -2
- package/dist/src/browser/analyze.d.ts +10 -99
- package/dist/src/browser/analyze.js +10 -226
- package/dist/src/browser/cdp.js +64 -7
- package/dist/src/browser/navigation-guard.d.ts +78 -0
- package/dist/src/browser/navigation-guard.js +142 -0
- package/dist/src/browser/navigation-guard.test.d.ts +1 -0
- package/dist/src/browser/url-policy.d.ts +89 -0
- package/dist/src/browser/url-policy.js +311 -0
- package/dist/src/browser/url-policy.test.d.ts +1 -0
- package/dist/src/browser/verify-fixture.js +2 -2
- package/dist/src/cli.js +10 -13
- package/dist/src/config-paths.d.ts +19 -0
- package/dist/src/config-paths.js +31 -0
- package/dist/src/daemon.js +240 -0
- package/dist/src/discovery.d.ts +5 -3
- package/dist/src/discovery.js +8 -6
- package/dist/src/execution.js +8 -3
- package/dist/src/main.js +18 -2
- package/dist/src/plugin.js +11 -8
- package/dist/src/recorder/highlevel/analyze.d.ts +29 -0
- package/dist/src/recorder/highlevel/analyze.js +91 -0
- package/dist/src/recorder/highlevel/init.d.ts +125 -0
- package/dist/src/recorder/highlevel/init.js +279 -0
- package/dist/src/recorder/highlevel/init.test.d.ts +1 -0
- package/dist/src/recorder/highlevel/verify.d.ts +73 -0
- package/dist/src/recorder/highlevel/verify.js +74 -0
- package/dist/src/recorder/http/analyze-runner.d.ts +28 -0
- package/dist/src/recorder/http/analyze-runner.js +78 -0
- package/dist/src/recorder/http/analyze-runner.test.d.ts +1 -0
- package/dist/src/recorder/http/handlers.d.ts +51 -0
- package/dist/src/recorder/http/handlers.js +219 -0
- package/dist/src/recorder/http/wrapper-config.d.ts +20 -0
- package/dist/src/recorder/http/wrapper-config.js +82 -0
- package/dist/src/recorder/http/wrapper-envelope.d.ts +31 -0
- package/dist/src/recorder/http/wrapper-envelope.js +51 -0
- package/dist/src/recorder/http/wrapper-gates.d.ts +16 -0
- package/dist/src/recorder/http/wrapper-gates.js +28 -0
- package/dist/src/recorder/http/wrapper-metrics.d.ts +1 -0
- package/dist/src/recorder/http/wrapper-metrics.js +5 -0
- package/dist/src/recorder/http/wrapper-registry.d.ts +50 -0
- package/dist/src/recorder/http/wrapper-registry.js +98 -0
- package/dist/src/recorder/http/wrapper-server.d.ts +59 -0
- package/dist/src/recorder/http/wrapper-server.js +240 -0
- package/dist/src/recorder/http/wrapper.test.d.ts +1 -0
- package/dist/src/recorder/observability/logger.d.ts +5 -0
- package/dist/src/recorder/observability/logger.js +13 -0
- package/dist/src/recorder/runner/config.d.ts +34 -0
- package/dist/src/recorder/runner/config.js +84 -0
- package/dist/src/recorder/runner/config.test.d.ts +1 -0
- package/dist/src/recorder/runner/reap.d.ts +79 -0
- package/dist/src/recorder/runner/reap.js +230 -0
- package/dist/src/recorder/runner/reap.test.d.ts +1 -0
- package/dist/src/recorder/runner/runner-port.d.ts +95 -0
- package/dist/src/recorder/runner/runner-port.js +518 -0
- package/dist/src/recorder/runner/runner-port.test.d.ts +1 -0
- package/dist/src/recorder/runner/session-keys.d.ts +28 -0
- package/dist/src/recorder/runner/session-keys.js +72 -0
- package/dist/src/recorder/runner/session-keys.test.d.ts +1 -0
- package/dist/src/recorder/runner/verify-runner-main.d.ts +95 -0
- package/dist/src/recorder/runner/verify-runner-main.js +275 -0
- package/dist/src/recorder/runner/verify-runner-main.test.d.ts +1 -0
- package/package.json +8 -5
- package/scripts/m1-daemon-navigate-drive.mjs +138 -0
- package/scripts/m1-extension-spike.mjs +237 -0
- package/scripts/m1-live-spike.mjs +156 -0
- package/scripts/m6b-connect-back-spike.sh +31 -0
- package/scripts/recorder.sh +186 -0
- package/clis/12306/utils.test.js +0 -331
- package/clis/1688/assets.test.js +0 -39
- package/clis/1688/download.test.js +0 -31
- package/clis/1688/item.test.js +0 -67
- package/clis/1688/search.test.js +0 -75
- package/clis/1688/shared.test.js +0 -57
- package/clis/1688/store.test.js +0 -62
- package/clis/36kr/article.test.js +0 -46
- package/clis/36kr/hot.test.js +0 -15
- package/clis/36kr/news.test.js +0 -85
- package/clis/51job/utils.test.js +0 -69
- package/clis/_atlassian/shared.test.js +0 -170
- package/clis/aibase/news.test.js +0 -59
- package/clis/amazon/bestsellers.test.js +0 -29
- package/clis/amazon/discussion.test.js +0 -124
- package/clis/amazon/offer.test.js +0 -29
- package/clis/amazon/product.test.js +0 -24
- package/clis/amazon/rankings.test.js +0 -41
- package/clis/amazon/search.test.js +0 -22
- package/clis/amazon/shared.test.js +0 -44
- package/clis/apple-podcasts/commands.test.js +0 -119
- package/clis/apple-podcasts/utils.test.js +0 -57
- package/clis/arxiv/arxiv.test.js +0 -112
- package/clis/baidu-scholar/search.test.js +0 -23
- package/clis/barchart/greeks.test.js +0 -138
- package/clis/bilibili/comment.test.js +0 -153
- package/clis/bilibili/comments.test.js +0 -142
- package/clis/bilibili/dynamic.test.js +0 -68
- package/clis/bilibili/hot.test.js +0 -17
- package/clis/bilibili/subtitle.test.js +0 -196
- package/clis/bilibili/summary.test.js +0 -210
- package/clis/bilibili/utils.test.js +0 -61
- package/clis/bilibili/video.test.js +0 -132
- package/clis/binance/commands.test.js +0 -70
- package/clis/bloomberg/utils.test.js +0 -129
- package/clis/booking/booking.test.js +0 -356
- package/clis/boss/chatlist.test.js +0 -211
- package/clis/boss/chatmsg.test.js +0 -230
- package/clis/boss/search.test.js +0 -78
- package/clis/boss/utils.test.js +0 -34
- package/clis/brave/search.test.js +0 -76
- package/clis/chaoxing/utils.test.js +0 -45
- package/clis/chatgpt/commands.test.js +0 -45
- package/clis/chatgpt/envelope.test.js +0 -108
- package/clis/chatgpt/image.test.js +0 -188
- package/clis/chatgpt/utils.test.js +0 -330
- package/clis/chatgpt-app/ax.test.js +0 -35
- package/clis/chatwise/composer.test.js +0 -186
- package/clis/chess/analyze.test.js +0 -79
- package/clis/chess/game.test.js +0 -178
- package/clis/chess/games.test.js +0 -164
- package/clis/chess/stats.test.js +0 -79
- package/clis/chess/utils.test.js +0 -230
- package/clis/claude/ask.test.js +0 -338
- package/clis/claude/commands.test.js +0 -118
- package/clis/claude/utils.test.js +0 -148
- package/clis/cnki/search.test.js +0 -18
- package/clis/codex/sidebar.test.js +0 -329
- package/clis/coingecko/coingecko.test.js +0 -109
- package/clis/confluence/commands.test.js +0 -195
- package/clis/coupang/coupang.test.js +0 -159
- package/clis/coupang/utils.test.js +0 -62
- package/clis/ctrip/ctrip.test.js +0 -719
- package/clis/dblp/dblp.test.js +0 -397
- package/clis/deepseek/ask.test.js +0 -312
- package/clis/deepseek/detail.test.js +0 -81
- package/clis/deepseek/send.test.js +0 -107
- package/clis/deepseek/utils.test.js +0 -370
- package/clis/defillama/defillama.test.js +0 -99
- package/clis/devto/devto.test.js +0 -236
- package/clis/dianping/dianping.test.js +0 -578
- package/clis/douban/download.test.js +0 -170
- package/clis/douban/movie-hot.test.js +0 -14
- package/clis/douban/search.test.js +0 -11
- package/clis/douban/subject.test.js +0 -11
- package/clis/douban/utils.test.js +0 -438
- package/clis/doubao/detail.test.js +0 -42
- package/clis/doubao/history.test.js +0 -37
- package/clis/doubao/utils.test.js +0 -344
- package/clis/douyin/_shared/browser-fetch.test.js +0 -64
- package/clis/douyin/_shared/creation-id.test.js +0 -22
- package/clis/douyin/_shared/imagex-upload.test.js +0 -87
- package/clis/douyin/_shared/sts2.test.js +0 -27
- package/clis/douyin/_shared/text-extra.test.js +0 -37
- package/clis/douyin/_shared/timing.test.js +0 -28
- package/clis/douyin/_shared/tos-upload-short-read.test.js +0 -83
- package/clis/douyin/_shared/tos-upload.test.js +0 -229
- package/clis/douyin/_shared/transcode.test.js +0 -91
- package/clis/douyin/_shared/vod-upload.test.js +0 -38
- package/clis/douyin/activities.test.js +0 -55
- package/clis/douyin/collections.test.js +0 -45
- package/clis/douyin/delete.test.js +0 -100
- package/clis/douyin/draft.test.js +0 -335
- package/clis/douyin/drafts.test.js +0 -11
- package/clis/douyin/hashtag.test.js +0 -171
- package/clis/douyin/location.test.js +0 -23
- package/clis/douyin/profile.test.js +0 -46
- package/clis/douyin/publish-upload-id.test.js +0 -170
- package/clis/douyin/publish.test.js +0 -38
- package/clis/douyin/stats.test.js +0 -22
- package/clis/douyin/update.test.js +0 -11
- package/clis/douyin/user-videos.test.js +0 -151
- package/clis/douyin/videos.test.js +0 -54
- package/clis/duckduckgo/search.test.js +0 -128
- package/clis/duckduckgo/suggest.test.js +0 -66
- package/clis/eastmoney/hot-rank.test.js +0 -59
- package/clis/endoflife/endoflife.test.js +0 -51
- package/clis/facebook/feed.test.js +0 -169
- package/clis/facebook/marketplace.test.js +0 -91
- package/clis/facebook/notifications.test.js +0 -458
- package/clis/facebook/search.test.js +0 -55
- package/clis/flathub/flathub.test.js +0 -90
- package/clis/flomo/memos.test.js +0 -144
- package/clis/gemini/ask.test.js +0 -100
- package/clis/gemini/deep-research-result.test.js +0 -155
- package/clis/gemini/deep-research.test.js +0 -181
- package/clis/gemini/reply-state.test.js +0 -671
- package/clis/gemini/utils.test.js +0 -198
- package/clis/geogebra/geogebra.test.js +0 -175
- package/clis/gitee/search.test.js +0 -65
- package/clis/gitee/user.test.js +0 -63
- package/clis/google/utils.test.js +0 -75
- package/clis/google-scholar/cite.test.js +0 -47
- package/clis/google-scholar/profile.test.js +0 -49
- package/clis/google-scholar/search.test.js +0 -71
- package/clis/goproxy/goproxy.test.js +0 -103
- package/clis/gov-law/commands.test.js +0 -39
- package/clis/gov-policy/gov-policy.test.js +0 -224
- package/clis/grok/ask.test.js +0 -29
- package/clis/grok/utils.test.js +0 -103
- package/clis/hackernews/hackernews.test.js +0 -132
- package/clis/hf/hf.test.js +0 -16
- package/clis/hupu/hot.test.js +0 -224
- package/clis/imdb/utils.test.js +0 -88
- package/clis/indeed/indeed.test.js +0 -375
- package/clis/instagram/_shared/private-publish.test.js +0 -705
- package/clis/instagram/_shared/protocol-capture.test.js +0 -114
- package/clis/instagram/download.test.js +0 -118
- package/clis/instagram/note.test.js +0 -53
- package/clis/instagram/post.test.js +0 -1647
- package/clis/instagram/reel.test.js +0 -167
- package/clis/instagram/story.test.js +0 -134
- package/clis/jd/commands.test.js +0 -41
- package/clis/jd/item.test.js +0 -354
- package/clis/jianyu/search.test.js +0 -153
- package/clis/jianyu/shared/procurement-contract.test.js +0 -72
- package/clis/jianyu/shared/procurement-detail.test.js +0 -84
- package/clis/jike/read.test.js +0 -86
- package/clis/jira/commands.test.js +0 -287
- package/clis/juejin/search.js +0 -123
- package/clis/juejin/search.test.js +0 -98
- package/clis/lesswrong/frontpage.test.js +0 -37
- package/clis/lichess/lichess.test.js +0 -85
- package/clis/linkedin/connect.test.js +0 -213
- package/clis/linkedin/inbox.test.js +0 -152
- package/clis/linkedin/job-detail.test.js +0 -38
- package/clis/linkedin/jobs-preferences.test.js +0 -43
- package/clis/linkedin/people-search.test.js +0 -216
- package/clis/linkedin/post-analytics.test.js +0 -40
- package/clis/linkedin/posts.test.js +0 -40
- package/clis/linkedin/profile-analytics.test.js +0 -67
- package/clis/linkedin/profile-experience.test.js +0 -152
- package/clis/linkedin/profile-projects.test.js +0 -111
- package/clis/linkedin/profile-read.test.js +0 -77
- package/clis/linkedin/safe-send.test.js +0 -204
- package/clis/linkedin/salesnav-inbox.test.js +0 -113
- package/clis/linkedin/salesnav-message.test.js +0 -172
- package/clis/linkedin/salesnav-search.test.js +0 -76
- package/clis/linkedin/salesnav-thread.test.js +0 -79
- package/clis/linkedin/search.test.js +0 -222
- package/clis/linkedin/sent-invitations.test.js +0 -62
- package/clis/linkedin/services-read.test.js +0 -105
- package/clis/linkedin/thread-snapshot.test.js +0 -89
- package/clis/linkedin/timeline.test.js +0 -81
- package/clis/linkedin-learning/course.test.js +0 -114
- package/clis/linkedin-learning/search.test.js +0 -144
- package/clis/linkedin-learning/trending.test.js +0 -123
- package/clis/linux-do/feed.test.js +0 -153
- package/clis/linux-do/topic-content.test.js +0 -59
- package/clis/lobsters/lobsters.test.js +0 -169
- package/clis/notebooklm/add-source.test.js +0 -97
- package/clis/notebooklm/compat.test.js +0 -16
- package/clis/notebooklm/create.test.js +0 -58
- package/clis/notebooklm/generate-audio.test.js +0 -63
- package/clis/notebooklm/generate-slides.test.js +0 -75
- package/clis/notebooklm/history.test.js +0 -58
- package/clis/notebooklm/note-list.test.js +0 -56
- package/clis/notebooklm/notes-get.test.js +0 -72
- package/clis/notebooklm/open.test.js +0 -63
- package/clis/notebooklm/rpc.test.js +0 -131
- package/clis/notebooklm/source-fulltext.test.js +0 -106
- package/clis/notebooklm/source-get.test.js +0 -84
- package/clis/notebooklm/source-guide.test.js +0 -104
- package/clis/notebooklm/summary.test.js +0 -78
- package/clis/notebooklm/utils.test.js +0 -449
- package/clis/notebooklm/write-note.test.js +0 -70
- package/clis/nuget/nuget.test.js +0 -111
- package/clis/oeis/oeis.test.js +0 -88
- package/clis/ones/task-helpers.test.js +0 -12
- package/clis/ones/worklog.test.js +0 -20
- package/clis/openfda/openfda.test.js +0 -114
- package/clis/openreview/openreview.test.js +0 -427
- package/clis/osv/osv.test.js +0 -97
- package/clis/paperreview/commands.test.js +0 -243
- package/clis/paperreview/utils.test.js +0 -49
- package/clis/pixiv/download.test.js +0 -87
- package/clis/pixiv/illusts.test.js +0 -99
- package/clis/pixiv/search.test.js +0 -83
- package/clis/powerchina/search.test.js +0 -93
- package/clis/producthunt/utils.test.js +0 -64
- package/clis/pubmed/pubmed.test.js +0 -276
- package/clis/quark/utils.test.js +0 -58
- package/clis/qwen/utils.test.js +0 -45
- package/clis/reddit/extract-media.test.js +0 -149
- package/clis/reddit/frontpage.test.js +0 -34
- package/clis/reddit/home.test.js +0 -170
- package/clis/reddit/hot.test.js +0 -32
- package/clis/reddit/popular.test.js +0 -26
- package/clis/reddit/read.test.js +0 -335
- package/clis/reddit/reply.test.js +0 -89
- package/clis/reddit/search.test.js +0 -26
- package/clis/reddit/subreddit-info.test.js +0 -163
- package/clis/reddit/subreddit.test.js +0 -31
- package/clis/reddit/subscribed.test.js +0 -168
- package/clis/reddit/whoami.test.js +0 -105
- package/clis/rednote/navigation.test.js +0 -26
- package/clis/rednote/rednote.test.js +0 -222
- package/clis/rest-countries/rest-countries.test.js +0 -83
- package/clis/reuters/reuters.test.js +0 -299
- package/clis/rfc/rfc.test.js +0 -74
- package/clis/sinafinance/stock.test.js +0 -59
- package/clis/spotify/utils.test.js +0 -67
- package/clis/stackoverflow/stackoverflow.test.js +0 -404
- package/clis/steam/steam.test.js +0 -46
- package/clis/substack/utils.test.js +0 -44
- package/clis/suno/commands.test.js +0 -188
- package/clis/suno/download.test.js +0 -151
- package/clis/suno/generate.test.js +0 -252
- package/clis/suno/utils.test.js +0 -329
- package/clis/taobao/commands.test.js +0 -41
- package/clis/tdx/hot-rank.test.js +0 -59
- package/clis/ths/hot-rank.test.js +0 -64
- package/clis/tieba/commands.test.js +0 -79
- package/clis/tieba/utils.test.js +0 -290
- package/clis/tiktok/creator-videos.test.js +0 -113
- package/clis/tiktok/refactor.test.js +0 -389
- package/clis/tiktok/write-refactor.test.js +0 -370
- package/clis/toutiao/toutiao.test.js +0 -378
- package/clis/tvmaze/tvmaze.test.js +0 -93
- package/clis/twitter/bookmark-folder.test.js +0 -395
- package/clis/twitter/bookmark-folders.test.js +0 -152
- package/clis/twitter/bookmark.test.js +0 -74
- package/clis/twitter/bookmarks.test.js +0 -205
- package/clis/twitter/delete.test.js +0 -85
- package/clis/twitter/device-follow.test.js +0 -287
- package/clis/twitter/download.test.js +0 -457
- package/clis/twitter/followers.test.js +0 -62
- package/clis/twitter/following.test.js +0 -338
- package/clis/twitter/hide-reply.test.js +0 -76
- package/clis/twitter/like.test.js +0 -73
- package/clis/twitter/likes.test.js +0 -220
- package/clis/twitter/list-add.test.js +0 -133
- package/clis/twitter/list-create.test.js +0 -169
- package/clis/twitter/list-remove.test.js +0 -110
- package/clis/twitter/list-tweets.test.js +0 -196
- package/clis/twitter/lists.test.js +0 -253
- package/clis/twitter/post.test.js +0 -258
- package/clis/twitter/profile.test.js +0 -39
- package/clis/twitter/quote.test.js +0 -194
- package/clis/twitter/reply.test.js +0 -237
- package/clis/twitter/retweet.test.js +0 -73
- package/clis/twitter/search.test.js +0 -349
- package/clis/twitter/shared.test.js +0 -794
- package/clis/twitter/thread.test.js +0 -58
- package/clis/twitter/timeline.test.js +0 -104
- package/clis/twitter/trending.test.js +0 -15
- package/clis/twitter/tweets.test.js +0 -362
- package/clis/twitter/unbookmark.test.js +0 -73
- package/clis/twitter/unlike.test.js +0 -75
- package/clis/twitter/unretweet.test.js +0 -73
- package/clis/twitter/utils.test.js +0 -169
- package/clis/uisdc/news.test.js +0 -66
- package/clis/uiverse/_shared.test.js +0 -62
- package/clis/uiverse/navigation.test.js +0 -12
- package/clis/upwork/upwork.test.js +0 -566
- package/clis/wanfang/search.test.js +0 -23
- package/clis/web/read.test.js +0 -392
- package/clis/weibo/delete.test.js +0 -94
- package/clis/weibo/envelope.test.js +0 -85
- package/clis/weibo/favorites.test.js +0 -114
- package/clis/weibo/publish.test.js +0 -192
- package/clis/weibo/user-posts.test.js +0 -92
- package/clis/weibo/utils.test.js +0 -36
- package/clis/weixin/drafts.test.js +0 -69
- package/clis/weixin/search.test.js +0 -227
- package/clis/weread/ai-outline.test.js +0 -83
- package/clis/weread/book-search.test.js +0 -242
- package/clis/weread/commands.test.js +0 -398
- package/clis/weread/private-api-regression.test.js +0 -369
- package/clis/weread/search-regression.test.js +0 -494
- package/clis/weread/utils.test.js +0 -128
- package/clis/weread-official/commands.test.js +0 -385
- package/clis/weread-official/utils.test.js +0 -242
- package/clis/wikidata/wikidata.test.js +0 -83
- package/clis/wikipedia/trending.test.js +0 -57
- package/clis/wttr/wttr.test.js +0 -84
- package/clis/xianyu/chat.test.js +0 -84
- package/clis/xianyu/im.test.js +0 -253
- package/clis/xianyu/item.test.js +0 -56
- package/clis/xianyu/publish.test.js +0 -220
- package/clis/xianyu/search.test.js +0 -17
- package/clis/xiaoe/xiaoe.test.js +0 -486
- package/clis/xiaohongshu/comments.test.js +0 -258
- package/clis/xiaohongshu/creator-note-detail.test.js +0 -451
- package/clis/xiaohongshu/creator-notes-summary.test.js +0 -94
- package/clis/xiaohongshu/creator-notes.test.js +0 -281
- package/clis/xiaohongshu/creator-stats.test.js +0 -24
- package/clis/xiaohongshu/delete-note.test.js +0 -172
- package/clis/xiaohongshu/download.test.js +0 -316
- package/clis/xiaohongshu/navigation.test.js +0 -34
- package/clis/xiaohongshu/note.test.js +0 -249
- package/clis/xiaohongshu/publish.test.js +0 -608
- package/clis/xiaohongshu/search.test.js +0 -325
- package/clis/xiaohongshu/user-helpers.test.js +0 -160
- package/clis/xiaoyuzhou/auth.test.js +0 -124
- package/clis/xiaoyuzhou/download.test.js +0 -135
- package/clis/xiaoyuzhou/podcast-episodes.test.js +0 -78
- package/clis/xiaoyuzhou/transcript.test.js +0 -195
- package/clis/xiaoyuzhou/utils.test.js +0 -39
- package/clis/xueqiu/comments.test.js +0 -696
- package/clis/xueqiu/danjuan-utils.test.js +0 -41
- package/clis/xueqiu/utils.test.js +0 -26
- package/clis/yahoo/search.test.js +0 -94
- package/clis/youdao/note.test.js +0 -99
- package/clis/youtube/channel.test.js +0 -59
- package/clis/youtube/feed.test.js +0 -131
- package/clis/youtube/transcript-group.test.js +0 -99
- package/clis/youtube/transcript.test.js +0 -296
- package/clis/youtube/utils.test.js +0 -84
- package/clis/yuanbao/ask.test.js +0 -124
- package/clis/yuanbao/new.test.js +0 -30
- package/clis/yuanbao/shared.test.js +0 -80
- package/clis/zhihu/answer-comments.test.js +0 -287
- package/clis/zhihu/answer-detail.test.js +0 -338
- package/clis/zhihu/answer.test.js +0 -54
- package/clis/zhihu/collection.test.js +0 -337
- package/clis/zhihu/collections.test.js +0 -182
- package/clis/zhihu/comment.test.js +0 -50
- package/clis/zhihu/download.test.js +0 -12
- package/clis/zhihu/favorite.test.js +0 -59
- package/clis/zhihu/follow.test.js +0 -42
- package/clis/zhihu/like.test.js +0 -41
- package/clis/zhihu/question.test.js +0 -251
- package/clis/zhihu/recommend.test.js +0 -143
- package/clis/zhihu/search.test.js +0 -198
- package/clis/zhihu/target.test.js +0 -77
- package/clis/zhihu/text.test.js +0 -24
- package/clis/zhihu/write-shared.test.js +0 -176
- package/clis/zlibrary/commands.test.js +0 -65
- package/clis/zsxq/search.test.js +0 -24
- package/clis/zsxq/topic.test.js +0 -30
- package/clis/zsxq/topics.test.js +0 -24
- package/clis/zsxq/utils.test.js +0 -31
- package/dist/src/adapter-shadow.test.js +0 -49
- package/dist/src/adapter-source.test.js +0 -29
- package/dist/src/browser/analyze.test.js +0 -166
- package/dist/src/browser/article-extract.e2e.test.js +0 -105
- package/dist/src/browser/article-extract.test.js +0 -94
- package/dist/src/browser/ax-snapshot.test.js +0 -91
- package/dist/src/browser/base-page.test.js +0 -758
- package/dist/src/browser/cdp-click-fixture.test.js +0 -87
- package/dist/src/browser/cdp.test.js +0 -79
- package/dist/src/browser/compound.test.js +0 -175
- package/dist/src/browser/daemon-client.test.js +0 -199
- package/dist/src/browser/dom-helpers.test.js +0 -106
- package/dist/src/browser/dom-snapshot.test.js +0 -313
- package/dist/src/browser/errors.test.js +0 -60
- package/dist/src/browser/extract.test.js +0 -129
- package/dist/src/browser/find.test.js +0 -180
- package/dist/src/browser/html-tree.test.js +0 -181
- package/dist/src/browser/network-cache.test.js +0 -58
- package/dist/src/browser/network-key.test.js +0 -49
- package/dist/src/browser/page.test.js +0 -378
- package/dist/src/browser/shape-filter.test.js +0 -101
- package/dist/src/browser/shape.test.js +0 -82
- package/dist/src/browser/stealth.test.js +0 -134
- package/dist/src/browser/target-errors.test.js +0 -94
- package/dist/src/browser/target-resolver.test.js +0 -118
- package/dist/src/browser/utils.test.js +0 -29
- package/dist/src/browser/verify-fixture.test.js +0 -219
- package/dist/src/browser.test.js +0 -232
- package/dist/src/build-manifest.test.js +0 -341
- package/dist/src/capabilityRouting.test.js +0 -63
- package/dist/src/cli-argv-preprocess.test.js +0 -209
- package/dist/src/cli.test.js +0 -2803
- package/dist/src/commanderAdapter.test.js +0 -352
- package/dist/src/commands/daemon.test.js +0 -219
- package/dist/src/completion.test.js +0 -24
- package/dist/src/convention-audit.test.js +0 -247
- package/dist/src/daemon.test.js +0 -60
- package/dist/src/doctor.test.js +0 -263
- package/dist/src/download/article-download.test.js +0 -235
- package/dist/src/download/index.test.js +0 -140
- package/dist/src/download/media-download.test.js +0 -112
- package/dist/src/download/progress.test.js +0 -36
- package/dist/src/electron-apps.test.js +0 -49
- package/dist/src/engine.test.js +0 -312
- package/dist/src/errors.test.js +0 -109
- package/dist/src/execution.test.js +0 -623
- package/dist/src/extension-manifest-regression.test.js +0 -12
- package/dist/src/external.test.js +0 -146
- package/dist/src/help.test.js +0 -58
- package/dist/src/hooks.test.js +0 -92
- package/dist/src/interceptor.test.js +0 -81
- package/dist/src/launcher.test.js +0 -138
- package/dist/src/node-network.test.js +0 -55
- package/dist/src/observation/artifact.test.js +0 -121
- package/dist/src/observation/manager.test.js +0 -13
- package/dist/src/observation/redaction.test.js +0 -32
- package/dist/src/observation/retention.test.js +0 -118
- package/dist/src/observation/ring-buffer.test.js +0 -22
- package/dist/src/output.test.js +0 -53
- package/dist/src/package-exports.test.js +0 -112
- package/dist/src/pipeline/executor.test.js +0 -182
- package/dist/src/pipeline/steps/download.test.js +0 -102
- package/dist/src/pipeline/steps/fetch.test.js +0 -117
- package/dist/src/pipeline/template.test.js +0 -175
- package/dist/src/pipeline/transform.test.js +0 -133
- package/dist/src/plugin-manifest.test.js +0 -179
- package/dist/src/plugin-scaffold.test.js +0 -83
- package/dist/src/plugin.test.js +0 -1324
- package/dist/src/registry.test.js +0 -209
- package/dist/src/runtime-detect.test.js +0 -39
- package/dist/src/serialization.test.js +0 -78
- package/dist/src/snapshotFormatter.test.js +0 -521
- package/dist/src/tui.test.js +0 -19
- package/dist/src/update-check.test.js +0 -116
- package/dist/src/utils.test.js +0 -155
- package/dist/src/validate.test.js +0 -90
- package/dist/src/weixin-download.test.js +0 -45
package/cli-manifest.json
CHANGED
|
@@ -14863,67 +14863,6 @@
|
|
|
14863
14863
|
"modulePath": "jira/search.js",
|
|
14864
14864
|
"sourceFile": "jira/search.js"
|
|
14865
14865
|
},
|
|
14866
|
-
{
|
|
14867
|
-
"site": "juejin",
|
|
14868
|
-
"name": "search",
|
|
14869
|
-
"description": "掘金搜索",
|
|
14870
|
-
"access": "read",
|
|
14871
|
-
"domain": "juejin.cn",
|
|
14872
|
-
"strategy": "public",
|
|
14873
|
-
"browser": true,
|
|
14874
|
-
"args": [
|
|
14875
|
-
{
|
|
14876
|
-
"name": "query",
|
|
14877
|
-
"type": "str",
|
|
14878
|
-
"required": true,
|
|
14879
|
-
"positional": true,
|
|
14880
|
-
"help": "搜索关键词"
|
|
14881
|
-
},
|
|
14882
|
-
{
|
|
14883
|
-
"name": "limit",
|
|
14884
|
-
"type": "int",
|
|
14885
|
-
"default": 10,
|
|
14886
|
-
"required": false,
|
|
14887
|
-
"help": "返回结果数量 (max 20)"
|
|
14888
|
-
},
|
|
14889
|
-
{
|
|
14890
|
-
"name": "type",
|
|
14891
|
-
"type": "int",
|
|
14892
|
-
"default": 0,
|
|
14893
|
-
"required": false,
|
|
14894
|
-
"help": "搜索类型:0=综合搜索,2=文章搜索,12=课程搜索,9=标签搜索,1=用户搜索",
|
|
14895
|
-
"choices": [
|
|
14896
|
-
"0",
|
|
14897
|
-
"2",
|
|
14898
|
-
"12",
|
|
14899
|
-
"9",
|
|
14900
|
-
"1"
|
|
14901
|
-
]
|
|
14902
|
-
},
|
|
14903
|
-
{
|
|
14904
|
-
"name": "sort",
|
|
14905
|
-
"type": "int",
|
|
14906
|
-
"default": 0,
|
|
14907
|
-
"required": false,
|
|
14908
|
-
"help": "排序类型:0=综合排序,1=最新优先,2=最热优先",
|
|
14909
|
-
"choices": [
|
|
14910
|
-
"0",
|
|
14911
|
-
"1",
|
|
14912
|
-
"2"
|
|
14913
|
-
]
|
|
14914
|
-
}
|
|
14915
|
-
],
|
|
14916
|
-
"columns": [
|
|
14917
|
-
"rank",
|
|
14918
|
-
"title",
|
|
14919
|
-
"author",
|
|
14920
|
-
"summary",
|
|
14921
|
-
"url"
|
|
14922
|
-
],
|
|
14923
|
-
"type": "js",
|
|
14924
|
-
"modulePath": "juejin/search.js",
|
|
14925
|
-
"sourceFile": "juejin/search.js"
|
|
14926
|
-
},
|
|
14927
14866
|
{
|
|
14928
14867
|
"site": "ke",
|
|
14929
14868
|
"name": "chengjiao",
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import * as fs from 'node:fs';
|
|
2
|
-
import * as os from 'node:os';
|
|
3
2
|
import * as path from 'node:path';
|
|
3
|
+
import { getUserClisDir } from './config-paths.js';
|
|
4
4
|
import { fileURLToPath } from 'node:url';
|
|
5
5
|
import { findPackageRoot, getCliManifestPath } from './package-paths.js';
|
|
6
6
|
function defaultBuiltinClisDir() {
|
|
@@ -31,7 +31,7 @@ function loadBuiltinCommandFiles(builtinClisDir) {
|
|
|
31
31
|
}
|
|
32
32
|
}
|
|
33
33
|
export function findShadowedUserAdapters(opts = {}) {
|
|
34
|
-
const userClisDir = opts.userClisDir ??
|
|
34
|
+
const userClisDir = opts.userClisDir ?? getUserClisDir();
|
|
35
35
|
const builtinClisDir = opts.builtinClisDir ?? defaultBuiltinClisDir();
|
|
36
36
|
const builtinCommandFiles = loadBuiltinCommandFiles(builtinClisDir);
|
|
37
37
|
const shadows = [];
|
|
@@ -1,103 +1,14 @@
|
|
|
1
1
|
/**
|
|
2
|
-
* `browser analyze <url>` —
|
|
2
|
+
* `browser analyze <url>` — site-recon classification.
|
|
3
3
|
*
|
|
4
|
-
*
|
|
5
|
-
*
|
|
6
|
-
*
|
|
7
|
-
*
|
|
8
|
-
*
|
|
4
|
+
* The pure analysis logic moved to the shared package `@sovovs/bycli-recorder-core`
|
|
5
|
+
* (M5a, same pure-domain split as rank/M4) so dashboard-be can reuse it too. This
|
|
6
|
+
* module re-exports it so existing main-repo importers (cli.ts, the high-level
|
|
7
|
+
* analyze module) are unchanged. The CLI wrapper drives a real page, feeds the
|
|
8
|
+
* resulting PageSignals here, and prints the verdict.
|
|
9
9
|
*
|
|
10
|
-
*
|
|
11
|
-
*
|
|
10
|
+
* Note: the package types the registry as a structural `AdapterRef` ({site,name,
|
|
11
|
+
* domain?}); the main-repo `CliCommand` is assignable to it, so callers can keep
|
|
12
|
+
* passing `getRegistry()` (Map<string, CliCommand>) unchanged.
|
|
12
13
|
*/
|
|
13
|
-
|
|
14
|
-
export interface PageSignals {
|
|
15
|
-
/** URL we navigated to (may redirect; both fields help agents notice that). */
|
|
16
|
-
requestedUrl: string;
|
|
17
|
-
finalUrl: string;
|
|
18
|
-
/** document.cookie split into names; value not needed for detection. */
|
|
19
|
-
cookieNames: string[];
|
|
20
|
-
/**
|
|
21
|
-
* Response bodies captured during the navigation + first few seconds.
|
|
22
|
-
* We only need enough body text to spot WAF markers; the CLI truncates
|
|
23
|
-
* per-entry before feeding us.
|
|
24
|
-
*/
|
|
25
|
-
networkEntries: Array<{
|
|
26
|
-
url: string;
|
|
27
|
-
status: number;
|
|
28
|
-
contentType: string;
|
|
29
|
-
/** First N chars of body; null when not available. */
|
|
30
|
-
bodyPreview: string | null;
|
|
31
|
-
}>;
|
|
32
|
-
/**
|
|
33
|
-
* Which globals the page exposes on `window`. We don't care about the values,
|
|
34
|
-
* just presence — distinguishes Pattern B (SSR state) from Pattern A.
|
|
35
|
-
*/
|
|
36
|
-
initialState: {
|
|
37
|
-
__INITIAL_STATE__: boolean;
|
|
38
|
-
__NUXT__: boolean;
|
|
39
|
-
__NEXT_DATA__: boolean;
|
|
40
|
-
__APOLLO_STATE__: boolean;
|
|
41
|
-
};
|
|
42
|
-
/** Document title — only for the human-debug `summary` field. */
|
|
43
|
-
title: string;
|
|
44
|
-
}
|
|
45
|
-
export type AntiBotVendor = 'aliyun_waf' | 'cloudflare' | 'akamai' | 'geetest' | 'unknown';
|
|
46
|
-
export interface AntiBotVerdict {
|
|
47
|
-
detected: boolean;
|
|
48
|
-
vendor: AntiBotVendor | null;
|
|
49
|
-
evidence: string[];
|
|
50
|
-
/** One-line imperative instruction for the agent. */
|
|
51
|
-
implication: string;
|
|
52
|
-
}
|
|
53
|
-
export declare function detectAntiBot(signals: PageSignals): AntiBotVerdict;
|
|
54
|
-
export type Pattern = 'A' | 'B' | 'C' | 'D' | 'E' | 'unknown';
|
|
55
|
-
export interface PatternVerdict {
|
|
56
|
-
pattern: Pattern;
|
|
57
|
-
reason: string;
|
|
58
|
-
/** How many JSON XHR/fetch responses we saw during navigation. */
|
|
59
|
-
json_responses: number;
|
|
60
|
-
/** Count of non-2xx API responses — hint for token-gated (Pattern D). */
|
|
61
|
-
auth_failures: number;
|
|
62
|
-
}
|
|
63
|
-
/**
|
|
64
|
-
* Apply the decision tree from `site-recon.md` mechanically.
|
|
65
|
-
*
|
|
66
|
-
* B beats A when initial-state globals are present: even if the page fetches
|
|
67
|
-
* more data via XHR afterwards, the SSR payload is the highest-leverage source.
|
|
68
|
-
* D (token-gated) dominates when we see 401/403 on what looks like API
|
|
69
|
-
* endpoints — without that, an authenticated route looks identical to A.
|
|
70
|
-
*/
|
|
71
|
-
export declare function classifyPattern(signals: PageSignals): PatternVerdict;
|
|
72
|
-
export interface NearestAdapter {
|
|
73
|
-
site: string;
|
|
74
|
-
example_commands: string[];
|
|
75
|
-
reason: string;
|
|
76
|
-
}
|
|
77
|
-
/**
|
|
78
|
-
* Find existing adapters that target the same site.
|
|
79
|
-
*
|
|
80
|
-
* Keep the hostname match simple — agents extend naming conventions
|
|
81
|
-
* differently per site, so we match on the registered `domain` field and fall
|
|
82
|
-
* back to site-name containment. Returning `null` is fine; agents can always
|
|
83
|
-
* read site-memory docs.
|
|
84
|
-
*/
|
|
85
|
-
export declare function findNearestAdapter(finalUrl: string, registry: Map<string, CliCommand>): NearestAdapter | null;
|
|
86
|
-
export interface AnalyzeReport {
|
|
87
|
-
requested_url: string;
|
|
88
|
-
final_url: string;
|
|
89
|
-
title: string;
|
|
90
|
-
pattern: PatternVerdict;
|
|
91
|
-
anti_bot: AntiBotVerdict;
|
|
92
|
-
initial_state: PageSignals['initialState'];
|
|
93
|
-
nearest_adapter: NearestAdapter | null;
|
|
94
|
-
recommended_next_step: string;
|
|
95
|
-
}
|
|
96
|
-
/**
|
|
97
|
-
* Synthesize the verdict from collected signals + registry.
|
|
98
|
-
*
|
|
99
|
-
* The `recommended_next_step` is deliberately a single imperative
|
|
100
|
-
* sentence — agents act on it directly instead of re-deriving advice from
|
|
101
|
-
* the structured fields.
|
|
102
|
-
*/
|
|
103
|
-
export declare function analyzeSite(signals: PageSignals, registry: Map<string, CliCommand>): AnalyzeReport;
|
|
14
|
+
export { analyzeSite, detectAntiBot, classifyPattern, findNearestAdapter, type AdapterRef, type PageSignals, type AnalyzeReport, type AntiBotVerdict, type AntiBotVendor, type PatternVerdict, type Pattern, type NearestAdapter, } from '@sovovs/bycli-recorder-core';
|
|
@@ -1,230 +1,14 @@
|
|
|
1
1
|
/**
|
|
2
|
-
* `browser analyze <url>` —
|
|
2
|
+
* `browser analyze <url>` — site-recon classification.
|
|
3
3
|
*
|
|
4
|
-
*
|
|
5
|
-
*
|
|
6
|
-
*
|
|
7
|
-
*
|
|
8
|
-
*
|
|
4
|
+
* The pure analysis logic moved to the shared package `@sovovs/bycli-recorder-core`
|
|
5
|
+
* (M5a, same pure-domain split as rank/M4) so dashboard-be can reuse it too. This
|
|
6
|
+
* module re-exports it so existing main-repo importers (cli.ts, the high-level
|
|
7
|
+
* analyze module) are unchanged. The CLI wrapper drives a real page, feeds the
|
|
8
|
+
* resulting PageSignals here, and prints the verdict.
|
|
9
9
|
*
|
|
10
|
-
*
|
|
11
|
-
*
|
|
10
|
+
* Note: the package types the registry as a structural `AdapterRef` ({site,name,
|
|
11
|
+
* domain?}); the main-repo `CliCommand` is assignable to it, so callers can keep
|
|
12
|
+
* passing `getRegistry()` (Map<string, CliCommand>) unchanged.
|
|
12
13
|
*/
|
|
13
|
-
|
|
14
|
-
* WAF vendors we can reliably detect from cookies + response body markers
|
|
15
|
-
* alone. Signals are orthogonal per vendor — so when two vendors match
|
|
16
|
-
* simultaneously (rare), we keep all evidence and report the higher-signal
|
|
17
|
-
* vendor first.
|
|
18
|
-
*/
|
|
19
|
-
const WAF_SIGNATURES = [
|
|
20
|
-
{
|
|
21
|
-
vendor: 'aliyun_waf',
|
|
22
|
-
cookiePatterns: [/^acw_sc__v2$/, /^acw_tc$/, /^ssxmod_itna/],
|
|
23
|
-
bodyPatterns: [/arg1\s*=\s*['"][0-9A-F]{30,}/, /\/ntc_captcha\//i],
|
|
24
|
-
implication: 'Direct Node-side fetch/curl will return the slider HTML. Validate the endpoint in browser context first; HTML COOKIE adapters still finish with Node-side fetch + page.getCookies.',
|
|
25
|
-
},
|
|
26
|
-
{
|
|
27
|
-
vendor: 'cloudflare',
|
|
28
|
-
cookiePatterns: [/^__cf_bm$/, /^cf_clearance$/, /^__cfduid$/],
|
|
29
|
-
bodyPatterns: [/Cloudflare Ray ID/i, /Checking your browser before accessing/i, /cf-chl-/i],
|
|
30
|
-
implication: 'Cloudflare bot check. Start from a real browser session; probe in browser context first. HTML COOKIE adapters still finish with Node-side fetch + page.getCookies.',
|
|
31
|
-
},
|
|
32
|
-
{
|
|
33
|
-
vendor: 'akamai',
|
|
34
|
-
cookiePatterns: [/^_abck$/, /^bm_sz$/, /^bm_sv$/],
|
|
35
|
-
bodyPatterns: [/akamai/i],
|
|
36
|
-
implication: 'Akamai Bot Manager. Probe in browser context first; keep final HTML COOKIE adapters on Node-side fetch + page.getCookies.',
|
|
37
|
-
},
|
|
38
|
-
{
|
|
39
|
-
vendor: 'geetest',
|
|
40
|
-
cookiePatterns: [],
|
|
41
|
-
bodyPatterns: [/geetest/i, /gt_captcha/i],
|
|
42
|
-
implication: 'Geetest slider/puzzle captcha. Agent cannot bypass programmatically — requires UI strategy or human-in-loop.',
|
|
43
|
-
},
|
|
44
|
-
];
|
|
45
|
-
export function detectAntiBot(signals) {
|
|
46
|
-
const evidence = [];
|
|
47
|
-
let match = null;
|
|
48
|
-
for (const sig of WAF_SIGNATURES) {
|
|
49
|
-
const hits = [];
|
|
50
|
-
for (const pat of sig.cookiePatterns) {
|
|
51
|
-
const hit = signals.cookieNames.find((c) => pat.test(c));
|
|
52
|
-
if (hit)
|
|
53
|
-
hits.push(`cookie:${hit}`);
|
|
54
|
-
}
|
|
55
|
-
for (const pat of sig.bodyPatterns) {
|
|
56
|
-
for (const entry of signals.networkEntries) {
|
|
57
|
-
if (entry.bodyPreview && pat.test(entry.bodyPreview)) {
|
|
58
|
-
hits.push(`body:${entry.url}`);
|
|
59
|
-
break;
|
|
60
|
-
}
|
|
61
|
-
}
|
|
62
|
-
}
|
|
63
|
-
if (hits.length > 0 && !match) {
|
|
64
|
-
match = sig;
|
|
65
|
-
evidence.push(...hits);
|
|
66
|
-
}
|
|
67
|
-
}
|
|
68
|
-
if (!match) {
|
|
69
|
-
return {
|
|
70
|
-
detected: false,
|
|
71
|
-
vendor: null,
|
|
72
|
-
evidence: [],
|
|
73
|
-
implication: 'No known anti-bot signatures. Try Node-side COOKIE fetch first; if endpoint validation is blocked, retry from browser context.',
|
|
74
|
-
};
|
|
75
|
-
}
|
|
76
|
-
return {
|
|
77
|
-
detected: true,
|
|
78
|
-
vendor: match.vendor,
|
|
79
|
-
evidence,
|
|
80
|
-
implication: match.implication,
|
|
81
|
-
};
|
|
82
|
-
}
|
|
83
|
-
/**
|
|
84
|
-
* Apply the decision tree from `site-recon.md` mechanically.
|
|
85
|
-
*
|
|
86
|
-
* B beats A when initial-state globals are present: even if the page fetches
|
|
87
|
-
* more data via XHR afterwards, the SSR payload is the highest-leverage source.
|
|
88
|
-
* D (token-gated) dominates when we see 401/403 on what looks like API
|
|
89
|
-
* endpoints — without that, an authenticated route looks identical to A.
|
|
90
|
-
*/
|
|
91
|
-
export function classifyPattern(signals) {
|
|
92
|
-
const jsonEntries = signals.networkEntries.filter((e) => /json/i.test(e.contentType));
|
|
93
|
-
const authFailures = signals.networkEntries.filter((e) => e.status === 401 || e.status === 403).length;
|
|
94
|
-
const hasInitialState = signals.initialState.__INITIAL_STATE__ ||
|
|
95
|
-
signals.initialState.__NUXT__ ||
|
|
96
|
-
signals.initialState.__NEXT_DATA__ ||
|
|
97
|
-
signals.initialState.__APOLLO_STATE__;
|
|
98
|
-
if (authFailures >= 2 && jsonEntries.length >= 1) {
|
|
99
|
-
return {
|
|
100
|
-
pattern: 'D',
|
|
101
|
-
reason: `${authFailures} auth-failing API responses seen — endpoint is token-gated`,
|
|
102
|
-
json_responses: jsonEntries.length,
|
|
103
|
-
auth_failures: authFailures,
|
|
104
|
-
};
|
|
105
|
-
}
|
|
106
|
-
if (hasInitialState) {
|
|
107
|
-
const which = Object.entries(signals.initialState)
|
|
108
|
-
.filter(([, v]) => v)
|
|
109
|
-
.map(([k]) => k);
|
|
110
|
-
return {
|
|
111
|
-
pattern: 'B',
|
|
112
|
-
reason: `SSR state global present: ${which.join(', ')}`,
|
|
113
|
-
json_responses: jsonEntries.length,
|
|
114
|
-
auth_failures: authFailures,
|
|
115
|
-
};
|
|
116
|
-
}
|
|
117
|
-
if (jsonEntries.length >= 1) {
|
|
118
|
-
return {
|
|
119
|
-
pattern: 'A',
|
|
120
|
-
reason: `${jsonEntries.length} JSON XHR/fetch responses observed — classic API pattern`,
|
|
121
|
-
json_responses: jsonEntries.length,
|
|
122
|
-
auth_failures: authFailures,
|
|
123
|
-
};
|
|
124
|
-
}
|
|
125
|
-
// No API, no SSR state — probably static HTML or a bundled SPA that lazy-loads.
|
|
126
|
-
// Pattern C (HTML scrape) is the default fallback; E (streaming) we can't
|
|
127
|
-
// reliably detect without watching WebSocket frames, so we label 'C' and
|
|
128
|
-
// leave the agent to upgrade to E manually if they see WS traffic.
|
|
129
|
-
return {
|
|
130
|
-
pattern: 'C',
|
|
131
|
-
reason: 'No JSON XHR and no SSR state — HTML scrape (Pattern C); escalate to E manually if WebSocket traffic appears',
|
|
132
|
-
json_responses: jsonEntries.length,
|
|
133
|
-
auth_failures: authFailures,
|
|
134
|
-
};
|
|
135
|
-
}
|
|
136
|
-
/**
|
|
137
|
-
* Find existing adapters that target the same site.
|
|
138
|
-
*
|
|
139
|
-
* Keep the hostname match simple — agents extend naming conventions
|
|
140
|
-
* differently per site, so we match on the registered `domain` field and fall
|
|
141
|
-
* back to site-name containment. Returning `null` is fine; agents can always
|
|
142
|
-
* read site-memory docs.
|
|
143
|
-
*/
|
|
144
|
-
export function findNearestAdapter(finalUrl, registry) {
|
|
145
|
-
let host;
|
|
146
|
-
try {
|
|
147
|
-
host = new URL(finalUrl).hostname;
|
|
148
|
-
}
|
|
149
|
-
catch {
|
|
150
|
-
return null;
|
|
151
|
-
}
|
|
152
|
-
// Strip leading www.; 'www' as a site identifier is never what an adapter uses.
|
|
153
|
-
const cleanedHost = host.replace(/^www\./, '');
|
|
154
|
-
// Extract apex (xx.com) and registrable parts for fuzzy match.
|
|
155
|
-
const parts = cleanedHost.split('.');
|
|
156
|
-
const apex = parts.slice(-2).join('.');
|
|
157
|
-
const siteKey = parts.length > 1 ? parts[parts.length - 2] : cleanedHost;
|
|
158
|
-
const hits = new Map();
|
|
159
|
-
for (const cmd of registry.values()) {
|
|
160
|
-
const domain = cmd.domain?.toLowerCase();
|
|
161
|
-
const siteMatches = (domain && (cleanedHost.endsWith(domain) || domain.endsWith(apex))) ||
|
|
162
|
-
cmd.site.toLowerCase() === siteKey?.toLowerCase() ||
|
|
163
|
-
cleanedHost.includes(cmd.site.toLowerCase());
|
|
164
|
-
if (siteMatches) {
|
|
165
|
-
const list = hits.get(cmd.site) ?? [];
|
|
166
|
-
list.push(cmd);
|
|
167
|
-
hits.set(cmd.site, list);
|
|
168
|
-
}
|
|
169
|
-
}
|
|
170
|
-
if (hits.size === 0)
|
|
171
|
-
return null;
|
|
172
|
-
// Pick the site with the most commands — likely the most-developed adapter,
|
|
173
|
-
// and the best reference for a new command on the same host.
|
|
174
|
-
let best = null;
|
|
175
|
-
for (const entry of hits) {
|
|
176
|
-
if (!best || entry[1].length > best[1].length)
|
|
177
|
-
best = entry;
|
|
178
|
-
}
|
|
179
|
-
if (!best)
|
|
180
|
-
return null;
|
|
181
|
-
return {
|
|
182
|
-
site: best[0],
|
|
183
|
-
example_commands: best[1].slice(0, 5).map((c) => `${c.site} ${c.name}`),
|
|
184
|
-
reason: `${best[1].length} existing adapter${best[1].length === 1 ? '' : 's'} target this site — reuse strategy/cookie config`,
|
|
185
|
-
};
|
|
186
|
-
}
|
|
187
|
-
/**
|
|
188
|
-
* Synthesize the verdict from collected signals + registry.
|
|
189
|
-
*
|
|
190
|
-
* The `recommended_next_step` is deliberately a single imperative
|
|
191
|
-
* sentence — agents act on it directly instead of re-deriving advice from
|
|
192
|
-
* the structured fields.
|
|
193
|
-
*/
|
|
194
|
-
export function analyzeSite(signals, registry) {
|
|
195
|
-
const pattern = classifyPattern(signals);
|
|
196
|
-
const antiBot = detectAntiBot(signals);
|
|
197
|
-
const nearest = findNearestAdapter(signals.finalUrl, registry);
|
|
198
|
-
let next;
|
|
199
|
-
if (antiBot.detected) {
|
|
200
|
-
next = antiBot.implication;
|
|
201
|
-
}
|
|
202
|
-
else if (pattern.pattern === 'A') {
|
|
203
|
-
next = 'Pick the most specific JSON endpoint from `bycli browser network` and try a bare Node fetch with cookies; escalate to browser-context fetch only if blocked.';
|
|
204
|
-
}
|
|
205
|
-
else if (pattern.pattern === 'B') {
|
|
206
|
-
next = 'Read the SSR global via `bycli browser eval "JSON.stringify(window.__INITIAL_STATE__ ?? window.__NUXT__ ?? window.__NEXT_DATA__ ?? window.__APOLLO_STATE__)"` — no API needed.';
|
|
207
|
-
}
|
|
208
|
-
else if (pattern.pattern === 'C') {
|
|
209
|
-
next = 'No API visible — use SSR HTML scrape (e.g. `bycli browser extract`) against the rendered page.';
|
|
210
|
-
}
|
|
211
|
-
else if (pattern.pattern === 'D') {
|
|
212
|
-
next = 'Endpoints need auth. Re-open the page from a signed-in session, then retry analyze; see `field-decode-playbook` §4 for token tracing.';
|
|
213
|
-
}
|
|
214
|
-
else if (pattern.pattern === 'E') {
|
|
215
|
-
next = 'WebSocket stream detected — find the underlying HTTP poll/long-poll endpoint; raw WS is not supported.';
|
|
216
|
-
}
|
|
217
|
-
else {
|
|
218
|
-
next = 'No strong signal. Manually inspect `bycli browser network --all` and pick a pattern.';
|
|
219
|
-
}
|
|
220
|
-
return {
|
|
221
|
-
requested_url: signals.requestedUrl,
|
|
222
|
-
final_url: signals.finalUrl,
|
|
223
|
-
title: signals.title,
|
|
224
|
-
pattern,
|
|
225
|
-
anti_bot: antiBot,
|
|
226
|
-
initial_state: signals.initialState,
|
|
227
|
-
nearest_adapter: nearest,
|
|
228
|
-
recommended_next_step: next,
|
|
229
|
-
};
|
|
230
|
-
}
|
|
14
|
+
export { analyzeSite, detectAntiBot, classifyPattern, findNearestAdapter, } from '@sovovs/bycli-recorder-core';
|
package/dist/src/browser/cdp.js
CHANGED
|
@@ -13,9 +13,36 @@ import { request as httpsRequest } from 'node:https';
|
|
|
13
13
|
import { buildEvaluateExpression } from './utils.js';
|
|
14
14
|
import { generateStealthJs } from './stealth.js';
|
|
15
15
|
import { waitForDomStableJs } from './dom-helpers.js';
|
|
16
|
+
import { armNavigationGuard, NavigationBlockedError } from './navigation-guard.js';
|
|
17
|
+
import { checkUrlPolicy } from './url-policy.js';
|
|
16
18
|
import { isRecord, saveBase64ToFile } from '../utils.js';
|
|
17
19
|
import { getAllElectronApps } from '../electron-apps.js';
|
|
18
20
|
import { BasePage } from './base-page.js';
|
|
21
|
+
/** Lazily-built DnsResolver backed by node:dns/promises (for guarded navigation). */
|
|
22
|
+
let _dnsResolver;
|
|
23
|
+
function getDefaultDnsResolver() {
|
|
24
|
+
if (_dnsResolver)
|
|
25
|
+
return _dnsResolver;
|
|
26
|
+
// Imported lazily so the module stays usable in environments without node:dns.
|
|
27
|
+
const dns = require('node:dns').promises;
|
|
28
|
+
_dnsResolver = {
|
|
29
|
+
resolve4: (h) => dns.resolve4(h),
|
|
30
|
+
resolve6: (h) => dns.resolve6(h),
|
|
31
|
+
};
|
|
32
|
+
return _dnsResolver;
|
|
33
|
+
}
|
|
34
|
+
/** Resolve once the guard records a blocked main-frame request (polled). */
|
|
35
|
+
function waitForBlocked(guard, intervalMs = 100, timeoutMs = 30_000) {
|
|
36
|
+
return new Promise((resolve) => {
|
|
37
|
+
const start = Date.now();
|
|
38
|
+
const timer = setInterval(() => {
|
|
39
|
+
if (guard.blocked.length > 0 || Date.now() - start > timeoutMs) {
|
|
40
|
+
clearInterval(timer);
|
|
41
|
+
resolve();
|
|
42
|
+
}
|
|
43
|
+
}, intervalMs);
|
|
44
|
+
});
|
|
45
|
+
}
|
|
19
46
|
const CDP_SEND_TIMEOUT = 30_000;
|
|
20
47
|
// Memory guard for in-process capture. The 4k cap we used to apply everywhere
|
|
21
48
|
// silently truncated JSON so `JSON.parse` failed or gave partial objects — the
|
|
@@ -172,13 +199,43 @@ class CDPPage extends BasePage {
|
|
|
172
199
|
await this.bridge.send('Page.enable');
|
|
173
200
|
this._pageEnabled = true;
|
|
174
201
|
}
|
|
175
|
-
|
|
176
|
-
|
|
177
|
-
|
|
178
|
-
|
|
179
|
-
|
|
180
|
-
|
|
181
|
-
|
|
202
|
+
// Opt-in guarded navigation (M1 P0-1, direct-CDP side). Only the recorder path
|
|
203
|
+
// sets guardNavigation; general automation (incl. localhost fixtures) is
|
|
204
|
+
// unchanged. Pre-checks the target (syntax + DNS) and arms Fetch before-send
|
|
205
|
+
// interception before Page.navigate, so redirects to forbidden targets are
|
|
206
|
+
// failRequest'd before send. ip-observed-only per ADR-0006.
|
|
207
|
+
let guard;
|
|
208
|
+
if (options?.guardNavigation) {
|
|
209
|
+
const resolver = getDefaultDnsResolver();
|
|
210
|
+
const pre = await checkUrlPolicy(url, resolver);
|
|
211
|
+
if (!pre.ok) {
|
|
212
|
+
throw new NavigationBlockedError(url, pre.reason, pre.detail);
|
|
213
|
+
}
|
|
214
|
+
guard = await armNavigationGuard(this.bridge, resolver, { observeIp: true });
|
|
215
|
+
}
|
|
216
|
+
try {
|
|
217
|
+
const loadPromise = this.bridge.waitForEvent('Page.loadEventFired', 30_000).catch(() => { });
|
|
218
|
+
await this.bridge.send('Page.navigate', { url });
|
|
219
|
+
if (guard) {
|
|
220
|
+
// A blocked main-frame request never fires loadEventFired — race the load
|
|
221
|
+
// against a short poll of the guard's blocked list so we don't wait 30s.
|
|
222
|
+
await Promise.race([loadPromise, waitForBlocked(guard)]);
|
|
223
|
+
if (guard.blocked.length > 0) {
|
|
224
|
+
throw new NavigationBlockedError(url, 'forbidden_ip', `redirect blocked: ${guard.blocked.map((b) => b.url).join(', ')}`);
|
|
225
|
+
}
|
|
226
|
+
}
|
|
227
|
+
else {
|
|
228
|
+
await loadPromise;
|
|
229
|
+
}
|
|
230
|
+
this._lastUrl = url;
|
|
231
|
+
if (options?.waitUntil !== 'none') {
|
|
232
|
+
const maxMs = options?.settleMs ?? 1000;
|
|
233
|
+
await this.evaluate(waitForDomStableJs(maxMs, Math.min(500, maxMs)));
|
|
234
|
+
}
|
|
235
|
+
}
|
|
236
|
+
finally {
|
|
237
|
+
if (guard)
|
|
238
|
+
await guard.dispose();
|
|
182
239
|
}
|
|
183
240
|
}
|
|
184
241
|
async evaluate(input, ...args) {
|
|
@@ -0,0 +1,78 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Navigation interception guard (M1 spike, direct-CDP / Node side).
|
|
3
|
+
*
|
|
4
|
+
* Closes the P0 hole where navigation fired before request interception was armed.
|
|
5
|
+
* The contract (ADR-0002, ADR-0006, 04-security-model.md) is:
|
|
6
|
+
*
|
|
7
|
+
* 1. Pre-check the target URL (syntax + DNS precheck) BEFORE any navigation.
|
|
8
|
+
* 2. Arm Fetch interception (resourceType Document, Request stage) and confirm the
|
|
9
|
+
* handler is registered BEFORE the single Page.navigate.
|
|
10
|
+
* 3. Re-run the full URL policy on every paused main-frame Document request
|
|
11
|
+
* (initial + every redirect / secondary main-frame request) and failRequest
|
|
12
|
+
* anything that does not pass — so a forbidden redirect target receives 0 bytes.
|
|
13
|
+
* 4. If interception cannot be armed, FAIL CLOSED — never fall back to bare
|
|
14
|
+
* navigation. Callers surface `navigation_redirect_requires_interception`.
|
|
15
|
+
*
|
|
16
|
+
* remoteIPAddress observed on Network.responseReceived is logged only — it is
|
|
17
|
+
* post-connect and therefore `ip-observed-only`, NOT a security boundary (ADR-0006).
|
|
18
|
+
*
|
|
19
|
+
* This module is runtime-bound only through the small InterceptBridge interface,
|
|
20
|
+
* so it is unit-testable with a fake bridge (no real CDP / network).
|
|
21
|
+
*/
|
|
22
|
+
import { type DnsResolver, type UrlPolicyReason } from './url-policy.js';
|
|
23
|
+
/** Minimal CDP surface this guard needs (satisfied by CDPBridge). */
|
|
24
|
+
export interface InterceptBridge {
|
|
25
|
+
send(method: string, params?: Record<string, unknown>): Promise<unknown>;
|
|
26
|
+
on(event: string, handler: (params: unknown) => void): void;
|
|
27
|
+
off(event: string, handler: (params: unknown) => void): void;
|
|
28
|
+
}
|
|
29
|
+
export declare class InterceptionUnavailableError extends Error {
|
|
30
|
+
readonly code = "navigation_redirect_requires_interception";
|
|
31
|
+
constructor(detail: string);
|
|
32
|
+
}
|
|
33
|
+
export declare class NavigationBlockedError extends Error {
|
|
34
|
+
readonly url: string;
|
|
35
|
+
readonly reason: UrlPolicyReason;
|
|
36
|
+
readonly code = "navigation_blocked_by_policy";
|
|
37
|
+
constructor(url: string, reason: UrlPolicyReason, detail: string);
|
|
38
|
+
}
|
|
39
|
+
export interface BlockedRecord {
|
|
40
|
+
url: string;
|
|
41
|
+
reason: UrlPolicyReason;
|
|
42
|
+
}
|
|
43
|
+
export interface NavigationGuard {
|
|
44
|
+
/** URLs that were failRequest'd by policy (initial + redirects). */
|
|
45
|
+
readonly blocked: ReadonlyArray<BlockedRecord>;
|
|
46
|
+
/** remoteIPAddress values observed post-connect (logged, not a boundary). */
|
|
47
|
+
readonly observedIps: ReadonlyArray<string>;
|
|
48
|
+
/** Tear down interception + listeners. Idempotent. */
|
|
49
|
+
dispose(): Promise<void>;
|
|
50
|
+
}
|
|
51
|
+
/**
|
|
52
|
+
* Arm Fetch interception and policy re-check, fail-closed. Must be called and
|
|
53
|
+
* resolved BEFORE the caller issues Page.navigate. Throws
|
|
54
|
+
* InterceptionUnavailableError if Fetch cannot be enabled.
|
|
55
|
+
*/
|
|
56
|
+
export declare function armNavigationGuard(bridge: InterceptBridge, resolver: DnsResolver, opts?: {
|
|
57
|
+
allowLiteralIp?: boolean;
|
|
58
|
+
observeIp?: boolean;
|
|
59
|
+
}): Promise<NavigationGuard>;
|
|
60
|
+
/**
|
|
61
|
+
* Full guarded navigation for the direct-CDP path. Pre-checks the URL, arms the
|
|
62
|
+
* guard, runs the caller-supplied navigate thunk (which issues Page.navigate), and
|
|
63
|
+
* keeps the guard live for redirects until the caller disposes it.
|
|
64
|
+
*
|
|
65
|
+
* Returns the live guard so the caller can inspect blocked/observedIps and dispose
|
|
66
|
+
* after the page settles. Throws NavigationBlockedError if the initial target fails
|
|
67
|
+
* policy (we never arm/navigate to a known-bad target), or
|
|
68
|
+
* InterceptionUnavailableError if interception cannot be armed.
|
|
69
|
+
*/
|
|
70
|
+
export declare function guardedNavigate(bridge: InterceptBridge, url: string, resolver: DnsResolver, navigate: () => Promise<void>, opts?: {
|
|
71
|
+
allowLiteralIp?: boolean;
|
|
72
|
+
observeIp?: boolean;
|
|
73
|
+
}): Promise<NavigationGuard>;
|
|
74
|
+
/** Convenience: synchronous pre-screen without DNS, for callers that only have the
|
|
75
|
+
* syntax layer available (e.g. the extension service worker, which cannot resolve). */
|
|
76
|
+
export declare function screenSyntaxOnly(url: string, opts?: {
|
|
77
|
+
allowLiteralIp?: boolean;
|
|
78
|
+
}): import("./url-policy.js").UrlPolicyResult;
|