@jackwener/opencli 1.3.1 → 1.3.3

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (241) hide show
  1. package/CHANGELOG.md +128 -0
  2. package/README.md +48 -9
  3. package/README.zh-CN.md +48 -9
  4. package/SKILL.md +317 -6
  5. package/TESTING.md +4 -4
  6. package/dist/browser/cdp.js +10 -1
  7. package/dist/browser/daemon-client.js +2 -1
  8. package/dist/browser/discover.js +2 -1
  9. package/dist/browser/errors.d.ts +2 -1
  10. package/dist/browser/errors.js +10 -10
  11. package/dist/browser/index.d.ts +1 -0
  12. package/dist/browser/index.js +1 -0
  13. package/dist/browser/page.js +12 -0
  14. package/dist/browser/stealth.d.ts +18 -0
  15. package/dist/browser/stealth.js +140 -0
  16. package/dist/browser.test.js +47 -1
  17. package/dist/build-manifest.js +1 -3
  18. package/dist/cli-manifest.json +2573 -989
  19. package/dist/cli.js +42 -2
  20. package/dist/clis/bilibili/download.js +20 -65
  21. package/dist/clis/bilibili/utils.js +2 -1
  22. package/dist/clis/chaoxing/assignments.js +2 -1
  23. package/dist/clis/doubao/ask.d.ts +1 -0
  24. package/dist/clis/doubao/ask.js +35 -0
  25. package/dist/clis/doubao/common.d.ts +23 -0
  26. package/dist/clis/doubao/common.js +564 -0
  27. package/dist/clis/doubao/new.d.ts +1 -0
  28. package/dist/clis/doubao/new.js +20 -0
  29. package/dist/clis/doubao/read.d.ts +1 -0
  30. package/dist/clis/doubao/read.js +19 -0
  31. package/dist/clis/doubao/send.d.ts +1 -0
  32. package/dist/clis/doubao/send.js +22 -0
  33. package/dist/clis/doubao/status.d.ts +1 -0
  34. package/dist/clis/doubao/status.js +24 -0
  35. package/dist/clis/doubao-app/ask.d.ts +1 -0
  36. package/dist/clis/doubao-app/ask.js +53 -0
  37. package/dist/clis/doubao-app/common.d.ts +37 -0
  38. package/dist/clis/doubao-app/common.js +110 -0
  39. package/dist/clis/doubao-app/dump.d.ts +1 -0
  40. package/dist/clis/doubao-app/dump.js +24 -0
  41. package/dist/clis/doubao-app/new.d.ts +1 -0
  42. package/dist/clis/doubao-app/new.js +20 -0
  43. package/dist/clis/doubao-app/read.d.ts +1 -0
  44. package/dist/clis/doubao-app/read.js +18 -0
  45. package/dist/clis/doubao-app/screenshot.d.ts +1 -0
  46. package/dist/clis/doubao-app/screenshot.js +18 -0
  47. package/dist/clis/doubao-app/send.d.ts +1 -0
  48. package/dist/clis/doubao-app/send.js +27 -0
  49. package/dist/clis/doubao-app/status.d.ts +1 -0
  50. package/dist/clis/doubao-app/status.js +16 -0
  51. package/dist/clis/hackernews/ask.yaml +38 -0
  52. package/dist/clis/hackernews/best.yaml +38 -0
  53. package/dist/clis/hackernews/jobs.yaml +36 -0
  54. package/dist/clis/hackernews/new.yaml +38 -0
  55. package/dist/clis/hackernews/search.yaml +44 -0
  56. package/dist/clis/hackernews/show.yaml +38 -0
  57. package/dist/clis/hackernews/top.yaml +3 -1
  58. package/dist/clis/hackernews/user.yaml +25 -0
  59. package/dist/clis/twitter/download.js +13 -97
  60. package/dist/clis/twitter/thread.js +2 -1
  61. package/dist/clis/v2ex/member.yaml +29 -0
  62. package/dist/clis/v2ex/node.yaml +34 -0
  63. package/dist/clis/v2ex/nodes.yaml +31 -0
  64. package/dist/clis/v2ex/replies.yaml +32 -0
  65. package/dist/clis/v2ex/user.yaml +34 -0
  66. package/dist/clis/weibo/search.d.ts +1 -0
  67. package/dist/clis/weibo/search.js +73 -0
  68. package/dist/clis/weixin/download.d.ts +12 -0
  69. package/dist/clis/weixin/download.js +183 -0
  70. package/dist/clis/xiaohongshu/download.js +12 -60
  71. package/dist/clis/xiaohongshu/publish.d.ts +18 -0
  72. package/dist/clis/xiaohongshu/publish.js +352 -0
  73. package/dist/clis/xiaohongshu/search.js +47 -15
  74. package/dist/clis/xiaohongshu/search.test.d.ts +1 -0
  75. package/dist/clis/xiaohongshu/search.test.js +114 -0
  76. package/dist/clis/yollomi/background.d.ts +4 -0
  77. package/dist/clis/yollomi/background.js +45 -0
  78. package/dist/clis/yollomi/edit.d.ts +5 -0
  79. package/dist/clis/yollomi/edit.js +56 -0
  80. package/dist/clis/yollomi/face-swap.d.ts +5 -0
  81. package/dist/clis/yollomi/face-swap.js +43 -0
  82. package/dist/clis/yollomi/generate.d.ts +9 -0
  83. package/dist/clis/yollomi/generate.js +100 -0
  84. package/dist/clis/yollomi/models.d.ts +1 -0
  85. package/dist/clis/yollomi/models.js +33 -0
  86. package/dist/clis/yollomi/object-remover.d.ts +4 -0
  87. package/dist/clis/yollomi/object-remover.js +42 -0
  88. package/dist/clis/yollomi/remove-bg.d.ts +4 -0
  89. package/dist/clis/yollomi/remove-bg.js +38 -0
  90. package/dist/clis/yollomi/restore.d.ts +4 -0
  91. package/dist/clis/yollomi/restore.js +38 -0
  92. package/dist/clis/yollomi/try-on.d.ts +4 -0
  93. package/dist/clis/yollomi/try-on.js +46 -0
  94. package/dist/clis/yollomi/upload.d.ts +7 -0
  95. package/dist/clis/yollomi/upload.js +71 -0
  96. package/dist/clis/yollomi/upscale.d.ts +4 -0
  97. package/dist/clis/yollomi/upscale.js +53 -0
  98. package/dist/clis/yollomi/utils.d.ts +45 -0
  99. package/dist/clis/yollomi/utils.js +180 -0
  100. package/dist/clis/yollomi/video.d.ts +5 -0
  101. package/dist/clis/yollomi/video.js +56 -0
  102. package/dist/clis/zhihu/download.d.ts +1 -5
  103. package/dist/clis/zhihu/download.js +20 -126
  104. package/dist/clis/zhihu/download.test.js +7 -5
  105. package/dist/clis/zhihu/question.js +2 -1
  106. package/dist/commanderAdapter.js +4 -6
  107. package/dist/constants.d.ts +2 -0
  108. package/dist/constants.js +2 -0
  109. package/dist/daemon.js +7 -3
  110. package/dist/discovery.js +10 -10
  111. package/dist/doctor.js +2 -1
  112. package/dist/download/article-download.d.ts +59 -0
  113. package/dist/download/article-download.js +178 -0
  114. package/dist/download/media-download.d.ts +49 -0
  115. package/dist/download/media-download.js +112 -0
  116. package/dist/errors.d.ts +23 -2
  117. package/dist/errors.js +58 -2
  118. package/dist/errors.test.d.ts +1 -0
  119. package/dist/errors.test.js +59 -0
  120. package/dist/execution.js +9 -10
  121. package/dist/explore.js +4 -2
  122. package/dist/external.d.ts +15 -0
  123. package/dist/external.js +48 -2
  124. package/dist/external.test.d.ts +1 -0
  125. package/dist/external.test.js +64 -0
  126. package/dist/main.js +10 -0
  127. package/dist/plugin.d.ts +4 -0
  128. package/dist/plugin.js +45 -23
  129. package/dist/plugin.test.js +6 -1
  130. package/dist/record.d.ts +47 -0
  131. package/dist/record.js +545 -0
  132. package/dist/registry.d.ts +7 -2
  133. package/dist/registry.js +2 -6
  134. package/dist/runtime.d.ts +3 -1
  135. package/dist/runtime.js +10 -3
  136. package/dist/validate.js +1 -3
  137. package/docs/.vitepress/config.mts +1 -0
  138. package/docs/adapters/browser/douban.md +18 -8
  139. package/docs/adapters/browser/doubao.md +35 -0
  140. package/docs/adapters/browser/hackernews.md +20 -4
  141. package/docs/adapters/browser/tiktok.md +1 -1
  142. package/docs/adapters/browser/v2ex.md +31 -10
  143. package/docs/adapters/browser/weibo.md +4 -0
  144. package/docs/adapters/browser/weixin.md +33 -0
  145. package/docs/adapters/browser/wikipedia.md +0 -9
  146. package/docs/adapters/browser/xiaohongshu.md +8 -6
  147. package/docs/adapters/browser/yollomi.md +69 -0
  148. package/docs/adapters/desktop/antigravity.md +0 -3
  149. package/docs/adapters/desktop/doubao-app.md +35 -0
  150. package/docs/adapters/index.md +19 -8
  151. package/docs/advanced/download.md +4 -0
  152. package/package.json +3 -1
  153. package/src/browser/cdp.ts +9 -1
  154. package/src/browser/daemon-client.ts +4 -3
  155. package/src/browser/discover.ts +2 -1
  156. package/src/browser/errors.ts +18 -11
  157. package/src/browser/index.ts +1 -0
  158. package/src/browser/page.ts +11 -0
  159. package/src/browser/stealth.ts +142 -0
  160. package/src/browser.test.ts +51 -1
  161. package/src/build-manifest.ts +1 -3
  162. package/src/cli.ts +45 -2
  163. package/src/clis/bilibili/download.ts +25 -83
  164. package/src/clis/bilibili/utils.ts +2 -1
  165. package/src/clis/chaoxing/assignments.ts +2 -1
  166. package/src/clis/doubao/ask.ts +40 -0
  167. package/src/clis/doubao/common.ts +619 -0
  168. package/src/clis/doubao/new.ts +22 -0
  169. package/src/clis/doubao/read.ts +20 -0
  170. package/src/clis/doubao/send.ts +25 -0
  171. package/src/clis/doubao/status.ts +27 -0
  172. package/src/clis/doubao-app/ask.ts +60 -0
  173. package/src/clis/doubao-app/common.ts +116 -0
  174. package/src/clis/doubao-app/dump.ts +28 -0
  175. package/src/clis/doubao-app/new.ts +21 -0
  176. package/src/clis/doubao-app/read.ts +21 -0
  177. package/src/clis/doubao-app/screenshot.ts +19 -0
  178. package/src/clis/doubao-app/send.ts +30 -0
  179. package/src/clis/doubao-app/status.ts +17 -0
  180. package/src/clis/hackernews/ask.yaml +38 -0
  181. package/src/clis/hackernews/best.yaml +38 -0
  182. package/src/clis/hackernews/jobs.yaml +36 -0
  183. package/src/clis/hackernews/new.yaml +38 -0
  184. package/src/clis/hackernews/search.yaml +44 -0
  185. package/src/clis/hackernews/show.yaml +38 -0
  186. package/src/clis/hackernews/top.yaml +3 -1
  187. package/src/clis/hackernews/user.yaml +25 -0
  188. package/src/clis/twitter/download.ts +13 -111
  189. package/src/clis/twitter/thread.ts +2 -1
  190. package/src/clis/v2ex/member.yaml +29 -0
  191. package/src/clis/v2ex/node.yaml +34 -0
  192. package/src/clis/v2ex/nodes.yaml +31 -0
  193. package/src/clis/v2ex/replies.yaml +32 -0
  194. package/src/clis/v2ex/user.yaml +34 -0
  195. package/src/clis/weibo/search.ts +78 -0
  196. package/src/clis/weixin/download.ts +199 -0
  197. package/src/clis/xiaohongshu/download.ts +12 -71
  198. package/src/clis/xiaohongshu/publish.ts +392 -0
  199. package/src/clis/xiaohongshu/search.test.ts +134 -0
  200. package/src/clis/xiaohongshu/search.ts +49 -15
  201. package/src/clis/yollomi/background.ts +48 -0
  202. package/src/clis/yollomi/edit.ts +58 -0
  203. package/src/clis/yollomi/face-swap.ts +45 -0
  204. package/src/clis/yollomi/generate.ts +95 -0
  205. package/src/clis/yollomi/models.ts +38 -0
  206. package/src/clis/yollomi/object-remover.ts +44 -0
  207. package/src/clis/yollomi/remove-bg.ts +40 -0
  208. package/src/clis/yollomi/restore.ts +40 -0
  209. package/src/clis/yollomi/try-on.ts +48 -0
  210. package/src/clis/yollomi/upload.ts +78 -0
  211. package/src/clis/yollomi/upscale.ts +49 -0
  212. package/src/clis/yollomi/utils.ts +202 -0
  213. package/src/clis/yollomi/video.ts +61 -0
  214. package/src/clis/zhihu/download.test.ts +7 -5
  215. package/src/clis/zhihu/download.ts +23 -158
  216. package/src/clis/zhihu/question.ts +2 -1
  217. package/src/commanderAdapter.ts +4 -7
  218. package/src/constants.ts +3 -0
  219. package/src/daemon.ts +7 -3
  220. package/src/discovery.ts +26 -26
  221. package/src/doctor.ts +2 -1
  222. package/src/download/article-download.ts +272 -0
  223. package/src/download/media-download.ts +178 -0
  224. package/src/errors.test.ts +79 -0
  225. package/src/errors.ts +92 -2
  226. package/src/execution.ts +14 -10
  227. package/src/explore.ts +4 -2
  228. package/src/external.test.ts +88 -0
  229. package/src/external.ts +56 -2
  230. package/src/generate.ts +2 -1
  231. package/src/main.ts +10 -0
  232. package/src/plugin.test.ts +7 -1
  233. package/src/plugin.ts +49 -25
  234. package/src/record.ts +617 -0
  235. package/src/registry.ts +9 -5
  236. package/src/runtime.ts +16 -4
  237. package/src/validate.ts +1 -3
  238. package/tests/e2e/browser-auth.test.ts +10 -1
  239. package/tests/e2e/browser-public.test.ts +13 -8
  240. package/tests/e2e/public-commands.test.ts +209 -21
  241. package/tests/smoke/api-health.test.ts +65 -6
package/src/record.ts ADDED
@@ -0,0 +1,617 @@
1
+ /**
2
+ * Record mode — capture API calls from a live browser session.
3
+ *
4
+ * Flow:
5
+ * 1. Navigate to the target URL in an automation tab
6
+ * 2. Inject a full-capture fetch/XHR interceptor (records url + method + body)
7
+ * 3. Poll every 2s and print newly captured requests
8
+ * 4. User operates the page; press Enter to stop
9
+ * 5. Analyze captured requests → infer capabilities → write YAML candidates
10
+ *
11
+ * Design: no new daemon endpoints, no extension changes.
12
+ * Uses existing exec + navigate actions only.
13
+ */
14
+
15
+ import * as fs from 'node:fs';
16
+ import * as path from 'node:path';
17
+ import * as readline from 'node:readline';
18
+ import chalk from 'chalk';
19
+ import yaml from 'js-yaml';
20
+ import { sendCommand } from './browser/daemon-client.js';
21
+ import type { IPage } from './types.js';
22
+ import {
23
+ VOLATILE_PARAMS,
24
+ SEARCH_PARAMS,
25
+ PAGINATION_PARAMS,
26
+ FIELD_ROLES,
27
+ } from './constants.js';
28
+
29
+ // ── Types ──────────────────────────────────────────────────────────────────
30
+
31
+ export interface RecordedRequest {
32
+ url: string;
33
+ method: string;
34
+ status: number | null;
35
+ contentType: string;
36
+ body: unknown;
37
+ capturedAt: number;
38
+ }
39
+
40
+ export interface RecordResult {
41
+ site: string;
42
+ url: string;
43
+ requests: RecordedRequest[];
44
+ outDir: string;
45
+ candidateCount: number;
46
+ candidates: Array<{ name: string; path: string; strategy: string }>;
47
+ }
48
+
49
+ // ── Interceptor JS ─────────────────────────────────────────────────────────
50
+
51
+ /**
52
+ * Generates a full-capture interceptor that stores {url, method, status, body}
53
+ * for every JSON response. No URL pattern filter — captures everything.
54
+ */
55
+ function generateFullCaptureInterceptorJs(): string {
56
+ return `
57
+ (() => {
58
+ // Restore original fetch/XHR if previously patched, then re-patch (idempotent injection)
59
+ if (window.__opencli_record_patched) {
60
+ if (window.__opencli_orig_fetch) window.fetch = window.__opencli_orig_fetch;
61
+ if (window.__opencli_orig_xhr_open) XMLHttpRequest.prototype.open = window.__opencli_orig_xhr_open;
62
+ if (window.__opencli_orig_xhr_send) XMLHttpRequest.prototype.send = window.__opencli_orig_xhr_send;
63
+ window.__opencli_record_patched = false;
64
+ }
65
+ // Preserve existing capture buffer across re-injections
66
+ window.__opencli_record = window.__opencli_record || [];
67
+
68
+ const _push = (url, method, body) => {
69
+ try {
70
+ // Only capture JSON-like responses
71
+ if (typeof body !== 'object' || body === null) return;
72
+ // Skip tiny/trivial responses (tracking pixels, empty acks)
73
+ const keys = Object.keys(body);
74
+ if (keys.length < 2) return;
75
+ window.__opencli_record.push({
76
+ url: String(url),
77
+ method: String(method).toUpperCase(),
78
+ status: null,
79
+ body,
80
+ ts: Date.now(),
81
+ });
82
+ } catch {}
83
+ };
84
+
85
+ // Patch fetch — save original for future restore
86
+ window.__opencli_orig_fetch = window.fetch;
87
+ window.fetch = async function(...args) {
88
+ const req = args[0];
89
+ const reqUrl = typeof req === 'string' ? req : (req instanceof Request ? req.url : String(req));
90
+ const method = (args[1]?.method || (req instanceof Request ? req.method : 'GET') || 'GET');
91
+ const res = await window.__opencli_orig_fetch.apply(this, args);
92
+ const ct = res.headers.get('content-type') || '';
93
+ if (ct.includes('json')) {
94
+ try {
95
+ const body = await res.clone().json();
96
+ _push(reqUrl, method, body);
97
+ } catch {}
98
+ }
99
+ return res;
100
+ };
101
+
102
+ // Patch XHR — save originals for future restore
103
+ const _XHR = XMLHttpRequest.prototype;
104
+ window.__opencli_orig_xhr_open = _XHR.open;
105
+ window.__opencli_orig_xhr_send = _XHR.send;
106
+ _XHR.open = function(method, url) {
107
+ this.__rec_url = String(url);
108
+ this.__rec_method = String(method);
109
+ this.__rec_listener_added = false; // reset per open() call
110
+ return window.__opencli_orig_xhr_open.apply(this, arguments);
111
+ };
112
+ _XHR.send = function() {
113
+ // Guard: only add one listener per XHR instance to prevent duplicate captures
114
+ if (!this.__rec_listener_added) {
115
+ this.__rec_listener_added = true;
116
+ this.addEventListener('load', function() {
117
+ const ct = this.getResponseHeader?.('content-type') || '';
118
+ if (ct.includes('json')) {
119
+ try { _push(this.__rec_url, this.__rec_method || 'GET', JSON.parse(this.responseText)); } catch {}
120
+ }
121
+ });
122
+ }
123
+ return window.__opencli_orig_xhr_send.apply(this, arguments);
124
+ };
125
+
126
+ window.__opencli_record_patched = true;
127
+ return 1;
128
+ })()
129
+ `;
130
+ }
131
+
132
+ /** Read and clear captured requests from the page */
133
+ function generateReadRecordedJs(): string {
134
+ return `
135
+ (() => {
136
+ const data = window.__opencli_record || [];
137
+ window.__opencli_record = [];
138
+ return data;
139
+ })()
140
+ `;
141
+ }
142
+
143
+ // ── Analysis helpers ───────────────────────────────────────────────────────
144
+
145
+ function urlToPattern(url: string): string {
146
+ try {
147
+ const p = new URL(url);
148
+ const pathNorm = p.pathname
149
+ .replace(/\/\d+/g, '/{id}')
150
+ .replace(/\/[0-9a-fA-F]{8,}/g, '/{hex}')
151
+ .replace(/\/BV[a-zA-Z0-9]{10}/g, '/{bvid}');
152
+ const params: string[] = [];
153
+ p.searchParams.forEach((_v, k) => { if (!VOLATILE_PARAMS.has(k)) params.push(k); });
154
+ return `${p.host}${pathNorm}${params.length ? '?' + params.sort().map(k => `${k}={}`).join('&') : ''}`;
155
+ } catch { return url; }
156
+ }
157
+
158
+ function detectAuthIndicators(url: string, body: unknown): string[] {
159
+ const indicators: string[] = [];
160
+ // Heuristic: if body contains sign/w_rid fields, it's likely signed
161
+ if (body && typeof body === 'object') {
162
+ const keys = Object.keys(body as object).map(k => k.toLowerCase());
163
+ if (keys.some(k => k.includes('sign') || k === 'w_rid' || k.includes('token'))) {
164
+ indicators.push('signature');
165
+ }
166
+ }
167
+ // Check URL for common auth patterns
168
+ if (url.includes('/wbi/') || url.includes('w_rid=')) indicators.push('signature');
169
+ if (url.includes('bearer') || url.includes('access_token')) indicators.push('bearer');
170
+ return indicators;
171
+ }
172
+
173
+ function findArrayPath(obj: unknown, depth = 0): { path: string; items: unknown[] } | null {
174
+ if (depth > 5 || !obj || typeof obj !== 'object') return null;
175
+ if (Array.isArray(obj)) {
176
+ if (obj.length >= 2 && obj.some(i => i && typeof i === 'object' && !Array.isArray(i))) {
177
+ return { path: '', items: obj };
178
+ }
179
+ return null;
180
+ }
181
+ let best: { path: string; items: unknown[] } | null = null;
182
+ for (const [key, val] of Object.entries(obj as Record<string, unknown>)) {
183
+ const found = findArrayPath(val, depth + 1);
184
+ if (found) {
185
+ const fullPath = found.path ? `${key}.${found.path}` : key;
186
+ const candidate = { path: fullPath, items: found.items };
187
+ if (!best || candidate.items.length > best.items.length) best = candidate;
188
+ }
189
+ }
190
+ return best;
191
+ }
192
+
193
+ function inferCapabilityName(url: string): string {
194
+ const u = url.toLowerCase();
195
+ if (u.includes('hot') || u.includes('popular') || u.includes('ranking') || u.includes('trending')) return 'hot';
196
+ if (u.includes('search')) return 'search';
197
+ if (u.includes('feed') || u.includes('timeline') || u.includes('dynamic')) return 'feed';
198
+ if (u.includes('comment') || u.includes('reply')) return 'comments';
199
+ if (u.includes('history')) return 'history';
200
+ if (u.includes('profile') || u.includes('me')) return 'me';
201
+ if (u.includes('favorite') || u.includes('collect') || u.includes('bookmark')) return 'favorite';
202
+ try {
203
+ const segs = new URL(url).pathname
204
+ .split('/')
205
+ .filter(s => s && !s.match(/^\d+$/) && !s.match(/^[0-9a-f]{8,}$/i) && !s.match(/^v\d+$/));
206
+ if (segs.length) return segs[segs.length - 1].replace(/[^a-z0-9]/gi, '_').toLowerCase();
207
+ } catch {}
208
+ return 'data';
209
+ }
210
+
211
+ function inferStrategy(authIndicators: string[]): string {
212
+ if (authIndicators.includes('signature')) return 'intercept';
213
+ if (authIndicators.includes('bearer') || authIndicators.includes('csrf')) return 'header';
214
+ return 'cookie';
215
+ }
216
+
217
+ function scoreRequest(req: RecordedRequest, arrayResult: ReturnType<typeof findArrayPath> | null): number {
218
+ let s = 0;
219
+ if (arrayResult) {
220
+ s += 10;
221
+ s += Math.min(arrayResult.items.length, 10);
222
+ // Bonus for detected semantic fields
223
+ const sample = arrayResult.items[0];
224
+ if (sample && typeof sample === 'object') {
225
+ const keys = Object.keys(sample as object).map(k => k.toLowerCase());
226
+ for (const aliases of Object.values(FIELD_ROLES)) {
227
+ if (aliases.some(a => keys.includes(a))) s += 2;
228
+ }
229
+ }
230
+ }
231
+ if (req.url.includes('/api/')) s += 3;
232
+ // Penalize likely tracking / analytics endpoints
233
+ if (req.url.match(/\/(track|log|analytics|beacon|pixel|stats|metric)/i)) s -= 10;
234
+ if (req.url.match(/\/(ping|heartbeat|keep.?alive)/i)) s -= 10;
235
+ return s;
236
+ }
237
+
238
+ // ── YAML generation ────────────────────────────────────────────────────────
239
+
240
+ function buildRecordedYaml(
241
+ site: string,
242
+ pageUrl: string,
243
+ req: RecordedRequest,
244
+ capName: string,
245
+ arrayResult: ReturnType<typeof findArrayPath>,
246
+ authIndicators: string[],
247
+ ): { name: string; yaml: unknown } {
248
+ const strategy = inferStrategy(authIndicators);
249
+ const domain = (() => { try { return new URL(pageUrl).hostname; } catch { return ''; } })();
250
+
251
+ // Detect fields from first array item
252
+ const detectedFields: Record<string, string> = {};
253
+ if (arrayResult?.items[0] && typeof arrayResult.items[0] === 'object') {
254
+ const sampleKeys = Object.keys(arrayResult.items[0] as object).map(k => k.toLowerCase());
255
+ for (const [role, aliases] of Object.entries(FIELD_ROLES)) {
256
+ const match = aliases.find(a => sampleKeys.includes(a));
257
+ if (match) detectedFields[role] = match;
258
+ }
259
+ }
260
+
261
+ const itemPath = arrayResult?.path ?? null;
262
+ // When path is '' (root-level array), access data directly; otherwise chain with optional chaining
263
+ const pathChain = itemPath === null
264
+ ? ''
265
+ : itemPath === ''
266
+ ? ''
267
+ : itemPath.split('.').map(p => `?.${p}`).join('');
268
+
269
+ // Detect search/limit/page params (must be before fetch URL building to use hasSearch/hasPage)
270
+ const qp: string[] = [];
271
+ try { new URL(req.url).searchParams.forEach((_v, k) => { if (!VOLATILE_PARAMS.has(k)) qp.push(k); }); } catch {}
272
+ const hasSearch = qp.some(p => SEARCH_PARAMS.has(p));
273
+ const hasPage = qp.some(p => PAGINATION_PARAMS.has(p));
274
+
275
+ // Build evaluate script
276
+ const mapLines = Object.entries(detectedFields)
277
+ .map(([role, field]) => ` ${role}: item?.${field}`)
278
+ .join(',\n');
279
+ const mapExpr = mapLines
280
+ ? `.map(item => ({\n${mapLines}\n }))`
281
+ : '';
282
+
283
+ // Build fetch URL — for search/page args, replace query param values with template vars
284
+ let fetchUrl = req.url;
285
+ try {
286
+ const u = new URL(req.url);
287
+ if (hasSearch) {
288
+ for (const p of SEARCH_PARAMS) {
289
+ if (u.searchParams.has(p)) { u.searchParams.set(p, '{{args.keyword}}'); break; }
290
+ }
291
+ }
292
+ if (hasPage) {
293
+ for (const p of PAGINATION_PARAMS) {
294
+ if (u.searchParams.has(p)) { u.searchParams.set(p, '{{args.page | default(1)}}'); break; }
295
+ }
296
+ }
297
+ fetchUrl = u.toString();
298
+ } catch {}
299
+
300
+ // When itemPath is empty, the array IS the response root; otherwise chain with ?.
301
+ const dataAccess = pathChain ? `data${pathChain}` : 'data';
302
+
303
+ const evaluateScript = [
304
+ '(async () => {',
305
+ ` const res = await fetch(${JSON.stringify(fetchUrl)}, { credentials: 'include' });`,
306
+ ' const data = await res.json();',
307
+ ` return (${dataAccess} || [])${mapExpr};`,
308
+ '})()',
309
+ ].join('\n');
310
+
311
+ const args: Record<string, unknown> = {};
312
+ if (hasSearch) args['keyword'] = { type: 'str', required: true, description: 'Search keyword', positional: true };
313
+ args['limit'] = { type: 'int', default: 20, description: 'Number of items' };
314
+ if (hasPage) args['page'] = { type: 'int', default: 1, description: 'Page number' };
315
+
316
+ const columns = ['rank', ...Object.keys(detectedFields).length ? Object.keys(detectedFields) : ['title', 'url']];
317
+
318
+ const mapStep: Record<string, string> = { rank: '${{ index + 1 }}' };
319
+ for (const col of columns.filter(c => c !== 'rank')) {
320
+ mapStep[col] = `\${{ item.${col} }}`;
321
+ }
322
+
323
+ const pipeline: unknown[] = [
324
+ { navigate: pageUrl },
325
+ { evaluate: evaluateScript },
326
+ { map: mapStep },
327
+ { limit: '${{ args.limit | default(20) }}' },
328
+ ];
329
+
330
+ return {
331
+ name: capName,
332
+ yaml: {
333
+ site,
334
+ name: capName,
335
+ description: `${site} ${capName} (recorded)`,
336
+ domain,
337
+ strategy,
338
+ browser: true,
339
+ args,
340
+ pipeline,
341
+ columns,
342
+ },
343
+ };
344
+ }
345
+
346
+ // ── Main record function ───────────────────────────────────────────────────
347
+
348
+ export interface RecordOptions {
349
+ BrowserFactory: new () => { connect(o?: unknown): Promise<IPage>; close(): Promise<void> };
350
+ site?: string;
351
+ url: string;
352
+ outDir?: string;
353
+ pollMs?: number;
354
+ timeoutMs?: number;
355
+ }
356
+
357
+ export async function recordSession(opts: RecordOptions): Promise<RecordResult> {
358
+ const pollMs = opts.pollMs ?? 2000;
359
+ const timeoutMs = opts.timeoutMs ?? 60_000;
360
+ const allRequests: RecordedRequest[] = [];
361
+ // Track which tabIds have already had the interceptor injected
362
+ const injectedTabs = new Set<number>();
363
+
364
+ // Infer site name from URL
365
+ const site = opts.site ?? (() => {
366
+ try {
367
+ const host = new URL(opts.url).hostname.toLowerCase().replace(/^www\./, '');
368
+ return host.split('.')[0] ?? 'site';
369
+ } catch { return 'site'; }
370
+ })();
371
+
372
+ const workspace = `record:${site}`;
373
+
374
+ console.log(chalk.bold.cyan('\n opencli record'));
375
+ console.log(chalk.dim(` Site: ${site} URL: ${opts.url}`));
376
+ console.log(chalk.dim(` Timeout: ${timeoutMs / 1000}s Poll: ${pollMs}ms`));
377
+ console.log(chalk.dim(' Navigating…'));
378
+
379
+ const factory = new opts.BrowserFactory();
380
+ const page = await factory.connect({ timeout: 30, workspace });
381
+
382
+ try {
383
+ // Navigate to target
384
+ await page.goto(opts.url);
385
+
386
+ // Inject into initial tab
387
+ const initialTabs = await listTabs(workspace);
388
+ for (const tab of initialTabs) {
389
+ await injectIntoTab(workspace, tab.tabId, injectedTabs);
390
+ }
391
+
392
+ console.log(chalk.bold('\n Recording. Operate the page in the automation window.'));
393
+ console.log(chalk.dim(` Will auto-stop after ${timeoutMs / 1000}s, or press Enter to stop now.\n`));
394
+
395
+ // Race: Enter key vs timeout
396
+ let stopped = false;
397
+ const stop = () => { stopped = true; };
398
+
399
+ const { promise: enterPromise, cleanup: cleanupEnter } = waitForEnter();
400
+ const enterRace = enterPromise.then(stop);
401
+ const timeoutPromise = new Promise<void>(r => setTimeout(() => {
402
+ stop();
403
+ cleanupEnter(); // close readline to prevent process from hanging
404
+ r();
405
+ }, timeoutMs));
406
+
407
+ // Poll loop: drain captured data + inject interceptor into any new tabs
408
+ const pollInterval = setInterval(async () => {
409
+ if (stopped) return;
410
+ try {
411
+ // Discover and inject into any new tabs
412
+ const tabs = await listTabs(workspace);
413
+ for (const tab of tabs) {
414
+ await injectIntoTab(workspace, tab.tabId, injectedTabs);
415
+ }
416
+
417
+ // Drain captured data from all known tabs
418
+ for (const tabId of injectedTabs) {
419
+ const batch = await execOnTab(workspace, tabId, generateReadRecordedJs()) as RecordedRequest[] | null;
420
+ if (Array.isArray(batch) && batch.length > 0) {
421
+ for (const r of batch) allRequests.push(r);
422
+ console.log(chalk.dim(` [tab:${tabId}] +${batch.length} captured — total: ${allRequests.length}`));
423
+ }
424
+ }
425
+ } catch {
426
+ // Tab may have navigated; keep going
427
+ }
428
+ }, pollMs);
429
+
430
+ await Promise.race([enterPromise, timeoutPromise]);
431
+ clearInterval(pollInterval);
432
+
433
+ // Final drain from all known tabs
434
+ for (const tabId of injectedTabs) {
435
+ try {
436
+ const last = await execOnTab(workspace, tabId, generateReadRecordedJs()) as RecordedRequest[] | null;
437
+ if (Array.isArray(last) && last.length > 0) {
438
+ for (const r of last) allRequests.push(r);
439
+ }
440
+ } catch {}
441
+ }
442
+
443
+ console.log(chalk.dim(`\n Stopped. Analyzing ${allRequests.length} captured requests…`));
444
+
445
+ const result = analyzeAndWrite(site, opts.url, allRequests, opts.outDir);
446
+ await factory.close().catch(() => {});
447
+ return result;
448
+ } catch (err) {
449
+ await factory.close().catch(() => {});
450
+ throw err;
451
+ }
452
+ }
453
+
454
+ // ── Tab helpers ────────────────────────────────────────────────────────────
455
+
456
+ interface TabInfo { tabId: number; url?: string }
457
+
458
+ async function listTabs(workspace: string): Promise<TabInfo[]> {
459
+ try {
460
+ const result = await sendCommand('tabs', { op: 'list', workspace }) as TabInfo[] | null;
461
+ return Array.isArray(result) ? result.filter(t => t.tabId != null) : [];
462
+ } catch { return []; }
463
+ }
464
+
465
+ async function execOnTab(workspace: string, tabId: number, code: string): Promise<unknown> {
466
+ return sendCommand('exec', { code, workspace, tabId });
467
+ }
468
+
469
+ async function injectIntoTab(workspace: string, tabId: number, injectedTabs: Set<number>): Promise<void> {
470
+ try {
471
+ await execOnTab(workspace, tabId, generateFullCaptureInterceptorJs());
472
+ if (!injectedTabs.has(tabId)) {
473
+ injectedTabs.add(tabId);
474
+ console.log(chalk.green(` ✓ Interceptor injected into tab:${tabId}`));
475
+ }
476
+ } catch {
477
+ // Tab not debuggable (e.g. chrome:// pages) — skip silently
478
+ }
479
+ }
480
+
481
+ /**
482
+ * Wait for user to press Enter on stdin.
483
+ * Returns both a promise and a cleanup fn so the caller can close the interface
484
+ * when a timeout fires (preventing the process from hanging on stdin).
485
+ */
486
+ function waitForEnter(): { promise: Promise<void>; cleanup: () => void } {
487
+ let rl: readline.Interface | null = null;
488
+ const promise = new Promise<void>((resolve) => {
489
+ rl = readline.createInterface({ input: process.stdin });
490
+ rl.once('line', () => { rl?.close(); rl = null; resolve(); });
491
+ // Handle Ctrl+C gracefully
492
+ rl.once('SIGINT', () => { rl?.close(); rl = null; resolve(); });
493
+ });
494
+ return {
495
+ promise,
496
+ cleanup: () => { rl?.close(); rl = null; },
497
+ };
498
+ }
499
+
500
+ // ── Analysis + output ──────────────────────────────────────────────────────
501
+
502
+ function analyzeAndWrite(
503
+ site: string,
504
+ pageUrl: string,
505
+ requests: RecordedRequest[],
506
+ outDir?: string,
507
+ ): RecordResult {
508
+ const targetDir = outDir ?? path.join('.opencli', 'record', site);
509
+ fs.mkdirSync(targetDir, { recursive: true });
510
+
511
+ if (requests.length === 0) {
512
+ console.log(chalk.yellow(' No API requests captured.'));
513
+ return { site, url: pageUrl, requests: [], outDir: targetDir, candidateCount: 0, candidates: [] };
514
+ }
515
+
516
+ // Deduplicate by pattern
517
+ const seen = new Map<string, RecordedRequest>();
518
+ for (const req of requests) {
519
+ const pattern = urlToPattern(req.url);
520
+ if (!seen.has(pattern)) seen.set(pattern, req);
521
+ }
522
+
523
+ // Score and rank unique requests
524
+ type ScoredEntry = {
525
+ req: RecordedRequest;
526
+ pattern: string;
527
+ arrayResult: ReturnType<typeof findArrayPath>;
528
+ authIndicators: string[];
529
+ score: number;
530
+ };
531
+
532
+ const scored: ScoredEntry[] = [];
533
+ for (const [pattern, req] of seen) {
534
+ const arrayResult = findArrayPath(req.body);
535
+ const authIndicators = detectAuthIndicators(req.url, req.body);
536
+ const score = scoreRequest(req, arrayResult);
537
+ if (score > 0) {
538
+ scored.push({ req, pattern, arrayResult, authIndicators, score });
539
+ }
540
+ }
541
+ scored.sort((a, b) => b.score - a.score);
542
+
543
+ // Save raw captured data
544
+ fs.writeFileSync(
545
+ path.join(targetDir, 'captured.json'),
546
+ JSON.stringify({ site, url: pageUrl, capturedAt: new Date().toISOString(), requests }, null, 2),
547
+ );
548
+
549
+ // Generate candidate YAMLs (top 5)
550
+ const candidates: RecordResult['candidates'] = [];
551
+ const usedNames = new Set<string>();
552
+
553
+ console.log(chalk.bold('\n Captured endpoints (scored):\n'));
554
+
555
+ for (const entry of scored.slice(0, 8)) {
556
+ const itemCount = entry.arrayResult?.items.length ?? 0;
557
+ const strategy = inferStrategy(entry.authIndicators);
558
+ const marker = entry.score >= 15 ? chalk.green('★') : entry.score >= 8 ? chalk.yellow('◆') : chalk.dim('·');
559
+ console.log(
560
+ ` ${marker} ${chalk.white(entry.pattern)}` +
561
+ chalk.dim(` [${strategy}]`) +
562
+ (itemCount ? chalk.cyan(` ← ${itemCount} items`) : ''),
563
+ );
564
+ }
565
+
566
+ console.log();
567
+
568
+ const topCandidates = scored.filter(e => e.arrayResult && e.score >= 8).slice(0, 5);
569
+ const candidatesDir = path.join(targetDir, 'candidates');
570
+ fs.mkdirSync(candidatesDir, { recursive: true });
571
+
572
+ for (const entry of topCandidates) {
573
+ let capName = inferCapabilityName(entry.req.url);
574
+ if (usedNames.has(capName)) capName = `${capName}_${usedNames.size + 1}`;
575
+ usedNames.add(capName);
576
+
577
+ const strategy = inferStrategy(entry.authIndicators);
578
+ const candidate = buildRecordedYaml(site, pageUrl, entry.req, capName, entry.arrayResult!, entry.authIndicators);
579
+ const filePath = path.join(candidatesDir, `${capName}.yaml`);
580
+ fs.writeFileSync(filePath, yaml.dump(candidate.yaml, { sortKeys: false, lineWidth: 120 }));
581
+ candidates.push({ name: capName, path: filePath, strategy });
582
+
583
+ console.log(chalk.green(` ✓ Generated: ${chalk.bold(capName)}.yaml [${strategy}]`));
584
+ console.log(chalk.dim(` → ${filePath}`));
585
+ }
586
+
587
+ if (candidates.length === 0) {
588
+ console.log(chalk.yellow(' No high-confidence candidates found.'));
589
+ console.log(chalk.dim(' Tip: make sure you triggered JSON API calls (open lists, search, scroll).'));
590
+ }
591
+
592
+ return {
593
+ site,
594
+ url: pageUrl,
595
+ requests,
596
+ outDir: targetDir,
597
+ candidateCount: candidates.length,
598
+ candidates,
599
+ };
600
+ }
601
+
602
+ export function renderRecordSummary(result: RecordResult): string {
603
+ const lines = [
604
+ `\n opencli record: ${result.candidateCount > 0 ? chalk.green('OK') : chalk.yellow('no candidates')}`,
605
+ ` Site: ${result.site}`,
606
+ ` Captured: ${result.requests.length} requests`,
607
+ ` Candidates: ${result.candidateCount}`,
608
+ ];
609
+ for (const c of result.candidates) {
610
+ lines.push(` • ${c.name} [${c.strategy}] → ${c.path}`);
611
+ }
612
+ if (result.candidateCount > 0) {
613
+ lines.push('');
614
+ lines.push(chalk.dim(` Copy a candidate to src/clis/${result.site}/ and run: npm run build`));
615
+ }
616
+ return lines.join('\n');
617
+ }
package/src/registry.ts CHANGED
@@ -22,6 +22,9 @@ export interface Arg {
22
22
  choices?: string[];
23
23
  }
24
24
 
25
+ // eslint-disable-next-line @typescript-eslint/no-explicit-any -- kwargs from CLI parsing are inherently untyped
26
+ export type CommandArgs = Record<string, any>;
27
+
25
28
  export interface CliCommand {
26
29
  site: string;
27
30
  name: string;
@@ -31,11 +34,12 @@ export interface CliCommand {
31
34
  browser?: boolean;
32
35
  args: Arg[];
33
36
  columns?: string[];
34
- func?: (page: IPage, kwargs: Record<string, any>, debug?: boolean) => Promise<unknown>;
37
+ func?: (page: IPage, kwargs: CommandArgs, debug?: boolean) => Promise<unknown>;
35
38
  pipeline?: Record<string, unknown>[];
36
39
  timeoutSeconds?: number;
40
+ /** Origin of this command: 'yaml', 'ts', or plugin name. */
37
41
  source?: string;
38
- footerExtra?: (kwargs: Record<string, any>) => string | undefined;
42
+ footerExtra?: (kwargs: CommandArgs) => string | undefined;
39
43
  /**
40
44
  * Control pre-navigation for cookie/header context before command execution.
41
45
  *
@@ -64,9 +68,9 @@ export interface CliOptions extends Partial<Omit<CliCommand, 'args' | 'descripti
64
68
  // Use globalThis to ensure a single shared registry across all module instances.
65
69
  // This is critical for TS plugins loaded via npm link / peerDependency — without
66
70
  // this, the plugin's import creates a separate module instance with its own Map.
67
- const REGISTRY_KEY = '__opencli_registry__';
71
+ declare global { var __opencli_registry__: Map<string, CliCommand> | undefined; }
68
72
  const _registry: Map<string, CliCommand> =
69
- (globalThis as any)[REGISTRY_KEY] ??= new Map<string, CliCommand>();
73
+ globalThis.__opencli_registry__ ??= new Map<string, CliCommand>();
70
74
 
71
75
  export function cli(opts: CliOptions): CliCommand {
72
76
  const strategy = opts.strategy ?? (opts.browser === false ? Strategy.PUBLIC : Strategy.COOKIE);
@@ -101,7 +105,7 @@ export function fullName(cmd: CliCommand): string {
101
105
  }
102
106
 
103
107
  export function strategyLabel(cmd: CliCommand): string {
104
- return cmd.strategy ?? 'public';
108
+ return cmd.strategy ?? Strategy.PUBLIC;
105
109
  }
106
110
 
107
111
  export function registerCommand(cmd: CliCommand): void {