@jackwener/opencli 1.7.4 → 1.7.6

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (181) hide show
  1. package/README.md +76 -51
  2. package/README.zh-CN.md +78 -62
  3. package/cli-manifest.json +4558 -2979
  4. package/clis/antigravity/serve.js +71 -25
  5. package/clis/baidu-scholar/search.js +87 -0
  6. package/clis/baidu-scholar/search.test.js +23 -0
  7. package/clis/bilibili/video.js +61 -0
  8. package/clis/bilibili/video.test.js +81 -0
  9. package/clis/deepseek/ask.js +94 -0
  10. package/clis/deepseek/ask.test.js +73 -0
  11. package/clis/deepseek/history.js +25 -0
  12. package/clis/deepseek/new.js +20 -0
  13. package/clis/deepseek/read.js +22 -0
  14. package/clis/deepseek/status.js +24 -0
  15. package/clis/deepseek/utils.js +291 -0
  16. package/clis/deepseek/utils.test.js +37 -0
  17. package/clis/eastmoney/_secid.js +78 -0
  18. package/clis/eastmoney/announcement.js +52 -0
  19. package/clis/eastmoney/convertible.js +73 -0
  20. package/clis/eastmoney/etf.js +65 -0
  21. package/clis/eastmoney/holders.js +78 -0
  22. package/clis/eastmoney/index-board.js +96 -0
  23. package/clis/eastmoney/kline.js +87 -0
  24. package/clis/eastmoney/kuaixun.js +54 -0
  25. package/clis/eastmoney/longhu.js +67 -0
  26. package/clis/eastmoney/money-flow.js +78 -0
  27. package/clis/eastmoney/northbound.js +57 -0
  28. package/clis/eastmoney/quote.js +107 -0
  29. package/clis/eastmoney/rank.js +94 -0
  30. package/clis/eastmoney/sectors.js +76 -0
  31. package/clis/google-scholar/search.js +58 -0
  32. package/clis/google-scholar/search.test.js +23 -0
  33. package/clis/gov-law/commands.test.js +39 -0
  34. package/clis/gov-law/recent.js +22 -0
  35. package/clis/gov-law/search.js +41 -0
  36. package/clis/gov-law/shared.js +51 -0
  37. package/clis/gov-policy/commands.test.js +27 -0
  38. package/clis/gov-policy/recent.js +47 -0
  39. package/clis/gov-policy/search.js +48 -0
  40. package/clis/jianyu/search.js +139 -3
  41. package/clis/jianyu/search.test.js +25 -0
  42. package/clis/jianyu/shared/procurement-detail.js +15 -0
  43. package/clis/jianyu/shared/procurement-detail.test.js +12 -0
  44. package/clis/nowcoder/companies.js +23 -0
  45. package/clis/nowcoder/creators.js +27 -0
  46. package/clis/nowcoder/detail.js +61 -0
  47. package/clis/nowcoder/experience.js +36 -0
  48. package/clis/nowcoder/hot.js +24 -0
  49. package/clis/nowcoder/jobs.js +21 -0
  50. package/clis/nowcoder/notifications.js +29 -0
  51. package/clis/nowcoder/papers.js +40 -0
  52. package/clis/nowcoder/practice.js +37 -0
  53. package/clis/nowcoder/recommend.js +30 -0
  54. package/clis/nowcoder/referral.js +39 -0
  55. package/clis/nowcoder/salary.js +40 -0
  56. package/clis/nowcoder/search.js +49 -0
  57. package/clis/nowcoder/suggest.js +33 -0
  58. package/clis/nowcoder/topics.js +27 -0
  59. package/clis/nowcoder/trending.js +25 -0
  60. package/clis/twitter/list-add.js +337 -0
  61. package/clis/twitter/list-add.test.js +15 -0
  62. package/clis/twitter/list-remove.js +297 -0
  63. package/clis/twitter/list-remove.test.js +14 -0
  64. package/clis/twitter/list-tweets.js +185 -0
  65. package/clis/twitter/list-tweets.test.js +108 -0
  66. package/clis/twitter/lists.js +134 -47
  67. package/clis/twitter/lists.test.js +105 -38
  68. package/clis/twitter/shared.js +7 -2
  69. package/clis/twitter/tweets.js +218 -0
  70. package/clis/twitter/tweets.test.js +125 -0
  71. package/clis/wanfang/search.js +66 -0
  72. package/clis/wanfang/search.test.js +23 -0
  73. package/clis/web/read.js +1 -1
  74. package/clis/weixin/download.js +3 -2
  75. package/clis/xiaohongshu/publish.js +149 -28
  76. package/clis/xiaohongshu/publish.test.js +319 -6
  77. package/clis/xiaoyuzhou/download.js +8 -4
  78. package/clis/xiaoyuzhou/download.test.js +23 -13
  79. package/clis/xiaoyuzhou/episode.js +9 -4
  80. package/clis/xiaoyuzhou/podcast-episodes.js +15 -11
  81. package/clis/xiaoyuzhou/podcast.js +9 -4
  82. package/clis/xiaoyuzhou/utils.js +0 -40
  83. package/clis/xiaoyuzhou/utils.test.js +15 -75
  84. package/clis/youtube/channel.js +35 -0
  85. package/clis/zsxq/dynamics.js +1 -1
  86. package/clis/zsxq/utils.js +6 -3
  87. package/clis/zsxq/utils.test.js +31 -0
  88. package/dist/src/browser/base-page.d.ts +14 -4
  89. package/dist/src/browser/base-page.js +35 -25
  90. package/dist/src/browser/bridge.d.ts +1 -0
  91. package/dist/src/browser/bridge.js +1 -1
  92. package/dist/src/browser/cdp.d.ts +1 -0
  93. package/dist/src/browser/cdp.js +13 -4
  94. package/dist/src/browser/compound.d.ts +59 -0
  95. package/dist/src/browser/compound.js +112 -0
  96. package/dist/src/browser/compound.test.js +175 -0
  97. package/dist/src/browser/daemon-client.d.ts +6 -4
  98. package/dist/src/browser/daemon-client.js +6 -1
  99. package/dist/src/browser/daemon-client.test.js +40 -1
  100. package/dist/src/browser/dom-snapshot.d.ts +7 -0
  101. package/dist/src/browser/dom-snapshot.js +83 -5
  102. package/dist/src/browser/dom-snapshot.test.js +65 -0
  103. package/dist/src/browser/extract.d.ts +69 -0
  104. package/dist/src/browser/extract.js +132 -0
  105. package/dist/src/browser/extract.test.js +129 -0
  106. package/dist/src/browser/find.d.ts +76 -0
  107. package/dist/src/browser/find.js +179 -0
  108. package/dist/src/browser/find.test.js +120 -0
  109. package/dist/src/browser/html-tree.d.ts +75 -0
  110. package/dist/src/browser/html-tree.js +112 -0
  111. package/dist/src/browser/html-tree.test.d.ts +1 -0
  112. package/dist/src/browser/html-tree.test.js +181 -0
  113. package/dist/src/browser/network-cache.d.ts +48 -0
  114. package/dist/src/browser/network-cache.js +66 -0
  115. package/dist/src/browser/network-cache.test.d.ts +1 -0
  116. package/dist/src/browser/network-cache.test.js +58 -0
  117. package/dist/src/browser/network-key.d.ts +22 -0
  118. package/dist/src/browser/network-key.js +66 -0
  119. package/dist/src/browser/network-key.test.d.ts +1 -0
  120. package/dist/src/browser/network-key.test.js +49 -0
  121. package/dist/src/browser/page.d.ts +14 -4
  122. package/dist/src/browser/page.js +48 -7
  123. package/dist/src/browser/page.test.js +97 -0
  124. package/dist/src/browser/shape-filter.d.ts +52 -0
  125. package/dist/src/browser/shape-filter.js +101 -0
  126. package/dist/src/browser/shape-filter.test.d.ts +1 -0
  127. package/dist/src/browser/shape-filter.test.js +101 -0
  128. package/dist/src/browser/shape.d.ts +23 -0
  129. package/dist/src/browser/shape.js +95 -0
  130. package/dist/src/browser/shape.test.d.ts +1 -0
  131. package/dist/src/browser/shape.test.js +82 -0
  132. package/dist/src/browser/target-errors.d.ts +14 -1
  133. package/dist/src/browser/target-errors.js +13 -0
  134. package/dist/src/browser/target-errors.test.js +39 -6
  135. package/dist/src/browser/target-resolver.d.ts +57 -10
  136. package/dist/src/browser/target-resolver.js +195 -75
  137. package/dist/src/browser/target-resolver.test.js +80 -5
  138. package/dist/src/cli.js +849 -267
  139. package/dist/src/cli.test.js +961 -90
  140. package/dist/src/commanderAdapter.d.ts +0 -1
  141. package/dist/src/commanderAdapter.js +2 -16
  142. package/dist/src/commanderAdapter.test.js +1 -1
  143. package/dist/src/completion-shared.js +2 -5
  144. package/dist/src/daemon.js +8 -0
  145. package/dist/src/download/article-download.d.ts +1 -0
  146. package/dist/src/download/article-download.js +3 -0
  147. package/dist/src/download/article-download.test.d.ts +1 -0
  148. package/dist/src/download/article-download.test.js +39 -0
  149. package/dist/src/execution.js +7 -2
  150. package/dist/src/execution.test.js +54 -0
  151. package/dist/src/main.js +16 -0
  152. package/dist/src/plugin.d.ts +1 -8
  153. package/dist/src/plugin.js +1 -27
  154. package/dist/src/plugin.test.js +1 -59
  155. package/dist/src/registry.d.ts +1 -0
  156. package/dist/src/registry.js +3 -2
  157. package/dist/src/registry.test.js +22 -0
  158. package/dist/src/types.d.ts +32 -8
  159. package/package.json +1 -1
  160. package/clis/twitter/lists-parser.js +0 -77
  161. package/clis/twitter/lists.d.ts +0 -5
  162. package/dist/src/cascade.d.ts +0 -46
  163. package/dist/src/cascade.js +0 -135
  164. package/dist/src/explore.d.ts +0 -99
  165. package/dist/src/explore.js +0 -402
  166. package/dist/src/generate-verified.d.ts +0 -105
  167. package/dist/src/generate-verified.js +0 -696
  168. package/dist/src/generate-verified.test.js +0 -925
  169. package/dist/src/generate.d.ts +0 -46
  170. package/dist/src/generate.js +0 -117
  171. package/dist/src/record.d.ts +0 -96
  172. package/dist/src/record.js +0 -657
  173. package/dist/src/record.test.js +0 -293
  174. package/dist/src/skill-generate.d.ts +0 -30
  175. package/dist/src/skill-generate.js +0 -75
  176. package/dist/src/skill-generate.test.js +0 -173
  177. package/dist/src/synthesize.d.ts +0 -97
  178. package/dist/src/synthesize.js +0 -208
  179. /package/dist/src/{generate-verified.test.d.ts → browser/compound.test.d.ts} +0 -0
  180. /package/dist/src/{record.test.d.ts → browser/extract.test.d.ts} +0 -0
  181. /package/dist/src/{skill-generate.test.d.ts → browser/find.test.d.ts} +0 -0
@@ -1,657 +0,0 @@
1
- /**
2
- * Record mode — capture API calls from a live browser session.
3
- *
4
- * Flow:
5
- * 1. Navigate to the target URL in an automation tab
6
- * 2. Inject a full-capture fetch/XHR interceptor (records url + method + body)
7
- * 3. Poll every 2s and print newly captured requests
8
- * 4. User operates the page; press Enter to stop
9
- * 5. Analyze captured requests → infer capabilities → write YAML candidates
10
- *
11
- * Design: no new daemon endpoints, no extension changes.
12
- * Uses existing exec + navigate actions only.
13
- */
14
- import * as fs from 'node:fs';
15
- import * as path from 'node:path';
16
- import * as readline from 'node:readline';
17
- import { styleText } from 'node:util';
18
- import { sendCommand } from './browser/daemon-client.js';
19
- import { SEARCH_PARAMS, PAGINATION_PARAMS, FIELD_ROLES } from './constants.js';
20
- import { urlToPattern, findArrayPath, inferCapabilityName, inferStrategy, detectAuthFromContent, classifyQueryParams, isNoiseUrl, } from './analysis.js';
21
- /** Keep the later candidate when multiple recordings share one bucket (prefer fresher data). */
22
- function preferRecordedCandidate(_current, next) {
23
- return next;
24
- }
25
- /** Build a candidate-level dedupe key. */
26
- function getRecordedCandidateKey(candidate) {
27
- return `${candidate.kind} ${getRecordedRequestKey(candidate.req)}`;
28
- }
29
- /** Build a request dedupe key from method and URL pattern. */
30
- function getRecordedRequestKey(req) {
31
- return `${req.method.toUpperCase()} ${urlToPattern(req.url)}`;
32
- }
33
- /** Deduplicate recorded requests by method and URL pattern. */
34
- function dedupeRecordedRequests(requests) {
35
- const deduped = new Map();
36
- for (const req of requests) {
37
- deduped.set(getRecordedRequestKey(req), req);
38
- }
39
- return [...deduped.values()];
40
- }
41
- /** Check whether a content type should be treated as JSON. */
42
- function isJsonContentType(contentType) {
43
- const normalized = contentType?.toLowerCase() ?? '';
44
- return normalized.includes('application/json') || normalized.includes('+json');
45
- }
46
- /** Parse a captured request body only when the request advertises JSON. */
47
- function parseJsonBodyText(contentType, raw) {
48
- if (!isJsonContentType(contentType))
49
- return null;
50
- if (!raw || !raw.trim())
51
- return null;
52
- try {
53
- return JSON.parse(raw);
54
- }
55
- catch {
56
- return null;
57
- }
58
- }
59
- /** Build one normalized recorded entry from captured request and response values. */
60
- export function createRecordedEntry(input) {
61
- const requestBody = parseJsonBodyText(input.requestContentType ?? null, input.requestBodyText ?? null);
62
- const responseContentType = input.responseContentType ?? 'application/json';
63
- return {
64
- url: input.url,
65
- method: input.method.toUpperCase(),
66
- status: input.status ?? null,
67
- requestContentType: input.requestContentType ?? null,
68
- responseContentType,
69
- requestBody,
70
- responseBody: input.responseBody,
71
- // Keep legacy fields in sync until the analyzer/template path is migrated.
72
- contentType: responseContentType,
73
- body: input.responseBody,
74
- capturedAt: input.capturedAt ?? Date.now(),
75
- };
76
- }
77
- // ── Interceptor JS ─────────────────────────────────────────────────────────
78
- /**
79
- * Generates a full-capture interceptor that stores {url, method, status, body}
80
- * for every JSON response. No URL pattern filter — captures everything.
81
- */
82
- export function generateFullCaptureInterceptorJs() {
83
- return `
84
- (() => {
85
- // Restore original fetch/XHR if previously patched, then re-patch (idempotent injection)
86
- if (window.__opencli_record_patched) {
87
- if (window.__opencli_orig_fetch) window.fetch = window.__opencli_orig_fetch;
88
- if (window.__opencli_orig_xhr_open) XMLHttpRequest.prototype.open = window.__opencli_orig_xhr_open;
89
- if (window.__opencli_orig_xhr_send) XMLHttpRequest.prototype.send = window.__opencli_orig_xhr_send;
90
- if (window.__opencli_orig_xhr_set_request_header) XMLHttpRequest.prototype.setRequestHeader = window.__opencli_orig_xhr_set_request_header;
91
- window.__opencli_record_patched = false;
92
- }
93
- // Preserve existing capture buffer across re-injections
94
- window.__opencli_record = window.__opencli_record || [];
95
-
96
- const _tryParseJson = (contentType, raw) => {
97
- try {
98
- const normalized = String(contentType || '').toLowerCase();
99
- if (!normalized.includes('application/json') && !normalized.includes('+json')) return null;
100
- if (typeof raw !== 'string' || !raw.trim()) return null;
101
- return JSON.parse(raw);
102
- } catch {
103
- return null;
104
- }
105
- };
106
-
107
- const _push = (entry) => {
108
- try {
109
- const responseBody = entry.responseBody;
110
- if (typeof responseBody !== 'object' || responseBody === null) return;
111
- const isReplayableWrite = ['POST', 'PUT', 'PATCH'].includes(String(entry.method).toUpperCase())
112
- && (() => {
113
- const normalized = String(entry.requestContentType || '').toLowerCase();
114
- return normalized.includes('application/json') || normalized.includes('+json');
115
- })()
116
- && entry.requestBody
117
- && typeof entry.requestBody === 'object';
118
- const keys = Object.keys(responseBody);
119
- if (keys.length < 2 && !isReplayableWrite) return;
120
- window.__opencli_record.push({
121
- url: String(entry.url),
122
- method: String(entry.method).toUpperCase(),
123
- status: null,
124
- requestContentType: entry.requestContentType || null,
125
- responseContentType: entry.responseContentType || 'application/json',
126
- requestBody: entry.requestBody || null,
127
- responseBody,
128
- contentType: entry.responseContentType || 'application/json',
129
- body: responseBody,
130
- capturedAt: Date.now(),
131
- });
132
- } catch {}
133
- };
134
-
135
- // Patch fetch — save original for future restore
136
- window.__opencli_orig_fetch = window.fetch;
137
- window.fetch = async function(...args) {
138
- const req = args[0];
139
- const init = args[1] || {};
140
- const reqUrl = typeof req === 'string' ? req : (req instanceof Request ? req.url : String(req));
141
- const method = (init?.method || (req instanceof Request ? req.method : 'GET') || 'GET');
142
- const requestContentType = (() => {
143
- if (init?.headers) {
144
- try {
145
- const headers = new Headers(init.headers);
146
- const value = headers.get('content-type');
147
- if (value) return value;
148
- } catch {}
149
- }
150
- if (req instanceof Request) {
151
- return req.headers.get('content-type');
152
- }
153
- return null;
154
- })();
155
- const requestBodyText = (() => {
156
- if (typeof init?.body === 'string') return init.body;
157
- return null;
158
- })();
159
- const shouldReadRequestBodyFromRequest = req instanceof Request
160
- && !requestBodyText
161
- && ['POST', 'PUT', 'PATCH'].includes(String(method).toUpperCase())
162
- && (() => {
163
- const normalized = String(requestContentType || '').toLowerCase();
164
- return normalized.includes('application/json') || normalized.includes('+json');
165
- })();
166
- let requestBodyTextFromRequest = null;
167
- if (shouldReadRequestBodyFromRequest) {
168
- try {
169
- requestBodyTextFromRequest = await req.clone().text();
170
- } catch {}
171
- }
172
- const requestBody = _tryParseJson(requestContentType, requestBodyText || requestBodyTextFromRequest);
173
- const res = await window.__opencli_orig_fetch.apply(this, args);
174
- const ct = res.headers.get('content-type') || '';
175
- if (ct.includes('json')) {
176
- try {
177
- const responseBody = await res.clone().json();
178
- _push({
179
- url: reqUrl,
180
- method,
181
- requestContentType,
182
- requestBody,
183
- responseContentType: ct,
184
- responseBody,
185
- });
186
- } catch {}
187
- }
188
- return res;
189
- };
190
-
191
- // Patch XHR — save originals for future restore
192
- const _XHR = XMLHttpRequest.prototype;
193
- window.__opencli_orig_xhr_open = _XHR.open;
194
- window.__opencli_orig_xhr_send = _XHR.send;
195
- window.__opencli_orig_xhr_set_request_header = _XHR.setRequestHeader;
196
- _XHR.open = function(method, url) {
197
- this.__rec_url = String(url);
198
- this.__rec_method = String(method);
199
- this.__rec_request_content_type = null;
200
- this.__rec_listener_added = false; // reset per open() call
201
- return window.__opencli_orig_xhr_open.apply(this, arguments);
202
- };
203
- _XHR.setRequestHeader = function(name, value) {
204
- if (String(name).toLowerCase() === 'content-type') {
205
- this.__rec_request_content_type = String(value);
206
- }
207
- return window.__opencli_orig_xhr_set_request_header.apply(this, arguments);
208
- };
209
- _XHR.send = function() {
210
- const requestBody = _tryParseJson(this.__rec_request_content_type, typeof arguments[0] === 'string' ? arguments[0] : null);
211
- // Guard: only add one listener per XHR instance to prevent duplicate captures
212
- if (!this.__rec_listener_added) {
213
- this.__rec_listener_added = true;
214
- this.addEventListener('load', function() {
215
- const ct = this.getResponseHeader?.('content-type') || '';
216
- if (ct.includes('json')) {
217
- try {
218
- _push({
219
- url: this.__rec_url,
220
- method: this.__rec_method || 'GET',
221
- requestContentType: this.__rec_request_content_type,
222
- requestBody,
223
- responseContentType: ct,
224
- responseBody: JSON.parse(this.responseText),
225
- });
226
- } catch {}
227
- }
228
- });
229
- }
230
- return window.__opencli_orig_xhr_send.apply(this, arguments);
231
- };
232
-
233
- window.__opencli_record_patched = true;
234
- return 1;
235
- })()
236
- `;
237
- }
238
- /** Read and clear captured requests from the page */
239
- function generateReadRecordedJs() {
240
- return `
241
- (() => {
242
- const data = window.__opencli_record || [];
243
- window.__opencli_record = [];
244
- return data;
245
- })()
246
- `;
247
- }
248
- // ── Analysis helpers ───────────────────────────────────────────────────────
249
- /** Check whether one recorded request is safe to treat as a write candidate. */
250
- function isWriteCandidate(req) {
251
- return ['POST', 'PUT', 'PATCH'].includes(req.method)
252
- && isJsonContentType(req.requestContentType)
253
- && !!req.requestBody
254
- && typeof req.requestBody === 'object'
255
- && !Array.isArray(req.requestBody)
256
- && !!req.responseBody
257
- && typeof req.responseBody === 'object'
258
- && !Array.isArray(req.responseBody);
259
- }
260
- /** Analyze recorded requests into read and write candidates, filtering out noise. */
261
- export function analyzeRecordedRequests(requests) {
262
- const candidates = [];
263
- for (const req of requests) {
264
- if (isNoiseUrl(req.url))
265
- continue;
266
- const arrayResult = findArrayPath(req.responseBody);
267
- if (isWriteCandidate(req)) {
268
- candidates.push({ kind: 'write', req, arrayResult: null });
269
- continue;
270
- }
271
- if (arrayResult) {
272
- candidates.push({ kind: 'read', req, arrayResult });
273
- }
274
- }
275
- return { candidates };
276
- }
277
- // ── YAML generation ────────────────────────────────────────────────────────
278
- function buildRecordedYaml(site, pageUrl, req, capName, arrayResult, authIndicators) {
279
- const strategy = inferStrategy(authIndicators);
280
- const domain = (() => { try {
281
- return new URL(pageUrl).hostname;
282
- }
283
- catch {
284
- return '';
285
- } })();
286
- // Detect fields from first array item
287
- const detectedFields = {};
288
- if (arrayResult?.items[0] && typeof arrayResult.items[0] === 'object') {
289
- const sampleKeys = Object.keys(arrayResult.items[0]).map(k => k.toLowerCase());
290
- for (const [role, aliases] of Object.entries(FIELD_ROLES)) {
291
- const match = aliases.find(a => sampleKeys.includes(a));
292
- if (match)
293
- detectedFields[role] = match;
294
- }
295
- }
296
- const itemPath = arrayResult?.path ?? null;
297
- // When path is '' (root-level array), access data directly; otherwise chain with optional chaining
298
- const pathChain = itemPath === null
299
- ? ''
300
- : itemPath === ''
301
- ? ''
302
- : itemPath.split('.').map(p => `?.${p}`).join('');
303
- // Detect search/limit/page params (must be before fetch URL building to use hasSearch/hasPage)
304
- const { hasSearch, hasPagination: hasPage } = classifyQueryParams(req.url);
305
- // Build evaluate script
306
- const mapLines = Object.entries(detectedFields)
307
- .map(([role, field]) => ` ${role}: item?.${field}`)
308
- .join(',\n');
309
- const mapExpr = mapLines
310
- ? `.map(item => ({\n${mapLines}\n }))`
311
- : '';
312
- // Build fetch URL — for search/page args, replace query param values with template vars
313
- let fetchUrl = req.url;
314
- try {
315
- const u = new URL(req.url);
316
- if (hasSearch) {
317
- for (const p of SEARCH_PARAMS) {
318
- if (u.searchParams.has(p)) {
319
- u.searchParams.set(p, '${{ args.keyword }}');
320
- break;
321
- }
322
- }
323
- }
324
- if (hasPage) {
325
- for (const p of PAGINATION_PARAMS) {
326
- if (u.searchParams.has(p)) {
327
- u.searchParams.set(p, '${{ args.page | default(1) }}');
328
- break;
329
- }
330
- }
331
- }
332
- fetchUrl = u.toString();
333
- fetchUrl = fetchUrl
334
- .replaceAll(encodeURIComponent('${{ args.keyword }}'), '${{ args.keyword }}')
335
- .replaceAll('%24%7B%7B+args.keyword+%7D%7D', '${{ args.keyword }}')
336
- .replaceAll(encodeURIComponent('${{ args.page | default(1) }}'), '${{ args.page | default(1) }}');
337
- fetchUrl = fetchUrl.replaceAll('%24%7B%7B+args.page+%7C+default%281%29+%7D%7D', '${{ args.page | default(1) }}');
338
- }
339
- catch { }
340
- // When itemPath is empty, the array IS the response root; otherwise chain with ?.
341
- const dataAccess = pathChain ? `data${pathChain}` : 'data';
342
- const evaluateScript = [
343
- '(async () => {',
344
- ` const res = await fetch(${JSON.stringify(fetchUrl)}, { credentials: 'include' });`,
345
- ' const data = await res.json();',
346
- ` return (${dataAccess} || [])${mapExpr};`,
347
- '})()',
348
- ].join('\n');
349
- const args = {};
350
- if (hasSearch)
351
- args['keyword'] = { type: 'str', required: true, description: 'Search keyword', positional: true };
352
- args['limit'] = { type: 'int', default: 20, description: 'Number of items' };
353
- if (hasPage)
354
- args['page'] = { type: 'int', default: 1, description: 'Page number' };
355
- const columns = ['rank', ...Object.keys(detectedFields).length ? Object.keys(detectedFields) : ['title', 'url']];
356
- const mapStep = { rank: '${{ index + 1 }}' };
357
- for (const col of columns.filter(c => c !== 'rank')) {
358
- mapStep[col] = `\${{ item.${col} }}`;
359
- }
360
- const pipeline = [
361
- { navigate: pageUrl },
362
- { evaluate: evaluateScript },
363
- { map: mapStep },
364
- { limit: '${{ args.limit | default(20) }}' },
365
- ];
366
- return {
367
- name: capName,
368
- yaml: {
369
- site,
370
- name: capName,
371
- description: `${site} ${capName} (recorded)`,
372
- domain,
373
- strategy,
374
- browser: true,
375
- args,
376
- pipeline,
377
- columns,
378
- },
379
- };
380
- }
381
- /** Build a minimal YAML candidate for replayable JSON write requests. */
382
- export function buildWriteRecordedYaml(site, pageUrl, req, capName) {
383
- const responseColumns = req.responseBody && typeof req.responseBody === 'object' && !Array.isArray(req.responseBody)
384
- ? Object.keys(req.responseBody).slice(0, 6)
385
- : ['ok'];
386
- const evaluateScript = [
387
- '(async () => {',
388
- ` const res = await fetch(${JSON.stringify(req.url)}, {`,
389
- ` method: ${JSON.stringify(req.method)},`,
390
- ` credentials: 'include',`,
391
- ` headers: { 'content-type': ${JSON.stringify(req.requestContentType ?? 'application/json')} },`,
392
- ` body: JSON.stringify(${JSON.stringify(req.requestBody)}),`,
393
- ' });',
394
- ' return await res.json();',
395
- '})()',
396
- ].join('\n');
397
- return {
398
- name: capName,
399
- yaml: {
400
- site,
401
- name: capName,
402
- description: `${site} ${capName} (recorded write)`,
403
- domain: (() => { try {
404
- return new URL(pageUrl).hostname;
405
- }
406
- catch {
407
- return '';
408
- } })(),
409
- strategy: 'cookie',
410
- browser: true,
411
- args: {},
412
- pipeline: [
413
- { navigate: pageUrl },
414
- { evaluate: evaluateScript },
415
- ],
416
- columns: responseColumns.length ? responseColumns : ['ok'],
417
- },
418
- };
419
- }
420
- /** Turn recorded requests into YAML-ready read and write candidates. */
421
- export function generateRecordedCandidates(site, pageUrl, requests) {
422
- const analysis = analyzeRecordedRequests(dedupeRecordedRequests(requests));
423
- const deduped = new Map();
424
- for (const candidate of analysis.candidates) {
425
- const key = getRecordedCandidateKey(candidate);
426
- const current = deduped.get(key);
427
- deduped.set(key, current ? preferRecordedCandidate(current, candidate) : candidate);
428
- }
429
- // Sort reads by array item count (richer data first), then take top 5
430
- const selected = [...deduped.values()]
431
- .sort((a, b) => (b.arrayResult?.items.length ?? 0) - (a.arrayResult?.items.length ?? 0))
432
- .slice(0, 5);
433
- const usedNames = new Set();
434
- return selected.map((candidate) => {
435
- let capName = inferCapabilityName(candidate.req.url);
436
- if (usedNames.has(capName))
437
- capName = `${capName}_${usedNames.size + 1}`;
438
- usedNames.add(capName);
439
- const authIndicators = detectAuthFromContent(candidate.req.url, candidate.req.responseBody);
440
- const strategy = candidate.kind === 'write' ? 'cookie' : inferStrategy(authIndicators);
441
- const yamlCandidate = candidate.kind === 'write'
442
- ? buildWriteRecordedYaml(site, pageUrl, candidate.req, capName)
443
- : buildRecordedYaml(site, pageUrl, candidate.req, capName, candidate.arrayResult, authIndicators);
444
- return {
445
- kind: candidate.kind,
446
- name: yamlCandidate.name,
447
- strategy,
448
- yaml: yamlCandidate.yaml,
449
- };
450
- });
451
- }
452
- export async function recordSession(opts) {
453
- const pollMs = opts.pollMs ?? 2000;
454
- const timeoutMs = opts.timeoutMs ?? 60_000;
455
- const allRequests = [];
456
- // Track which pages (targetIds) have already had the interceptor injected
457
- const injectedPages = new Set();
458
- // Infer site name from URL
459
- const site = opts.site ?? (() => {
460
- try {
461
- const host = new URL(opts.url).hostname.toLowerCase().replace(/^www\./, '');
462
- return host.split('.')[0] ?? 'site';
463
- }
464
- catch {
465
- return 'site';
466
- }
467
- })();
468
- const workspace = `record:${site}`;
469
- console.log(styleText(['bold', 'cyan'], '\n opencli record'));
470
- console.log(styleText('dim', ` Site: ${site} URL: ${opts.url}`));
471
- console.log(styleText('dim', ` Timeout: ${timeoutMs / 1000}s Poll: ${pollMs}ms`));
472
- console.log(styleText('dim', ' Navigating…'));
473
- const factory = new opts.BrowserFactory();
474
- const page = await factory.connect({ timeout: 30, workspace });
475
- try {
476
- // Navigate to target
477
- await page.goto(opts.url);
478
- // Inject into initial tab
479
- const initialTabs = await listTabs(workspace);
480
- for (const tab of initialTabs) {
481
- if (tab.page)
482
- await injectIntoPage(workspace, tab.page, injectedPages);
483
- }
484
- console.log(styleText('bold', '\n Recording. Use the page in the browser automation window.'));
485
- console.log(styleText('dim', ` Will auto-stop after ${timeoutMs / 1000}s, or press Enter to stop now.\n`));
486
- // Race: Enter key vs timeout
487
- let stopped = false;
488
- const stop = () => { stopped = true; };
489
- const { promise: enterPromise, cleanup: cleanupEnter } = waitForEnter();
490
- enterPromise.then(stop);
491
- const timeoutPromise = new Promise(r => setTimeout(() => {
492
- stop();
493
- r();
494
- }, timeoutMs));
495
- // Poll loop: drain captured data + inject interceptor into any new tabs
496
- const pollInterval = setInterval(async () => {
497
- if (stopped)
498
- return;
499
- try {
500
- // Discover and inject into any new tabs
501
- const tabs = await listTabs(workspace);
502
- for (const tab of tabs) {
503
- if (tab.page)
504
- await injectIntoPage(workspace, tab.page, injectedPages);
505
- }
506
- // Drain captured data from all known pages
507
- for (const page of injectedPages) {
508
- const batch = await execOnPage(workspace, page, generateReadRecordedJs());
509
- if (Array.isArray(batch) && batch.length > 0) {
510
- for (const r of batch)
511
- allRequests.push(r);
512
- console.log(styleText('dim', ` [page:${page.slice(0, 8)}] +${batch.length} captured — total: ${allRequests.length}`));
513
- }
514
- }
515
- }
516
- catch {
517
- // Tab may have navigated; keep going
518
- }
519
- }, pollMs);
520
- await Promise.race([enterPromise, timeoutPromise]);
521
- cleanupEnter(); // Always clean up readline to prevent process from hanging
522
- clearInterval(pollInterval);
523
- // Final drain from all known pages
524
- for (const page of injectedPages) {
525
- try {
526
- const last = await execOnPage(workspace, page, generateReadRecordedJs());
527
- if (Array.isArray(last) && last.length > 0) {
528
- for (const r of last)
529
- allRequests.push(r);
530
- }
531
- }
532
- catch { }
533
- }
534
- console.log(styleText('dim', `\n Stopped. Analyzing ${allRequests.length} captured requests…`));
535
- const result = analyzeAndWrite(site, opts.url, allRequests, opts.outDir);
536
- await factory.close().catch(() => { });
537
- return result;
538
- }
539
- catch (err) {
540
- await factory.close().catch(() => { });
541
- throw err;
542
- }
543
- }
544
- async function listTabs(workspace) {
545
- try {
546
- const result = await sendCommand('tabs', { op: 'list', workspace });
547
- return Array.isArray(result) ? result.filter(t => t.page != null) : [];
548
- }
549
- catch {
550
- return [];
551
- }
552
- }
553
- async function execOnPage(workspace, page, code) {
554
- return sendCommand('exec', { code, workspace, page });
555
- }
556
- async function injectIntoPage(workspace, page, injectedPages) {
557
- try {
558
- await execOnPage(workspace, page, generateFullCaptureInterceptorJs());
559
- if (!injectedPages.has(page)) {
560
- injectedPages.add(page);
561
- console.log(styleText('green', ` ✓ Interceptor injected into page:${page.slice(0, 8)}`));
562
- }
563
- }
564
- catch {
565
- // Page not debuggable (e.g. chrome:// pages) — skip silently
566
- }
567
- }
568
- /**
569
- * Wait for user to press Enter on stdin.
570
- * Returns both a promise and a cleanup fn so the caller can close the interface
571
- * when a timeout fires (preventing the process from hanging on stdin).
572
- */
573
- function waitForEnter() {
574
- let rl = null;
575
- const promise = new Promise((resolve) => {
576
- rl = readline.createInterface({ input: process.stdin });
577
- rl.once('line', () => { rl?.close(); rl = null; resolve(); });
578
- // Handle Ctrl+C gracefully
579
- rl.once('SIGINT', () => { rl?.close(); rl = null; resolve(); });
580
- });
581
- return {
582
- promise,
583
- cleanup: () => { rl?.close(); rl = null; },
584
- };
585
- }
586
- // ── Analysis + output ──────────────────────────────────────────────────────
587
- function analyzeAndWrite(site, pageUrl, requests, outDir) {
588
- const targetDir = outDir ?? path.join('.opencli', 'record', site);
589
- fs.mkdirSync(targetDir, { recursive: true });
590
- if (requests.length === 0) {
591
- console.log(styleText('yellow', ' No API requests captured.'));
592
- return { site, url: pageUrl, requests: [], outDir: targetDir, candidateCount: 0, candidates: [] };
593
- }
594
- // Score and rank deduplicated requests for console output and candidate generation.
595
- const analysisRequests = dedupeRecordedRequests(requests);
596
- const analysis = analyzeRecordedRequests(analysisRequests);
597
- // Save raw captured data
598
- fs.writeFileSync(path.join(targetDir, 'captured.json'), JSON.stringify({ site, url: pageUrl, capturedAt: new Date().toISOString(), requests }, null, 2));
599
- // Generate candidate YAMLs (top 5)
600
- const candidates = [];
601
- const usedNames = new Set();
602
- console.log(styleText('bold', '\n Captured endpoints:\n'));
603
- for (const entry of analysis.candidates.sort((a, b) => (b.arrayResult?.items.length ?? 0) - (a.arrayResult?.items.length ?? 0)).slice(0, 8)) {
604
- const itemCount = entry.arrayResult?.items.length ?? 0;
605
- const strategy = entry.kind === 'write'
606
- ? 'cookie'
607
- : inferStrategy(detectAuthFromContent(entry.req.url, entry.req.responseBody));
608
- const marker = entry.kind === 'write' ? styleText('magenta', '✎') : itemCount > 5 ? styleText('green', '★') : styleText('dim', '·');
609
- console.log(` ${marker} ${styleText('white', urlToPattern(entry.req.url))}` +
610
- styleText('dim', ` [${strategy}]`) +
611
- (entry.kind === 'write'
612
- ? styleText('magenta', ' ← write')
613
- : itemCount ? styleText('cyan', ` ← ${itemCount} items`) : ''));
614
- }
615
- console.log();
616
- const topCandidates = generateRecordedCandidates(site, pageUrl, analysisRequests);
617
- const candidatesDir = path.join(targetDir, 'candidates');
618
- fs.mkdirSync(candidatesDir, { recursive: true });
619
- for (const entry of topCandidates) {
620
- if (usedNames.has(entry.name))
621
- continue;
622
- usedNames.add(entry.name);
623
- const filePath = path.join(candidatesDir, `${entry.name}.json`);
624
- fs.writeFileSync(filePath, JSON.stringify(entry.yaml, null, 2));
625
- candidates.push({ name: entry.name, path: filePath, strategy: entry.strategy });
626
- console.log(styleText('green', ` ✓ Generated: ${styleText('bold', entry.name)}.json [${entry.strategy}]`));
627
- console.log(styleText('dim', ` → ${filePath}`));
628
- }
629
- if (candidates.length === 0) {
630
- console.log(styleText('yellow', ' No candidates found.'));
631
- console.log(styleText('dim', ' Tip: make sure you triggered JSON API calls (open lists, search, scroll).'));
632
- }
633
- return {
634
- site,
635
- url: pageUrl,
636
- requests,
637
- outDir: targetDir,
638
- candidateCount: candidates.length,
639
- candidates,
640
- };
641
- }
642
- export function renderRecordSummary(result) {
643
- const lines = [
644
- `\n opencli record: ${result.candidateCount > 0 ? styleText('green', 'OK') : styleText('yellow', 'no candidates')}`,
645
- ` Site: ${result.site}`,
646
- ` Captured: ${result.requests.length} requests`,
647
- ` Candidates: ${result.candidateCount}`,
648
- ];
649
- for (const c of result.candidates) {
650
- lines.push(` • ${c.name} [${c.strategy}] → ${c.path}`);
651
- }
652
- if (result.candidateCount > 0) {
653
- lines.push('');
654
- lines.push(styleText('dim', ` Copy a candidate to clis/${result.site}/ and run: npm run build`));
655
- }
656
- return lines.join('\n');
657
- }