@jackwener/opencli 1.5.8 → 1.6.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (220) hide show
  1. package/CHANGELOG.md +42 -0
  2. package/README.md +35 -1
  3. package/README.zh-CN.md +17 -1
  4. package/SKILL.md +31 -851
  5. package/autoresearch/baseline-browse.txt +1 -0
  6. package/autoresearch/baseline-skill.txt +1 -0
  7. package/autoresearch/browse-tasks.json +688 -0
  8. package/autoresearch/eval-browse.ts +185 -0
  9. package/autoresearch/eval-skill.ts +248 -0
  10. package/autoresearch/run-browse.sh +9 -0
  11. package/autoresearch/run-skill.sh +9 -0
  12. package/dist/browser/base-page.d.ts +48 -0
  13. package/dist/browser/base-page.js +160 -0
  14. package/dist/browser/cdp.js +4 -106
  15. package/dist/browser/daemon-client.d.ts +20 -7
  16. package/dist/browser/daemon-client.js +39 -39
  17. package/dist/browser/daemon-client.test.js +77 -0
  18. package/dist/browser/discover.d.ts +1 -4
  19. package/dist/browser/discover.js +9 -23
  20. package/dist/browser/errors.d.ts +4 -0
  21. package/dist/browser/errors.js +20 -0
  22. package/dist/browser/index.d.ts +1 -1
  23. package/dist/browser/index.js +1 -1
  24. package/dist/browser/page.d.ts +10 -35
  25. package/dist/browser/page.js +55 -187
  26. package/dist/browser/tabs.js +5 -5
  27. package/dist/browser.test.js +15 -15
  28. package/dist/cli-manifest.json +294 -22
  29. package/dist/cli.js +392 -0
  30. package/dist/clis/amazon/bestsellers.d.ts +21 -0
  31. package/dist/clis/amazon/bestsellers.js +130 -0
  32. package/dist/clis/amazon/bestsellers.test.js +20 -0
  33. package/dist/clis/amazon/discussion.d.ts +20 -0
  34. package/dist/clis/amazon/discussion.js +91 -0
  35. package/dist/clis/amazon/discussion.test.d.ts +1 -0
  36. package/dist/clis/amazon/discussion.test.js +36 -0
  37. package/dist/clis/amazon/offer.d.ts +23 -0
  38. package/dist/clis/amazon/offer.js +140 -0
  39. package/dist/clis/amazon/offer.test.d.ts +1 -0
  40. package/dist/clis/amazon/offer.test.js +29 -0
  41. package/dist/clis/amazon/product.d.ts +18 -0
  42. package/dist/clis/amazon/product.js +92 -0
  43. package/dist/clis/amazon/product.test.d.ts +1 -0
  44. package/dist/clis/amazon/product.test.js +24 -0
  45. package/dist/clis/amazon/search.d.ts +18 -0
  46. package/dist/clis/amazon/search.js +87 -0
  47. package/dist/clis/amazon/search.test.d.ts +1 -0
  48. package/dist/clis/amazon/search.test.js +22 -0
  49. package/dist/clis/amazon/shared.d.ts +64 -0
  50. package/dist/clis/amazon/shared.js +255 -0
  51. package/dist/clis/amazon/shared.test.d.ts +1 -0
  52. package/dist/clis/amazon/shared.test.js +33 -0
  53. package/dist/clis/gemini/ask.d.ts +1 -0
  54. package/dist/clis/gemini/ask.js +40 -0
  55. package/dist/clis/gemini/image.d.ts +1 -0
  56. package/dist/clis/gemini/image.js +105 -0
  57. package/dist/clis/gemini/new.d.ts +1 -0
  58. package/dist/clis/gemini/new.js +20 -0
  59. package/dist/clis/gemini/utils.d.ts +34 -0
  60. package/dist/clis/gemini/utils.js +463 -0
  61. package/dist/clis/gemini/utils.test.d.ts +1 -0
  62. package/dist/clis/gemini/utils.test.js +31 -0
  63. package/dist/clis/notebooklm/compat.test.d.ts +1 -1
  64. package/dist/clis/notebooklm/compat.test.js +3 -3
  65. package/dist/clis/notebooklm/current.js +2 -3
  66. package/dist/clis/notebooklm/get.js +2 -3
  67. package/dist/clis/notebooklm/history.js +2 -3
  68. package/dist/clis/notebooklm/note-list.js +2 -3
  69. package/dist/clis/notebooklm/notes-get.js +2 -3
  70. package/dist/clis/notebooklm/open.d.ts +1 -0
  71. package/dist/clis/notebooklm/open.js +41 -0
  72. package/dist/clis/notebooklm/open.test.d.ts +1 -0
  73. package/dist/clis/notebooklm/open.test.js +63 -0
  74. package/dist/clis/notebooklm/source-fulltext.js +2 -3
  75. package/dist/clis/notebooklm/source-get.js +2 -3
  76. package/dist/clis/notebooklm/source-guide.js +2 -3
  77. package/dist/clis/notebooklm/source-list.js +2 -3
  78. package/dist/clis/notebooklm/status.js +1 -2
  79. package/dist/clis/notebooklm/summary.js +2 -3
  80. package/dist/clis/notebooklm/utils.d.ts +2 -1
  81. package/dist/clis/notebooklm/utils.js +20 -21
  82. package/dist/clis/twitter/article.js +28 -1
  83. package/dist/clis/xiaohongshu/creator-note-detail.test.js +11 -11
  84. package/dist/clis/xiaohongshu/creator-notes-summary.test.js +6 -6
  85. package/dist/clis/xiaohongshu/creator-notes.test.js +22 -22
  86. package/dist/clis/xiaohongshu/note.js +11 -0
  87. package/dist/clis/xiaohongshu/note.test.js +49 -0
  88. package/dist/commanderAdapter.js +7 -4
  89. package/dist/commanderAdapter.test.js +76 -0
  90. package/dist/commands/daemon.js +8 -47
  91. package/dist/commands/daemon.test.js +45 -70
  92. package/dist/discovery.js +27 -0
  93. package/dist/doctor.d.ts +1 -2
  94. package/dist/doctor.js +7 -8
  95. package/dist/explore.js +1 -1
  96. package/dist/output.js +28 -0
  97. package/dist/output.test.js +15 -0
  98. package/dist/pipeline/executor.js +2 -7
  99. package/dist/pipeline/steps/browser.js +1 -1
  100. package/dist/pipeline/template.js +25 -3
  101. package/dist/record.d.ts +50 -0
  102. package/dist/record.js +298 -57
  103. package/dist/record.test.d.ts +1 -0
  104. package/dist/record.test.js +293 -0
  105. package/dist/registry.d.ts +2 -0
  106. package/dist/registry.js +1 -0
  107. package/dist/registry.test.js +10 -0
  108. package/dist/runtime.js +3 -3
  109. package/dist/snapshotFormatter.d.ts +1 -1
  110. package/dist/snapshotFormatter.js +4 -4
  111. package/dist/snapshotFormatter.test.d.ts +1 -1
  112. package/dist/snapshotFormatter.test.js +2 -2
  113. package/dist/types.d.ts +11 -1
  114. package/dist/types.js +1 -1
  115. package/docs/.vitepress/config.mts +2 -0
  116. package/docs/adapters/browser/amazon.md +53 -0
  117. package/docs/adapters/browser/gemini.md +72 -0
  118. package/docs/adapters/browser/notebooklm.md +5 -5
  119. package/docs/adapters/index.md +3 -1
  120. package/docs/guide/getting-started.md +21 -0
  121. package/docs/superpowers/specs/2026-04-02-browse-skill-testing-design.md +144 -0
  122. package/docs/zh/guide/getting-started.md +21 -0
  123. package/extension/package-lock.json +2 -2
  124. package/extension/src/background.test.ts +7 -163
  125. package/extension/src/background.ts +58 -161
  126. package/extension/src/cdp.ts +77 -124
  127. package/extension/src/protocol.ts +5 -5
  128. package/package.json +1 -1
  129. package/skills/opencli-explorer/SKILL.md +853 -0
  130. package/skills/opencli-oneshot/SKILL.md +222 -0
  131. package/skills/opencli-operate/SKILL.md +213 -0
  132. package/skills/opencli-usage/SKILL.md +152 -0
  133. package/skills/opencli-usage/browser.md +429 -0
  134. package/skills/opencli-usage/desktop.md +118 -0
  135. package/skills/opencli-usage/plugins.md +82 -0
  136. package/skills/opencli-usage/public-api.md +149 -0
  137. package/src/browser/base-page.ts +197 -0
  138. package/src/browser/cdp.ts +7 -131
  139. package/src/browser/daemon-client.test.ts +103 -0
  140. package/src/browser/daemon-client.ts +55 -43
  141. package/src/browser/discover.ts +9 -21
  142. package/src/browser/errors.ts +22 -0
  143. package/src/browser/index.ts +1 -1
  144. package/src/browser/page.ts +57 -209
  145. package/src/browser/tabs.ts +5 -5
  146. package/src/browser.test.ts +15 -15
  147. package/src/cli.ts +392 -0
  148. package/src/clis/amazon/bestsellers.test.ts +22 -0
  149. package/src/clis/amazon/bestsellers.ts +180 -0
  150. package/src/clis/amazon/discussion.test.ts +38 -0
  151. package/src/clis/amazon/discussion.ts +131 -0
  152. package/src/clis/amazon/offer.test.ts +35 -0
  153. package/src/clis/amazon/offer.ts +185 -0
  154. package/src/clis/amazon/product.test.ts +26 -0
  155. package/src/clis/amazon/product.ts +131 -0
  156. package/src/clis/amazon/search.test.ts +24 -0
  157. package/src/clis/amazon/search.ts +128 -0
  158. package/src/clis/amazon/shared.test.ts +37 -0
  159. package/src/clis/amazon/shared.ts +316 -0
  160. package/src/clis/gemini/ask.ts +46 -0
  161. package/src/clis/gemini/image.ts +115 -0
  162. package/src/clis/gemini/new.ts +22 -0
  163. package/src/clis/gemini/utils.test.ts +36 -0
  164. package/src/clis/gemini/utils.ts +523 -0
  165. package/src/clis/notebooklm/compat.test.ts +3 -3
  166. package/src/clis/notebooklm/current.ts +2 -3
  167. package/src/clis/notebooklm/get.ts +1 -3
  168. package/src/clis/notebooklm/history.ts +1 -3
  169. package/src/clis/notebooklm/note-list.ts +1 -3
  170. package/src/clis/notebooklm/notes-get.ts +1 -3
  171. package/src/clis/notebooklm/open.test.ts +78 -0
  172. package/src/clis/notebooklm/open.ts +61 -0
  173. package/src/clis/notebooklm/source-fulltext.ts +1 -3
  174. package/src/clis/notebooklm/source-get.ts +1 -3
  175. package/src/clis/notebooklm/source-guide.ts +1 -3
  176. package/src/clis/notebooklm/source-list.ts +1 -3
  177. package/src/clis/notebooklm/status.ts +1 -2
  178. package/src/clis/notebooklm/summary.ts +1 -3
  179. package/src/clis/notebooklm/utils.ts +29 -20
  180. package/src/clis/twitter/article.ts +31 -1
  181. package/src/clis/xiaohongshu/creator-note-detail.test.ts +11 -11
  182. package/src/clis/xiaohongshu/creator-notes-summary.test.ts +6 -6
  183. package/src/clis/xiaohongshu/creator-notes.test.ts +22 -22
  184. package/src/clis/xiaohongshu/note.test.ts +51 -0
  185. package/src/clis/xiaohongshu/note.ts +18 -0
  186. package/src/commanderAdapter.test.ts +109 -0
  187. package/src/commanderAdapter.ts +8 -4
  188. package/src/commands/daemon.test.ts +50 -84
  189. package/src/commands/daemon.ts +8 -56
  190. package/src/discovery.ts +22 -0
  191. package/src/doctor.ts +8 -9
  192. package/src/explore.ts +1 -1
  193. package/src/output.test.ts +17 -0
  194. package/src/output.ts +27 -0
  195. package/src/pipeline/executor.ts +2 -7
  196. package/src/pipeline/steps/browser.ts +1 -1
  197. package/src/pipeline/template.ts +27 -4
  198. package/src/record.test.ts +362 -0
  199. package/src/record.ts +341 -62
  200. package/src/registry.test.ts +12 -0
  201. package/src/registry.ts +3 -0
  202. package/src/runtime.ts +3 -3
  203. package/src/snapshotFormatter.test.ts +2 -2
  204. package/src/snapshotFormatter.ts +4 -4
  205. package/src/types.ts +11 -1
  206. package/.agents/skills/cross-project-adapter-migration/SKILL.md +0 -249
  207. package/.agents/workflows/cross-project-adapter-migration.md +0 -54
  208. package/dist/clis/notebooklm/bind-current.js +0 -29
  209. package/dist/clis/notebooklm/bind-current.test.d.ts +0 -1
  210. package/dist/clis/notebooklm/bind-current.test.js +0 -35
  211. package/dist/clis/notebooklm/binding.test.js +0 -44
  212. package/extension/dist/background.js +0 -819
  213. package/src/clis/notebooklm/bind-current.test.ts +0 -43
  214. package/src/clis/notebooklm/bind-current.ts +0 -36
  215. package/src/clis/notebooklm/binding.test.ts +0 -53
  216. /package/dist/browser/{mcp.d.ts → bridge.d.ts} +0 -0
  217. /package/dist/browser/{mcp.js → bridge.js} +0 -0
  218. /package/dist/{clis/notebooklm/bind-current.d.ts → browser/daemon-client.test.d.ts} +0 -0
  219. /package/dist/clis/{notebooklm/binding.test.d.ts → amazon/bestsellers.test.d.ts} +0 -0
  220. /package/src/browser/{mcp.ts → bridge.ts} +0 -0
@@ -62,7 +62,7 @@ export async function stepSnapshot(page, params, _data, _args) {
62
62
  export async function stepEvaluate(page, params, data, args) {
63
63
  const js = String(render(params, { args, data }));
64
64
  let result = await page.evaluate(js);
65
- // MCP may return JSON as a string — auto-parse it
65
+ // Browser may return JSON as a string — auto-parse it
66
66
  if (typeof result === 'string') {
67
67
  const trimmed = result.trim();
68
68
  if ((trimmed.startsWith('[') && trimmed.endsWith(']')) || (trimmed.startsWith('{') && trimmed.endsWith('}'))) {
@@ -179,6 +179,10 @@ export function resolvePath(pathStr, ctx) {
179
179
  /**
180
180
  * Evaluate arbitrary JS expressions as a last-resort fallback.
181
181
  * Runs inside a `node:vm` sandbox with dynamic code generation disabled.
182
+ *
183
+ * Compiled functions are cached by expression string to avoid re-creating
184
+ * VM contexts on every invocation — critical for loops where the same
185
+ * expression is evaluated hundreds of times.
182
186
  */
183
187
  const FORBIDDEN_EXPR_PATTERNS = /\b(constructor|__proto__|prototype|globalThis|process|require|import|eval)\b/;
184
188
  /**
@@ -197,6 +201,23 @@ function sanitizeContext(obj) {
197
201
  return {};
198
202
  }
199
203
  }
204
+ /** LRU-bounded cache for compiled VM scripts — prevents unbounded memory growth. */
205
+ const MAX_VM_CACHE_SIZE = 256;
206
+ const _vmCache = new Map();
207
+ function getOrCompileScript(expr) {
208
+ let script = _vmCache.get(expr);
209
+ if (script)
210
+ return script;
211
+ // Evict oldest entry when cache is full
212
+ if (_vmCache.size >= MAX_VM_CACHE_SIZE) {
213
+ const firstKey = _vmCache.keys().next().value;
214
+ if (firstKey !== undefined)
215
+ _vmCache.delete(firstKey);
216
+ }
217
+ script = new vm.Script(`(${expr})`);
218
+ _vmCache.set(expr, script);
219
+ return script;
220
+ }
200
221
  function evalJsExpr(expr, ctx) {
201
222
  // Guard against absurdly long expressions that could indicate injection.
202
223
  if (expr.length > 2000)
@@ -209,7 +230,8 @@ function evalJsExpr(expr, ctx) {
209
230
  const data = sanitizeContext(ctx.data);
210
231
  const index = ctx.index ?? 0;
211
232
  try {
212
- return vm.runInNewContext(`(${expr})`, {
233
+ const script = getOrCompileScript(expr);
234
+ const sandbox = vm.createContext({
213
235
  args,
214
236
  item,
215
237
  data,
@@ -224,12 +246,12 @@ function evalJsExpr(expr, ctx) {
224
246
  Array,
225
247
  Date,
226
248
  }, {
227
- timeout: 50,
228
- contextCodeGeneration: {
249
+ codeGeneration: {
229
250
  strings: false,
230
251
  wasm: false,
231
252
  },
232
253
  });
254
+ return script.runInContext(sandbox, { timeout: 50 });
233
255
  }
234
256
  catch {
235
257
  return undefined;
package/dist/record.d.ts CHANGED
@@ -12,10 +12,19 @@
12
12
  * Uses existing exec + navigate actions only.
13
13
  */
14
14
  import type { IPage } from './types.js';
15
+ import { findArrayPath } from './analysis.js';
15
16
  export interface RecordedRequest {
16
17
  url: string;
17
18
  method: string;
18
19
  status: number | null;
20
+ /** Request content type captured at record time, if available. */
21
+ requestContentType: string | null;
22
+ /** Response content type captured at record time, if available. */
23
+ responseContentType: string | null;
24
+ /** Parsed JSON request body for replayable write requests. */
25
+ requestBody: unknown;
26
+ /** Parsed JSON response body captured from the network call. */
27
+ responseBody: unknown;
19
28
  contentType: string;
20
29
  body: unknown;
21
30
  capturedAt: number;
@@ -32,6 +41,46 @@ export interface RecordResult {
32
41
  strategy: string;
33
42
  }>;
34
43
  }
44
+ type RecordedCandidateKind = 'read' | 'write';
45
+ export interface RecordedCandidate {
46
+ kind: RecordedCandidateKind;
47
+ req: RecordedRequest;
48
+ score: number;
49
+ arrayResult: ReturnType<typeof findArrayPath> | null;
50
+ }
51
+ interface GeneratedRecordedCandidate {
52
+ kind: RecordedCandidateKind;
53
+ name: string;
54
+ strategy: string;
55
+ yaml: unknown;
56
+ }
57
+ /** Build one normalized recorded entry from captured request and response values. */
58
+ export declare function createRecordedEntry(input: {
59
+ url: string;
60
+ method: string;
61
+ requestContentType?: string | null;
62
+ requestBodyText?: string | null;
63
+ responseBody: unknown;
64
+ responseContentType?: string | null;
65
+ status?: number | null;
66
+ capturedAt?: number;
67
+ }): RecordedRequest;
68
+ /**
69
+ * Generates a full-capture interceptor that stores {url, method, status, body}
70
+ * for every JSON response. No URL pattern filter — captures everything.
71
+ */
72
+ export declare function generateFullCaptureInterceptorJs(): string;
73
+ /** Analyze recorded requests into read and write candidates. */
74
+ export declare function analyzeRecordedRequests(requests: RecordedRequest[]): {
75
+ candidates: RecordedCandidate[];
76
+ };
77
+ /** Build a minimal YAML candidate for replayable JSON write requests. */
78
+ export declare function buildWriteRecordedYaml(site: string, pageUrl: string, req: RecordedRequest, capName: string): {
79
+ name: string;
80
+ yaml: unknown;
81
+ };
82
+ /** Turn recorded requests into YAML-ready read and write candidates. */
83
+ export declare function generateRecordedCandidates(site: string, pageUrl: string, requests: RecordedRequest[]): GeneratedRecordedCandidate[];
35
84
  export interface RecordOptions {
36
85
  BrowserFactory: new () => {
37
86
  connect(o?: unknown): Promise<IPage>;
@@ -45,3 +94,4 @@ export interface RecordOptions {
45
94
  }
46
95
  export declare function recordSession(opts: RecordOptions): Promise<RecordResult>;
47
96
  export declare function renderRecordSummary(result: RecordResult): string;
97
+ export {};
package/dist/record.js CHANGED
@@ -19,12 +19,83 @@ import yaml from 'js-yaml';
19
19
  import { sendCommand } from './browser/daemon-client.js';
20
20
  import { SEARCH_PARAMS, PAGINATION_PARAMS, FIELD_ROLES } from './constants.js';
21
21
  import { urlToPattern, findArrayPath, inferCapabilityName, inferStrategy, detectAuthFromContent, classifyQueryParams, } from './analysis.js';
22
+ /** Keep the stronger candidate when multiple recordings share one bucket. */
23
+ function preferRecordedCandidate(current, next) {
24
+ if (next.score > current.score)
25
+ return next;
26
+ if (next.score < current.score)
27
+ return current;
28
+ return next;
29
+ }
30
+ /** Apply shared endpoint score tweaks. */
31
+ function applyCommonEndpointScoreAdjustments(req, score) {
32
+ let adjusted = score;
33
+ if (req.url.includes('/api/'))
34
+ adjusted += 3;
35
+ if (req.url.match(/\/(track|log|analytics|beacon|pixel|stats|metric)/i))
36
+ adjusted -= 10;
37
+ if (req.url.match(/\/(ping|heartbeat|keep.?alive)/i))
38
+ adjusted -= 10;
39
+ return adjusted;
40
+ }
41
+ /** Build a candidate-level dedupe key. */
42
+ function getRecordedCandidateKey(candidate) {
43
+ return `${candidate.kind} ${getRecordedRequestKey(candidate.req)}`;
44
+ }
45
+ /** Build a request dedupe key from method and URL pattern. */
46
+ function getRecordedRequestKey(req) {
47
+ return `${req.method.toUpperCase()} ${urlToPattern(req.url)}`;
48
+ }
49
+ /** Deduplicate recorded requests by method and URL pattern. */
50
+ function dedupeRecordedRequests(requests) {
51
+ const deduped = new Map();
52
+ for (const req of requests) {
53
+ deduped.set(getRecordedRequestKey(req), req);
54
+ }
55
+ return [...deduped.values()];
56
+ }
57
+ /** Check whether a content type should be treated as JSON. */
58
+ function isJsonContentType(contentType) {
59
+ const normalized = contentType?.toLowerCase() ?? '';
60
+ return normalized.includes('application/json') || normalized.includes('+json');
61
+ }
62
+ /** Parse a captured request body only when the request advertises JSON. */
63
+ function parseJsonBodyText(contentType, raw) {
64
+ if (!isJsonContentType(contentType))
65
+ return null;
66
+ if (!raw || !raw.trim())
67
+ return null;
68
+ try {
69
+ return JSON.parse(raw);
70
+ }
71
+ catch {
72
+ return null;
73
+ }
74
+ }
75
+ /** Build one normalized recorded entry from captured request and response values. */
76
+ export function createRecordedEntry(input) {
77
+ const requestBody = parseJsonBodyText(input.requestContentType ?? null, input.requestBodyText ?? null);
78
+ const responseContentType = input.responseContentType ?? 'application/json';
79
+ return {
80
+ url: input.url,
81
+ method: input.method.toUpperCase(),
82
+ status: input.status ?? null,
83
+ requestContentType: input.requestContentType ?? null,
84
+ responseContentType,
85
+ requestBody,
86
+ responseBody: input.responseBody,
87
+ // Keep legacy fields in sync until the analyzer/template path is migrated.
88
+ contentType: responseContentType,
89
+ body: input.responseBody,
90
+ capturedAt: input.capturedAt ?? Date.now(),
91
+ };
92
+ }
22
93
  // ── Interceptor JS ─────────────────────────────────────────────────────────
23
94
  /**
24
95
  * Generates a full-capture interceptor that stores {url, method, status, body}
25
96
  * for every JSON response. No URL pattern filter — captures everything.
26
97
  */
27
- function generateFullCaptureInterceptorJs() {
98
+ export function generateFullCaptureInterceptorJs() {
28
99
  return `
29
100
  (() => {
30
101
  // Restore original fetch/XHR if previously patched, then re-patch (idempotent injection)
@@ -32,24 +103,47 @@ function generateFullCaptureInterceptorJs() {
32
103
  if (window.__opencli_orig_fetch) window.fetch = window.__opencli_orig_fetch;
33
104
  if (window.__opencli_orig_xhr_open) XMLHttpRequest.prototype.open = window.__opencli_orig_xhr_open;
34
105
  if (window.__opencli_orig_xhr_send) XMLHttpRequest.prototype.send = window.__opencli_orig_xhr_send;
106
+ if (window.__opencli_orig_xhr_set_request_header) XMLHttpRequest.prototype.setRequestHeader = window.__opencli_orig_xhr_set_request_header;
35
107
  window.__opencli_record_patched = false;
36
108
  }
37
109
  // Preserve existing capture buffer across re-injections
38
110
  window.__opencli_record = window.__opencli_record || [];
39
111
 
40
- const _push = (url, method, body) => {
112
+ const _tryParseJson = (contentType, raw) => {
113
+ try {
114
+ const normalized = String(contentType || '').toLowerCase();
115
+ if (!normalized.includes('application/json') && !normalized.includes('+json')) return null;
116
+ if (typeof raw !== 'string' || !raw.trim()) return null;
117
+ return JSON.parse(raw);
118
+ } catch {
119
+ return null;
120
+ }
121
+ };
122
+
123
+ const _push = (entry) => {
41
124
  try {
42
- // Only capture JSON-like responses
43
- if (typeof body !== 'object' || body === null) return;
44
- // Skip tiny/trivial responses (tracking pixels, empty acks)
45
- const keys = Object.keys(body);
46
- if (keys.length < 2) return;
125
+ const responseBody = entry.responseBody;
126
+ if (typeof responseBody !== 'object' || responseBody === null) return;
127
+ const isReplayableWrite = ['POST', 'PUT', 'PATCH'].includes(String(entry.method).toUpperCase())
128
+ && (() => {
129
+ const normalized = String(entry.requestContentType || '').toLowerCase();
130
+ return normalized.includes('application/json') || normalized.includes('+json');
131
+ })()
132
+ && entry.requestBody
133
+ && typeof entry.requestBody === 'object';
134
+ const keys = Object.keys(responseBody);
135
+ if (keys.length < 2 && !isReplayableWrite) return;
47
136
  window.__opencli_record.push({
48
- url: String(url),
49
- method: String(method).toUpperCase(),
137
+ url: String(entry.url),
138
+ method: String(entry.method).toUpperCase(),
50
139
  status: null,
51
- body,
52
- ts: Date.now(),
140
+ requestContentType: entry.requestContentType || null,
141
+ responseContentType: entry.responseContentType || 'application/json',
142
+ requestBody: entry.requestBody || null,
143
+ responseBody,
144
+ contentType: entry.responseContentType || 'application/json',
145
+ body: responseBody,
146
+ capturedAt: Date.now(),
53
147
  });
54
148
  } catch {}
55
149
  };
@@ -58,14 +152,53 @@ function generateFullCaptureInterceptorJs() {
58
152
  window.__opencli_orig_fetch = window.fetch;
59
153
  window.fetch = async function(...args) {
60
154
  const req = args[0];
155
+ const init = args[1] || {};
61
156
  const reqUrl = typeof req === 'string' ? req : (req instanceof Request ? req.url : String(req));
62
- const method = (args[1]?.method || (req instanceof Request ? req.method : 'GET') || 'GET');
157
+ const method = (init?.method || (req instanceof Request ? req.method : 'GET') || 'GET');
158
+ const requestContentType = (() => {
159
+ if (init?.headers) {
160
+ try {
161
+ const headers = new Headers(init.headers);
162
+ const value = headers.get('content-type');
163
+ if (value) return value;
164
+ } catch {}
165
+ }
166
+ if (req instanceof Request) {
167
+ return req.headers.get('content-type');
168
+ }
169
+ return null;
170
+ })();
171
+ const requestBodyText = (() => {
172
+ if (typeof init?.body === 'string') return init.body;
173
+ return null;
174
+ })();
175
+ const shouldReadRequestBodyFromRequest = req instanceof Request
176
+ && !requestBodyText
177
+ && ['POST', 'PUT', 'PATCH'].includes(String(method).toUpperCase())
178
+ && (() => {
179
+ const normalized = String(requestContentType || '').toLowerCase();
180
+ return normalized.includes('application/json') || normalized.includes('+json');
181
+ })();
182
+ let requestBodyTextFromRequest = null;
183
+ if (shouldReadRequestBodyFromRequest) {
184
+ try {
185
+ requestBodyTextFromRequest = await req.clone().text();
186
+ } catch {}
187
+ }
188
+ const requestBody = _tryParseJson(requestContentType, requestBodyText || requestBodyTextFromRequest);
63
189
  const res = await window.__opencli_orig_fetch.apply(this, args);
64
190
  const ct = res.headers.get('content-type') || '';
65
191
  if (ct.includes('json')) {
66
192
  try {
67
- const body = await res.clone().json();
68
- _push(reqUrl, method, body);
193
+ const responseBody = await res.clone().json();
194
+ _push({
195
+ url: reqUrl,
196
+ method,
197
+ requestContentType,
198
+ requestBody,
199
+ responseContentType: ct,
200
+ responseBody,
201
+ });
69
202
  } catch {}
70
203
  }
71
204
  return res;
@@ -75,20 +208,38 @@ function generateFullCaptureInterceptorJs() {
75
208
  const _XHR = XMLHttpRequest.prototype;
76
209
  window.__opencli_orig_xhr_open = _XHR.open;
77
210
  window.__opencli_orig_xhr_send = _XHR.send;
211
+ window.__opencli_orig_xhr_set_request_header = _XHR.setRequestHeader;
78
212
  _XHR.open = function(method, url) {
79
213
  this.__rec_url = String(url);
80
214
  this.__rec_method = String(method);
215
+ this.__rec_request_content_type = null;
81
216
  this.__rec_listener_added = false; // reset per open() call
82
217
  return window.__opencli_orig_xhr_open.apply(this, arguments);
83
218
  };
219
+ _XHR.setRequestHeader = function(name, value) {
220
+ if (String(name).toLowerCase() === 'content-type') {
221
+ this.__rec_request_content_type = String(value);
222
+ }
223
+ return window.__opencli_orig_xhr_set_request_header.apply(this, arguments);
224
+ };
84
225
  _XHR.send = function() {
226
+ const requestBody = _tryParseJson(this.__rec_request_content_type, typeof arguments[0] === 'string' ? arguments[0] : null);
85
227
  // Guard: only add one listener per XHR instance to prevent duplicate captures
86
228
  if (!this.__rec_listener_added) {
87
229
  this.__rec_listener_added = true;
88
230
  this.addEventListener('load', function() {
89
231
  const ct = this.getResponseHeader?.('content-type') || '';
90
232
  if (ct.includes('json')) {
91
- try { _push(this.__rec_url, this.__rec_method || 'GET', JSON.parse(this.responseText)); } catch {}
233
+ try {
234
+ _push({
235
+ url: this.__rec_url,
236
+ method: this.__rec_method || 'GET',
237
+ requestContentType: this.__rec_request_content_type,
238
+ requestBody,
239
+ responseContentType: ct,
240
+ responseBody: JSON.parse(this.responseText),
241
+ });
242
+ } catch {}
92
243
  }
93
244
  });
94
245
  }
@@ -126,14 +277,41 @@ function scoreRequest(req, arrayResult) {
126
277
  }
127
278
  }
128
279
  }
129
- if (req.url.includes('/api/'))
130
- s += 3;
131
- // Penalize likely tracking / analytics endpoints
132
- if (req.url.match(/\/(track|log|analytics|beacon|pixel|stats|metric)/i))
133
- s -= 10;
134
- if (req.url.match(/\/(ping|heartbeat|keep.?alive)/i))
135
- s -= 10;
136
- return s;
280
+ return applyCommonEndpointScoreAdjustments(req, s);
281
+ }
282
+ /** Check whether one recorded request is safe to treat as a write candidate. */
283
+ function isWriteCandidate(req) {
284
+ return ['POST', 'PUT', 'PATCH'].includes(req.method)
285
+ && isJsonContentType(req.requestContentType)
286
+ && !!req.requestBody
287
+ && typeof req.requestBody === 'object'
288
+ && !Array.isArray(req.requestBody)
289
+ && !!req.responseBody
290
+ && typeof req.responseBody === 'object'
291
+ && !Array.isArray(req.responseBody);
292
+ }
293
+ /** Score replayable write requests while keeping tracking and heartbeat traffic suppressed. */
294
+ function scoreWriteRequest(req) {
295
+ return applyCommonEndpointScoreAdjustments(req, 6);
296
+ }
297
+ /** Analyze recorded requests into read and write candidates. */
298
+ export function analyzeRecordedRequests(requests) {
299
+ const candidates = [];
300
+ for (const req of requests) {
301
+ const arrayResult = findArrayPath(req.responseBody);
302
+ if (isWriteCandidate(req)) {
303
+ const score = scoreWriteRequest(req);
304
+ if (score > 0)
305
+ candidates.push({ kind: 'write', req, score, arrayResult: null });
306
+ continue;
307
+ }
308
+ if (arrayResult) {
309
+ const score = scoreRequest(req, arrayResult);
310
+ if (score > 0)
311
+ candidates.push({ kind: 'read', req, score, arrayResult });
312
+ }
313
+ }
314
+ return { candidates };
137
315
  }
138
316
  // ── YAML generation ────────────────────────────────────────────────────────
139
317
  function buildRecordedYaml(site, pageUrl, req, capName, arrayResult, authIndicators) {
@@ -177,7 +355,7 @@ function buildRecordedYaml(site, pageUrl, req, capName, arrayResult, authIndicat
177
355
  if (hasSearch) {
178
356
  for (const p of SEARCH_PARAMS) {
179
357
  if (u.searchParams.has(p)) {
180
- u.searchParams.set(p, '{{args.keyword}}');
358
+ u.searchParams.set(p, '${{ args.keyword }}');
181
359
  break;
182
360
  }
183
361
  }
@@ -185,12 +363,17 @@ function buildRecordedYaml(site, pageUrl, req, capName, arrayResult, authIndicat
185
363
  if (hasPage) {
186
364
  for (const p of PAGINATION_PARAMS) {
187
365
  if (u.searchParams.has(p)) {
188
- u.searchParams.set(p, '{{args.page | default(1)}}');
366
+ u.searchParams.set(p, '${{ args.page | default(1) }}');
189
367
  break;
190
368
  }
191
369
  }
192
370
  }
193
371
  fetchUrl = u.toString();
372
+ fetchUrl = fetchUrl
373
+ .replaceAll(encodeURIComponent('${{ args.keyword }}'), '${{ args.keyword }}')
374
+ .replaceAll('%24%7B%7B+args.keyword+%7D%7D', '${{ args.keyword }}')
375
+ .replaceAll(encodeURIComponent('${{ args.page | default(1) }}'), '${{ args.page | default(1) }}');
376
+ fetchUrl = fetchUrl.replaceAll('%24%7B%7B+args.page+%7C+default%281%29+%7D%7D', '${{ args.page | default(1) }}');
194
377
  }
195
378
  catch { }
196
379
  // When itemPath is empty, the array IS the response root; otherwise chain with ?.
@@ -234,6 +417,77 @@ function buildRecordedYaml(site, pageUrl, req, capName, arrayResult, authIndicat
234
417
  },
235
418
  };
236
419
  }
420
+ /** Build a minimal YAML candidate for replayable JSON write requests. */
421
+ export function buildWriteRecordedYaml(site, pageUrl, req, capName) {
422
+ const responseColumns = req.responseBody && typeof req.responseBody === 'object' && !Array.isArray(req.responseBody)
423
+ ? Object.keys(req.responseBody).slice(0, 6)
424
+ : ['ok'];
425
+ const evaluateScript = [
426
+ '(async () => {',
427
+ ` const res = await fetch(${JSON.stringify(req.url)}, {`,
428
+ ` method: ${JSON.stringify(req.method)},`,
429
+ ` credentials: 'include',`,
430
+ ` headers: { 'content-type': ${JSON.stringify(req.requestContentType ?? 'application/json')} },`,
431
+ ` body: JSON.stringify(${JSON.stringify(req.requestBody)}),`,
432
+ ' });',
433
+ ' return await res.json();',
434
+ '})()',
435
+ ].join('\n');
436
+ return {
437
+ name: capName,
438
+ yaml: {
439
+ site,
440
+ name: capName,
441
+ description: `${site} ${capName} (recorded write)`,
442
+ domain: (() => { try {
443
+ return new URL(pageUrl).hostname;
444
+ }
445
+ catch {
446
+ return '';
447
+ } })(),
448
+ strategy: 'cookie',
449
+ browser: true,
450
+ args: {},
451
+ pipeline: [
452
+ { navigate: pageUrl },
453
+ { evaluate: evaluateScript },
454
+ ],
455
+ columns: responseColumns.length ? responseColumns : ['ok'],
456
+ },
457
+ };
458
+ }
459
+ /** Turn recorded requests into YAML-ready read and write candidates. */
460
+ export function generateRecordedCandidates(site, pageUrl, requests) {
461
+ const analysis = analyzeRecordedRequests(dedupeRecordedRequests(requests));
462
+ const deduped = new Map();
463
+ for (const candidate of analysis.candidates) {
464
+ const key = getRecordedCandidateKey(candidate);
465
+ const current = deduped.get(key);
466
+ deduped.set(key, current ? preferRecordedCandidate(current, candidate) : candidate);
467
+ }
468
+ const selected = [...deduped.values()]
469
+ .filter((candidate) => candidate.kind === 'read' ? candidate.score >= 8 : candidate.score >= 6)
470
+ .sort((a, b) => b.score - a.score)
471
+ .slice(0, 5);
472
+ const usedNames = new Set();
473
+ return selected.map((candidate) => {
474
+ let capName = inferCapabilityName(candidate.req.url);
475
+ if (usedNames.has(capName))
476
+ capName = `${capName}_${usedNames.size + 1}`;
477
+ usedNames.add(capName);
478
+ const authIndicators = detectAuthFromContent(candidate.req.url, candidate.req.responseBody);
479
+ const strategy = candidate.kind === 'write' ? 'cookie' : inferStrategy(authIndicators);
480
+ const yamlCandidate = candidate.kind === 'write'
481
+ ? buildWriteRecordedYaml(site, pageUrl, candidate.req, capName)
482
+ : buildRecordedYaml(site, pageUrl, candidate.req, capName, candidate.arrayResult, authIndicators);
483
+ return {
484
+ kind: candidate.kind,
485
+ name: yamlCandidate.name,
486
+ strategy,
487
+ yaml: yamlCandidate.yaml,
488
+ };
489
+ });
490
+ }
237
491
  export async function recordSession(opts) {
238
492
  const pollMs = opts.pollMs ?? 2000;
239
493
  const timeoutMs = opts.timeoutMs ?? 60_000;
@@ -374,52 +628,39 @@ function analyzeAndWrite(site, pageUrl, requests, outDir) {
374
628
  console.log(chalk.yellow(' No API requests captured.'));
375
629
  return { site, url: pageUrl, requests: [], outDir: targetDir, candidateCount: 0, candidates: [] };
376
630
  }
377
- // Deduplicate by pattern
378
- const seen = new Map();
379
- for (const req of requests) {
380
- const pattern = urlToPattern(req.url);
381
- if (!seen.has(pattern))
382
- seen.set(pattern, req);
383
- }
384
- const scored = [];
385
- for (const [pattern, req] of seen) {
386
- const arrayResult = findArrayPath(req.body);
387
- const authIndicators = detectAuthFromContent(req.url, req.body);
388
- const score = scoreRequest(req, arrayResult);
389
- if (score > 0) {
390
- scored.push({ req, pattern, arrayResult, authIndicators, score });
391
- }
392
- }
393
- scored.sort((a, b) => b.score - a.score);
631
+ // Score and rank deduplicated requests for console output and candidate generation.
632
+ const analysisRequests = dedupeRecordedRequests(requests);
633
+ const analysis = analyzeRecordedRequests(analysisRequests);
394
634
  // Save raw captured data
395
635
  fs.writeFileSync(path.join(targetDir, 'captured.json'), JSON.stringify({ site, url: pageUrl, capturedAt: new Date().toISOString(), requests }, null, 2));
396
636
  // Generate candidate YAMLs (top 5)
397
637
  const candidates = [];
398
638
  const usedNames = new Set();
399
639
  console.log(chalk.bold('\n Captured endpoints (scored):\n'));
400
- for (const entry of scored.slice(0, 8)) {
640
+ for (const entry of analysis.candidates.sort((a, b) => b.score - a.score).slice(0, 8)) {
401
641
  const itemCount = entry.arrayResult?.items.length ?? 0;
402
- const strategy = inferStrategy(entry.authIndicators);
642
+ const strategy = entry.kind === 'write'
643
+ ? 'cookie'
644
+ : inferStrategy(detectAuthFromContent(entry.req.url, entry.req.responseBody));
403
645
  const marker = entry.score >= 15 ? chalk.green('★') : entry.score >= 8 ? chalk.yellow('◆') : chalk.dim('·');
404
- console.log(` ${marker} ${chalk.white(entry.pattern)}` +
646
+ console.log(` ${marker} ${chalk.white(urlToPattern(entry.req.url))}` +
405
647
  chalk.dim(` [${strategy}]`) +
406
- (itemCount ? chalk.cyan(` ${itemCount} items`) : ''));
648
+ (entry.kind === 'write'
649
+ ? chalk.magenta(' ← write')
650
+ : itemCount ? chalk.cyan(` ← ${itemCount} items`) : ''));
407
651
  }
408
652
  console.log();
409
- const topCandidates = scored.filter(e => e.arrayResult && e.score >= 8).slice(0, 5);
653
+ const topCandidates = generateRecordedCandidates(site, pageUrl, analysisRequests);
410
654
  const candidatesDir = path.join(targetDir, 'candidates');
411
655
  fs.mkdirSync(candidatesDir, { recursive: true });
412
656
  for (const entry of topCandidates) {
413
- let capName = inferCapabilityName(entry.req.url);
414
- if (usedNames.has(capName))
415
- capName = `${capName}_${usedNames.size + 1}`;
416
- usedNames.add(capName);
417
- const strategy = inferStrategy(entry.authIndicators);
418
- const candidate = buildRecordedYaml(site, pageUrl, entry.req, capName, entry.arrayResult, entry.authIndicators);
419
- const filePath = path.join(candidatesDir, `${capName}.yaml`);
420
- fs.writeFileSync(filePath, yaml.dump(candidate.yaml, { sortKeys: false, lineWidth: 120 }));
421
- candidates.push({ name: capName, path: filePath, strategy });
422
- console.log(chalk.green(` ✓ Generated: ${chalk.bold(capName)}.yaml [${strategy}]`));
657
+ if (usedNames.has(entry.name))
658
+ continue;
659
+ usedNames.add(entry.name);
660
+ const filePath = path.join(candidatesDir, `${entry.name}.yaml`);
661
+ fs.writeFileSync(filePath, yaml.dump(entry.yaml, { sortKeys: false, lineWidth: 120 }));
662
+ candidates.push({ name: entry.name, path: filePath, strategy: entry.strategy });
663
+ console.log(chalk.green(` ✓ Generated: ${chalk.bold(entry.name)}.yaml [${entry.strategy}]`));
423
664
  console.log(chalk.dim(` → ${filePath}`));
424
665
  }
425
666
  if (candidates.length === 0) {
@@ -0,0 +1 @@
1
+ export {};