@jackwener/opencli 1.5.8 → 1.6.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +42 -0
- package/README.md +35 -1
- package/README.zh-CN.md +17 -1
- package/SKILL.md +31 -851
- package/autoresearch/baseline-browse.txt +1 -0
- package/autoresearch/baseline-skill.txt +1 -0
- package/autoresearch/browse-tasks.json +688 -0
- package/autoresearch/eval-browse.ts +185 -0
- package/autoresearch/eval-skill.ts +248 -0
- package/autoresearch/run-browse.sh +9 -0
- package/autoresearch/run-skill.sh +9 -0
- package/dist/browser/base-page.d.ts +48 -0
- package/dist/browser/base-page.js +160 -0
- package/dist/browser/cdp.js +4 -106
- package/dist/browser/daemon-client.d.ts +20 -7
- package/dist/browser/daemon-client.js +39 -39
- package/dist/browser/daemon-client.test.js +77 -0
- package/dist/browser/discover.d.ts +1 -4
- package/dist/browser/discover.js +9 -23
- package/dist/browser/errors.d.ts +4 -0
- package/dist/browser/errors.js +20 -0
- package/dist/browser/index.d.ts +1 -1
- package/dist/browser/index.js +1 -1
- package/dist/browser/page.d.ts +10 -35
- package/dist/browser/page.js +55 -187
- package/dist/browser/tabs.js +5 -5
- package/dist/browser.test.js +15 -15
- package/dist/cli-manifest.json +294 -22
- package/dist/cli.js +392 -0
- package/dist/clis/amazon/bestsellers.d.ts +21 -0
- package/dist/clis/amazon/bestsellers.js +130 -0
- package/dist/clis/amazon/bestsellers.test.js +20 -0
- package/dist/clis/amazon/discussion.d.ts +20 -0
- package/dist/clis/amazon/discussion.js +91 -0
- package/dist/clis/amazon/discussion.test.d.ts +1 -0
- package/dist/clis/amazon/discussion.test.js +36 -0
- package/dist/clis/amazon/offer.d.ts +23 -0
- package/dist/clis/amazon/offer.js +140 -0
- package/dist/clis/amazon/offer.test.d.ts +1 -0
- package/dist/clis/amazon/offer.test.js +29 -0
- package/dist/clis/amazon/product.d.ts +18 -0
- package/dist/clis/amazon/product.js +92 -0
- package/dist/clis/amazon/product.test.d.ts +1 -0
- package/dist/clis/amazon/product.test.js +24 -0
- package/dist/clis/amazon/search.d.ts +18 -0
- package/dist/clis/amazon/search.js +87 -0
- package/dist/clis/amazon/search.test.d.ts +1 -0
- package/dist/clis/amazon/search.test.js +22 -0
- package/dist/clis/amazon/shared.d.ts +64 -0
- package/dist/clis/amazon/shared.js +255 -0
- package/dist/clis/amazon/shared.test.d.ts +1 -0
- package/dist/clis/amazon/shared.test.js +33 -0
- package/dist/clis/gemini/ask.d.ts +1 -0
- package/dist/clis/gemini/ask.js +40 -0
- package/dist/clis/gemini/image.d.ts +1 -0
- package/dist/clis/gemini/image.js +105 -0
- package/dist/clis/gemini/new.d.ts +1 -0
- package/dist/clis/gemini/new.js +20 -0
- package/dist/clis/gemini/utils.d.ts +34 -0
- package/dist/clis/gemini/utils.js +463 -0
- package/dist/clis/gemini/utils.test.d.ts +1 -0
- package/dist/clis/gemini/utils.test.js +31 -0
- package/dist/clis/notebooklm/compat.test.d.ts +1 -1
- package/dist/clis/notebooklm/compat.test.js +3 -3
- package/dist/clis/notebooklm/current.js +2 -3
- package/dist/clis/notebooklm/get.js +2 -3
- package/dist/clis/notebooklm/history.js +2 -3
- package/dist/clis/notebooklm/note-list.js +2 -3
- package/dist/clis/notebooklm/notes-get.js +2 -3
- package/dist/clis/notebooklm/open.d.ts +1 -0
- package/dist/clis/notebooklm/open.js +41 -0
- package/dist/clis/notebooklm/open.test.d.ts +1 -0
- package/dist/clis/notebooklm/open.test.js +63 -0
- package/dist/clis/notebooklm/source-fulltext.js +2 -3
- package/dist/clis/notebooklm/source-get.js +2 -3
- package/dist/clis/notebooklm/source-guide.js +2 -3
- package/dist/clis/notebooklm/source-list.js +2 -3
- package/dist/clis/notebooklm/status.js +1 -2
- package/dist/clis/notebooklm/summary.js +2 -3
- package/dist/clis/notebooklm/utils.d.ts +2 -1
- package/dist/clis/notebooklm/utils.js +20 -21
- package/dist/clis/twitter/article.js +28 -1
- package/dist/clis/xiaohongshu/creator-note-detail.test.js +11 -11
- package/dist/clis/xiaohongshu/creator-notes-summary.test.js +6 -6
- package/dist/clis/xiaohongshu/creator-notes.test.js +22 -22
- package/dist/clis/xiaohongshu/note.js +11 -0
- package/dist/clis/xiaohongshu/note.test.js +49 -0
- package/dist/commanderAdapter.js +7 -4
- package/dist/commanderAdapter.test.js +76 -0
- package/dist/commands/daemon.js +8 -47
- package/dist/commands/daemon.test.js +45 -70
- package/dist/discovery.js +27 -0
- package/dist/doctor.d.ts +1 -2
- package/dist/doctor.js +7 -8
- package/dist/explore.js +1 -1
- package/dist/output.js +28 -0
- package/dist/output.test.js +15 -0
- package/dist/pipeline/executor.js +2 -7
- package/dist/pipeline/steps/browser.js +1 -1
- package/dist/pipeline/template.js +25 -3
- package/dist/record.d.ts +50 -0
- package/dist/record.js +298 -57
- package/dist/record.test.d.ts +1 -0
- package/dist/record.test.js +293 -0
- package/dist/registry.d.ts +2 -0
- package/dist/registry.js +1 -0
- package/dist/registry.test.js +10 -0
- package/dist/runtime.js +3 -3
- package/dist/snapshotFormatter.d.ts +1 -1
- package/dist/snapshotFormatter.js +4 -4
- package/dist/snapshotFormatter.test.d.ts +1 -1
- package/dist/snapshotFormatter.test.js +2 -2
- package/dist/types.d.ts +11 -1
- package/dist/types.js +1 -1
- package/docs/.vitepress/config.mts +2 -0
- package/docs/adapters/browser/amazon.md +53 -0
- package/docs/adapters/browser/gemini.md +72 -0
- package/docs/adapters/browser/notebooklm.md +5 -5
- package/docs/adapters/index.md +3 -1
- package/docs/guide/getting-started.md +21 -0
- package/docs/superpowers/specs/2026-04-02-browse-skill-testing-design.md +144 -0
- package/docs/zh/guide/getting-started.md +21 -0
- package/extension/package-lock.json +2 -2
- package/extension/src/background.test.ts +7 -163
- package/extension/src/background.ts +58 -161
- package/extension/src/cdp.ts +77 -124
- package/extension/src/protocol.ts +5 -5
- package/package.json +1 -1
- package/skills/opencli-explorer/SKILL.md +853 -0
- package/skills/opencli-oneshot/SKILL.md +222 -0
- package/skills/opencli-operate/SKILL.md +213 -0
- package/skills/opencli-usage/SKILL.md +152 -0
- package/skills/opencli-usage/browser.md +429 -0
- package/skills/opencli-usage/desktop.md +118 -0
- package/skills/opencli-usage/plugins.md +82 -0
- package/skills/opencli-usage/public-api.md +149 -0
- package/src/browser/base-page.ts +197 -0
- package/src/browser/cdp.ts +7 -131
- package/src/browser/daemon-client.test.ts +103 -0
- package/src/browser/daemon-client.ts +55 -43
- package/src/browser/discover.ts +9 -21
- package/src/browser/errors.ts +22 -0
- package/src/browser/index.ts +1 -1
- package/src/browser/page.ts +57 -209
- package/src/browser/tabs.ts +5 -5
- package/src/browser.test.ts +15 -15
- package/src/cli.ts +392 -0
- package/src/clis/amazon/bestsellers.test.ts +22 -0
- package/src/clis/amazon/bestsellers.ts +180 -0
- package/src/clis/amazon/discussion.test.ts +38 -0
- package/src/clis/amazon/discussion.ts +131 -0
- package/src/clis/amazon/offer.test.ts +35 -0
- package/src/clis/amazon/offer.ts +185 -0
- package/src/clis/amazon/product.test.ts +26 -0
- package/src/clis/amazon/product.ts +131 -0
- package/src/clis/amazon/search.test.ts +24 -0
- package/src/clis/amazon/search.ts +128 -0
- package/src/clis/amazon/shared.test.ts +37 -0
- package/src/clis/amazon/shared.ts +316 -0
- package/src/clis/gemini/ask.ts +46 -0
- package/src/clis/gemini/image.ts +115 -0
- package/src/clis/gemini/new.ts +22 -0
- package/src/clis/gemini/utils.test.ts +36 -0
- package/src/clis/gemini/utils.ts +523 -0
- package/src/clis/notebooklm/compat.test.ts +3 -3
- package/src/clis/notebooklm/current.ts +2 -3
- package/src/clis/notebooklm/get.ts +1 -3
- package/src/clis/notebooklm/history.ts +1 -3
- package/src/clis/notebooklm/note-list.ts +1 -3
- package/src/clis/notebooklm/notes-get.ts +1 -3
- package/src/clis/notebooklm/open.test.ts +78 -0
- package/src/clis/notebooklm/open.ts +61 -0
- package/src/clis/notebooklm/source-fulltext.ts +1 -3
- package/src/clis/notebooklm/source-get.ts +1 -3
- package/src/clis/notebooklm/source-guide.ts +1 -3
- package/src/clis/notebooklm/source-list.ts +1 -3
- package/src/clis/notebooklm/status.ts +1 -2
- package/src/clis/notebooklm/summary.ts +1 -3
- package/src/clis/notebooklm/utils.ts +29 -20
- package/src/clis/twitter/article.ts +31 -1
- package/src/clis/xiaohongshu/creator-note-detail.test.ts +11 -11
- package/src/clis/xiaohongshu/creator-notes-summary.test.ts +6 -6
- package/src/clis/xiaohongshu/creator-notes.test.ts +22 -22
- package/src/clis/xiaohongshu/note.test.ts +51 -0
- package/src/clis/xiaohongshu/note.ts +18 -0
- package/src/commanderAdapter.test.ts +109 -0
- package/src/commanderAdapter.ts +8 -4
- package/src/commands/daemon.test.ts +50 -84
- package/src/commands/daemon.ts +8 -56
- package/src/discovery.ts +22 -0
- package/src/doctor.ts +8 -9
- package/src/explore.ts +1 -1
- package/src/output.test.ts +17 -0
- package/src/output.ts +27 -0
- package/src/pipeline/executor.ts +2 -7
- package/src/pipeline/steps/browser.ts +1 -1
- package/src/pipeline/template.ts +27 -4
- package/src/record.test.ts +362 -0
- package/src/record.ts +341 -62
- package/src/registry.test.ts +12 -0
- package/src/registry.ts +3 -0
- package/src/runtime.ts +3 -3
- package/src/snapshotFormatter.test.ts +2 -2
- package/src/snapshotFormatter.ts +4 -4
- package/src/types.ts +11 -1
- package/.agents/skills/cross-project-adapter-migration/SKILL.md +0 -249
- package/.agents/workflows/cross-project-adapter-migration.md +0 -54
- package/dist/clis/notebooklm/bind-current.js +0 -29
- package/dist/clis/notebooklm/bind-current.test.d.ts +0 -1
- package/dist/clis/notebooklm/bind-current.test.js +0 -35
- package/dist/clis/notebooklm/binding.test.js +0 -44
- package/extension/dist/background.js +0 -819
- package/src/clis/notebooklm/bind-current.test.ts +0 -43
- package/src/clis/notebooklm/bind-current.ts +0 -36
- package/src/clis/notebooklm/binding.test.ts +0 -53
- /package/dist/browser/{mcp.d.ts → bridge.d.ts} +0 -0
- /package/dist/browser/{mcp.js → bridge.js} +0 -0
- /package/dist/{clis/notebooklm/bind-current.d.ts → browser/daemon-client.test.d.ts} +0 -0
- /package/dist/clis/{notebooklm/binding.test.d.ts → amazon/bestsellers.test.d.ts} +0 -0
- /package/src/browser/{mcp.ts → bridge.ts} +0 -0
|
@@ -62,7 +62,7 @@ export async function stepSnapshot(page, params, _data, _args) {
|
|
|
62
62
|
export async function stepEvaluate(page, params, data, args) {
|
|
63
63
|
const js = String(render(params, { args, data }));
|
|
64
64
|
let result = await page.evaluate(js);
|
|
65
|
-
//
|
|
65
|
+
// Browser may return JSON as a string — auto-parse it
|
|
66
66
|
if (typeof result === 'string') {
|
|
67
67
|
const trimmed = result.trim();
|
|
68
68
|
if ((trimmed.startsWith('[') && trimmed.endsWith(']')) || (trimmed.startsWith('{') && trimmed.endsWith('}'))) {
|
|
@@ -179,6 +179,10 @@ export function resolvePath(pathStr, ctx) {
|
|
|
179
179
|
/**
|
|
180
180
|
* Evaluate arbitrary JS expressions as a last-resort fallback.
|
|
181
181
|
* Runs inside a `node:vm` sandbox with dynamic code generation disabled.
|
|
182
|
+
*
|
|
183
|
+
* Compiled functions are cached by expression string to avoid re-creating
|
|
184
|
+
* VM contexts on every invocation — critical for loops where the same
|
|
185
|
+
* expression is evaluated hundreds of times.
|
|
182
186
|
*/
|
|
183
187
|
const FORBIDDEN_EXPR_PATTERNS = /\b(constructor|__proto__|prototype|globalThis|process|require|import|eval)\b/;
|
|
184
188
|
/**
|
|
@@ -197,6 +201,23 @@ function sanitizeContext(obj) {
|
|
|
197
201
|
return {};
|
|
198
202
|
}
|
|
199
203
|
}
|
|
204
|
+
/** LRU-bounded cache for compiled VM scripts — prevents unbounded memory growth. */
|
|
205
|
+
const MAX_VM_CACHE_SIZE = 256;
|
|
206
|
+
const _vmCache = new Map();
|
|
207
|
+
function getOrCompileScript(expr) {
|
|
208
|
+
let script = _vmCache.get(expr);
|
|
209
|
+
if (script)
|
|
210
|
+
return script;
|
|
211
|
+
// Evict oldest entry when cache is full
|
|
212
|
+
if (_vmCache.size >= MAX_VM_CACHE_SIZE) {
|
|
213
|
+
const firstKey = _vmCache.keys().next().value;
|
|
214
|
+
if (firstKey !== undefined)
|
|
215
|
+
_vmCache.delete(firstKey);
|
|
216
|
+
}
|
|
217
|
+
script = new vm.Script(`(${expr})`);
|
|
218
|
+
_vmCache.set(expr, script);
|
|
219
|
+
return script;
|
|
220
|
+
}
|
|
200
221
|
function evalJsExpr(expr, ctx) {
|
|
201
222
|
// Guard against absurdly long expressions that could indicate injection.
|
|
202
223
|
if (expr.length > 2000)
|
|
@@ -209,7 +230,8 @@ function evalJsExpr(expr, ctx) {
|
|
|
209
230
|
const data = sanitizeContext(ctx.data);
|
|
210
231
|
const index = ctx.index ?? 0;
|
|
211
232
|
try {
|
|
212
|
-
|
|
233
|
+
const script = getOrCompileScript(expr);
|
|
234
|
+
const sandbox = vm.createContext({
|
|
213
235
|
args,
|
|
214
236
|
item,
|
|
215
237
|
data,
|
|
@@ -224,12 +246,12 @@ function evalJsExpr(expr, ctx) {
|
|
|
224
246
|
Array,
|
|
225
247
|
Date,
|
|
226
248
|
}, {
|
|
227
|
-
|
|
228
|
-
contextCodeGeneration: {
|
|
249
|
+
codeGeneration: {
|
|
229
250
|
strings: false,
|
|
230
251
|
wasm: false,
|
|
231
252
|
},
|
|
232
253
|
});
|
|
254
|
+
return script.runInContext(sandbox, { timeout: 50 });
|
|
233
255
|
}
|
|
234
256
|
catch {
|
|
235
257
|
return undefined;
|
package/dist/record.d.ts
CHANGED
|
@@ -12,10 +12,19 @@
|
|
|
12
12
|
* Uses existing exec + navigate actions only.
|
|
13
13
|
*/
|
|
14
14
|
import type { IPage } from './types.js';
|
|
15
|
+
import { findArrayPath } from './analysis.js';
|
|
15
16
|
export interface RecordedRequest {
|
|
16
17
|
url: string;
|
|
17
18
|
method: string;
|
|
18
19
|
status: number | null;
|
|
20
|
+
/** Request content type captured at record time, if available. */
|
|
21
|
+
requestContentType: string | null;
|
|
22
|
+
/** Response content type captured at record time, if available. */
|
|
23
|
+
responseContentType: string | null;
|
|
24
|
+
/** Parsed JSON request body for replayable write requests. */
|
|
25
|
+
requestBody: unknown;
|
|
26
|
+
/** Parsed JSON response body captured from the network call. */
|
|
27
|
+
responseBody: unknown;
|
|
19
28
|
contentType: string;
|
|
20
29
|
body: unknown;
|
|
21
30
|
capturedAt: number;
|
|
@@ -32,6 +41,46 @@ export interface RecordResult {
|
|
|
32
41
|
strategy: string;
|
|
33
42
|
}>;
|
|
34
43
|
}
|
|
44
|
+
type RecordedCandidateKind = 'read' | 'write';
|
|
45
|
+
export interface RecordedCandidate {
|
|
46
|
+
kind: RecordedCandidateKind;
|
|
47
|
+
req: RecordedRequest;
|
|
48
|
+
score: number;
|
|
49
|
+
arrayResult: ReturnType<typeof findArrayPath> | null;
|
|
50
|
+
}
|
|
51
|
+
interface GeneratedRecordedCandidate {
|
|
52
|
+
kind: RecordedCandidateKind;
|
|
53
|
+
name: string;
|
|
54
|
+
strategy: string;
|
|
55
|
+
yaml: unknown;
|
|
56
|
+
}
|
|
57
|
+
/** Build one normalized recorded entry from captured request and response values. */
|
|
58
|
+
export declare function createRecordedEntry(input: {
|
|
59
|
+
url: string;
|
|
60
|
+
method: string;
|
|
61
|
+
requestContentType?: string | null;
|
|
62
|
+
requestBodyText?: string | null;
|
|
63
|
+
responseBody: unknown;
|
|
64
|
+
responseContentType?: string | null;
|
|
65
|
+
status?: number | null;
|
|
66
|
+
capturedAt?: number;
|
|
67
|
+
}): RecordedRequest;
|
|
68
|
+
/**
|
|
69
|
+
* Generates a full-capture interceptor that stores {url, method, status, body}
|
|
70
|
+
* for every JSON response. No URL pattern filter — captures everything.
|
|
71
|
+
*/
|
|
72
|
+
export declare function generateFullCaptureInterceptorJs(): string;
|
|
73
|
+
/** Analyze recorded requests into read and write candidates. */
|
|
74
|
+
export declare function analyzeRecordedRequests(requests: RecordedRequest[]): {
|
|
75
|
+
candidates: RecordedCandidate[];
|
|
76
|
+
};
|
|
77
|
+
/** Build a minimal YAML candidate for replayable JSON write requests. */
|
|
78
|
+
export declare function buildWriteRecordedYaml(site: string, pageUrl: string, req: RecordedRequest, capName: string): {
|
|
79
|
+
name: string;
|
|
80
|
+
yaml: unknown;
|
|
81
|
+
};
|
|
82
|
+
/** Turn recorded requests into YAML-ready read and write candidates. */
|
|
83
|
+
export declare function generateRecordedCandidates(site: string, pageUrl: string, requests: RecordedRequest[]): GeneratedRecordedCandidate[];
|
|
35
84
|
export interface RecordOptions {
|
|
36
85
|
BrowserFactory: new () => {
|
|
37
86
|
connect(o?: unknown): Promise<IPage>;
|
|
@@ -45,3 +94,4 @@ export interface RecordOptions {
|
|
|
45
94
|
}
|
|
46
95
|
export declare function recordSession(opts: RecordOptions): Promise<RecordResult>;
|
|
47
96
|
export declare function renderRecordSummary(result: RecordResult): string;
|
|
97
|
+
export {};
|
package/dist/record.js
CHANGED
|
@@ -19,12 +19,83 @@ import yaml from 'js-yaml';
|
|
|
19
19
|
import { sendCommand } from './browser/daemon-client.js';
|
|
20
20
|
import { SEARCH_PARAMS, PAGINATION_PARAMS, FIELD_ROLES } from './constants.js';
|
|
21
21
|
import { urlToPattern, findArrayPath, inferCapabilityName, inferStrategy, detectAuthFromContent, classifyQueryParams, } from './analysis.js';
|
|
22
|
+
/** Keep the stronger candidate when multiple recordings share one bucket. */
|
|
23
|
+
function preferRecordedCandidate(current, next) {
|
|
24
|
+
if (next.score > current.score)
|
|
25
|
+
return next;
|
|
26
|
+
if (next.score < current.score)
|
|
27
|
+
return current;
|
|
28
|
+
return next;
|
|
29
|
+
}
|
|
30
|
+
/** Apply shared endpoint score tweaks. */
|
|
31
|
+
function applyCommonEndpointScoreAdjustments(req, score) {
|
|
32
|
+
let adjusted = score;
|
|
33
|
+
if (req.url.includes('/api/'))
|
|
34
|
+
adjusted += 3;
|
|
35
|
+
if (req.url.match(/\/(track|log|analytics|beacon|pixel|stats|metric)/i))
|
|
36
|
+
adjusted -= 10;
|
|
37
|
+
if (req.url.match(/\/(ping|heartbeat|keep.?alive)/i))
|
|
38
|
+
adjusted -= 10;
|
|
39
|
+
return adjusted;
|
|
40
|
+
}
|
|
41
|
+
/** Build a candidate-level dedupe key. */
|
|
42
|
+
function getRecordedCandidateKey(candidate) {
|
|
43
|
+
return `${candidate.kind} ${getRecordedRequestKey(candidate.req)}`;
|
|
44
|
+
}
|
|
45
|
+
/** Build a request dedupe key from method and URL pattern. */
|
|
46
|
+
function getRecordedRequestKey(req) {
|
|
47
|
+
return `${req.method.toUpperCase()} ${urlToPattern(req.url)}`;
|
|
48
|
+
}
|
|
49
|
+
/** Deduplicate recorded requests by method and URL pattern. */
|
|
50
|
+
function dedupeRecordedRequests(requests) {
|
|
51
|
+
const deduped = new Map();
|
|
52
|
+
for (const req of requests) {
|
|
53
|
+
deduped.set(getRecordedRequestKey(req), req);
|
|
54
|
+
}
|
|
55
|
+
return [...deduped.values()];
|
|
56
|
+
}
|
|
57
|
+
/** Check whether a content type should be treated as JSON. */
|
|
58
|
+
function isJsonContentType(contentType) {
|
|
59
|
+
const normalized = contentType?.toLowerCase() ?? '';
|
|
60
|
+
return normalized.includes('application/json') || normalized.includes('+json');
|
|
61
|
+
}
|
|
62
|
+
/** Parse a captured request body only when the request advertises JSON. */
|
|
63
|
+
function parseJsonBodyText(contentType, raw) {
|
|
64
|
+
if (!isJsonContentType(contentType))
|
|
65
|
+
return null;
|
|
66
|
+
if (!raw || !raw.trim())
|
|
67
|
+
return null;
|
|
68
|
+
try {
|
|
69
|
+
return JSON.parse(raw);
|
|
70
|
+
}
|
|
71
|
+
catch {
|
|
72
|
+
return null;
|
|
73
|
+
}
|
|
74
|
+
}
|
|
75
|
+
/** Build one normalized recorded entry from captured request and response values. */
|
|
76
|
+
export function createRecordedEntry(input) {
|
|
77
|
+
const requestBody = parseJsonBodyText(input.requestContentType ?? null, input.requestBodyText ?? null);
|
|
78
|
+
const responseContentType = input.responseContentType ?? 'application/json';
|
|
79
|
+
return {
|
|
80
|
+
url: input.url,
|
|
81
|
+
method: input.method.toUpperCase(),
|
|
82
|
+
status: input.status ?? null,
|
|
83
|
+
requestContentType: input.requestContentType ?? null,
|
|
84
|
+
responseContentType,
|
|
85
|
+
requestBody,
|
|
86
|
+
responseBody: input.responseBody,
|
|
87
|
+
// Keep legacy fields in sync until the analyzer/template path is migrated.
|
|
88
|
+
contentType: responseContentType,
|
|
89
|
+
body: input.responseBody,
|
|
90
|
+
capturedAt: input.capturedAt ?? Date.now(),
|
|
91
|
+
};
|
|
92
|
+
}
|
|
22
93
|
// ── Interceptor JS ─────────────────────────────────────────────────────────
|
|
23
94
|
/**
|
|
24
95
|
* Generates a full-capture interceptor that stores {url, method, status, body}
|
|
25
96
|
* for every JSON response. No URL pattern filter — captures everything.
|
|
26
97
|
*/
|
|
27
|
-
function generateFullCaptureInterceptorJs() {
|
|
98
|
+
export function generateFullCaptureInterceptorJs() {
|
|
28
99
|
return `
|
|
29
100
|
(() => {
|
|
30
101
|
// Restore original fetch/XHR if previously patched, then re-patch (idempotent injection)
|
|
@@ -32,24 +103,47 @@ function generateFullCaptureInterceptorJs() {
|
|
|
32
103
|
if (window.__opencli_orig_fetch) window.fetch = window.__opencli_orig_fetch;
|
|
33
104
|
if (window.__opencli_orig_xhr_open) XMLHttpRequest.prototype.open = window.__opencli_orig_xhr_open;
|
|
34
105
|
if (window.__opencli_orig_xhr_send) XMLHttpRequest.prototype.send = window.__opencli_orig_xhr_send;
|
|
106
|
+
if (window.__opencli_orig_xhr_set_request_header) XMLHttpRequest.prototype.setRequestHeader = window.__opencli_orig_xhr_set_request_header;
|
|
35
107
|
window.__opencli_record_patched = false;
|
|
36
108
|
}
|
|
37
109
|
// Preserve existing capture buffer across re-injections
|
|
38
110
|
window.__opencli_record = window.__opencli_record || [];
|
|
39
111
|
|
|
40
|
-
const
|
|
112
|
+
const _tryParseJson = (contentType, raw) => {
|
|
113
|
+
try {
|
|
114
|
+
const normalized = String(contentType || '').toLowerCase();
|
|
115
|
+
if (!normalized.includes('application/json') && !normalized.includes('+json')) return null;
|
|
116
|
+
if (typeof raw !== 'string' || !raw.trim()) return null;
|
|
117
|
+
return JSON.parse(raw);
|
|
118
|
+
} catch {
|
|
119
|
+
return null;
|
|
120
|
+
}
|
|
121
|
+
};
|
|
122
|
+
|
|
123
|
+
const _push = (entry) => {
|
|
41
124
|
try {
|
|
42
|
-
|
|
43
|
-
if (typeof
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
125
|
+
const responseBody = entry.responseBody;
|
|
126
|
+
if (typeof responseBody !== 'object' || responseBody === null) return;
|
|
127
|
+
const isReplayableWrite = ['POST', 'PUT', 'PATCH'].includes(String(entry.method).toUpperCase())
|
|
128
|
+
&& (() => {
|
|
129
|
+
const normalized = String(entry.requestContentType || '').toLowerCase();
|
|
130
|
+
return normalized.includes('application/json') || normalized.includes('+json');
|
|
131
|
+
})()
|
|
132
|
+
&& entry.requestBody
|
|
133
|
+
&& typeof entry.requestBody === 'object';
|
|
134
|
+
const keys = Object.keys(responseBody);
|
|
135
|
+
if (keys.length < 2 && !isReplayableWrite) return;
|
|
47
136
|
window.__opencli_record.push({
|
|
48
|
-
url: String(url),
|
|
49
|
-
method: String(method).toUpperCase(),
|
|
137
|
+
url: String(entry.url),
|
|
138
|
+
method: String(entry.method).toUpperCase(),
|
|
50
139
|
status: null,
|
|
51
|
-
|
|
52
|
-
|
|
140
|
+
requestContentType: entry.requestContentType || null,
|
|
141
|
+
responseContentType: entry.responseContentType || 'application/json',
|
|
142
|
+
requestBody: entry.requestBody || null,
|
|
143
|
+
responseBody,
|
|
144
|
+
contentType: entry.responseContentType || 'application/json',
|
|
145
|
+
body: responseBody,
|
|
146
|
+
capturedAt: Date.now(),
|
|
53
147
|
});
|
|
54
148
|
} catch {}
|
|
55
149
|
};
|
|
@@ -58,14 +152,53 @@ function generateFullCaptureInterceptorJs() {
|
|
|
58
152
|
window.__opencli_orig_fetch = window.fetch;
|
|
59
153
|
window.fetch = async function(...args) {
|
|
60
154
|
const req = args[0];
|
|
155
|
+
const init = args[1] || {};
|
|
61
156
|
const reqUrl = typeof req === 'string' ? req : (req instanceof Request ? req.url : String(req));
|
|
62
|
-
const method = (
|
|
157
|
+
const method = (init?.method || (req instanceof Request ? req.method : 'GET') || 'GET');
|
|
158
|
+
const requestContentType = (() => {
|
|
159
|
+
if (init?.headers) {
|
|
160
|
+
try {
|
|
161
|
+
const headers = new Headers(init.headers);
|
|
162
|
+
const value = headers.get('content-type');
|
|
163
|
+
if (value) return value;
|
|
164
|
+
} catch {}
|
|
165
|
+
}
|
|
166
|
+
if (req instanceof Request) {
|
|
167
|
+
return req.headers.get('content-type');
|
|
168
|
+
}
|
|
169
|
+
return null;
|
|
170
|
+
})();
|
|
171
|
+
const requestBodyText = (() => {
|
|
172
|
+
if (typeof init?.body === 'string') return init.body;
|
|
173
|
+
return null;
|
|
174
|
+
})();
|
|
175
|
+
const shouldReadRequestBodyFromRequest = req instanceof Request
|
|
176
|
+
&& !requestBodyText
|
|
177
|
+
&& ['POST', 'PUT', 'PATCH'].includes(String(method).toUpperCase())
|
|
178
|
+
&& (() => {
|
|
179
|
+
const normalized = String(requestContentType || '').toLowerCase();
|
|
180
|
+
return normalized.includes('application/json') || normalized.includes('+json');
|
|
181
|
+
})();
|
|
182
|
+
let requestBodyTextFromRequest = null;
|
|
183
|
+
if (shouldReadRequestBodyFromRequest) {
|
|
184
|
+
try {
|
|
185
|
+
requestBodyTextFromRequest = await req.clone().text();
|
|
186
|
+
} catch {}
|
|
187
|
+
}
|
|
188
|
+
const requestBody = _tryParseJson(requestContentType, requestBodyText || requestBodyTextFromRequest);
|
|
63
189
|
const res = await window.__opencli_orig_fetch.apply(this, args);
|
|
64
190
|
const ct = res.headers.get('content-type') || '';
|
|
65
191
|
if (ct.includes('json')) {
|
|
66
192
|
try {
|
|
67
|
-
const
|
|
68
|
-
_push(
|
|
193
|
+
const responseBody = await res.clone().json();
|
|
194
|
+
_push({
|
|
195
|
+
url: reqUrl,
|
|
196
|
+
method,
|
|
197
|
+
requestContentType,
|
|
198
|
+
requestBody,
|
|
199
|
+
responseContentType: ct,
|
|
200
|
+
responseBody,
|
|
201
|
+
});
|
|
69
202
|
} catch {}
|
|
70
203
|
}
|
|
71
204
|
return res;
|
|
@@ -75,20 +208,38 @@ function generateFullCaptureInterceptorJs() {
|
|
|
75
208
|
const _XHR = XMLHttpRequest.prototype;
|
|
76
209
|
window.__opencli_orig_xhr_open = _XHR.open;
|
|
77
210
|
window.__opencli_orig_xhr_send = _XHR.send;
|
|
211
|
+
window.__opencli_orig_xhr_set_request_header = _XHR.setRequestHeader;
|
|
78
212
|
_XHR.open = function(method, url) {
|
|
79
213
|
this.__rec_url = String(url);
|
|
80
214
|
this.__rec_method = String(method);
|
|
215
|
+
this.__rec_request_content_type = null;
|
|
81
216
|
this.__rec_listener_added = false; // reset per open() call
|
|
82
217
|
return window.__opencli_orig_xhr_open.apply(this, arguments);
|
|
83
218
|
};
|
|
219
|
+
_XHR.setRequestHeader = function(name, value) {
|
|
220
|
+
if (String(name).toLowerCase() === 'content-type') {
|
|
221
|
+
this.__rec_request_content_type = String(value);
|
|
222
|
+
}
|
|
223
|
+
return window.__opencli_orig_xhr_set_request_header.apply(this, arguments);
|
|
224
|
+
};
|
|
84
225
|
_XHR.send = function() {
|
|
226
|
+
const requestBody = _tryParseJson(this.__rec_request_content_type, typeof arguments[0] === 'string' ? arguments[0] : null);
|
|
85
227
|
// Guard: only add one listener per XHR instance to prevent duplicate captures
|
|
86
228
|
if (!this.__rec_listener_added) {
|
|
87
229
|
this.__rec_listener_added = true;
|
|
88
230
|
this.addEventListener('load', function() {
|
|
89
231
|
const ct = this.getResponseHeader?.('content-type') || '';
|
|
90
232
|
if (ct.includes('json')) {
|
|
91
|
-
try {
|
|
233
|
+
try {
|
|
234
|
+
_push({
|
|
235
|
+
url: this.__rec_url,
|
|
236
|
+
method: this.__rec_method || 'GET',
|
|
237
|
+
requestContentType: this.__rec_request_content_type,
|
|
238
|
+
requestBody,
|
|
239
|
+
responseContentType: ct,
|
|
240
|
+
responseBody: JSON.parse(this.responseText),
|
|
241
|
+
});
|
|
242
|
+
} catch {}
|
|
92
243
|
}
|
|
93
244
|
});
|
|
94
245
|
}
|
|
@@ -126,14 +277,41 @@ function scoreRequest(req, arrayResult) {
|
|
|
126
277
|
}
|
|
127
278
|
}
|
|
128
279
|
}
|
|
129
|
-
|
|
130
|
-
|
|
131
|
-
|
|
132
|
-
|
|
133
|
-
|
|
134
|
-
|
|
135
|
-
|
|
136
|
-
|
|
280
|
+
return applyCommonEndpointScoreAdjustments(req, s);
|
|
281
|
+
}
|
|
282
|
+
/** Check whether one recorded request is safe to treat as a write candidate. */
|
|
283
|
+
function isWriteCandidate(req) {
|
|
284
|
+
return ['POST', 'PUT', 'PATCH'].includes(req.method)
|
|
285
|
+
&& isJsonContentType(req.requestContentType)
|
|
286
|
+
&& !!req.requestBody
|
|
287
|
+
&& typeof req.requestBody === 'object'
|
|
288
|
+
&& !Array.isArray(req.requestBody)
|
|
289
|
+
&& !!req.responseBody
|
|
290
|
+
&& typeof req.responseBody === 'object'
|
|
291
|
+
&& !Array.isArray(req.responseBody);
|
|
292
|
+
}
|
|
293
|
+
/** Score replayable write requests while keeping tracking and heartbeat traffic suppressed. */
|
|
294
|
+
function scoreWriteRequest(req) {
|
|
295
|
+
return applyCommonEndpointScoreAdjustments(req, 6);
|
|
296
|
+
}
|
|
297
|
+
/** Analyze recorded requests into read and write candidates. */
|
|
298
|
+
export function analyzeRecordedRequests(requests) {
|
|
299
|
+
const candidates = [];
|
|
300
|
+
for (const req of requests) {
|
|
301
|
+
const arrayResult = findArrayPath(req.responseBody);
|
|
302
|
+
if (isWriteCandidate(req)) {
|
|
303
|
+
const score = scoreWriteRequest(req);
|
|
304
|
+
if (score > 0)
|
|
305
|
+
candidates.push({ kind: 'write', req, score, arrayResult: null });
|
|
306
|
+
continue;
|
|
307
|
+
}
|
|
308
|
+
if (arrayResult) {
|
|
309
|
+
const score = scoreRequest(req, arrayResult);
|
|
310
|
+
if (score > 0)
|
|
311
|
+
candidates.push({ kind: 'read', req, score, arrayResult });
|
|
312
|
+
}
|
|
313
|
+
}
|
|
314
|
+
return { candidates };
|
|
137
315
|
}
|
|
138
316
|
// ── YAML generation ────────────────────────────────────────────────────────
|
|
139
317
|
function buildRecordedYaml(site, pageUrl, req, capName, arrayResult, authIndicators) {
|
|
@@ -177,7 +355,7 @@ function buildRecordedYaml(site, pageUrl, req, capName, arrayResult, authIndicat
|
|
|
177
355
|
if (hasSearch) {
|
|
178
356
|
for (const p of SEARCH_PARAMS) {
|
|
179
357
|
if (u.searchParams.has(p)) {
|
|
180
|
-
u.searchParams.set(p, '{{args.keyword}}');
|
|
358
|
+
u.searchParams.set(p, '${{ args.keyword }}');
|
|
181
359
|
break;
|
|
182
360
|
}
|
|
183
361
|
}
|
|
@@ -185,12 +363,17 @@ function buildRecordedYaml(site, pageUrl, req, capName, arrayResult, authIndicat
|
|
|
185
363
|
if (hasPage) {
|
|
186
364
|
for (const p of PAGINATION_PARAMS) {
|
|
187
365
|
if (u.searchParams.has(p)) {
|
|
188
|
-
u.searchParams.set(p, '{{args.page | default(1)}}');
|
|
366
|
+
u.searchParams.set(p, '${{ args.page | default(1) }}');
|
|
189
367
|
break;
|
|
190
368
|
}
|
|
191
369
|
}
|
|
192
370
|
}
|
|
193
371
|
fetchUrl = u.toString();
|
|
372
|
+
fetchUrl = fetchUrl
|
|
373
|
+
.replaceAll(encodeURIComponent('${{ args.keyword }}'), '${{ args.keyword }}')
|
|
374
|
+
.replaceAll('%24%7B%7B+args.keyword+%7D%7D', '${{ args.keyword }}')
|
|
375
|
+
.replaceAll(encodeURIComponent('${{ args.page | default(1) }}'), '${{ args.page | default(1) }}');
|
|
376
|
+
fetchUrl = fetchUrl.replaceAll('%24%7B%7B+args.page+%7C+default%281%29+%7D%7D', '${{ args.page | default(1) }}');
|
|
194
377
|
}
|
|
195
378
|
catch { }
|
|
196
379
|
// When itemPath is empty, the array IS the response root; otherwise chain with ?.
|
|
@@ -234,6 +417,77 @@ function buildRecordedYaml(site, pageUrl, req, capName, arrayResult, authIndicat
|
|
|
234
417
|
},
|
|
235
418
|
};
|
|
236
419
|
}
|
|
420
|
+
/** Build a minimal YAML candidate for replayable JSON write requests. */
|
|
421
|
+
export function buildWriteRecordedYaml(site, pageUrl, req, capName) {
|
|
422
|
+
const responseColumns = req.responseBody && typeof req.responseBody === 'object' && !Array.isArray(req.responseBody)
|
|
423
|
+
? Object.keys(req.responseBody).slice(0, 6)
|
|
424
|
+
: ['ok'];
|
|
425
|
+
const evaluateScript = [
|
|
426
|
+
'(async () => {',
|
|
427
|
+
` const res = await fetch(${JSON.stringify(req.url)}, {`,
|
|
428
|
+
` method: ${JSON.stringify(req.method)},`,
|
|
429
|
+
` credentials: 'include',`,
|
|
430
|
+
` headers: { 'content-type': ${JSON.stringify(req.requestContentType ?? 'application/json')} },`,
|
|
431
|
+
` body: JSON.stringify(${JSON.stringify(req.requestBody)}),`,
|
|
432
|
+
' });',
|
|
433
|
+
' return await res.json();',
|
|
434
|
+
'})()',
|
|
435
|
+
].join('\n');
|
|
436
|
+
return {
|
|
437
|
+
name: capName,
|
|
438
|
+
yaml: {
|
|
439
|
+
site,
|
|
440
|
+
name: capName,
|
|
441
|
+
description: `${site} ${capName} (recorded write)`,
|
|
442
|
+
domain: (() => { try {
|
|
443
|
+
return new URL(pageUrl).hostname;
|
|
444
|
+
}
|
|
445
|
+
catch {
|
|
446
|
+
return '';
|
|
447
|
+
} })(),
|
|
448
|
+
strategy: 'cookie',
|
|
449
|
+
browser: true,
|
|
450
|
+
args: {},
|
|
451
|
+
pipeline: [
|
|
452
|
+
{ navigate: pageUrl },
|
|
453
|
+
{ evaluate: evaluateScript },
|
|
454
|
+
],
|
|
455
|
+
columns: responseColumns.length ? responseColumns : ['ok'],
|
|
456
|
+
},
|
|
457
|
+
};
|
|
458
|
+
}
|
|
459
|
+
/** Turn recorded requests into YAML-ready read and write candidates. */
|
|
460
|
+
export function generateRecordedCandidates(site, pageUrl, requests) {
|
|
461
|
+
const analysis = analyzeRecordedRequests(dedupeRecordedRequests(requests));
|
|
462
|
+
const deduped = new Map();
|
|
463
|
+
for (const candidate of analysis.candidates) {
|
|
464
|
+
const key = getRecordedCandidateKey(candidate);
|
|
465
|
+
const current = deduped.get(key);
|
|
466
|
+
deduped.set(key, current ? preferRecordedCandidate(current, candidate) : candidate);
|
|
467
|
+
}
|
|
468
|
+
const selected = [...deduped.values()]
|
|
469
|
+
.filter((candidate) => candidate.kind === 'read' ? candidate.score >= 8 : candidate.score >= 6)
|
|
470
|
+
.sort((a, b) => b.score - a.score)
|
|
471
|
+
.slice(0, 5);
|
|
472
|
+
const usedNames = new Set();
|
|
473
|
+
return selected.map((candidate) => {
|
|
474
|
+
let capName = inferCapabilityName(candidate.req.url);
|
|
475
|
+
if (usedNames.has(capName))
|
|
476
|
+
capName = `${capName}_${usedNames.size + 1}`;
|
|
477
|
+
usedNames.add(capName);
|
|
478
|
+
const authIndicators = detectAuthFromContent(candidate.req.url, candidate.req.responseBody);
|
|
479
|
+
const strategy = candidate.kind === 'write' ? 'cookie' : inferStrategy(authIndicators);
|
|
480
|
+
const yamlCandidate = candidate.kind === 'write'
|
|
481
|
+
? buildWriteRecordedYaml(site, pageUrl, candidate.req, capName)
|
|
482
|
+
: buildRecordedYaml(site, pageUrl, candidate.req, capName, candidate.arrayResult, authIndicators);
|
|
483
|
+
return {
|
|
484
|
+
kind: candidate.kind,
|
|
485
|
+
name: yamlCandidate.name,
|
|
486
|
+
strategy,
|
|
487
|
+
yaml: yamlCandidate.yaml,
|
|
488
|
+
};
|
|
489
|
+
});
|
|
490
|
+
}
|
|
237
491
|
export async function recordSession(opts) {
|
|
238
492
|
const pollMs = opts.pollMs ?? 2000;
|
|
239
493
|
const timeoutMs = opts.timeoutMs ?? 60_000;
|
|
@@ -374,52 +628,39 @@ function analyzeAndWrite(site, pageUrl, requests, outDir) {
|
|
|
374
628
|
console.log(chalk.yellow(' No API requests captured.'));
|
|
375
629
|
return { site, url: pageUrl, requests: [], outDir: targetDir, candidateCount: 0, candidates: [] };
|
|
376
630
|
}
|
|
377
|
-
//
|
|
378
|
-
const
|
|
379
|
-
|
|
380
|
-
const pattern = urlToPattern(req.url);
|
|
381
|
-
if (!seen.has(pattern))
|
|
382
|
-
seen.set(pattern, req);
|
|
383
|
-
}
|
|
384
|
-
const scored = [];
|
|
385
|
-
for (const [pattern, req] of seen) {
|
|
386
|
-
const arrayResult = findArrayPath(req.body);
|
|
387
|
-
const authIndicators = detectAuthFromContent(req.url, req.body);
|
|
388
|
-
const score = scoreRequest(req, arrayResult);
|
|
389
|
-
if (score > 0) {
|
|
390
|
-
scored.push({ req, pattern, arrayResult, authIndicators, score });
|
|
391
|
-
}
|
|
392
|
-
}
|
|
393
|
-
scored.sort((a, b) => b.score - a.score);
|
|
631
|
+
// Score and rank deduplicated requests for console output and candidate generation.
|
|
632
|
+
const analysisRequests = dedupeRecordedRequests(requests);
|
|
633
|
+
const analysis = analyzeRecordedRequests(analysisRequests);
|
|
394
634
|
// Save raw captured data
|
|
395
635
|
fs.writeFileSync(path.join(targetDir, 'captured.json'), JSON.stringify({ site, url: pageUrl, capturedAt: new Date().toISOString(), requests }, null, 2));
|
|
396
636
|
// Generate candidate YAMLs (top 5)
|
|
397
637
|
const candidates = [];
|
|
398
638
|
const usedNames = new Set();
|
|
399
639
|
console.log(chalk.bold('\n Captured endpoints (scored):\n'));
|
|
400
|
-
for (const entry of
|
|
640
|
+
for (const entry of analysis.candidates.sort((a, b) => b.score - a.score).slice(0, 8)) {
|
|
401
641
|
const itemCount = entry.arrayResult?.items.length ?? 0;
|
|
402
|
-
const strategy =
|
|
642
|
+
const strategy = entry.kind === 'write'
|
|
643
|
+
? 'cookie'
|
|
644
|
+
: inferStrategy(detectAuthFromContent(entry.req.url, entry.req.responseBody));
|
|
403
645
|
const marker = entry.score >= 15 ? chalk.green('★') : entry.score >= 8 ? chalk.yellow('◆') : chalk.dim('·');
|
|
404
|
-
console.log(` ${marker} ${chalk.white(entry.
|
|
646
|
+
console.log(` ${marker} ${chalk.white(urlToPattern(entry.req.url))}` +
|
|
405
647
|
chalk.dim(` [${strategy}]`) +
|
|
406
|
-
(
|
|
648
|
+
(entry.kind === 'write'
|
|
649
|
+
? chalk.magenta(' ← write')
|
|
650
|
+
: itemCount ? chalk.cyan(` ← ${itemCount} items`) : ''));
|
|
407
651
|
}
|
|
408
652
|
console.log();
|
|
409
|
-
const topCandidates =
|
|
653
|
+
const topCandidates = generateRecordedCandidates(site, pageUrl, analysisRequests);
|
|
410
654
|
const candidatesDir = path.join(targetDir, 'candidates');
|
|
411
655
|
fs.mkdirSync(candidatesDir, { recursive: true });
|
|
412
656
|
for (const entry of topCandidates) {
|
|
413
|
-
|
|
414
|
-
|
|
415
|
-
|
|
416
|
-
|
|
417
|
-
|
|
418
|
-
|
|
419
|
-
|
|
420
|
-
fs.writeFileSync(filePath, yaml.dump(candidate.yaml, { sortKeys: false, lineWidth: 120 }));
|
|
421
|
-
candidates.push({ name: capName, path: filePath, strategy });
|
|
422
|
-
console.log(chalk.green(` ✓ Generated: ${chalk.bold(capName)}.yaml [${strategy}]`));
|
|
657
|
+
if (usedNames.has(entry.name))
|
|
658
|
+
continue;
|
|
659
|
+
usedNames.add(entry.name);
|
|
660
|
+
const filePath = path.join(candidatesDir, `${entry.name}.yaml`);
|
|
661
|
+
fs.writeFileSync(filePath, yaml.dump(entry.yaml, { sortKeys: false, lineWidth: 120 }));
|
|
662
|
+
candidates.push({ name: entry.name, path: filePath, strategy: entry.strategy });
|
|
663
|
+
console.log(chalk.green(` ✓ Generated: ${chalk.bold(entry.name)}.yaml [${entry.strategy}]`));
|
|
423
664
|
console.log(chalk.dim(` → ${filePath}`));
|
|
424
665
|
}
|
|
425
666
|
if (candidates.length === 0) {
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
export {};
|