@jackwener/opencli 1.0.1 → 1.0.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.github/workflows/build-extension.yml +62 -0
- package/.github/workflows/ci.yml +6 -6
- package/.github/workflows/e2e-headed.yml +2 -2
- package/.github/workflows/pkg-pr-new.yml +2 -2
- package/.github/workflows/release.yml +2 -5
- package/.github/workflows/security.yml +2 -2
- package/CDP.md +1 -1
- package/CDP.zh-CN.md +1 -1
- package/README.md +15 -7
- package/README.zh-CN.md +15 -7
- package/SKILL.md +3 -5
- package/dist/browser/cdp.d.ts +27 -0
- package/dist/browser/cdp.js +295 -0
- package/dist/browser/index.d.ts +3 -0
- package/dist/browser/index.js +4 -0
- package/dist/browser/page.js +2 -23
- package/dist/browser/utils.d.ts +10 -0
- package/dist/browser/utils.js +27 -0
- package/dist/browser.test.js +42 -1
- package/dist/chaoxing.d.ts +58 -0
- package/dist/chaoxing.js +225 -0
- package/dist/chaoxing.test.d.ts +1 -0
- package/dist/chaoxing.test.js +38 -0
- package/dist/cli-manifest.json +203 -0
- package/dist/cli.d.ts +1 -0
- package/dist/cli.js +197 -0
- package/dist/clis/boss/chatlist.d.ts +1 -0
- package/dist/clis/boss/chatlist.js +50 -0
- package/dist/clis/boss/chatmsg.d.ts +1 -0
- package/dist/clis/boss/chatmsg.js +73 -0
- package/dist/clis/boss/send.d.ts +1 -0
- package/dist/clis/boss/send.js +176 -0
- package/dist/clis/chaoxing/assignments.d.ts +1 -0
- package/dist/clis/chaoxing/assignments.js +74 -0
- package/dist/clis/chaoxing/exams.d.ts +1 -0
- package/dist/clis/chaoxing/exams.js +74 -0
- package/dist/clis/chatgpt/ask.js +15 -14
- package/dist/clis/chatgpt/ax.d.ts +1 -0
- package/dist/clis/chatgpt/ax.js +78 -0
- package/dist/clis/chatgpt/read.js +5 -6
- package/dist/clis/twitter/post.js +9 -2
- package/dist/clis/twitter/search.js +14 -33
- package/dist/clis/xiaohongshu/download.d.ts +1 -1
- package/dist/clis/xiaohongshu/download.js +1 -1
- package/dist/engine.js +24 -13
- package/dist/explore.js +46 -101
- package/dist/main.js +4 -193
- package/dist/output.d.ts +1 -1
- package/dist/registry.d.ts +3 -3
- package/dist/scripts/framework.d.ts +4 -0
- package/dist/scripts/framework.js +21 -0
- package/dist/scripts/interact.d.ts +4 -0
- package/dist/scripts/interact.js +20 -0
- package/dist/scripts/store.d.ts +9 -0
- package/dist/scripts/store.js +44 -0
- package/dist/synthesize.js +1 -1
- package/extension/dist/background.js +338 -430
- package/extension/manifest.json +2 -2
- package/extension/src/background.ts +2 -2
- package/package.json +1 -1
- package/src/browser/cdp.ts +295 -0
- package/src/browser/index.ts +4 -0
- package/src/browser/page.ts +2 -24
- package/src/browser/utils.ts +27 -0
- package/src/browser.test.ts +46 -0
- package/src/chaoxing.test.ts +45 -0
- package/src/chaoxing.ts +268 -0
- package/src/cli.ts +185 -0
- package/src/clis/antigravity/SKILL.md +5 -0
- package/src/clis/boss/chatlist.ts +50 -0
- package/src/clis/boss/chatmsg.ts +70 -0
- package/src/clis/boss/send.ts +193 -0
- package/src/clis/chaoxing/README.md +36 -0
- package/src/clis/chaoxing/README.zh-CN.md +35 -0
- package/src/clis/chaoxing/assignments.ts +88 -0
- package/src/clis/chaoxing/exams.ts +88 -0
- package/src/clis/chatgpt/ask.ts +14 -15
- package/src/clis/chatgpt/ax.ts +81 -0
- package/src/clis/chatgpt/read.ts +5 -7
- package/src/clis/twitter/post.ts +9 -2
- package/src/clis/twitter/search.ts +15 -33
- package/src/clis/xiaohongshu/download.ts +1 -1
- package/src/engine.ts +20 -13
- package/src/explore.ts +51 -100
- package/src/main.ts +4 -180
- package/src/output.ts +12 -12
- package/src/registry.ts +3 -3
- package/src/scripts/framework.ts +20 -0
- package/src/scripts/interact.ts +22 -0
- package/src/scripts/store.ts +40 -0
- package/src/synthesize.ts +1 -1
|
@@ -24,46 +24,28 @@ cli({
|
|
|
24
24
|
// fetch will capture the SearchTimeline API call.
|
|
25
25
|
await page.installInterceptor('SearchTimeline');
|
|
26
26
|
|
|
27
|
-
// 3.
|
|
28
|
-
//
|
|
27
|
+
// 3. Trigger SPA navigation to search results via history API.
|
|
28
|
+
// pushState + popstate triggers React Router's listener without
|
|
29
|
+
// a full page reload, so the interceptor stays alive.
|
|
30
|
+
// Note: the previous approach (nativeSetter + Enter keydown on the
|
|
31
|
+
// search input) does not reliably trigger Twitter's form submission.
|
|
32
|
+
const searchUrl = JSON.stringify(`/search?q=${encodeURIComponent(query)}&f=top`);
|
|
29
33
|
await page.evaluate(`
|
|
30
34
|
(() => {
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
input.focus();
|
|
34
|
-
const nativeSetter = Object.getOwnPropertyDescriptor(
|
|
35
|
-
HTMLInputElement.prototype, 'value'
|
|
36
|
-
).set;
|
|
37
|
-
nativeSetter.call(input, ${JSON.stringify(query)});
|
|
38
|
-
input.dispatchEvent(new Event('input', { bubbles: true }));
|
|
39
|
-
})()
|
|
40
|
-
`);
|
|
41
|
-
await page.wait(0.5);
|
|
42
|
-
// Press Enter to submit
|
|
43
|
-
await page.evaluate(`
|
|
44
|
-
(() => {
|
|
45
|
-
const input = document.querySelector('input[data-testid="SearchBox_Search_Input"]');
|
|
46
|
-
if (!input) throw new Error('Search input not found');
|
|
47
|
-
input.dispatchEvent(new KeyboardEvent('keydown', { key: 'Enter', code: 'Enter', keyCode: 13, bubbles: true }));
|
|
35
|
+
window.history.pushState({}, '', ${searchUrl});
|
|
36
|
+
window.dispatchEvent(new PopStateEvent('popstate', { state: {} }));
|
|
48
37
|
})()
|
|
49
38
|
`);
|
|
50
39
|
await page.wait(5);
|
|
51
40
|
|
|
52
|
-
//
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
(
|
|
56
|
-
|
|
57
|
-
for (const tab of tabs) {
|
|
58
|
-
if (tab.textContent.trim() === 'Top') { tab.click(); break; }
|
|
59
|
-
}
|
|
60
|
-
})()
|
|
61
|
-
`);
|
|
62
|
-
await page.wait(2);
|
|
63
|
-
} catch { /* ignore if tab not found */ }
|
|
41
|
+
// Verify SPA navigation succeeded
|
|
42
|
+
const currentPath = await page.evaluate('() => window.location.pathname');
|
|
43
|
+
if (!currentPath?.startsWith('/search')) {
|
|
44
|
+
throw new Error('SPA navigation to /search failed. Twitter may have changed its routing.');
|
|
45
|
+
}
|
|
64
46
|
|
|
65
|
-
//
|
|
66
|
-
await page.autoScroll({ times:
|
|
47
|
+
// 4. Scroll to trigger additional pagination
|
|
48
|
+
await page.autoScroll({ times: 3, delayMs: 2000 });
|
|
67
49
|
|
|
68
50
|
// 6. Retrieve captured data
|
|
69
51
|
const requests = await page.getInterceptedRequests();
|
package/src/engine.ts
CHANGED
|
@@ -28,12 +28,14 @@ export async function discoverClis(...dirs: string[]): Promise<void> {
|
|
|
28
28
|
// Fast path: try manifest first (production / post-build)
|
|
29
29
|
for (const dir of dirs) {
|
|
30
30
|
const manifestPath = path.resolve(dir, '..', 'cli-manifest.json');
|
|
31
|
-
|
|
32
|
-
|
|
31
|
+
try {
|
|
32
|
+
await fs.promises.access(manifestPath);
|
|
33
|
+
await loadFromManifest(manifestPath, dir);
|
|
33
34
|
continue; // Skip filesystem scan for this directory
|
|
35
|
+
} catch {
|
|
36
|
+
// Fallback: runtime filesystem scan (development)
|
|
37
|
+
await discoverClisFromFs(dir);
|
|
34
38
|
}
|
|
35
|
-
// Fallback: runtime filesystem scan (development)
|
|
36
|
-
await discoverClisFromFs(dir);
|
|
37
39
|
}
|
|
38
40
|
}
|
|
39
41
|
|
|
@@ -42,9 +44,10 @@ export async function discoverClis(...dirs: string[]): Promise<void> {
|
|
|
42
44
|
* YAML pipelines are inlined — zero YAML parsing at runtime.
|
|
43
45
|
* TS modules are deferred — loaded lazily on first execution.
|
|
44
46
|
*/
|
|
45
|
-
function loadFromManifest(manifestPath: string, clisDir: string): void {
|
|
47
|
+
async function loadFromManifest(manifestPath: string, clisDir: string): Promise<void> {
|
|
46
48
|
try {
|
|
47
|
-
const
|
|
49
|
+
const raw = await fs.promises.readFile(manifestPath, 'utf-8');
|
|
50
|
+
const manifest = JSON.parse(raw) as any[];
|
|
48
51
|
for (const entry of manifest) {
|
|
49
52
|
if (entry.type === 'yaml') {
|
|
50
53
|
// YAML pipelines fully inlined in manifest — register directly
|
|
@@ -94,15 +97,19 @@ function loadFromManifest(manifestPath: string, clisDir: string): void {
|
|
|
94
97
|
* Fallback: traditional filesystem scan (used during development with tsx).
|
|
95
98
|
*/
|
|
96
99
|
async function discoverClisFromFs(dir: string): Promise<void> {
|
|
97
|
-
|
|
100
|
+
try { await fs.promises.access(dir); } catch { return; }
|
|
98
101
|
const promises: Promise<any>[] = [];
|
|
99
|
-
|
|
102
|
+
const sites = await fs.promises.readdir(dir);
|
|
103
|
+
|
|
104
|
+
for (const site of sites) {
|
|
100
105
|
const siteDir = path.join(dir, site);
|
|
101
|
-
|
|
102
|
-
|
|
106
|
+
const stat = await fs.promises.stat(siteDir);
|
|
107
|
+
if (!stat.isDirectory()) continue;
|
|
108
|
+
const files = await fs.promises.readdir(siteDir);
|
|
109
|
+
for (const file of files) {
|
|
103
110
|
const filePath = path.join(siteDir, file);
|
|
104
111
|
if (file.endsWith('.yaml') || file.endsWith('.yml')) {
|
|
105
|
-
registerYamlCli(filePath, site);
|
|
112
|
+
promises.push(registerYamlCli(filePath, site));
|
|
106
113
|
} else if (
|
|
107
114
|
(file.endsWith('.js') && !file.endsWith('.d.js')) ||
|
|
108
115
|
(file.endsWith('.ts') && !file.endsWith('.d.ts') && !file.endsWith('.test.ts'))
|
|
@@ -118,9 +125,9 @@ async function discoverClisFromFs(dir: string): Promise<void> {
|
|
|
118
125
|
await Promise.all(promises);
|
|
119
126
|
}
|
|
120
127
|
|
|
121
|
-
function registerYamlCli(filePath: string, defaultSite: string): void {
|
|
128
|
+
async function registerYamlCli(filePath: string, defaultSite: string): Promise<void> {
|
|
122
129
|
try {
|
|
123
|
-
const raw = fs.
|
|
130
|
+
const raw = await fs.promises.readFile(filePath, 'utf-8');
|
|
124
131
|
const def = yaml.load(raw) as any;
|
|
125
132
|
if (!def || typeof def !== 'object') return;
|
|
126
133
|
|
package/src/explore.ts
CHANGED
|
@@ -10,6 +10,9 @@ import * as fs from 'node:fs';
|
|
|
10
10
|
import * as path from 'node:path';
|
|
11
11
|
import { DEFAULT_BROWSER_EXPLORE_TIMEOUT, browserSession, runWithTimeout } from './runtime.js';
|
|
12
12
|
import { VOLATILE_PARAMS, SEARCH_PARAMS, PAGINATION_PARAMS, LIMIT_PARAMS, FIELD_ROLES } from './constants.js';
|
|
13
|
+
import { detectFramework } from './scripts/framework.js';
|
|
14
|
+
import { discoverStores } from './scripts/store.js';
|
|
15
|
+
import { interactFuzz } from './scripts/interact.js';
|
|
13
16
|
|
|
14
17
|
// ── Site name detection ────────────────────────────────────────────────────
|
|
15
18
|
|
|
@@ -50,7 +53,7 @@ export function slugify(value: string): string {
|
|
|
50
53
|
|
|
51
54
|
interface NetworkEntry {
|
|
52
55
|
method: string; url: string; status: number | null;
|
|
53
|
-
contentType: string; responseBody?:
|
|
56
|
+
contentType: string; responseBody?: unknown; requestHeaders?: Record<string, string>;
|
|
54
57
|
}
|
|
55
58
|
|
|
56
59
|
interface AnalyzedEndpoint {
|
|
@@ -72,7 +75,7 @@ interface InferredCapability {
|
|
|
72
75
|
* Parse raw network output from Playwright MCP.
|
|
73
76
|
* Handles text format: [GET] url => [200]
|
|
74
77
|
*/
|
|
75
|
-
function parseNetworkRequests(raw:
|
|
78
|
+
function parseNetworkRequests(raw: unknown): NetworkEntry[] {
|
|
76
79
|
if (typeof raw === 'string') {
|
|
77
80
|
const entries: NetworkEntry[] = [];
|
|
78
81
|
for (const line of raw.split('\n')) {
|
|
@@ -120,11 +123,11 @@ function detectAuthIndicators(headers?: Record<string, string>): string[] {
|
|
|
120
123
|
return indicators;
|
|
121
124
|
}
|
|
122
125
|
|
|
123
|
-
function analyzeResponseBody(body:
|
|
126
|
+
function analyzeResponseBody(body: unknown): AnalyzedEndpoint['responseAnalysis'] {
|
|
124
127
|
if (!body || typeof body !== 'object') return null;
|
|
125
|
-
const candidates: Array<{ path: string; items:
|
|
128
|
+
const candidates: Array<{ path: string; items: unknown[] }> = [];
|
|
126
129
|
|
|
127
|
-
function findArrays(obj:
|
|
130
|
+
function findArrays(obj: unknown, path: string, depth: number) {
|
|
128
131
|
if (depth > 4) return;
|
|
129
132
|
if (Array.isArray(obj) && obj.length >= 2 && obj.some(item => item && typeof item === 'object' && !Array.isArray(item))) {
|
|
130
133
|
candidates.push({ path, items: obj });
|
|
@@ -151,13 +154,15 @@ function analyzeResponseBody(body: any): AnalyzedEndpoint['responseAnalysis'] {
|
|
|
151
154
|
return { itemPath: best.path || null, itemCount: best.items.length, detectedFields, sampleFields };
|
|
152
155
|
}
|
|
153
156
|
|
|
154
|
-
function flattenFields(obj:
|
|
157
|
+
function flattenFields(obj: unknown, prefix: string, maxDepth: number): string[] {
|
|
155
158
|
if (maxDepth <= 0 || !obj || typeof obj !== 'object') return [];
|
|
156
159
|
const names: string[] = [];
|
|
157
|
-
|
|
160
|
+
const record = obj as Record<string, unknown>;
|
|
161
|
+
for (const key of Object.keys(record)) {
|
|
158
162
|
const full = prefix ? `${prefix}.${key}` : key;
|
|
159
163
|
names.push(full);
|
|
160
|
-
|
|
164
|
+
const val = record[key];
|
|
165
|
+
if (val && typeof val === 'object' && !Array.isArray(val)) names.push(...flattenFields(val, full, maxDepth - 1));
|
|
161
166
|
}
|
|
162
167
|
return names;
|
|
163
168
|
}
|
|
@@ -201,63 +206,11 @@ function inferStrategy(authIndicators: string[]): string {
|
|
|
201
206
|
|
|
202
207
|
// ── Framework detection ────────────────────────────────────────────────────
|
|
203
208
|
|
|
204
|
-
const FRAMEWORK_DETECT_JS =
|
|
205
|
-
() => {
|
|
206
|
-
const r = {};
|
|
207
|
-
try {
|
|
208
|
-
const app = document.querySelector('#app');
|
|
209
|
-
r.vue3 = !!(app && app.__vue_app__);
|
|
210
|
-
r.vue2 = !!(app && app.__vue__);
|
|
211
|
-
r.react = !!window.__REACT_DEVTOOLS_GLOBAL_HOOK__ || !!document.querySelector('[data-reactroot]');
|
|
212
|
-
r.nextjs = !!window.__NEXT_DATA__;
|
|
213
|
-
r.nuxt = !!window.__NUXT__;
|
|
214
|
-
if (r.vue3 && app.__vue_app__) { const gp = app.__vue_app__.config?.globalProperties; r.pinia = !!(gp && gp.$pinia); r.vuex = !!(gp && gp.$store); }
|
|
215
|
-
} catch {}
|
|
216
|
-
return r;
|
|
217
|
-
}
|
|
218
|
-
`;
|
|
209
|
+
const FRAMEWORK_DETECT_JS = detectFramework.toString();
|
|
219
210
|
|
|
220
211
|
// ── Store discovery ────────────────────────────────────────────────────────
|
|
221
212
|
|
|
222
|
-
const STORE_DISCOVER_JS =
|
|
223
|
-
() => {
|
|
224
|
-
const stores = [];
|
|
225
|
-
try {
|
|
226
|
-
const app = document.querySelector('#app');
|
|
227
|
-
if (!app?.__vue_app__) return stores;
|
|
228
|
-
const gp = app.__vue_app__.config?.globalProperties;
|
|
229
|
-
|
|
230
|
-
// Pinia stores
|
|
231
|
-
const pinia = gp?.$pinia;
|
|
232
|
-
if (pinia?._s) {
|
|
233
|
-
pinia._s.forEach((store, id) => {
|
|
234
|
-
const actions = [];
|
|
235
|
-
const stateKeys = [];
|
|
236
|
-
for (const k in store) {
|
|
237
|
-
try {
|
|
238
|
-
if (k.startsWith('$') || k.startsWith('_')) continue;
|
|
239
|
-
if (typeof store[k] === 'function') actions.push(k);
|
|
240
|
-
else stateKeys.push(k);
|
|
241
|
-
} catch {}
|
|
242
|
-
}
|
|
243
|
-
stores.push({ type: 'pinia', id, actions: actions.slice(0, 20), stateKeys: stateKeys.slice(0, 15) });
|
|
244
|
-
});
|
|
245
|
-
}
|
|
246
|
-
|
|
247
|
-
// Vuex store modules
|
|
248
|
-
const vuex = gp?.$store;
|
|
249
|
-
if (vuex?._modules?.root?._children) {
|
|
250
|
-
const children = vuex._modules.root._children;
|
|
251
|
-
for (const [modName, mod] of Object.entries(children)) {
|
|
252
|
-
const actions = Object.keys(mod._rawModule?.actions ?? {}).slice(0, 20);
|
|
253
|
-
const stateKeys = Object.keys(mod.state ?? {}).slice(0, 15);
|
|
254
|
-
stores.push({ type: 'vuex', id: modName, actions, stateKeys });
|
|
255
|
-
}
|
|
256
|
-
}
|
|
257
|
-
} catch {}
|
|
258
|
-
return stores;
|
|
259
|
-
}
|
|
260
|
-
`;
|
|
213
|
+
const STORE_DISCOVER_JS = discoverStores.toString();
|
|
261
214
|
|
|
262
215
|
export interface DiscoveredStore {
|
|
263
216
|
type: 'pinia' | 'vuex';
|
|
@@ -268,27 +221,7 @@ export interface DiscoveredStore {
|
|
|
268
221
|
|
|
269
222
|
// ── Auto-Interaction (Fuzzing) ─────────────────────────────────────────────
|
|
270
223
|
|
|
271
|
-
const INTERACT_FUZZ_JS =
|
|
272
|
-
async () => {
|
|
273
|
-
const sleep = ms => new Promise(r => setTimeout(r, ms));
|
|
274
|
-
const clickables = Array.from(document.querySelectorAll(
|
|
275
|
-
'button, [role="button"], [role="tab"], .tab, .btn, a[href="javascript:void(0)"], a[href="#"]'
|
|
276
|
-
)).slice(0, 15); // limit to 15 to avoid endless loops
|
|
277
|
-
|
|
278
|
-
let clicked = 0;
|
|
279
|
-
for (const el of clickables) {
|
|
280
|
-
try {
|
|
281
|
-
const rect = el.getBoundingClientRect();
|
|
282
|
-
if (rect.width > 0 && rect.height > 0) {
|
|
283
|
-
el.dispatchEvent(new MouseEvent('click', { bubbles: true, cancelable: true, view: window }));
|
|
284
|
-
clicked++;
|
|
285
|
-
await sleep(300); // give it time to trigger network
|
|
286
|
-
}
|
|
287
|
-
} catch {}
|
|
288
|
-
}
|
|
289
|
-
return clicked;
|
|
290
|
-
}
|
|
291
|
-
`;
|
|
224
|
+
const INTERACT_FUZZ_JS = interactFuzz.toString();
|
|
292
225
|
|
|
293
226
|
// ── Main explore function ──────────────────────────────────────────────────
|
|
294
227
|
|
|
@@ -310,8 +243,8 @@ export async function exploreUrl(
|
|
|
310
243
|
await page.goto(url);
|
|
311
244
|
await page.wait(waitSeconds);
|
|
312
245
|
|
|
313
|
-
// Step 2: Auto-scroll to trigger lazy loading
|
|
314
|
-
|
|
246
|
+
// Step 2: Auto-scroll to trigger lazy loading intelligently
|
|
247
|
+
await page.autoScroll({ times: 3, delayMs: 1500 }).catch(() => {});
|
|
315
248
|
|
|
316
249
|
// Step 2.5: Interactive Fuzzing (if requested)
|
|
317
250
|
if (opts.auto) {
|
|
@@ -319,11 +252,11 @@ export async function exploreUrl(
|
|
|
319
252
|
// First: targeted clicks by label (e.g. "字幕", "CC", "评论")
|
|
320
253
|
if (opts.clickLabels?.length) {
|
|
321
254
|
for (const label of opts.clickLabels) {
|
|
322
|
-
const safeLabel =
|
|
255
|
+
const safeLabel = JSON.stringify(label);
|
|
323
256
|
await page.evaluate(`
|
|
324
257
|
(() => {
|
|
325
258
|
const el = [...document.querySelectorAll('button, [role="button"], [role="tab"], a, span')]
|
|
326
|
-
.find(e => e.textContent && e.textContent.trim().includes(
|
|
259
|
+
.find(e => e.textContent && e.textContent.trim().includes(${safeLabel}));
|
|
327
260
|
if (el) el.click();
|
|
328
261
|
})()
|
|
329
262
|
`);
|
|
@@ -345,11 +278,27 @@ export async function exploreUrl(
|
|
|
345
278
|
const rawNetwork = await page.networkRequests(false);
|
|
346
279
|
const networkEntries = parseNetworkRequests(rawNetwork);
|
|
347
280
|
|
|
348
|
-
// Step 5: For JSON endpoints, re-fetch
|
|
349
|
-
const jsonEndpoints = networkEntries.filter(e => e.contentType.includes('json') && e.method === 'GET' && e.status === 200);
|
|
350
|
-
for (const ep of jsonEndpoints.slice(0,
|
|
281
|
+
// Step 5: For JSON endpoints missing a body, carefully re-fetch in-browser via a pristine iframe
|
|
282
|
+
const jsonEndpoints = networkEntries.filter(e => e.contentType.includes('json') && e.method === 'GET' && e.status === 200 && !e.responseBody);
|
|
283
|
+
for (const ep of jsonEndpoints.slice(0, 5)) {
|
|
351
284
|
try {
|
|
352
|
-
const body = await page.evaluate(`async () => {
|
|
285
|
+
const body = await page.evaluate(`async () => {
|
|
286
|
+
let iframe = null;
|
|
287
|
+
try {
|
|
288
|
+
iframe = document.createElement('iframe');
|
|
289
|
+
iframe.style.display = 'none';
|
|
290
|
+
document.body.appendChild(iframe);
|
|
291
|
+
const cleanFetch = iframe.contentWindow.fetch || window.fetch;
|
|
292
|
+
const r = await cleanFetch(${JSON.stringify(ep.url)}, { credentials: 'include' });
|
|
293
|
+
if (!r.ok) return null;
|
|
294
|
+
const d = await r.json();
|
|
295
|
+
return JSON.stringify(d).slice(0, 10000);
|
|
296
|
+
} catch {
|
|
297
|
+
return null;
|
|
298
|
+
} finally {
|
|
299
|
+
if (iframe && iframe.parentNode) iframe.parentNode.removeChild(iframe);
|
|
300
|
+
}
|
|
301
|
+
}`);
|
|
353
302
|
if (body && typeof body === 'string') { try { ep.responseBody = JSON.parse(body); } catch {} }
|
|
354
303
|
else if (body && typeof body === 'object') ep.responseBody = body;
|
|
355
304
|
} catch {}
|
|
@@ -455,7 +404,7 @@ export async function exploreUrl(
|
|
|
455
404
|
|
|
456
405
|
const siteName = opts.site ?? detectSiteName(metadata.url || url);
|
|
457
406
|
const targetDir = opts.outDir ?? path.join('.opencli', 'explore', siteName);
|
|
458
|
-
fs.
|
|
407
|
+
await fs.promises.mkdir(targetDir, { recursive: true });
|
|
459
408
|
|
|
460
409
|
const result = {
|
|
461
410
|
site: siteName, target_url: url, final_url: metadata.url, title: metadata.title,
|
|
@@ -466,24 +415,26 @@ export async function exploreUrl(
|
|
|
466
415
|
};
|
|
467
416
|
|
|
468
417
|
// Write artifacts
|
|
469
|
-
|
|
418
|
+
const writeTasks = [];
|
|
419
|
+
writeTasks.push(fs.promises.writeFile(path.join(targetDir, 'manifest.json'), JSON.stringify({
|
|
470
420
|
site: siteName, target_url: url, final_url: metadata.url, title: metadata.title,
|
|
471
421
|
framework, stores: stores.map(s => ({ type: s.type, id: s.id, actions: s.actions })),
|
|
472
422
|
top_strategy: topStrategy, explored_at: new Date().toISOString(),
|
|
473
|
-
}, null, 2));
|
|
474
|
-
fs.
|
|
423
|
+
}, null, 2)));
|
|
424
|
+
writeTasks.push(fs.promises.writeFile(path.join(targetDir, 'endpoints.json'), JSON.stringify(analyzedEndpoints.map(ep => ({
|
|
475
425
|
pattern: ep.pattern, method: ep.method, url: ep.url, status: ep.status,
|
|
476
426
|
contentType: ep.contentType, score: ep.score, queryParams: ep.queryParams,
|
|
477
427
|
itemPath: ep.responseAnalysis?.itemPath ?? null, itemCount: ep.responseAnalysis?.itemCount ?? 0,
|
|
478
428
|
detectedFields: ep.responseAnalysis?.detectedFields ?? {}, authIndicators: ep.authIndicators,
|
|
479
|
-
})), null, 2));
|
|
480
|
-
fs.
|
|
481
|
-
fs.
|
|
429
|
+
})), null, 2)));
|
|
430
|
+
writeTasks.push(fs.promises.writeFile(path.join(targetDir, 'capabilities.json'), JSON.stringify(capabilities, null, 2)));
|
|
431
|
+
writeTasks.push(fs.promises.writeFile(path.join(targetDir, 'auth.json'), JSON.stringify({
|
|
482
432
|
top_strategy: topStrategy, indicators: [...allAuth], framework,
|
|
483
|
-
}, null, 2));
|
|
433
|
+
}, null, 2)));
|
|
484
434
|
if (stores.length > 0) {
|
|
485
|
-
fs.
|
|
435
|
+
writeTasks.push(fs.promises.writeFile(path.join(targetDir, 'stores.json'), JSON.stringify(stores, null, 2)));
|
|
486
436
|
}
|
|
437
|
+
await Promise.all(writeTasks);
|
|
487
438
|
|
|
488
439
|
return { ...result, out_dir: targetDir };
|
|
489
440
|
})(), { timeout: exploreTimeout, label: `Explore ${url}` });
|
package/src/main.ts
CHANGED
|
@@ -6,16 +6,9 @@
|
|
|
6
6
|
import * as os from 'node:os';
|
|
7
7
|
import * as path from 'node:path';
|
|
8
8
|
import { fileURLToPath } from 'node:url';
|
|
9
|
-
import {
|
|
10
|
-
import
|
|
11
|
-
import {
|
|
12
|
-
import { Strategy, type CliCommand, fullName, getRegistry, strategyLabel } from './registry.js';
|
|
13
|
-
import { render as renderOutput } from './output.js';
|
|
14
|
-
import { BrowserBridge } from './browser/index.js';
|
|
15
|
-
import { browserSession, DEFAULT_BROWSER_COMMAND_TIMEOUT, runWithTimeout } from './runtime.js';
|
|
16
|
-
import { PKG_VERSION } from './version.js';
|
|
17
|
-
import { getCompletions, printCompletionScript } from './completion.js';
|
|
18
|
-
import { CliError } from './errors.js';
|
|
9
|
+
import { discoverClis } from './engine.js';
|
|
10
|
+
import { getCompletions } from './completion.js';
|
|
11
|
+
import { runCli } from './cli.js';
|
|
19
12
|
|
|
20
13
|
const __filename = fileURLToPath(import.meta.url);
|
|
21
14
|
const __dirname = path.dirname(__filename);
|
|
@@ -45,173 +38,4 @@ if (getCompIdx !== -1) {
|
|
|
45
38
|
process.exit(0);
|
|
46
39
|
}
|
|
47
40
|
|
|
48
|
-
|
|
49
|
-
program.name('opencli').description('Make any website your CLI. Zero setup. AI-powered.').version(PKG_VERSION);
|
|
50
|
-
|
|
51
|
-
// ── Built-in commands ──────────────────────────────────────────────────────
|
|
52
|
-
|
|
53
|
-
program.command('list').description('List all available CLI commands').option('-f, --format <fmt>', 'Output format: table, json, yaml, md, csv', 'table').option('--json', 'JSON output (deprecated)')
|
|
54
|
-
.action((opts) => {
|
|
55
|
-
const registry = getRegistry();
|
|
56
|
-
const commands = [...registry.values()].sort((a, b) => fullName(a).localeCompare(fullName(b)));
|
|
57
|
-
const rows = commands.map(c => ({
|
|
58
|
-
command: fullName(c),
|
|
59
|
-
site: c.site,
|
|
60
|
-
name: c.name,
|
|
61
|
-
description: c.description,
|
|
62
|
-
strategy: strategyLabel(c),
|
|
63
|
-
browser: c.browser,
|
|
64
|
-
args: c.args.map(a => a.name).join(', '),
|
|
65
|
-
}));
|
|
66
|
-
const fmt = opts.json && opts.format === 'table' ? 'json' : opts.format;
|
|
67
|
-
if (fmt !== 'table') {
|
|
68
|
-
renderOutput(rows, {
|
|
69
|
-
fmt,
|
|
70
|
-
columns: ['command', 'site', 'name', 'description', 'strategy', 'browser', 'args'],
|
|
71
|
-
title: 'opencli/list',
|
|
72
|
-
source: 'opencli list',
|
|
73
|
-
});
|
|
74
|
-
return;
|
|
75
|
-
}
|
|
76
|
-
const sites = new Map<string, CliCommand[]>();
|
|
77
|
-
for (const cmd of commands) { const g = sites.get(cmd.site) ?? []; g.push(cmd); sites.set(cmd.site, g); }
|
|
78
|
-
console.log(); console.log(chalk.bold(' opencli') + chalk.dim(' — available commands')); console.log();
|
|
79
|
-
for (const [site, cmds] of sites) {
|
|
80
|
-
console.log(chalk.bold.cyan(` ${site}`));
|
|
81
|
-
for (const cmd of cmds) { const tag = strategyLabel(cmd) === 'public' ? chalk.green('[public]') : chalk.yellow(`[${strategyLabel(cmd)}]`); console.log(` ${cmd.name} ${tag}${cmd.description ? chalk.dim(` — ${cmd.description}`) : ''}`); }
|
|
82
|
-
console.log();
|
|
83
|
-
}
|
|
84
|
-
console.log(chalk.dim(` ${commands.length} commands across ${sites.size} sites`)); console.log();
|
|
85
|
-
});
|
|
86
|
-
|
|
87
|
-
program.command('validate').description('Validate CLI definitions').argument('[target]', 'site or site/name')
|
|
88
|
-
.action(async (target) => {
|
|
89
|
-
const { validateClisWithTarget, renderValidationReport } = await import('./validate.js');
|
|
90
|
-
console.log(renderValidationReport(validateClisWithTarget([BUILTIN_CLIS, USER_CLIS], target)));
|
|
91
|
-
});
|
|
92
|
-
|
|
93
|
-
program.command('verify').description('Validate + smoke test').argument('[target]').option('--smoke', 'Run smoke tests', false)
|
|
94
|
-
.action(async (target, opts) => {
|
|
95
|
-
const { verifyClis, renderVerifyReport } = await import('./verify.js');
|
|
96
|
-
const r = await verifyClis({ builtinClis: BUILTIN_CLIS, userClis: USER_CLIS, target, smoke: opts.smoke });
|
|
97
|
-
console.log(renderVerifyReport(r));
|
|
98
|
-
process.exitCode = r.ok ? 0 : 1;
|
|
99
|
-
});
|
|
100
|
-
|
|
101
|
-
program.command('explore').alias('probe').description('Explore a website: discover APIs, stores, and recommend strategies').argument('<url>').option('--site <name>').option('--goal <text>').option('--wait <s>', '', '3').option('--auto', 'Enable interactive fuzzing (simulate clicks to trigger lazy APIs)').option('--click <labels>', 'Comma-separated labels to click before fuzzing (e.g. "字幕,CC,评论")')
|
|
102
|
-
.action(async (url, opts) => { const { exploreUrl, renderExploreSummary } = await import('./explore.js'); const clickLabels = opts.click ? opts.click.split(',').map((s: string) => s.trim()) : undefined; console.log(renderExploreSummary(await exploreUrl(url, { BrowserFactory: BrowserBridge, site: opts.site, goal: opts.goal, waitSeconds: parseFloat(opts.wait), auto: opts.auto, clickLabels }))); });
|
|
103
|
-
|
|
104
|
-
program.command('synthesize').description('Synthesize CLIs from explore').argument('<target>').option('--top <n>', '', '3')
|
|
105
|
-
.action(async (target, opts) => { const { synthesizeFromExplore, renderSynthesizeSummary } = await import('./synthesize.js'); console.log(renderSynthesizeSummary(synthesizeFromExplore(target, { top: parseInt(opts.top) }))); });
|
|
106
|
-
|
|
107
|
-
program.command('generate').description('One-shot: explore → synthesize → register').argument('<url>').option('--goal <text>').option('--site <name>')
|
|
108
|
-
.action(async (url, opts) => { const { generateCliFromUrl, renderGenerateSummary } = await import('./generate.js'); const r = await generateCliFromUrl({ url, BrowserFactory: BrowserBridge, builtinClis: BUILTIN_CLIS, userClis: USER_CLIS, goal: opts.goal, site: opts.site }); console.log(renderGenerateSummary(r)); process.exitCode = r.ok ? 0 : 1; });
|
|
109
|
-
|
|
110
|
-
program.command('cascade').description('Strategy cascade: find simplest working strategy').argument('<url>').option('--site <name>')
|
|
111
|
-
.action(async (url, opts) => {
|
|
112
|
-
const { cascadeProbe, renderCascadeResult } = await import('./cascade.js');
|
|
113
|
-
const result = await browserSession(BrowserBridge, async (page) => {
|
|
114
|
-
// Navigate to the site first for cookie context
|
|
115
|
-
try { const siteUrl = new URL(url); await page.goto(`${siteUrl.protocol}//${siteUrl.host}`); await page.wait(2); } catch {}
|
|
116
|
-
return cascadeProbe(page, url);
|
|
117
|
-
});
|
|
118
|
-
console.log(renderCascadeResult(result));
|
|
119
|
-
});
|
|
120
|
-
|
|
121
|
-
program.command('doctor')
|
|
122
|
-
.description('Diagnose opencli browser bridge connectivity')
|
|
123
|
-
.option('--live', 'Test browser connectivity (requires Chrome running)', false)
|
|
124
|
-
.action(async (opts) => {
|
|
125
|
-
const { runBrowserDoctor, renderBrowserDoctorReport } = await import('./doctor.js');
|
|
126
|
-
const report = await runBrowserDoctor({ live: opts.live, cliVersion: PKG_VERSION });
|
|
127
|
-
console.log(renderBrowserDoctorReport(report));
|
|
128
|
-
});
|
|
129
|
-
|
|
130
|
-
program.command('setup')
|
|
131
|
-
.description('Interactive setup: verify browser bridge connectivity')
|
|
132
|
-
.action(async () => {
|
|
133
|
-
const { runSetup } = await import('./setup.js');
|
|
134
|
-
await runSetup({ cliVersion: PKG_VERSION });
|
|
135
|
-
});
|
|
136
|
-
|
|
137
|
-
program.command('completion')
|
|
138
|
-
.description('Output shell completion script')
|
|
139
|
-
.argument('<shell>', 'Shell type: bash, zsh, or fish')
|
|
140
|
-
.action((shell) => {
|
|
141
|
-
printCompletionScript(shell);
|
|
142
|
-
});
|
|
143
|
-
|
|
144
|
-
// ── Dynamic site commands ──────────────────────────────────────────────────
|
|
145
|
-
|
|
146
|
-
const registry = getRegistry();
|
|
147
|
-
const siteGroups = new Map<string, Command>();
|
|
148
|
-
|
|
149
|
-
for (const [, cmd] of registry) {
|
|
150
|
-
let siteCmd = siteGroups.get(cmd.site);
|
|
151
|
-
if (!siteCmd) { siteCmd = program.command(cmd.site).description(`${cmd.site} commands`); siteGroups.set(cmd.site, siteCmd); }
|
|
152
|
-
const subCmd = siteCmd.command(cmd.name).description(cmd.description);
|
|
153
|
-
|
|
154
|
-
// Register positional args first, then named options
|
|
155
|
-
const positionalArgs: typeof cmd.args = [];
|
|
156
|
-
for (const arg of cmd.args) {
|
|
157
|
-
if (arg.positional) {
|
|
158
|
-
const bracket = arg.required ? `<${arg.name}>` : `[${arg.name}]`;
|
|
159
|
-
subCmd.argument(bracket, arg.help ?? '');
|
|
160
|
-
positionalArgs.push(arg);
|
|
161
|
-
} else {
|
|
162
|
-
const flag = arg.required ? `--${arg.name} <value>` : `--${arg.name} [value]`;
|
|
163
|
-
if (arg.required) subCmd.requiredOption(flag, arg.help ?? '');
|
|
164
|
-
else if (arg.default != null) subCmd.option(flag, arg.help ?? '', String(arg.default));
|
|
165
|
-
else subCmd.option(flag, arg.help ?? '');
|
|
166
|
-
}
|
|
167
|
-
}
|
|
168
|
-
subCmd.option('-f, --format <fmt>', 'Output format: table, json, yaml, md, csv', 'table').option('-v, --verbose', 'Debug output', false);
|
|
169
|
-
|
|
170
|
-
subCmd.action(async (...actionArgs: any[]) => {
|
|
171
|
-
// Commander passes positional args first, then options object, then the Command
|
|
172
|
-
const actionOpts = actionArgs[positionalArgs.length] ?? {};
|
|
173
|
-
const startTime = Date.now();
|
|
174
|
-
const kwargs: Record<string, any> = {};
|
|
175
|
-
|
|
176
|
-
// Collect positional args
|
|
177
|
-
for (let i = 0; i < positionalArgs.length; i++) {
|
|
178
|
-
const arg = positionalArgs[i];
|
|
179
|
-
const v = actionArgs[i];
|
|
180
|
-
if (v !== undefined) kwargs[arg.name] = v;
|
|
181
|
-
}
|
|
182
|
-
|
|
183
|
-
// Collect named options
|
|
184
|
-
for (const arg of cmd.args) {
|
|
185
|
-
if (arg.positional) continue;
|
|
186
|
-
const camelName = arg.name.replace(/-([a-z])/g, (_m, ch: string) => ch.toUpperCase());
|
|
187
|
-
const v = actionOpts[arg.name] ?? actionOpts[camelName];
|
|
188
|
-
if (v !== undefined) kwargs[arg.name] = v;
|
|
189
|
-
}
|
|
190
|
-
|
|
191
|
-
try {
|
|
192
|
-
if (actionOpts.verbose) process.env.OPENCLI_VERBOSE = '1';
|
|
193
|
-
let result: any;
|
|
194
|
-
if (cmd.browser) {
|
|
195
|
-
result = await browserSession(BrowserBridge, async (page) => {
|
|
196
|
-
return runWithTimeout(executeCommand(cmd, page, kwargs, actionOpts.verbose), { timeout: cmd.timeoutSeconds ?? DEFAULT_BROWSER_COMMAND_TIMEOUT, label: fullName(cmd) });
|
|
197
|
-
});
|
|
198
|
-
} else { result = await executeCommand(cmd, null, kwargs, actionOpts.verbose); }
|
|
199
|
-
if (actionOpts.verbose && (!result || (Array.isArray(result) && result.length === 0))) {
|
|
200
|
-
console.error(chalk.yellow(`[Verbose] Warning: Command returned an empty result. If the website structural API changed or requires authentication, check the network or update the adapter.`));
|
|
201
|
-
}
|
|
202
|
-
renderOutput(result, { fmt: actionOpts.format, columns: cmd.columns, title: `${cmd.site}/${cmd.name}`, elapsed: (Date.now() - startTime) / 1000, source: fullName(cmd) });
|
|
203
|
-
} catch (err: any) {
|
|
204
|
-
if (err instanceof CliError) {
|
|
205
|
-
console.error(chalk.red(`Error [${err.code}]: ${err.message}`));
|
|
206
|
-
if (err.hint) console.error(chalk.yellow(`Hint: ${err.hint}`));
|
|
207
|
-
} else if (actionOpts.verbose && err.stack) {
|
|
208
|
-
console.error(chalk.red(err.stack));
|
|
209
|
-
} else {
|
|
210
|
-
console.error(chalk.red(`Error: ${err.message ?? err}`));
|
|
211
|
-
}
|
|
212
|
-
process.exitCode = 1;
|
|
213
|
-
}
|
|
214
|
-
});
|
|
215
|
-
}
|
|
216
|
-
|
|
217
|
-
program.parse();
|
|
41
|
+
runCli(BUILTIN_CLIS, USER_CLIS);
|