@jackwener/opencli 1.0.1 → 1.0.3

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (91) hide show
  1. package/.github/workflows/build-extension.yml +62 -0
  2. package/.github/workflows/ci.yml +6 -6
  3. package/.github/workflows/e2e-headed.yml +2 -2
  4. package/.github/workflows/pkg-pr-new.yml +2 -2
  5. package/.github/workflows/release.yml +2 -5
  6. package/.github/workflows/security.yml +2 -2
  7. package/CDP.md +1 -1
  8. package/CDP.zh-CN.md +1 -1
  9. package/README.md +15 -7
  10. package/README.zh-CN.md +15 -7
  11. package/SKILL.md +3 -5
  12. package/dist/browser/cdp.d.ts +27 -0
  13. package/dist/browser/cdp.js +295 -0
  14. package/dist/browser/index.d.ts +3 -0
  15. package/dist/browser/index.js +4 -0
  16. package/dist/browser/page.js +2 -23
  17. package/dist/browser/utils.d.ts +10 -0
  18. package/dist/browser/utils.js +27 -0
  19. package/dist/browser.test.js +42 -1
  20. package/dist/chaoxing.d.ts +58 -0
  21. package/dist/chaoxing.js +225 -0
  22. package/dist/chaoxing.test.d.ts +1 -0
  23. package/dist/chaoxing.test.js +38 -0
  24. package/dist/cli-manifest.json +203 -0
  25. package/dist/cli.d.ts +1 -0
  26. package/dist/cli.js +197 -0
  27. package/dist/clis/boss/chatlist.d.ts +1 -0
  28. package/dist/clis/boss/chatlist.js +50 -0
  29. package/dist/clis/boss/chatmsg.d.ts +1 -0
  30. package/dist/clis/boss/chatmsg.js +73 -0
  31. package/dist/clis/boss/send.d.ts +1 -0
  32. package/dist/clis/boss/send.js +176 -0
  33. package/dist/clis/chaoxing/assignments.d.ts +1 -0
  34. package/dist/clis/chaoxing/assignments.js +74 -0
  35. package/dist/clis/chaoxing/exams.d.ts +1 -0
  36. package/dist/clis/chaoxing/exams.js +74 -0
  37. package/dist/clis/chatgpt/ask.js +15 -14
  38. package/dist/clis/chatgpt/ax.d.ts +1 -0
  39. package/dist/clis/chatgpt/ax.js +78 -0
  40. package/dist/clis/chatgpt/read.js +5 -6
  41. package/dist/clis/twitter/post.js +9 -2
  42. package/dist/clis/twitter/search.js +14 -33
  43. package/dist/clis/xiaohongshu/download.d.ts +1 -1
  44. package/dist/clis/xiaohongshu/download.js +1 -1
  45. package/dist/engine.js +24 -13
  46. package/dist/explore.js +46 -101
  47. package/dist/main.js +4 -193
  48. package/dist/output.d.ts +1 -1
  49. package/dist/registry.d.ts +3 -3
  50. package/dist/scripts/framework.d.ts +4 -0
  51. package/dist/scripts/framework.js +21 -0
  52. package/dist/scripts/interact.d.ts +4 -0
  53. package/dist/scripts/interact.js +20 -0
  54. package/dist/scripts/store.d.ts +9 -0
  55. package/dist/scripts/store.js +44 -0
  56. package/dist/synthesize.js +1 -1
  57. package/extension/dist/background.js +338 -430
  58. package/extension/manifest.json +2 -2
  59. package/extension/src/background.ts +2 -2
  60. package/package.json +1 -1
  61. package/src/browser/cdp.ts +295 -0
  62. package/src/browser/index.ts +4 -0
  63. package/src/browser/page.ts +2 -24
  64. package/src/browser/utils.ts +27 -0
  65. package/src/browser.test.ts +46 -0
  66. package/src/chaoxing.test.ts +45 -0
  67. package/src/chaoxing.ts +268 -0
  68. package/src/cli.ts +185 -0
  69. package/src/clis/antigravity/SKILL.md +5 -0
  70. package/src/clis/boss/chatlist.ts +50 -0
  71. package/src/clis/boss/chatmsg.ts +70 -0
  72. package/src/clis/boss/send.ts +193 -0
  73. package/src/clis/chaoxing/README.md +36 -0
  74. package/src/clis/chaoxing/README.zh-CN.md +35 -0
  75. package/src/clis/chaoxing/assignments.ts +88 -0
  76. package/src/clis/chaoxing/exams.ts +88 -0
  77. package/src/clis/chatgpt/ask.ts +14 -15
  78. package/src/clis/chatgpt/ax.ts +81 -0
  79. package/src/clis/chatgpt/read.ts +5 -7
  80. package/src/clis/twitter/post.ts +9 -2
  81. package/src/clis/twitter/search.ts +15 -33
  82. package/src/clis/xiaohongshu/download.ts +1 -1
  83. package/src/engine.ts +20 -13
  84. package/src/explore.ts +51 -100
  85. package/src/main.ts +4 -180
  86. package/src/output.ts +12 -12
  87. package/src/registry.ts +3 -3
  88. package/src/scripts/framework.ts +20 -0
  89. package/src/scripts/interact.ts +22 -0
  90. package/src/scripts/store.ts +40 -0
  91. package/src/synthesize.ts +1 -1
@@ -24,46 +24,28 @@ cli({
24
24
  // fetch will capture the SearchTimeline API call.
25
25
  await page.installInterceptor('SearchTimeline');
26
26
 
27
- // 3. Use the search input to submit the query (SPA, no full reload).
28
- // Find the search input, type the query, and submit.
27
+ // 3. Trigger SPA navigation to search results via history API.
28
+ // pushState + popstate triggers React Router's listener without
29
+ // a full page reload, so the interceptor stays alive.
30
+ // Note: the previous approach (nativeSetter + Enter keydown on the
31
+ // search input) does not reliably trigger Twitter's form submission.
32
+ const searchUrl = JSON.stringify(`/search?q=${encodeURIComponent(query)}&f=top`);
29
33
  await page.evaluate(`
30
34
  (() => {
31
- const input = document.querySelector('input[data-testid="SearchBox_Search_Input"]');
32
- if (!input) throw new Error('Search input not found');
33
- input.focus();
34
- const nativeSetter = Object.getOwnPropertyDescriptor(
35
- HTMLInputElement.prototype, 'value'
36
- ).set;
37
- nativeSetter.call(input, ${JSON.stringify(query)});
38
- input.dispatchEvent(new Event('input', { bubbles: true }));
39
- })()
40
- `);
41
- await page.wait(0.5);
42
- // Press Enter to submit
43
- await page.evaluate(`
44
- (() => {
45
- const input = document.querySelector('input[data-testid="SearchBox_Search_Input"]');
46
- if (!input) throw new Error('Search input not found');
47
- input.dispatchEvent(new KeyboardEvent('keydown', { key: 'Enter', code: 'Enter', keyCode: 13, bubbles: true }));
35
+ window.history.pushState({}, '', ${searchUrl});
36
+ window.dispatchEvent(new PopStateEvent('popstate', { state: {} }));
48
37
  })()
49
38
  `);
50
39
  await page.wait(5);
51
40
 
52
- // 4. Click "Top" tab if available (ensures we get top results)
53
- try {
54
- await page.evaluate(`
55
- (() => {
56
- const tabs = document.querySelectorAll('[role="tab"]');
57
- for (const tab of tabs) {
58
- if (tab.textContent.trim() === 'Top') { tab.click(); break; }
59
- }
60
- })()
61
- `);
62
- await page.wait(2);
63
- } catch { /* ignore if tab not found */ }
41
+ // Verify SPA navigation succeeded
42
+ const currentPath = await page.evaluate('() => window.location.pathname');
43
+ if (!currentPath?.startsWith('/search')) {
44
+ throw new Error('SPA navigation to /search failed. Twitter may have changed its routing.');
45
+ }
64
46
 
65
- // 5. Scroll to trigger additional pagination
66
- await page.autoScroll({ times: 2, delayMs: 2000 });
47
+ // 4. Scroll to trigger additional pagination
48
+ await page.autoScroll({ times: 3, delayMs: 2000 });
67
49
 
68
50
  // 6. Retrieve captured data
69
51
  const requests = await page.getInterceptedRequests();
@@ -2,7 +2,7 @@
2
2
  * Xiaohongshu download — download images and videos from a note.
3
3
  *
4
4
  * Usage:
5
- * opencli xiaohongshu download --note-id abc123 --output ./xhs
5
+ * opencli xiaohongshu download --note_id abc123 --output ./xhs
6
6
  */
7
7
 
8
8
  import * as fs from 'node:fs';
package/src/engine.ts CHANGED
@@ -28,12 +28,14 @@ export async function discoverClis(...dirs: string[]): Promise<void> {
28
28
  // Fast path: try manifest first (production / post-build)
29
29
  for (const dir of dirs) {
30
30
  const manifestPath = path.resolve(dir, '..', 'cli-manifest.json');
31
- if (fs.existsSync(manifestPath)) {
32
- loadFromManifest(manifestPath, dir);
31
+ try {
32
+ await fs.promises.access(manifestPath);
33
+ await loadFromManifest(manifestPath, dir);
33
34
  continue; // Skip filesystem scan for this directory
35
+ } catch {
36
+ // Fallback: runtime filesystem scan (development)
37
+ await discoverClisFromFs(dir);
34
38
  }
35
- // Fallback: runtime filesystem scan (development)
36
- await discoverClisFromFs(dir);
37
39
  }
38
40
  }
39
41
 
@@ -42,9 +44,10 @@ export async function discoverClis(...dirs: string[]): Promise<void> {
42
44
  * YAML pipelines are inlined — zero YAML parsing at runtime.
43
45
  * TS modules are deferred — loaded lazily on first execution.
44
46
  */
45
- function loadFromManifest(manifestPath: string, clisDir: string): void {
47
+ async function loadFromManifest(manifestPath: string, clisDir: string): Promise<void> {
46
48
  try {
47
- const manifest = JSON.parse(fs.readFileSync(manifestPath, 'utf-8')) as any[];
49
+ const raw = await fs.promises.readFile(manifestPath, 'utf-8');
50
+ const manifest = JSON.parse(raw) as any[];
48
51
  for (const entry of manifest) {
49
52
  if (entry.type === 'yaml') {
50
53
  // YAML pipelines fully inlined in manifest — register directly
@@ -94,15 +97,19 @@ function loadFromManifest(manifestPath: string, clisDir: string): void {
94
97
  * Fallback: traditional filesystem scan (used during development with tsx).
95
98
  */
96
99
  async function discoverClisFromFs(dir: string): Promise<void> {
97
- if (!fs.existsSync(dir)) return;
100
+ try { await fs.promises.access(dir); } catch { return; }
98
101
  const promises: Promise<any>[] = [];
99
- for (const site of fs.readdirSync(dir)) {
102
+ const sites = await fs.promises.readdir(dir);
103
+
104
+ for (const site of sites) {
100
105
  const siteDir = path.join(dir, site);
101
- if (!fs.statSync(siteDir).isDirectory()) continue;
102
- for (const file of fs.readdirSync(siteDir)) {
106
+ const stat = await fs.promises.stat(siteDir);
107
+ if (!stat.isDirectory()) continue;
108
+ const files = await fs.promises.readdir(siteDir);
109
+ for (const file of files) {
103
110
  const filePath = path.join(siteDir, file);
104
111
  if (file.endsWith('.yaml') || file.endsWith('.yml')) {
105
- registerYamlCli(filePath, site);
112
+ promises.push(registerYamlCli(filePath, site));
106
113
  } else if (
107
114
  (file.endsWith('.js') && !file.endsWith('.d.js')) ||
108
115
  (file.endsWith('.ts') && !file.endsWith('.d.ts') && !file.endsWith('.test.ts'))
@@ -118,9 +125,9 @@ async function discoverClisFromFs(dir: string): Promise<void> {
118
125
  await Promise.all(promises);
119
126
  }
120
127
 
121
- function registerYamlCli(filePath: string, defaultSite: string): void {
128
+ async function registerYamlCli(filePath: string, defaultSite: string): Promise<void> {
122
129
  try {
123
- const raw = fs.readFileSync(filePath, 'utf-8');
130
+ const raw = await fs.promises.readFile(filePath, 'utf-8');
124
131
  const def = yaml.load(raw) as any;
125
132
  if (!def || typeof def !== 'object') return;
126
133
 
package/src/explore.ts CHANGED
@@ -10,6 +10,9 @@ import * as fs from 'node:fs';
10
10
  import * as path from 'node:path';
11
11
  import { DEFAULT_BROWSER_EXPLORE_TIMEOUT, browserSession, runWithTimeout } from './runtime.js';
12
12
  import { VOLATILE_PARAMS, SEARCH_PARAMS, PAGINATION_PARAMS, LIMIT_PARAMS, FIELD_ROLES } from './constants.js';
13
+ import { detectFramework } from './scripts/framework.js';
14
+ import { discoverStores } from './scripts/store.js';
15
+ import { interactFuzz } from './scripts/interact.js';
13
16
 
14
17
  // ── Site name detection ────────────────────────────────────────────────────
15
18
 
@@ -50,7 +53,7 @@ export function slugify(value: string): string {
50
53
 
51
54
  interface NetworkEntry {
52
55
  method: string; url: string; status: number | null;
53
- contentType: string; responseBody?: any; requestHeaders?: Record<string, string>;
56
+ contentType: string; responseBody?: unknown; requestHeaders?: Record<string, string>;
54
57
  }
55
58
 
56
59
  interface AnalyzedEndpoint {
@@ -72,7 +75,7 @@ interface InferredCapability {
72
75
  * Parse raw network output from Playwright MCP.
73
76
  * Handles text format: [GET] url => [200]
74
77
  */
75
- function parseNetworkRequests(raw: any): NetworkEntry[] {
78
+ function parseNetworkRequests(raw: unknown): NetworkEntry[] {
76
79
  if (typeof raw === 'string') {
77
80
  const entries: NetworkEntry[] = [];
78
81
  for (const line of raw.split('\n')) {
@@ -120,11 +123,11 @@ function detectAuthIndicators(headers?: Record<string, string>): string[] {
120
123
  return indicators;
121
124
  }
122
125
 
123
- function analyzeResponseBody(body: any): AnalyzedEndpoint['responseAnalysis'] {
126
+ function analyzeResponseBody(body: unknown): AnalyzedEndpoint['responseAnalysis'] {
124
127
  if (!body || typeof body !== 'object') return null;
125
- const candidates: Array<{ path: string; items: any[] }> = [];
128
+ const candidates: Array<{ path: string; items: unknown[] }> = [];
126
129
 
127
- function findArrays(obj: any, path: string, depth: number) {
130
+ function findArrays(obj: unknown, path: string, depth: number) {
128
131
  if (depth > 4) return;
129
132
  if (Array.isArray(obj) && obj.length >= 2 && obj.some(item => item && typeof item === 'object' && !Array.isArray(item))) {
130
133
  candidates.push({ path, items: obj });
@@ -151,13 +154,15 @@ function analyzeResponseBody(body: any): AnalyzedEndpoint['responseAnalysis'] {
151
154
  return { itemPath: best.path || null, itemCount: best.items.length, detectedFields, sampleFields };
152
155
  }
153
156
 
154
- function flattenFields(obj: any, prefix: string, maxDepth: number): string[] {
157
+ function flattenFields(obj: unknown, prefix: string, maxDepth: number): string[] {
155
158
  if (maxDepth <= 0 || !obj || typeof obj !== 'object') return [];
156
159
  const names: string[] = [];
157
- for (const key of Object.keys(obj)) {
160
+ const record = obj as Record<string, unknown>;
161
+ for (const key of Object.keys(record)) {
158
162
  const full = prefix ? `${prefix}.${key}` : key;
159
163
  names.push(full);
160
- if (obj[key] && typeof obj[key] === 'object' && !Array.isArray(obj[key])) names.push(...flattenFields(obj[key], full, maxDepth - 1));
164
+ const val = record[key];
165
+ if (val && typeof val === 'object' && !Array.isArray(val)) names.push(...flattenFields(val, full, maxDepth - 1));
161
166
  }
162
167
  return names;
163
168
  }
@@ -201,63 +206,11 @@ function inferStrategy(authIndicators: string[]): string {
201
206
 
202
207
  // ── Framework detection ────────────────────────────────────────────────────
203
208
 
204
- const FRAMEWORK_DETECT_JS = `
205
- () => {
206
- const r = {};
207
- try {
208
- const app = document.querySelector('#app');
209
- r.vue3 = !!(app && app.__vue_app__);
210
- r.vue2 = !!(app && app.__vue__);
211
- r.react = !!window.__REACT_DEVTOOLS_GLOBAL_HOOK__ || !!document.querySelector('[data-reactroot]');
212
- r.nextjs = !!window.__NEXT_DATA__;
213
- r.nuxt = !!window.__NUXT__;
214
- if (r.vue3 && app.__vue_app__) { const gp = app.__vue_app__.config?.globalProperties; r.pinia = !!(gp && gp.$pinia); r.vuex = !!(gp && gp.$store); }
215
- } catch {}
216
- return r;
217
- }
218
- `;
209
+ const FRAMEWORK_DETECT_JS = detectFramework.toString();
219
210
 
220
211
  // ── Store discovery ────────────────────────────────────────────────────────
221
212
 
222
- const STORE_DISCOVER_JS = `
223
- () => {
224
- const stores = [];
225
- try {
226
- const app = document.querySelector('#app');
227
- if (!app?.__vue_app__) return stores;
228
- const gp = app.__vue_app__.config?.globalProperties;
229
-
230
- // Pinia stores
231
- const pinia = gp?.$pinia;
232
- if (pinia?._s) {
233
- pinia._s.forEach((store, id) => {
234
- const actions = [];
235
- const stateKeys = [];
236
- for (const k in store) {
237
- try {
238
- if (k.startsWith('$') || k.startsWith('_')) continue;
239
- if (typeof store[k] === 'function') actions.push(k);
240
- else stateKeys.push(k);
241
- } catch {}
242
- }
243
- stores.push({ type: 'pinia', id, actions: actions.slice(0, 20), stateKeys: stateKeys.slice(0, 15) });
244
- });
245
- }
246
-
247
- // Vuex store modules
248
- const vuex = gp?.$store;
249
- if (vuex?._modules?.root?._children) {
250
- const children = vuex._modules.root._children;
251
- for (const [modName, mod] of Object.entries(children)) {
252
- const actions = Object.keys(mod._rawModule?.actions ?? {}).slice(0, 20);
253
- const stateKeys = Object.keys(mod.state ?? {}).slice(0, 15);
254
- stores.push({ type: 'vuex', id: modName, actions, stateKeys });
255
- }
256
- }
257
- } catch {}
258
- return stores;
259
- }
260
- `;
213
+ const STORE_DISCOVER_JS = discoverStores.toString();
261
214
 
262
215
  export interface DiscoveredStore {
263
216
  type: 'pinia' | 'vuex';
@@ -268,27 +221,7 @@ export interface DiscoveredStore {
268
221
 
269
222
  // ── Auto-Interaction (Fuzzing) ─────────────────────────────────────────────
270
223
 
271
- const INTERACT_FUZZ_JS = `
272
- async () => {
273
- const sleep = ms => new Promise(r => setTimeout(r, ms));
274
- const clickables = Array.from(document.querySelectorAll(
275
- 'button, [role="button"], [role="tab"], .tab, .btn, a[href="javascript:void(0)"], a[href="#"]'
276
- )).slice(0, 15); // limit to 15 to avoid endless loops
277
-
278
- let clicked = 0;
279
- for (const el of clickables) {
280
- try {
281
- const rect = el.getBoundingClientRect();
282
- if (rect.width > 0 && rect.height > 0) {
283
- el.dispatchEvent(new MouseEvent('click', { bubbles: true, cancelable: true, view: window }));
284
- clicked++;
285
- await sleep(300); // give it time to trigger network
286
- }
287
- } catch {}
288
- }
289
- return clicked;
290
- }
291
- `;
224
+ const INTERACT_FUZZ_JS = interactFuzz.toString();
292
225
 
293
226
  // ── Main explore function ──────────────────────────────────────────────────
294
227
 
@@ -310,8 +243,8 @@ export async function exploreUrl(
310
243
  await page.goto(url);
311
244
  await page.wait(waitSeconds);
312
245
 
313
- // Step 2: Auto-scroll to trigger lazy loading (use keyboard since page.scroll may not exist)
314
- for (let i = 0; i < 3; i++) { try { await page.pressKey('End'); } catch {} await page.wait(1); }
246
+ // Step 2: Auto-scroll to trigger lazy loading intelligently
247
+ await page.autoScroll({ times: 3, delayMs: 1500 }).catch(() => {});
315
248
 
316
249
  // Step 2.5: Interactive Fuzzing (if requested)
317
250
  if (opts.auto) {
@@ -319,11 +252,11 @@ export async function exploreUrl(
319
252
  // First: targeted clicks by label (e.g. "字幕", "CC", "评论")
320
253
  if (opts.clickLabels?.length) {
321
254
  for (const label of opts.clickLabels) {
322
- const safeLabel = label.replace(/'/g, "\\'");
255
+ const safeLabel = JSON.stringify(label);
323
256
  await page.evaluate(`
324
257
  (() => {
325
258
  const el = [...document.querySelectorAll('button, [role="button"], [role="tab"], a, span')]
326
- .find(e => e.textContent && e.textContent.trim().includes('${safeLabel}'));
259
+ .find(e => e.textContent && e.textContent.trim().includes(${safeLabel}));
327
260
  if (el) el.click();
328
261
  })()
329
262
  `);
@@ -345,11 +278,27 @@ export async function exploreUrl(
345
278
  const rawNetwork = await page.networkRequests(false);
346
279
  const networkEntries = parseNetworkRequests(rawNetwork);
347
280
 
348
- // Step 5: For JSON endpoints, re-fetch response body in-browser
349
- const jsonEndpoints = networkEntries.filter(e => e.contentType.includes('json') && e.method === 'GET' && e.status === 200);
350
- for (const ep of jsonEndpoints.slice(0, 10)) {
281
+ // Step 5: For JSON endpoints missing a body, carefully re-fetch in-browser via a pristine iframe
282
+ const jsonEndpoints = networkEntries.filter(e => e.contentType.includes('json') && e.method === 'GET' && e.status === 200 && !e.responseBody);
283
+ for (const ep of jsonEndpoints.slice(0, 5)) {
351
284
  try {
352
- const body = await page.evaluate(`async () => { try { const r = await fetch(${JSON.stringify(ep.url)}, {credentials:'include'}); if (!r.ok) return null; const d = await r.json(); return JSON.stringify(d).slice(0,10000); } catch { return null; } }`);
285
+ const body = await page.evaluate(`async () => {
286
+ let iframe = null;
287
+ try {
288
+ iframe = document.createElement('iframe');
289
+ iframe.style.display = 'none';
290
+ document.body.appendChild(iframe);
291
+ const cleanFetch = iframe.contentWindow.fetch || window.fetch;
292
+ const r = await cleanFetch(${JSON.stringify(ep.url)}, { credentials: 'include' });
293
+ if (!r.ok) return null;
294
+ const d = await r.json();
295
+ return JSON.stringify(d).slice(0, 10000);
296
+ } catch {
297
+ return null;
298
+ } finally {
299
+ if (iframe && iframe.parentNode) iframe.parentNode.removeChild(iframe);
300
+ }
301
+ }`);
353
302
  if (body && typeof body === 'string') { try { ep.responseBody = JSON.parse(body); } catch {} }
354
303
  else if (body && typeof body === 'object') ep.responseBody = body;
355
304
  } catch {}
@@ -455,7 +404,7 @@ export async function exploreUrl(
455
404
 
456
405
  const siteName = opts.site ?? detectSiteName(metadata.url || url);
457
406
  const targetDir = opts.outDir ?? path.join('.opencli', 'explore', siteName);
458
- fs.mkdirSync(targetDir, { recursive: true });
407
+ await fs.promises.mkdir(targetDir, { recursive: true });
459
408
 
460
409
  const result = {
461
410
  site: siteName, target_url: url, final_url: metadata.url, title: metadata.title,
@@ -466,24 +415,26 @@ export async function exploreUrl(
466
415
  };
467
416
 
468
417
  // Write artifacts
469
- fs.writeFileSync(path.join(targetDir, 'manifest.json'), JSON.stringify({
418
+ const writeTasks = [];
419
+ writeTasks.push(fs.promises.writeFile(path.join(targetDir, 'manifest.json'), JSON.stringify({
470
420
  site: siteName, target_url: url, final_url: metadata.url, title: metadata.title,
471
421
  framework, stores: stores.map(s => ({ type: s.type, id: s.id, actions: s.actions })),
472
422
  top_strategy: topStrategy, explored_at: new Date().toISOString(),
473
- }, null, 2));
474
- fs.writeFileSync(path.join(targetDir, 'endpoints.json'), JSON.stringify(analyzedEndpoints.map(ep => ({
423
+ }, null, 2)));
424
+ writeTasks.push(fs.promises.writeFile(path.join(targetDir, 'endpoints.json'), JSON.stringify(analyzedEndpoints.map(ep => ({
475
425
  pattern: ep.pattern, method: ep.method, url: ep.url, status: ep.status,
476
426
  contentType: ep.contentType, score: ep.score, queryParams: ep.queryParams,
477
427
  itemPath: ep.responseAnalysis?.itemPath ?? null, itemCount: ep.responseAnalysis?.itemCount ?? 0,
478
428
  detectedFields: ep.responseAnalysis?.detectedFields ?? {}, authIndicators: ep.authIndicators,
479
- })), null, 2));
480
- fs.writeFileSync(path.join(targetDir, 'capabilities.json'), JSON.stringify(capabilities, null, 2));
481
- fs.writeFileSync(path.join(targetDir, 'auth.json'), JSON.stringify({
429
+ })), null, 2)));
430
+ writeTasks.push(fs.promises.writeFile(path.join(targetDir, 'capabilities.json'), JSON.stringify(capabilities, null, 2)));
431
+ writeTasks.push(fs.promises.writeFile(path.join(targetDir, 'auth.json'), JSON.stringify({
482
432
  top_strategy: topStrategy, indicators: [...allAuth], framework,
483
- }, null, 2));
433
+ }, null, 2)));
484
434
  if (stores.length > 0) {
485
- fs.writeFileSync(path.join(targetDir, 'stores.json'), JSON.stringify(stores, null, 2));
435
+ writeTasks.push(fs.promises.writeFile(path.join(targetDir, 'stores.json'), JSON.stringify(stores, null, 2)));
486
436
  }
437
+ await Promise.all(writeTasks);
487
438
 
488
439
  return { ...result, out_dir: targetDir };
489
440
  })(), { timeout: exploreTimeout, label: `Explore ${url}` });
package/src/main.ts CHANGED
@@ -6,16 +6,9 @@
6
6
  import * as os from 'node:os';
7
7
  import * as path from 'node:path';
8
8
  import { fileURLToPath } from 'node:url';
9
- import { Command } from 'commander';
10
- import chalk from 'chalk';
11
- import { discoverClis, executeCommand } from './engine.js';
12
- import { Strategy, type CliCommand, fullName, getRegistry, strategyLabel } from './registry.js';
13
- import { render as renderOutput } from './output.js';
14
- import { BrowserBridge } from './browser/index.js';
15
- import { browserSession, DEFAULT_BROWSER_COMMAND_TIMEOUT, runWithTimeout } from './runtime.js';
16
- import { PKG_VERSION } from './version.js';
17
- import { getCompletions, printCompletionScript } from './completion.js';
18
- import { CliError } from './errors.js';
9
+ import { discoverClis } from './engine.js';
10
+ import { getCompletions } from './completion.js';
11
+ import { runCli } from './cli.js';
19
12
 
20
13
  const __filename = fileURLToPath(import.meta.url);
21
14
  const __dirname = path.dirname(__filename);
@@ -45,173 +38,4 @@ if (getCompIdx !== -1) {
45
38
  process.exit(0);
46
39
  }
47
40
 
48
- const program = new Command();
49
- program.name('opencli').description('Make any website your CLI. Zero setup. AI-powered.').version(PKG_VERSION);
50
-
51
- // ── Built-in commands ──────────────────────────────────────────────────────
52
-
53
- program.command('list').description('List all available CLI commands').option('-f, --format <fmt>', 'Output format: table, json, yaml, md, csv', 'table').option('--json', 'JSON output (deprecated)')
54
- .action((opts) => {
55
- const registry = getRegistry();
56
- const commands = [...registry.values()].sort((a, b) => fullName(a).localeCompare(fullName(b)));
57
- const rows = commands.map(c => ({
58
- command: fullName(c),
59
- site: c.site,
60
- name: c.name,
61
- description: c.description,
62
- strategy: strategyLabel(c),
63
- browser: c.browser,
64
- args: c.args.map(a => a.name).join(', '),
65
- }));
66
- const fmt = opts.json && opts.format === 'table' ? 'json' : opts.format;
67
- if (fmt !== 'table') {
68
- renderOutput(rows, {
69
- fmt,
70
- columns: ['command', 'site', 'name', 'description', 'strategy', 'browser', 'args'],
71
- title: 'opencli/list',
72
- source: 'opencli list',
73
- });
74
- return;
75
- }
76
- const sites = new Map<string, CliCommand[]>();
77
- for (const cmd of commands) { const g = sites.get(cmd.site) ?? []; g.push(cmd); sites.set(cmd.site, g); }
78
- console.log(); console.log(chalk.bold(' opencli') + chalk.dim(' — available commands')); console.log();
79
- for (const [site, cmds] of sites) {
80
- console.log(chalk.bold.cyan(` ${site}`));
81
- for (const cmd of cmds) { const tag = strategyLabel(cmd) === 'public' ? chalk.green('[public]') : chalk.yellow(`[${strategyLabel(cmd)}]`); console.log(` ${cmd.name} ${tag}${cmd.description ? chalk.dim(` — ${cmd.description}`) : ''}`); }
82
- console.log();
83
- }
84
- console.log(chalk.dim(` ${commands.length} commands across ${sites.size} sites`)); console.log();
85
- });
86
-
87
- program.command('validate').description('Validate CLI definitions').argument('[target]', 'site or site/name')
88
- .action(async (target) => {
89
- const { validateClisWithTarget, renderValidationReport } = await import('./validate.js');
90
- console.log(renderValidationReport(validateClisWithTarget([BUILTIN_CLIS, USER_CLIS], target)));
91
- });
92
-
93
- program.command('verify').description('Validate + smoke test').argument('[target]').option('--smoke', 'Run smoke tests', false)
94
- .action(async (target, opts) => {
95
- const { verifyClis, renderVerifyReport } = await import('./verify.js');
96
- const r = await verifyClis({ builtinClis: BUILTIN_CLIS, userClis: USER_CLIS, target, smoke: opts.smoke });
97
- console.log(renderVerifyReport(r));
98
- process.exitCode = r.ok ? 0 : 1;
99
- });
100
-
101
- program.command('explore').alias('probe').description('Explore a website: discover APIs, stores, and recommend strategies').argument('<url>').option('--site <name>').option('--goal <text>').option('--wait <s>', '', '3').option('--auto', 'Enable interactive fuzzing (simulate clicks to trigger lazy APIs)').option('--click <labels>', 'Comma-separated labels to click before fuzzing (e.g. "字幕,CC,评论")')
102
- .action(async (url, opts) => { const { exploreUrl, renderExploreSummary } = await import('./explore.js'); const clickLabels = opts.click ? opts.click.split(',').map((s: string) => s.trim()) : undefined; console.log(renderExploreSummary(await exploreUrl(url, { BrowserFactory: BrowserBridge, site: opts.site, goal: opts.goal, waitSeconds: parseFloat(opts.wait), auto: opts.auto, clickLabels }))); });
103
-
104
- program.command('synthesize').description('Synthesize CLIs from explore').argument('<target>').option('--top <n>', '', '3')
105
- .action(async (target, opts) => { const { synthesizeFromExplore, renderSynthesizeSummary } = await import('./synthesize.js'); console.log(renderSynthesizeSummary(synthesizeFromExplore(target, { top: parseInt(opts.top) }))); });
106
-
107
- program.command('generate').description('One-shot: explore → synthesize → register').argument('<url>').option('--goal <text>').option('--site <name>')
108
- .action(async (url, opts) => { const { generateCliFromUrl, renderGenerateSummary } = await import('./generate.js'); const r = await generateCliFromUrl({ url, BrowserFactory: BrowserBridge, builtinClis: BUILTIN_CLIS, userClis: USER_CLIS, goal: opts.goal, site: opts.site }); console.log(renderGenerateSummary(r)); process.exitCode = r.ok ? 0 : 1; });
109
-
110
- program.command('cascade').description('Strategy cascade: find simplest working strategy').argument('<url>').option('--site <name>')
111
- .action(async (url, opts) => {
112
- const { cascadeProbe, renderCascadeResult } = await import('./cascade.js');
113
- const result = await browserSession(BrowserBridge, async (page) => {
114
- // Navigate to the site first for cookie context
115
- try { const siteUrl = new URL(url); await page.goto(`${siteUrl.protocol}//${siteUrl.host}`); await page.wait(2); } catch {}
116
- return cascadeProbe(page, url);
117
- });
118
- console.log(renderCascadeResult(result));
119
- });
120
-
121
- program.command('doctor')
122
- .description('Diagnose opencli browser bridge connectivity')
123
- .option('--live', 'Test browser connectivity (requires Chrome running)', false)
124
- .action(async (opts) => {
125
- const { runBrowserDoctor, renderBrowserDoctorReport } = await import('./doctor.js');
126
- const report = await runBrowserDoctor({ live: opts.live, cliVersion: PKG_VERSION });
127
- console.log(renderBrowserDoctorReport(report));
128
- });
129
-
130
- program.command('setup')
131
- .description('Interactive setup: verify browser bridge connectivity')
132
- .action(async () => {
133
- const { runSetup } = await import('./setup.js');
134
- await runSetup({ cliVersion: PKG_VERSION });
135
- });
136
-
137
- program.command('completion')
138
- .description('Output shell completion script')
139
- .argument('<shell>', 'Shell type: bash, zsh, or fish')
140
- .action((shell) => {
141
- printCompletionScript(shell);
142
- });
143
-
144
- // ── Dynamic site commands ──────────────────────────────────────────────────
145
-
146
- const registry = getRegistry();
147
- const siteGroups = new Map<string, Command>();
148
-
149
- for (const [, cmd] of registry) {
150
- let siteCmd = siteGroups.get(cmd.site);
151
- if (!siteCmd) { siteCmd = program.command(cmd.site).description(`${cmd.site} commands`); siteGroups.set(cmd.site, siteCmd); }
152
- const subCmd = siteCmd.command(cmd.name).description(cmd.description);
153
-
154
- // Register positional args first, then named options
155
- const positionalArgs: typeof cmd.args = [];
156
- for (const arg of cmd.args) {
157
- if (arg.positional) {
158
- const bracket = arg.required ? `<${arg.name}>` : `[${arg.name}]`;
159
- subCmd.argument(bracket, arg.help ?? '');
160
- positionalArgs.push(arg);
161
- } else {
162
- const flag = arg.required ? `--${arg.name} <value>` : `--${arg.name} [value]`;
163
- if (arg.required) subCmd.requiredOption(flag, arg.help ?? '');
164
- else if (arg.default != null) subCmd.option(flag, arg.help ?? '', String(arg.default));
165
- else subCmd.option(flag, arg.help ?? '');
166
- }
167
- }
168
- subCmd.option('-f, --format <fmt>', 'Output format: table, json, yaml, md, csv', 'table').option('-v, --verbose', 'Debug output', false);
169
-
170
- subCmd.action(async (...actionArgs: any[]) => {
171
- // Commander passes positional args first, then options object, then the Command
172
- const actionOpts = actionArgs[positionalArgs.length] ?? {};
173
- const startTime = Date.now();
174
- const kwargs: Record<string, any> = {};
175
-
176
- // Collect positional args
177
- for (let i = 0; i < positionalArgs.length; i++) {
178
- const arg = positionalArgs[i];
179
- const v = actionArgs[i];
180
- if (v !== undefined) kwargs[arg.name] = v;
181
- }
182
-
183
- // Collect named options
184
- for (const arg of cmd.args) {
185
- if (arg.positional) continue;
186
- const camelName = arg.name.replace(/-([a-z])/g, (_m, ch: string) => ch.toUpperCase());
187
- const v = actionOpts[arg.name] ?? actionOpts[camelName];
188
- if (v !== undefined) kwargs[arg.name] = v;
189
- }
190
-
191
- try {
192
- if (actionOpts.verbose) process.env.OPENCLI_VERBOSE = '1';
193
- let result: any;
194
- if (cmd.browser) {
195
- result = await browserSession(BrowserBridge, async (page) => {
196
- return runWithTimeout(executeCommand(cmd, page, kwargs, actionOpts.verbose), { timeout: cmd.timeoutSeconds ?? DEFAULT_BROWSER_COMMAND_TIMEOUT, label: fullName(cmd) });
197
- });
198
- } else { result = await executeCommand(cmd, null, kwargs, actionOpts.verbose); }
199
- if (actionOpts.verbose && (!result || (Array.isArray(result) && result.length === 0))) {
200
- console.error(chalk.yellow(`[Verbose] Warning: Command returned an empty result. If the website structural API changed or requires authentication, check the network or update the adapter.`));
201
- }
202
- renderOutput(result, { fmt: actionOpts.format, columns: cmd.columns, title: `${cmd.site}/${cmd.name}`, elapsed: (Date.now() - startTime) / 1000, source: fullName(cmd) });
203
- } catch (err: any) {
204
- if (err instanceof CliError) {
205
- console.error(chalk.red(`Error [${err.code}]: ${err.message}`));
206
- if (err.hint) console.error(chalk.yellow(`Hint: ${err.hint}`));
207
- } else if (actionOpts.verbose && err.stack) {
208
- console.error(chalk.red(err.stack));
209
- } else {
210
- console.error(chalk.red(`Error: ${err.message ?? err}`));
211
- }
212
- process.exitCode = 1;
213
- }
214
- });
215
- }
216
-
217
- program.parse();
41
+ runCli(BUILTIN_CLIS, USER_CLIS);