@jackwener/opencli 0.1.0 → 0.1.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (135) hide show
  1. package/CLI-CREATOR.md +594 -0
  2. package/README.md +124 -39
  3. package/README.zh-CN.md +151 -0
  4. package/SKILL.md +178 -102
  5. package/dist/bilibili.d.ts +6 -5
  6. package/dist/browser.d.ts +3 -1
  7. package/dist/browser.js +44 -2
  8. package/dist/cascade.d.ts +46 -0
  9. package/dist/cascade.js +180 -0
  10. package/dist/clis/bbc/news.js +42 -0
  11. package/dist/clis/bilibili/hot.yaml +38 -0
  12. package/dist/clis/boss/search.js +47 -0
  13. package/dist/clis/ctrip/search.d.ts +1 -0
  14. package/dist/clis/ctrip/search.js +62 -0
  15. package/dist/clis/hackernews/top.yaml +36 -0
  16. package/dist/clis/index.d.ts +10 -1
  17. package/dist/clis/index.js +19 -1
  18. package/dist/clis/reddit/hot.yaml +46 -0
  19. package/dist/clis/reuters/search.d.ts +1 -0
  20. package/dist/clis/reuters/search.js +52 -0
  21. package/dist/clis/smzdm/search.d.ts +1 -0
  22. package/dist/clis/smzdm/search.js +66 -0
  23. package/dist/clis/twitter/trending.yaml +40 -0
  24. package/dist/clis/v2ex/hot.yaml +25 -0
  25. package/dist/clis/v2ex/latest.yaml +25 -0
  26. package/dist/clis/v2ex/topic.yaml +27 -0
  27. package/dist/clis/weibo/hot.d.ts +1 -0
  28. package/dist/clis/weibo/hot.js +41 -0
  29. package/dist/clis/xiaohongshu/feed.yaml +32 -0
  30. package/dist/clis/xiaohongshu/notifications.yaml +38 -0
  31. package/dist/clis/xiaohongshu/search.d.ts +5 -0
  32. package/dist/clis/xiaohongshu/search.js +68 -0
  33. package/dist/clis/yahoo-finance/quote.d.ts +1 -0
  34. package/dist/clis/yahoo-finance/quote.js +74 -0
  35. package/dist/clis/youtube/search.d.ts +1 -0
  36. package/dist/clis/youtube/search.js +60 -0
  37. package/dist/clis/zhihu/hot.yaml +42 -0
  38. package/dist/clis/zhihu/question.d.ts +1 -0
  39. package/dist/clis/zhihu/question.js +39 -0
  40. package/dist/clis/zhihu/search.yaml +55 -0
  41. package/dist/engine.d.ts +2 -1
  42. package/dist/explore.d.ts +23 -13
  43. package/dist/explore.js +293 -422
  44. package/dist/generate.js +2 -1
  45. package/dist/main.js +21 -2
  46. package/dist/pipeline/executor.d.ts +9 -0
  47. package/dist/pipeline/executor.js +88 -0
  48. package/dist/pipeline/index.d.ts +5 -0
  49. package/dist/pipeline/index.js +5 -0
  50. package/dist/pipeline/steps/browser.d.ts +12 -0
  51. package/dist/pipeline/steps/browser.js +68 -0
  52. package/dist/pipeline/steps/fetch.d.ts +5 -0
  53. package/dist/pipeline/steps/fetch.js +50 -0
  54. package/dist/pipeline/steps/intercept.d.ts +5 -0
  55. package/dist/pipeline/steps/intercept.js +75 -0
  56. package/dist/pipeline/steps/tap.d.ts +12 -0
  57. package/dist/pipeline/steps/tap.js +130 -0
  58. package/dist/pipeline/steps/transform.d.ts +8 -0
  59. package/dist/pipeline/steps/transform.js +53 -0
  60. package/dist/pipeline/template.d.ts +16 -0
  61. package/dist/pipeline/template.js +115 -0
  62. package/dist/pipeline/template.test.d.ts +4 -0
  63. package/dist/pipeline/template.test.js +102 -0
  64. package/dist/pipeline/transform.test.d.ts +4 -0
  65. package/dist/pipeline/transform.test.js +90 -0
  66. package/dist/pipeline.d.ts +5 -7
  67. package/dist/pipeline.js +5 -313
  68. package/dist/registry.d.ts +3 -2
  69. package/dist/runtime.d.ts +2 -1
  70. package/dist/synthesize.d.ts +11 -8
  71. package/dist/synthesize.js +142 -118
  72. package/dist/types.d.ts +27 -0
  73. package/dist/types.js +7 -0
  74. package/package.json +9 -4
  75. package/src/bilibili.ts +9 -7
  76. package/src/browser.ts +41 -3
  77. package/src/cascade.ts +218 -0
  78. package/src/clis/bbc/news.ts +42 -0
  79. package/src/clis/boss/search.ts +47 -0
  80. package/src/clis/ctrip/search.ts +62 -0
  81. package/src/clis/index.ts +28 -1
  82. package/src/clis/reddit/hot.yaml +46 -0
  83. package/src/clis/reuters/search.ts +52 -0
  84. package/src/clis/smzdm/search.ts +66 -0
  85. package/src/clis/v2ex/hot.yaml +5 -9
  86. package/src/clis/v2ex/latest.yaml +5 -8
  87. package/src/clis/v2ex/topic.yaml +27 -0
  88. package/src/clis/weibo/hot.ts +41 -0
  89. package/src/clis/xiaohongshu/feed.yaml +32 -0
  90. package/src/clis/xiaohongshu/notifications.yaml +38 -0
  91. package/src/clis/xiaohongshu/search.ts +71 -0
  92. package/src/clis/yahoo-finance/quote.ts +74 -0
  93. package/src/clis/youtube/search.ts +60 -0
  94. package/src/clis/zhihu/hot.yaml +22 -8
  95. package/src/clis/zhihu/question.ts +45 -0
  96. package/src/clis/zhihu/search.yaml +55 -0
  97. package/src/engine.ts +2 -1
  98. package/src/explore.ts +303 -465
  99. package/src/generate.ts +3 -1
  100. package/src/main.ts +18 -2
  101. package/src/pipeline/executor.ts +98 -0
  102. package/src/pipeline/index.ts +6 -0
  103. package/src/pipeline/steps/browser.ts +67 -0
  104. package/src/pipeline/steps/fetch.ts +60 -0
  105. package/src/pipeline/steps/intercept.ts +78 -0
  106. package/src/pipeline/steps/tap.ts +137 -0
  107. package/src/pipeline/steps/transform.ts +50 -0
  108. package/src/pipeline/template.test.ts +107 -0
  109. package/src/pipeline/template.ts +101 -0
  110. package/src/pipeline/transform.test.ts +107 -0
  111. package/src/pipeline.ts +5 -292
  112. package/src/registry.ts +4 -2
  113. package/src/runtime.ts +3 -1
  114. package/src/synthesize.ts +142 -137
  115. package/src/types.ts +23 -0
  116. package/vitest.config.ts +7 -0
  117. package/dist/clis/github/search.js +0 -20
  118. package/dist/clis/zhihu/search.js +0 -58
  119. package/dist/promote.d.ts +0 -1
  120. package/dist/promote.js +0 -3
  121. package/dist/register.d.ts +0 -2
  122. package/dist/register.js +0 -2
  123. package/dist/scaffold.d.ts +0 -2
  124. package/dist/scaffold.js +0 -2
  125. package/dist/smoke.d.ts +0 -2
  126. package/dist/smoke.js +0 -2
  127. package/src/clis/github/search.ts +0 -21
  128. package/src/clis/github/trending.yaml +0 -58
  129. package/src/clis/zhihu/search.ts +0 -65
  130. package/src/promote.ts +0 -3
  131. package/src/register.ts +0 -2
  132. package/src/scaffold.ts +0 -2
  133. package/src/smoke.ts +0 -2
  134. /package/dist/clis/{github/search.d.ts → bbc/news.d.ts} +0 -0
  135. /package/dist/clis/{zhihu → boss}/search.d.ts +0 -0
package/dist/runtime.d.ts CHANGED
@@ -1,6 +1,7 @@
1
1
  /**
2
2
  * Runtime utilities: timeouts and browser session management.
3
3
  */
4
+ import type { IPage } from './types.js';
4
5
  export declare const DEFAULT_BROWSER_CONNECT_TIMEOUT: number;
5
6
  export declare const DEFAULT_BROWSER_COMMAND_TIMEOUT: number;
6
7
  export declare const DEFAULT_BROWSER_EXPLORE_TIMEOUT: number;
@@ -9,4 +10,4 @@ export declare function runWithTimeout<T>(promise: Promise<T>, opts: {
9
10
  timeout: number;
10
11
  label?: string;
11
12
  }): Promise<T>;
12
- export declare function browserSession<T>(BrowserFactory: new () => any, fn: (page: any) => Promise<T>): Promise<T>;
13
+ export declare function browserSession<T>(BrowserFactory: new () => any, fn: (page: IPage) => Promise<T>): Promise<T>;
@@ -1,10 +1,13 @@
1
1
  /**
2
- * Synthesize: turn explore capabilities into ready-to-use CLI definitions.
3
- *
4
- * Takes the structured capabilities from Deep Explore and generates
5
- * YAML pipeline files that can be directly registered as CLI commands.
6
- *
7
- * This is the bridge between discovery (explore) and usability (CLI).
2
+ * Synthesize candidate CLIs from explore artifacts.
3
+ * Generates evaluate-based YAML pipelines (matching hand-written adapter patterns).
8
4
  */
9
- export declare function synthesizeFromExplore(target: string, opts?: any): any;
10
- export declare function renderSynthesizeSummary(r: any): string;
5
+ export declare function synthesizeFromExplore(target: string, opts?: {
6
+ outDir?: string;
7
+ top?: number;
8
+ }): Record<string, any>;
9
+ export declare function renderSynthesizeSummary(result: Record<string, any>): string;
10
+ export declare function resolveExploreDir(target: string): string;
11
+ export declare function loadExploreBundle(exploreDir: string): Record<string, any>;
12
+ /** Backward-compatible export for scaffold.ts */
13
+ export declare function buildCandidate(site: string, targetUrl: string, cap: any, endpoint: any): any;
@@ -1,147 +1,181 @@
1
1
  /**
2
- * Synthesize: turn explore capabilities into ready-to-use CLI definitions.
3
- *
4
- * Takes the structured capabilities from Deep Explore and generates
5
- * YAML pipeline files that can be directly registered as CLI commands.
6
- *
7
- * This is the bridge between discovery (explore) and usability (CLI).
2
+ * Synthesize candidate CLIs from explore artifacts.
3
+ * Generates evaluate-based YAML pipelines (matching hand-written adapter patterns).
8
4
  */
9
5
  import * as fs from 'node:fs';
10
6
  import * as path from 'node:path';
11
7
  import yaml from 'js-yaml';
8
+ /** Volatile params to strip from generated URLs */
9
+ const VOLATILE_PARAMS = new Set(['w_rid', 'wts', 'callback', '_', 'timestamp', 't', 'nonce', 'sign']);
10
+ const SEARCH_PARAM_NAMES = new Set(['q', 'query', 'keyword', 'search', 'wd', 'kw', 'w', 'search_query']);
11
+ const LIMIT_PARAM_NAMES = new Set(['ps', 'page_size', 'limit', 'count', 'per_page', 'size', 'num']);
12
+ const PAGE_PARAM_NAMES = new Set(['pn', 'page', 'page_num', 'offset', 'cursor']);
12
13
  export function synthesizeFromExplore(target, opts = {}) {
13
- const exploreDir = fs.existsSync(target) ? target : path.join('.opencli', 'explore', target);
14
- if (!fs.existsSync(exploreDir))
15
- throw new Error(`Explore dir not found: ${target}`);
16
- const manifest = JSON.parse(fs.readFileSync(path.join(exploreDir, 'manifest.json'), 'utf-8'));
17
- const capabilities = JSON.parse(fs.readFileSync(path.join(exploreDir, 'capabilities.json'), 'utf-8'));
18
- const endpoints = JSON.parse(fs.readFileSync(path.join(exploreDir, 'endpoints.json'), 'utf-8'));
19
- const auth = JSON.parse(fs.readFileSync(path.join(exploreDir, 'auth.json'), 'utf-8'));
14
+ const exploreDir = resolveExploreDir(target);
15
+ const bundle = loadExploreBundle(exploreDir);
20
16
  const targetDir = opts.outDir ?? path.join(exploreDir, 'candidates');
21
17
  fs.mkdirSync(targetDir, { recursive: true });
22
- const site = manifest.site;
23
- const topN = opts.top ?? 5;
24
- const candidates = [];
25
- // Sort capabilities by confidence
26
- const sortedCaps = [...capabilities]
18
+ const site = bundle.manifest.site;
19
+ const capabilities = (bundle.capabilities ?? [])
27
20
  .sort((a, b) => (b.confidence ?? 0) - (a.confidence ?? 0))
28
- .slice(0, topN);
29
- for (const cap of sortedCaps) {
30
- // Find the matching endpoint for more detail
31
- const endpoint = endpoints.find((ep) => ep.pattern === cap.endpoint) ??
32
- endpoints[0];
33
- const candidate = buildCandidateYaml(site, manifest, cap, endpoint);
34
- const fileName = `${cap.name}.yaml`;
35
- const filePath = path.join(targetDir, fileName);
21
+ .slice(0, opts.top ?? 3);
22
+ const candidates = [];
23
+ for (const cap of capabilities) {
24
+ const endpoint = chooseEndpoint(cap, bundle.endpoints);
25
+ if (!endpoint)
26
+ continue;
27
+ const candidate = buildCandidateYaml(site, bundle.manifest, cap, endpoint);
28
+ const filePath = path.join(targetDir, `${candidate.name}.yaml`);
36
29
  fs.writeFileSync(filePath, yaml.dump(candidate.yaml, { sortKeys: false, lineWidth: 120 }));
37
- candidates.push({
38
- name: cap.name,
39
- path: filePath,
40
- strategy: cap.strategy,
41
- endpoint: cap.endpoint,
42
- confidence: cap.confidence,
43
- columns: candidate.yaml.columns,
44
- });
30
+ candidates.push({ name: candidate.name, path: filePath, strategy: cap.strategy, confidence: cap.confidence });
45
31
  }
46
- const index = {
47
- site,
48
- target_url: manifest.target_url,
49
- generated_from: exploreDir,
50
- candidate_count: candidates.length,
51
- candidates,
52
- };
32
+ const index = { site, target_url: bundle.manifest.target_url, generated_from: exploreDir, candidate_count: candidates.length, candidates };
53
33
  fs.writeFileSync(path.join(targetDir, 'candidates.json'), JSON.stringify(index, null, 2));
34
+ return { site, explore_dir: exploreDir, out_dir: targetDir, candidate_count: candidates.length, candidates };
35
+ }
36
+ export function renderSynthesizeSummary(result) {
37
+ const lines = ['opencli synthesize: OK', `Site: ${result.site}`, `Source: ${result.explore_dir}`, `Candidates: ${result.candidate_count}`];
38
+ for (const c of result.candidates ?? [])
39
+ lines.push(` • ${c.name} (${c.strategy}, ${((c.confidence ?? 0) * 100).toFixed(0)}% confidence) → ${c.path}`);
40
+ return lines.join('\n');
41
+ }
42
+ export function resolveExploreDir(target) {
43
+ if (fs.existsSync(target))
44
+ return target;
45
+ const candidate = path.join('.opencli', 'explore', target);
46
+ if (fs.existsSync(candidate))
47
+ return candidate;
48
+ throw new Error(`Explore directory not found: ${target}`);
49
+ }
50
+ export function loadExploreBundle(exploreDir) {
54
51
  return {
55
- site,
56
- explore_dir: exploreDir,
57
- out_dir: targetDir,
58
- candidate_count: candidates.length,
59
- candidates,
52
+ manifest: JSON.parse(fs.readFileSync(path.join(exploreDir, 'manifest.json'), 'utf-8')),
53
+ endpoints: JSON.parse(fs.readFileSync(path.join(exploreDir, 'endpoints.json'), 'utf-8')),
54
+ capabilities: JSON.parse(fs.readFileSync(path.join(exploreDir, 'capabilities.json'), 'utf-8')),
55
+ auth: JSON.parse(fs.readFileSync(path.join(exploreDir, 'auth.json'), 'utf-8')),
60
56
  };
61
57
  }
62
- /** Volatile params to strip from generated URLs */
63
- const VOLATILE_PARAMS = new Set(['w_rid', 'wts', 'callback', '_', 'timestamp', 't', 'nonce', 'sign']);
64
- const SEARCH_PARAM_NAMES = new Set(['q', 'query', 'keyword', 'search', 'wd', 'kw', 'w', 'search_query']);
65
- const LIMIT_PARAM_NAMES = new Set(['ps', 'page_size', 'limit', 'count', 'per_page', 'size', 'num']);
66
- const PAGE_PARAM_NAMES = new Set(['pn', 'page', 'page_num', 'offset', 'cursor']);
67
- /**
68
- * Build a clean templated URL from a raw API URL.
69
- * - Strips volatile params (w_rid, wts, etc.)
70
- * - Templates search, limit, and pagination params
71
- * - Builds URL string manually to avoid URL encoding of ${{ }} expressions
72
- */
73
- function buildTemplatedUrl(rawUrl, cap, endpoint) {
58
+ function chooseEndpoint(cap, endpoints) {
59
+ if (!endpoints.length)
60
+ return null;
61
+ // Match by endpoint pattern from capability
62
+ if (cap.endpoint) {
63
+ const match = endpoints.find((e) => e.pattern === cap.endpoint || e.url?.includes(cap.endpoint));
64
+ if (match)
65
+ return match;
66
+ }
67
+ return endpoints.sort((a, b) => (b.score ?? 0) - (a.score ?? 0))[0];
68
+ }
69
+ // ── URL templating ─────────────────────────────────────────────────────────
70
+ function buildTemplatedUrl(rawUrl, cap, _endpoint) {
74
71
  try {
75
72
  const u = new URL(rawUrl);
76
73
  const base = `${u.protocol}//${u.host}${u.pathname}`;
77
74
  const params = [];
78
75
  const hasKeyword = cap.recommendedArgs?.some((a) => a.name === 'keyword');
79
76
  u.searchParams.forEach((v, k) => {
80
- // Skip volatile params
81
77
  if (VOLATILE_PARAMS.has(k))
82
78
  return;
83
- // Template known param types
84
- if (hasKeyword && SEARCH_PARAM_NAMES.has(k)) {
79
+ if (hasKeyword && SEARCH_PARAM_NAMES.has(k))
85
80
  params.push([k, '${{ args.keyword }}']);
86
- }
87
- else if (LIMIT_PARAM_NAMES.has(k)) {
81
+ else if (LIMIT_PARAM_NAMES.has(k))
88
82
  params.push([k, '${{ args.limit | default(20) }}']);
89
- }
90
- else if (PAGE_PARAM_NAMES.has(k)) {
83
+ else if (PAGE_PARAM_NAMES.has(k))
91
84
  params.push([k, '${{ args.page | default(1) }}']);
92
- }
93
- else {
85
+ else
94
86
  params.push([k, v]);
95
- }
96
87
  });
97
- if (params.length === 0)
98
- return base;
99
- return base + '?' + params.map(([k, v]) => `${k}=${v}`).join('&');
88
+ return params.length ? base + '?' + params.map(([k, v]) => `${k}=${v}`).join('&') : base;
100
89
  }
101
90
  catch {
102
91
  return rawUrl;
103
92
  }
104
93
  }
105
94
  /**
106
- * Build a YAML pipeline definition from a capability + endpoint.
95
+ * Build inline evaluate script for browser-based fetch+parse.
96
+ * Follows patterns from bilibili/hot.yaml and twitter/trending.yaml.
107
97
  */
98
+ function buildEvaluateScript(url, itemPath, endpoint) {
99
+ const pathChain = itemPath.split('.').map((p) => `?.${p}`).join('');
100
+ const detectedFields = endpoint?.detectedFields ?? {};
101
+ const hasFields = Object.keys(detectedFields).length > 0;
102
+ let mapCode = '';
103
+ if (hasFields) {
104
+ const mappings = Object.entries(detectedFields)
105
+ .map(([role, field]) => ` ${role}: item${String(field).split('.').map(p => `?.${p}`).join('')}`)
106
+ .join(',\n');
107
+ mapCode = `.map((item) => ({\n${mappings}\n }))`;
108
+ }
109
+ return [
110
+ '(async () => {',
111
+ ` const res = await fetch('${url}', {`,
112
+ ` credentials: 'include'`,
113
+ ' });',
114
+ ' const data = await res.json();',
115
+ ` return (data${pathChain} || [])${mapCode};`,
116
+ '})()\n',
117
+ ].join('\n');
118
+ }
119
+ // ── YAML pipeline generation ───────────────────────────────────────────────
108
120
  function buildCandidateYaml(site, manifest, cap, endpoint) {
109
121
  const needsBrowser = cap.strategy !== 'public';
110
122
  const pipeline = [];
111
- // Step 1: Navigate (if browser-based)
112
- if (needsBrowser) {
123
+ const templatedUrl = buildTemplatedUrl(endpoint?.url ?? manifest.target_url, cap, endpoint);
124
+ let domain = '';
125
+ try {
126
+ domain = new URL(manifest.target_url).hostname;
127
+ }
128
+ catch { }
129
+ if (cap.strategy === 'store-action' && cap.storeHint) {
130
+ // Store Action: navigate + wait + tap (declarative, clean)
113
131
  pipeline.push({ navigate: manifest.target_url });
132
+ pipeline.push({ wait: 3 });
133
+ const tapStep = {
134
+ store: cap.storeHint.store,
135
+ action: cap.storeHint.action,
136
+ timeout: 8,
137
+ };
138
+ // Infer capture pattern from endpoint URL
139
+ if (endpoint?.url) {
140
+ try {
141
+ const epUrl = new URL(endpoint.url);
142
+ const pathParts = epUrl.pathname.split('/').filter((p) => p);
143
+ // Use last meaningful path segment as capture pattern
144
+ const capturePart = pathParts.filter((p) => !p.match(/^v\d+$/)).pop();
145
+ if (capturePart)
146
+ tapStep.capture = capturePart;
147
+ }
148
+ catch { }
149
+ }
150
+ if (cap.itemPath)
151
+ tapStep.select = cap.itemPath;
152
+ pipeline.push({ tap: tapStep });
153
+ }
154
+ else if (needsBrowser) {
155
+ // Browser-based: navigate + evaluate (like bilibili/hot.yaml, twitter/trending.yaml)
156
+ pipeline.push({ navigate: manifest.target_url });
157
+ const itemPath = cap.itemPath ?? 'data.data.list';
158
+ pipeline.push({ evaluate: buildEvaluateScript(templatedUrl, itemPath, endpoint) });
114
159
  }
115
- // Step 2: Fetch the API — build a clean URL with templates
116
- const rawUrl = endpoint?.url ?? manifest.target_url;
117
- const fetchStep = { url: buildTemplatedUrl(rawUrl, cap, endpoint) };
118
- pipeline.push({ fetch: fetchStep });
119
- // Step 3: Select the item path
120
- if (cap.itemPath) {
121
- pipeline.push({ select: cap.itemPath });
160
+ else {
161
+ // Public API: direct fetch (like hackernews/top.yaml)
162
+ pipeline.push({ fetch: { url: templatedUrl } });
163
+ if (cap.itemPath)
164
+ pipeline.push({ select: cap.itemPath });
122
165
  }
123
- // Step 4: Map fields to columns
166
+ // Map fields
124
167
  const mapStep = {};
125
168
  const columns = cap.recommendedColumns ?? ['title', 'url'];
126
- // Add a rank column if not doing search
127
- if (!cap.recommendedArgs?.some((a) => a.name === 'keyword')) {
169
+ if (!cap.recommendedArgs?.some((a) => a.name === 'keyword'))
128
170
  mapStep['rank'] = '${{ index + 1 }}';
129
- }
130
- // Build field mappings from the endpoint's detected fields
131
171
  const detectedFields = endpoint?.detectedFields ?? {};
132
172
  for (const col of columns) {
133
173
  const fieldPath = detectedFields[col];
134
- if (fieldPath) {
135
- mapStep[col] = `\${{ item.${fieldPath} }}`;
136
- }
137
- else {
138
- mapStep[col] = `\${{ item.${col} }}`;
139
- }
174
+ mapStep[col] = fieldPath ? `\${{ item.${fieldPath} }}` : `\${{ item.${col} }}`;
140
175
  }
141
176
  pipeline.push({ map: mapStep });
142
- // Step 5: Limit
143
177
  pipeline.push({ limit: '${{ args.limit | default(20) }}' });
144
- // Build args definition
178
+ // Args
145
179
  const argsDef = {};
146
180
  for (const arg of cap.recommendedArgs ?? []) {
147
181
  const def = { type: arg.type ?? 'str' };
@@ -157,35 +191,25 @@ function buildCandidateYaml(site, manifest, cap, endpoint) {
157
191
  def.description = 'Page number';
158
192
  argsDef[arg.name] = def;
159
193
  }
160
- // Ensure limit arg always exists
161
- if (!argsDef['limit']) {
194
+ if (!argsDef['limit'])
162
195
  argsDef['limit'] = { type: 'int', default: 20, description: 'Number of items to return' };
163
- }
164
- const allColumns = Object.keys(mapStep);
165
196
  return {
166
197
  name: cap.name,
167
198
  yaml: {
168
- site,
169
- name: cap.name,
170
- description: `${site} ${cap.name} (auto-generated)`,
171
- domain: manifest.final_url ? new URL(manifest.final_url).hostname : undefined,
172
- strategy: cap.strategy,
173
- browser: needsBrowser,
174
- args: argsDef,
175
- pipeline,
176
- columns: allColumns,
199
+ site, name: cap.name, description: `${cap.description || site + ' ' + cap.name} (auto-generated)`,
200
+ domain, strategy: cap.strategy, browser: needsBrowser,
201
+ args: argsDef, pipeline, columns: Object.keys(mapStep),
177
202
  },
178
203
  };
179
204
  }
180
- export function renderSynthesizeSummary(r) {
181
- const lines = [
182
- 'opencli synthesize: OK',
183
- `Site: ${r.site}`,
184
- `Source: ${r.explore_dir}`,
185
- `Candidates: ${r.candidate_count}`,
186
- ];
187
- for (const c of r.candidates ?? []) {
188
- lines.push(` • ${c.name} (${c.strategy}, ${(c.confidence * 100).toFixed(0)}% confidence) ${c.path}`);
189
- }
190
- return lines.join('\n');
205
+ /** Backward-compatible export for scaffold.ts */
206
+ export function buildCandidate(site, targetUrl, cap, endpoint) {
207
+ // Map old-style field names to new ones
208
+ const normalizedCap = {
209
+ ...cap,
210
+ recommendedArgs: cap.recommendedArgs ?? cap.recommended_args,
211
+ recommendedColumns: cap.recommendedColumns ?? cap.recommended_columns,
212
+ };
213
+ const manifest = { target_url: targetUrl, final_url: targetUrl };
214
+ return buildCandidateYaml(site, manifest, normalizedCap, endpoint);
191
215
  }
@@ -0,0 +1,27 @@
1
+ /**
2
+ * Page interface: type-safe abstraction over Playwright MCP browser page.
3
+ *
4
+ * All pipeline steps and CLI adapters should use this interface
5
+ * instead of `any` for browser interactions.
6
+ */
7
+ export interface IPage {
8
+ goto(url: string): Promise<void>;
9
+ evaluate(js: string): Promise<any>;
10
+ snapshot(opts?: {
11
+ interactive?: boolean;
12
+ compact?: boolean;
13
+ maxDepth?: number;
14
+ raw?: boolean;
15
+ }): Promise<any>;
16
+ click(ref: string): Promise<void>;
17
+ typeText(ref: string, text: string): Promise<void>;
18
+ pressKey(key: string): Promise<void>;
19
+ wait(seconds: number): Promise<void>;
20
+ tabs(): Promise<any>;
21
+ closeTab(index?: number): Promise<void>;
22
+ newTab(): Promise<void>;
23
+ selectTab(index: number): Promise<void>;
24
+ networkRequests(includeStatic?: boolean): Promise<any>;
25
+ consoleMessages(level?: string): Promise<any>;
26
+ scroll(direction?: string, amount?: number): Promise<void>;
27
+ }
package/dist/types.js ADDED
@@ -0,0 +1,7 @@
1
+ /**
2
+ * Page interface: type-safe abstraction over Playwright MCP browser page.
3
+ *
4
+ * All pipeline steps and CLI adapters should use this interface
5
+ * instead of `any` for browser interactions.
6
+ */
7
+ export {};
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@jackwener/opencli",
3
- "version": "0.1.0",
3
+ "version": "0.1.2",
4
4
  "publishConfig": {
5
5
  "access": "public"
6
6
  },
@@ -12,11 +12,15 @@
12
12
  },
13
13
  "scripts": {
14
14
  "dev": "tsx src/main.ts",
15
- "build": "tsc",
15
+ "build": "tsc && npm run clean-yaml && npm run copy-yaml",
16
+ "clean-yaml": "find dist/clis -name '*.yaml' -o -name '*.yml' 2>/dev/null | xargs rm -f",
17
+ "copy-yaml": "find src/clis -name '*.yaml' -o -name '*.yml' | while read f; do d=\"dist/${f#src/}\"; mkdir -p \"$(dirname \"$d\")\"; cp \"$f\" \"$d\"; done",
16
18
  "start": "node dist/main.js",
17
19
  "typecheck": "tsc --noEmit",
18
20
  "lint": "tsc --noEmit",
19
- "prepublishOnly": "npm run build"
21
+ "prepublishOnly": "npm run build",
22
+ "test": "vitest run",
23
+ "test:watch": "vitest"
20
24
  },
21
25
  "keywords": [
22
26
  "cli",
@@ -42,6 +46,7 @@
42
46
  "@types/js-yaml": "^4.0.9",
43
47
  "@types/node": "^22.13.10",
44
48
  "tsx": "^4.19.3",
45
- "typescript": "^5.8.2"
49
+ "typescript": "^5.8.2",
50
+ "vitest": "^4.1.0"
46
51
  }
47
52
  }
package/src/bilibili.ts CHANGED
@@ -2,6 +2,8 @@
2
2
  * Bilibili shared helpers: WBI signing, authenticated fetch, nav data, UID resolution.
3
3
  */
4
4
 
5
+ import type { IPage } from './types.js';
6
+
5
7
  const MIXIN_KEY_ENC_TAB = [
6
8
  46,47,18,2,53,8,23,32,15,50,10,31,58,3,45,35,27,43,5,49,
7
9
  33,9,42,19,29,28,14,39,12,38,41,13,37,48,7,16,24,55,40,
@@ -17,7 +19,7 @@ export function payloadData(payload: any): any {
17
19
  return payload?.data ?? payload;
18
20
  }
19
21
 
20
- async function getNavData(page: any): Promise<any> {
22
+ async function getNavData(page: IPage): Promise<any> {
21
23
  return page.evaluate(`
22
24
  async () => {
23
25
  const res = await fetch('https://api.bilibili.com/x/web-interface/nav', { credentials: 'include' });
@@ -26,7 +28,7 @@ async function getNavData(page: any): Promise<any> {
26
28
  `);
27
29
  }
28
30
 
29
- async function getWbiKeys(page: any): Promise<{ imgKey: string; subKey: string }> {
31
+ async function getWbiKeys(page: IPage): Promise<{ imgKey: string; subKey: string }> {
30
32
  const nav = await getNavData(page);
31
33
  const wbiImg = nav?.data?.wbi_img ?? {};
32
34
  const imgUrl = wbiImg.img_url ?? '';
@@ -47,7 +49,7 @@ async function md5(text: string): Promise<string> {
47
49
  }
48
50
 
49
51
  export async function wbiSign(
50
- page: any,
52
+ page: IPage,
51
53
  params: Record<string, any>,
52
54
  ): Promise<Record<string, string>> {
53
55
  const { imgKey, subKey } = await getWbiKeys(page);
@@ -65,7 +67,7 @@ export async function wbiSign(
65
67
  }
66
68
 
67
69
  export async function apiGet(
68
- page: any,
70
+ page: IPage,
69
71
  path: string,
70
72
  opts: { params?: Record<string, any>; signed?: boolean } = {},
71
73
  ): Promise<any> {
@@ -81,7 +83,7 @@ export async function apiGet(
81
83
  return fetchJson(page, url);
82
84
  }
83
85
 
84
- export async function fetchJson(page: any, url: string): Promise<any> {
86
+ export async function fetchJson(page: IPage, url: string): Promise<any> {
85
87
  const escapedUrl = url.replace(/"/g, '\\"');
86
88
  return page.evaluate(`
87
89
  async () => {
@@ -91,14 +93,14 @@ export async function fetchJson(page: any, url: string): Promise<any> {
91
93
  `);
92
94
  }
93
95
 
94
- export async function getSelfUid(page: any): Promise<string> {
96
+ export async function getSelfUid(page: IPage): Promise<string> {
95
97
  const nav = await getNavData(page);
96
98
  const mid = nav?.data?.mid;
97
99
  if (!mid) throw new Error('Not logged in to Bilibili');
98
100
  return String(mid);
99
101
  }
100
102
 
101
- export async function resolveUid(page: any, input: string): Promise<string> {
103
+ export async function resolveUid(page: IPage, input: string): Promise<string> {
102
104
  if (/^\d+$/.test(input)) return input;
103
105
  // Search for user by name
104
106
  const payload = await apiGet(page, '/x/web-interface/wbi/search/type', {
package/src/browser.ts CHANGED
@@ -10,6 +10,10 @@ import * as os from 'node:os';
10
10
  import * as path from 'node:path';
11
11
  import { formatSnapshot } from './snapshotFormatter.js';
12
12
 
13
+ // Read version from package.json (single source of truth)
14
+ const __browser_dirname = path.dirname(fileURLToPath(import.meta.url));
15
+ const PKG_VERSION = (() => { try { return JSON.parse(fs.readFileSync(path.resolve(__browser_dirname, '..', 'package.json'), 'utf-8')).version; } catch { return '0.0.0'; } })();
16
+
13
17
  const EXTENSION_LOCK_TIMEOUT = parseInt(process.env.OPENCLI_EXTENSION_LOCK_TIMEOUT ?? '120', 10);
14
18
  const EXTENSION_LOCK_POLL = parseInt(process.env.OPENCLI_EXTENSION_LOCK_POLL_INTERVAL ?? '1', 10);
15
19
  const CONNECT_TIMEOUT = parseInt(process.env.OPENCLI_BROWSER_CONNECT_TIMEOUT ?? '30', 10);
@@ -21,10 +25,12 @@ function jsonRpcRequest(method: string, params: Record<string, any> = {}): strin
21
25
  return JSON.stringify({ jsonrpc: '2.0', id: _nextId++, method, params }) + '\n';
22
26
  }
23
27
 
28
+ import type { IPage } from './types.js';
29
+
24
30
  /**
25
31
  * Page abstraction wrapping JSON-RPC calls to Playwright MCP.
26
32
  */
27
- export class Page {
33
+ export class Page implements IPage {
28
34
  constructor(private _send: (msg: string) => void, private _recv: () => Promise<any>) {}
29
35
 
30
36
  async call(method: string, params: Record<string, any> = {}): Promise<any> {
@@ -36,7 +42,18 @@ export class Page {
36
42
  if (result?.content) {
37
43
  const textParts = result.content.filter((c: any) => c.type === 'text');
38
44
  if (textParts.length === 1) {
39
- const text = textParts[0].text;
45
+ let text = textParts[0].text;
46
+ // MCP browser_evaluate returns: "[JSON]\n### Ran Playwright code\n```js\n...\n```"
47
+ // Strip the "### Ran Playwright code" suffix to get clean JSON
48
+ const codeMarker = text.indexOf('### Ran Playwright code');
49
+ if (codeMarker !== -1) {
50
+ text = text.slice(0, codeMarker).trim();
51
+ }
52
+ // Also handle "### Result\n[JSON]" format (some MCP versions)
53
+ const resultMarker = text.indexOf('### Result\n');
54
+ if (resultMarker !== -1) {
55
+ text = text.slice(resultMarker + '### Result\n'.length).trim();
56
+ }
40
57
  try { return JSON.parse(text); } catch { return text; }
41
58
  }
42
59
  }
@@ -115,6 +132,8 @@ export class PlaywrightMCP {
115
132
  private _lockAcquired = false;
116
133
  private _initialTabCount = 0;
117
134
 
135
+ private _page: Page | null = null;
136
+
118
137
  async connect(opts: { timeout?: number } = {}): Promise<Page> {
119
138
  await this._acquireLock();
120
139
  const timeout = opts.timeout ?? CONNECT_TIMEOUT;
@@ -137,6 +156,7 @@ export class PlaywrightMCP {
137
156
  (msg) => { if (this._proc?.stdin?.writable) this._proc.stdin.write(msg); },
138
157
  () => new Promise<any>((res) => { this._waiters.push(res); }),
139
158
  );
159
+ this._page = page;
140
160
 
141
161
  this._proc.stdout?.on('data', (chunk: Buffer) => {
142
162
  this._buffer += chunk.toString();
@@ -159,7 +179,7 @@ export class PlaywrightMCP {
159
179
  const initMsg = jsonRpcRequest('initialize', {
160
180
  protocolVersion: '2024-11-05',
161
181
  capabilities: {},
162
- clientInfo: { name: 'opencli', version: '0.1.0' },
182
+ clientInfo: { name: 'opencli', version: PKG_VERSION },
163
183
  });
164
184
  this._proc.stdin?.write(initMsg);
165
185
 
@@ -185,11 +205,29 @@ export class PlaywrightMCP {
185
205
 
186
206
  async close(): Promise<void> {
187
207
  try {
208
+ // Close tabs opened during this session (site tabs + extension tabs)
209
+ if (this._page && this._proc && !this._proc.killed) {
210
+ try {
211
+ const tabs = await this._page.tabs();
212
+ const tabStr = typeof tabs === 'string' ? tabs : JSON.stringify(tabs);
213
+ const allTabs = tabStr.match(/Tab (\d+)/g) || [];
214
+ const currentTabCount = allTabs.length;
215
+
216
+ // Close tabs in reverse order to avoid index shifting issues
217
+ // Keep the original tabs that existed before the command started
218
+ if (currentTabCount > this._initialTabCount && this._initialTabCount > 0) {
219
+ for (let i = currentTabCount - 1; i >= this._initialTabCount; i--) {
220
+ try { await this._page.closeTab(i); } catch {}
221
+ }
222
+ }
223
+ } catch {}
224
+ }
188
225
  if (this._proc && !this._proc.killed) {
189
226
  this._proc.kill('SIGTERM');
190
227
  await new Promise<void>((res) => { this._proc?.on('exit', () => res()); setTimeout(res, 3000); });
191
228
  }
192
229
  } finally {
230
+ this._page = null;
193
231
  this._releaseLock();
194
232
  }
195
233
  }