@jackwener/opencli 1.7.4 → 1.7.6

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (181) hide show
  1. package/README.md +76 -51
  2. package/README.zh-CN.md +78 -62
  3. package/cli-manifest.json +4558 -2979
  4. package/clis/antigravity/serve.js +71 -25
  5. package/clis/baidu-scholar/search.js +87 -0
  6. package/clis/baidu-scholar/search.test.js +23 -0
  7. package/clis/bilibili/video.js +61 -0
  8. package/clis/bilibili/video.test.js +81 -0
  9. package/clis/deepseek/ask.js +94 -0
  10. package/clis/deepseek/ask.test.js +73 -0
  11. package/clis/deepseek/history.js +25 -0
  12. package/clis/deepseek/new.js +20 -0
  13. package/clis/deepseek/read.js +22 -0
  14. package/clis/deepseek/status.js +24 -0
  15. package/clis/deepseek/utils.js +291 -0
  16. package/clis/deepseek/utils.test.js +37 -0
  17. package/clis/eastmoney/_secid.js +78 -0
  18. package/clis/eastmoney/announcement.js +52 -0
  19. package/clis/eastmoney/convertible.js +73 -0
  20. package/clis/eastmoney/etf.js +65 -0
  21. package/clis/eastmoney/holders.js +78 -0
  22. package/clis/eastmoney/index-board.js +96 -0
  23. package/clis/eastmoney/kline.js +87 -0
  24. package/clis/eastmoney/kuaixun.js +54 -0
  25. package/clis/eastmoney/longhu.js +67 -0
  26. package/clis/eastmoney/money-flow.js +78 -0
  27. package/clis/eastmoney/northbound.js +57 -0
  28. package/clis/eastmoney/quote.js +107 -0
  29. package/clis/eastmoney/rank.js +94 -0
  30. package/clis/eastmoney/sectors.js +76 -0
  31. package/clis/google-scholar/search.js +58 -0
  32. package/clis/google-scholar/search.test.js +23 -0
  33. package/clis/gov-law/commands.test.js +39 -0
  34. package/clis/gov-law/recent.js +22 -0
  35. package/clis/gov-law/search.js +41 -0
  36. package/clis/gov-law/shared.js +51 -0
  37. package/clis/gov-policy/commands.test.js +27 -0
  38. package/clis/gov-policy/recent.js +47 -0
  39. package/clis/gov-policy/search.js +48 -0
  40. package/clis/jianyu/search.js +139 -3
  41. package/clis/jianyu/search.test.js +25 -0
  42. package/clis/jianyu/shared/procurement-detail.js +15 -0
  43. package/clis/jianyu/shared/procurement-detail.test.js +12 -0
  44. package/clis/nowcoder/companies.js +23 -0
  45. package/clis/nowcoder/creators.js +27 -0
  46. package/clis/nowcoder/detail.js +61 -0
  47. package/clis/nowcoder/experience.js +36 -0
  48. package/clis/nowcoder/hot.js +24 -0
  49. package/clis/nowcoder/jobs.js +21 -0
  50. package/clis/nowcoder/notifications.js +29 -0
  51. package/clis/nowcoder/papers.js +40 -0
  52. package/clis/nowcoder/practice.js +37 -0
  53. package/clis/nowcoder/recommend.js +30 -0
  54. package/clis/nowcoder/referral.js +39 -0
  55. package/clis/nowcoder/salary.js +40 -0
  56. package/clis/nowcoder/search.js +49 -0
  57. package/clis/nowcoder/suggest.js +33 -0
  58. package/clis/nowcoder/topics.js +27 -0
  59. package/clis/nowcoder/trending.js +25 -0
  60. package/clis/twitter/list-add.js +337 -0
  61. package/clis/twitter/list-add.test.js +15 -0
  62. package/clis/twitter/list-remove.js +297 -0
  63. package/clis/twitter/list-remove.test.js +14 -0
  64. package/clis/twitter/list-tweets.js +185 -0
  65. package/clis/twitter/list-tweets.test.js +108 -0
  66. package/clis/twitter/lists.js +134 -47
  67. package/clis/twitter/lists.test.js +105 -38
  68. package/clis/twitter/shared.js +7 -2
  69. package/clis/twitter/tweets.js +218 -0
  70. package/clis/twitter/tweets.test.js +125 -0
  71. package/clis/wanfang/search.js +66 -0
  72. package/clis/wanfang/search.test.js +23 -0
  73. package/clis/web/read.js +1 -1
  74. package/clis/weixin/download.js +3 -2
  75. package/clis/xiaohongshu/publish.js +149 -28
  76. package/clis/xiaohongshu/publish.test.js +319 -6
  77. package/clis/xiaoyuzhou/download.js +8 -4
  78. package/clis/xiaoyuzhou/download.test.js +23 -13
  79. package/clis/xiaoyuzhou/episode.js +9 -4
  80. package/clis/xiaoyuzhou/podcast-episodes.js +15 -11
  81. package/clis/xiaoyuzhou/podcast.js +9 -4
  82. package/clis/xiaoyuzhou/utils.js +0 -40
  83. package/clis/xiaoyuzhou/utils.test.js +15 -75
  84. package/clis/youtube/channel.js +35 -0
  85. package/clis/zsxq/dynamics.js +1 -1
  86. package/clis/zsxq/utils.js +6 -3
  87. package/clis/zsxq/utils.test.js +31 -0
  88. package/dist/src/browser/base-page.d.ts +14 -4
  89. package/dist/src/browser/base-page.js +35 -25
  90. package/dist/src/browser/bridge.d.ts +1 -0
  91. package/dist/src/browser/bridge.js +1 -1
  92. package/dist/src/browser/cdp.d.ts +1 -0
  93. package/dist/src/browser/cdp.js +13 -4
  94. package/dist/src/browser/compound.d.ts +59 -0
  95. package/dist/src/browser/compound.js +112 -0
  96. package/dist/src/browser/compound.test.js +175 -0
  97. package/dist/src/browser/daemon-client.d.ts +6 -4
  98. package/dist/src/browser/daemon-client.js +6 -1
  99. package/dist/src/browser/daemon-client.test.js +40 -1
  100. package/dist/src/browser/dom-snapshot.d.ts +7 -0
  101. package/dist/src/browser/dom-snapshot.js +83 -5
  102. package/dist/src/browser/dom-snapshot.test.js +65 -0
  103. package/dist/src/browser/extract.d.ts +69 -0
  104. package/dist/src/browser/extract.js +132 -0
  105. package/dist/src/browser/extract.test.js +129 -0
  106. package/dist/src/browser/find.d.ts +76 -0
  107. package/dist/src/browser/find.js +179 -0
  108. package/dist/src/browser/find.test.js +120 -0
  109. package/dist/src/browser/html-tree.d.ts +75 -0
  110. package/dist/src/browser/html-tree.js +112 -0
  111. package/dist/src/browser/html-tree.test.d.ts +1 -0
  112. package/dist/src/browser/html-tree.test.js +181 -0
  113. package/dist/src/browser/network-cache.d.ts +48 -0
  114. package/dist/src/browser/network-cache.js +66 -0
  115. package/dist/src/browser/network-cache.test.d.ts +1 -0
  116. package/dist/src/browser/network-cache.test.js +58 -0
  117. package/dist/src/browser/network-key.d.ts +22 -0
  118. package/dist/src/browser/network-key.js +66 -0
  119. package/dist/src/browser/network-key.test.d.ts +1 -0
  120. package/dist/src/browser/network-key.test.js +49 -0
  121. package/dist/src/browser/page.d.ts +14 -4
  122. package/dist/src/browser/page.js +48 -7
  123. package/dist/src/browser/page.test.js +97 -0
  124. package/dist/src/browser/shape-filter.d.ts +52 -0
  125. package/dist/src/browser/shape-filter.js +101 -0
  126. package/dist/src/browser/shape-filter.test.d.ts +1 -0
  127. package/dist/src/browser/shape-filter.test.js +101 -0
  128. package/dist/src/browser/shape.d.ts +23 -0
  129. package/dist/src/browser/shape.js +95 -0
  130. package/dist/src/browser/shape.test.d.ts +1 -0
  131. package/dist/src/browser/shape.test.js +82 -0
  132. package/dist/src/browser/target-errors.d.ts +14 -1
  133. package/dist/src/browser/target-errors.js +13 -0
  134. package/dist/src/browser/target-errors.test.js +39 -6
  135. package/dist/src/browser/target-resolver.d.ts +57 -10
  136. package/dist/src/browser/target-resolver.js +195 -75
  137. package/dist/src/browser/target-resolver.test.js +80 -5
  138. package/dist/src/cli.js +849 -267
  139. package/dist/src/cli.test.js +961 -90
  140. package/dist/src/commanderAdapter.d.ts +0 -1
  141. package/dist/src/commanderAdapter.js +2 -16
  142. package/dist/src/commanderAdapter.test.js +1 -1
  143. package/dist/src/completion-shared.js +2 -5
  144. package/dist/src/daemon.js +8 -0
  145. package/dist/src/download/article-download.d.ts +1 -0
  146. package/dist/src/download/article-download.js +3 -0
  147. package/dist/src/download/article-download.test.d.ts +1 -0
  148. package/dist/src/download/article-download.test.js +39 -0
  149. package/dist/src/execution.js +7 -2
  150. package/dist/src/execution.test.js +54 -0
  151. package/dist/src/main.js +16 -0
  152. package/dist/src/plugin.d.ts +1 -8
  153. package/dist/src/plugin.js +1 -27
  154. package/dist/src/plugin.test.js +1 -59
  155. package/dist/src/registry.d.ts +1 -0
  156. package/dist/src/registry.js +3 -2
  157. package/dist/src/registry.test.js +22 -0
  158. package/dist/src/types.d.ts +32 -8
  159. package/package.json +1 -1
  160. package/clis/twitter/lists-parser.js +0 -77
  161. package/clis/twitter/lists.d.ts +0 -5
  162. package/dist/src/cascade.d.ts +0 -46
  163. package/dist/src/cascade.js +0 -135
  164. package/dist/src/explore.d.ts +0 -99
  165. package/dist/src/explore.js +0 -402
  166. package/dist/src/generate-verified.d.ts +0 -105
  167. package/dist/src/generate-verified.js +0 -696
  168. package/dist/src/generate-verified.test.js +0 -925
  169. package/dist/src/generate.d.ts +0 -46
  170. package/dist/src/generate.js +0 -117
  171. package/dist/src/record.d.ts +0 -96
  172. package/dist/src/record.js +0 -657
  173. package/dist/src/record.test.js +0 -293
  174. package/dist/src/skill-generate.d.ts +0 -30
  175. package/dist/src/skill-generate.js +0 -75
  176. package/dist/src/skill-generate.test.js +0 -173
  177. package/dist/src/synthesize.d.ts +0 -97
  178. package/dist/src/synthesize.js +0 -208
  179. /package/dist/src/{generate-verified.test.d.ts → browser/compound.test.d.ts} +0 -0
  180. /package/dist/src/{record.test.d.ts → browser/extract.test.d.ts} +0 -0
  181. /package/dist/src/{skill-generate.test.d.ts → browser/find.test.d.ts} +0 -0
@@ -1,696 +0,0 @@
1
- /**
2
- * Verified adapter generation:
3
- * discover → synthesize → candidate-bound probe → single-session verify.
4
- *
5
- * v1 contract keeps scope narrow:
6
- * - PUBLIC + COOKIE only
7
- * - read-only JSON API surfaces
8
- * - single best candidate only
9
- * - bounded repair: select/itemPath replacement once
10
- *
11
- * Contract design principles:
12
- * 1. machine-readable
13
- * 2. explicit + explainable
14
- * 3. testable + versioned
15
- * 4. taxonomy by skill decision needs (not internal error sources)
16
- * 5. early hint / terminal outcome share consistent decision language
17
- */
18
- import * as fs from 'node:fs';
19
- import * as path from 'node:path';
20
- import { exploreUrl } from './explore.js';
21
- import { loadExploreBundle, synthesizeFromExplore } from './synthesize.js';
22
- import { normalizeGoal, selectCandidate } from './generate.js';
23
- import { browserSession } from './runtime.js';
24
- import { executePipeline } from './pipeline/index.js';
25
- import { registerCommand, Strategy } from './registry.js';
26
- import { AuthRequiredError, BrowserConnectError, CommandExecutionError, SelectorError, TimeoutError, getErrorMessage, } from './errors.js';
27
- import { USER_CLIS_DIR } from './discovery.js';
28
- // ── Helpers ───────────────────────────────────────────────────────────────────
29
- function parseSupportedStrategy(value) {
30
- return value === Strategy.PUBLIC || value === Strategy.COOKIE ? value : null;
31
- }
32
- function commandName(site, name) {
33
- return `${site}/${name}`;
34
- }
35
- function buildStats(args) {
36
- return {
37
- endpoint_count: args.endpointCount,
38
- api_endpoint_count: args.apiEndpointCount,
39
- candidate_count: args.candidateCount,
40
- verified: args.verified ?? false,
41
- repair_attempted: args.repairAttempted ?? false,
42
- explore_dir: args.exploreDir,
43
- };
44
- }
45
- function readCandidateJson(filePath) {
46
- const loaded = JSON.parse(fs.readFileSync(filePath, 'utf-8'));
47
- if (!loaded || typeof loaded !== 'object') {
48
- throw new CommandExecutionError(`Generated candidate is invalid: ${filePath}`);
49
- }
50
- return loaded;
51
- }
52
- function chooseEndpoint(capability, endpoints) {
53
- if (!endpoints.length)
54
- return null;
55
- if (capability?.endpoint) {
56
- const endpointPattern = capability.endpoint;
57
- const exact = endpoints.find((endpoint) => endpoint.pattern === endpointPattern || endpoint.url.includes(endpointPattern));
58
- if (exact)
59
- return exact;
60
- }
61
- return [...endpoints].sort((a, b) => {
62
- const aScore = (a.itemCount ?? 0) * 10 + Object.keys(a.detectedFields ?? {}).length;
63
- const bScore = (b.itemCount ?? 0) * 10 + Object.keys(b.detectedFields ?? {}).length;
64
- return bScore - aScore;
65
- })[0] ?? null;
66
- }
67
- function cloneCandidate(candidate) {
68
- return JSON.parse(JSON.stringify(candidate));
69
- }
70
- function hasBrowserOnlyStep(pipeline) {
71
- return pipeline.some((step) => {
72
- const op = Object.keys(step)[0];
73
- return op === 'navigate' || op === 'wait' || op === 'evaluate' || op === 'click' || op === 'tap' || op === 'type' || op === 'press';
74
- });
75
- }
76
- function detectBrowserFlag(candidate) {
77
- return candidate.browser ?? hasBrowserOnlyStep(candidate.pipeline);
78
- }
79
- function candidateToCommand(candidate, source) {
80
- return {
81
- site: candidate.site,
82
- name: candidate.name,
83
- description: candidate.description,
84
- domain: candidate.domain,
85
- strategy: parseSupportedStrategy(candidate.strategy) ?? Strategy.COOKIE,
86
- browser: detectBrowserFlag(candidate),
87
- args: Object.entries(candidate.args ?? {}).map(([name, def]) => ({
88
- name,
89
- type: def.type,
90
- required: def.required,
91
- default: def.default,
92
- help: def.description,
93
- })),
94
- columns: candidate.columns,
95
- pipeline: candidate.pipeline,
96
- source,
97
- };
98
- }
99
- function buildDefaultArgs(candidate) {
100
- const args = {};
101
- for (const [name, def] of Object.entries(candidate.args ?? {})) {
102
- if (def.default !== undefined) {
103
- args[name] = def.default;
104
- continue;
105
- }
106
- if (def.type === 'int' || def.type === 'number') {
107
- args[name] = name === 'page' ? 1 : 20;
108
- continue;
109
- }
110
- if (def.type === 'boolean' || def.type === 'bool') {
111
- args[name] = false;
112
- continue;
113
- }
114
- if (name === 'keyword' || name === 'query') {
115
- args[name] = 'test';
116
- continue;
117
- }
118
- if (def.required)
119
- args[name] = 'test';
120
- }
121
- return args;
122
- }
123
- function getUnsupportedVerificationArgs(candidate) {
124
- return Object.entries(candidate.args ?? {})
125
- .filter(([name, def]) => {
126
- if (!def.required || def.default !== undefined)
127
- return false;
128
- if (def.type === 'int' || def.type === 'number')
129
- return false;
130
- if (def.type === 'boolean' || def.type === 'bool')
131
- return false;
132
- if (name === 'keyword' || name === 'query')
133
- return false;
134
- return true;
135
- })
136
- .map(([name]) => name);
137
- }
138
- function assessResult(result, expectedFields = []) {
139
- if (!Array.isArray(result))
140
- return { ok: false, reason: 'non-array-result' };
141
- if (result.length === 0)
142
- return { ok: false, reason: 'empty-result' };
143
- const sample = result[0];
144
- if (!sample || typeof sample !== 'object' || Array.isArray(sample)) {
145
- return { ok: false, reason: 'sparse-fields' };
146
- }
147
- const record = sample;
148
- const keys = Object.keys(record);
149
- const populated = keys.filter((key) => record[key] !== null && record[key] !== undefined && record[key] !== '');
150
- if (populated.length < 2)
151
- return { ok: false, reason: 'sparse-fields' };
152
- if (expectedFields.length > 0) {
153
- const matched = expectedFields.filter((field) => keys.includes(field));
154
- if (matched.length === 0)
155
- return { ok: false, reason: 'sparse-fields' };
156
- }
157
- return { ok: true };
158
- }
159
- function withItemPath(candidate, itemPath) {
160
- if (!itemPath)
161
- return null;
162
- const next = cloneCandidate(candidate);
163
- const selectIndex = next.pipeline.findIndex((step) => 'select' in step);
164
- if (selectIndex === -1)
165
- return null;
166
- const current = next.pipeline[selectIndex];
167
- if (current.select === itemPath)
168
- return null;
169
- next.pipeline[selectIndex] = { select: itemPath };
170
- return next;
171
- }
172
- function applyStrategy(candidate, strategy) {
173
- const next = cloneCandidate(candidate);
174
- next.strategy = strategy;
175
- if (strategy === Strategy.COOKIE)
176
- next.browser = true;
177
- return next;
178
- }
179
- // ── Escalation builders ───────────────────────────────────────────────────────
180
- function mapVerifyFailureToEscalation(reason) {
181
- return reason; // VerifyFailureReason is a subset of EscalationReason
182
- }
183
- function suggestAction(reason) {
184
- switch (reason) {
185
- case 'unsupported-required-args': return 'ask-for-sample-arg';
186
- case 'timeout': return 'inspect-with-browser';
187
- case 'selector-mismatch': return 'inspect-with-browser';
188
- case 'empty-result': return 'inspect-with-browser';
189
- case 'sparse-fields': return 'inspect-with-browser';
190
- case 'non-array-result': return 'inspect-with-browser';
191
- case 'verify-inconclusive': return 'manual-review';
192
- }
193
- }
194
- function buildEscalation(stage, reason, summary, site, opts) {
195
- return {
196
- stage,
197
- reason,
198
- confidence: opts?.confidence ?? 'medium',
199
- suggested_action: suggestAction(reason),
200
- candidate: {
201
- name: summary.name,
202
- command: commandName(site, summary.name),
203
- path: summary.path ?? null,
204
- reusability: opts?.reusability ?? 'unverified-candidate',
205
- },
206
- };
207
- }
208
- // ── Verification ──────────────────────────────────────────────────────────────
209
- async function verifyCandidate(page, candidate, expectedFields) {
210
- try {
211
- const result = await executePipeline(page, candidate.pipeline, {
212
- args: buildDefaultArgs(candidate),
213
- });
214
- return assessResult(result, expectedFields);
215
- }
216
- catch (error) {
217
- if (error instanceof BrowserConnectError) {
218
- return { ok: false, terminal: 'blocked', reason: 'execution-environment-unavailable', issue: getErrorMessage(error) };
219
- }
220
- if (error instanceof AuthRequiredError) {
221
- return { ok: false, terminal: 'blocked', reason: 'auth-too-complex', issue: getErrorMessage(error) };
222
- }
223
- if (error instanceof SelectorError) {
224
- return { ok: false, terminal: 'needs-human-check', escalationReason: 'selector-mismatch', issue: getErrorMessage(error) };
225
- }
226
- if (error instanceof TimeoutError) {
227
- return { ok: false, terminal: 'needs-human-check', escalationReason: 'timeout', issue: getErrorMessage(error) };
228
- }
229
- if (error instanceof CommandExecutionError) {
230
- return { ok: false, terminal: 'needs-human-check', escalationReason: 'verify-inconclusive', issue: getErrorMessage(error) };
231
- }
232
- return { ok: false, terminal: 'needs-human-check', escalationReason: 'verify-inconclusive', issue: getErrorMessage(error) };
233
- }
234
- }
235
- async function probeCandidateStrategy(page, endpointUrl) {
236
- const { cascadeProbe } = await import('./cascade.js');
237
- const result = await cascadeProbe(page, endpointUrl, { maxStrategy: Strategy.COOKIE });
238
- const success = result.probes.find((probe) => probe.success);
239
- return parseSupportedStrategy(success?.strategy);
240
- }
241
- // ── Artifact persistence ──────────────────────────────────────────────────────
242
- function candidateToJs(candidate) {
243
- const strategyMap = {
244
- public: 'Strategy.PUBLIC',
245
- cookie: 'Strategy.COOKIE',
246
- header: 'Strategy.HEADER',
247
- intercept: 'Strategy.INTERCEPT',
248
- ui: 'Strategy.UI',
249
- };
250
- const stratEnum = strategyMap[candidate.strategy?.toLowerCase()] ?? 'Strategy.COOKIE';
251
- const browser = detectBrowserFlag(candidate);
252
- const argsArray = Object.entries(candidate.args ?? {}).map(([name, def]) => {
253
- const parts = [`name: '${name.replace(/'/g, "\\'")}'`];
254
- if (def.type && def.type !== 'str')
255
- parts.push(`type: '${def.type.replace(/'/g, "\\'")}'`);
256
- if (def.required)
257
- parts.push('required: true');
258
- if (def.default !== undefined)
259
- parts.push(`default: ${JSON.stringify(def.default)}`);
260
- if (def.description)
261
- parts.push(`help: '${def.description.replace(/'/g, "\\'")}'`);
262
- return ` { ${parts.join(', ')} }`;
263
- });
264
- const formatStepValue = (v) => {
265
- if (typeof v === 'string') {
266
- if (v.includes('\n') || v.includes("'")) {
267
- return '`' + v.replace(/\\/g, '\\\\').replace(/`/g, '\\`').replace(/\$\{/g, '\\${') + '`';
268
- }
269
- return `'${v.replace(/\\/g, '\\\\')}'`;
270
- }
271
- if (typeof v === 'number' || typeof v === 'boolean')
272
- return String(v);
273
- if (v === null || v === undefined)
274
- return 'undefined';
275
- if (Array.isArray(v))
276
- return `[${v.map(formatStepValue).join(', ')}]`;
277
- if (typeof v === 'object') {
278
- const entries = Object.entries(v);
279
- const items = entries.map(([k, val]) => `${k}: ${formatStepValue(val)}`);
280
- return `{ ${items.join(', ')} }`;
281
- }
282
- return String(v);
283
- };
284
- const pipelineSteps = (candidate.pipeline ?? []).map((step) => {
285
- const entries = Object.entries(step);
286
- if (entries.length === 1) {
287
- const [op, value] = entries[0];
288
- return ` { ${op}: ${formatStepValue(value)} }`;
289
- }
290
- return ` ${formatStepValue(step)}`;
291
- });
292
- const lines = [];
293
- lines.push("import { cli, Strategy } from '@jackwener/opencli/registry';");
294
- lines.push('');
295
- lines.push('cli({');
296
- lines.push(` site: '${candidate.site.replace(/'/g, "\\'")}',`);
297
- lines.push(` name: '${candidate.name.replace(/'/g, "\\'")}',`);
298
- if (candidate.description)
299
- lines.push(` description: '${candidate.description.replace(/'/g, "\\'")}',`);
300
- if (candidate.domain)
301
- lines.push(` domain: '${candidate.domain.replace(/'/g, "\\'")}',`);
302
- lines.push(` strategy: ${stratEnum},`);
303
- lines.push(` browser: ${browser},`);
304
- if (argsArray.length > 0) {
305
- lines.push(` args: [`);
306
- lines.push(argsArray.join(',\n') + ',');
307
- lines.push(' ],');
308
- }
309
- if (candidate.columns?.length) {
310
- lines.push(` columns: [${candidate.columns.map(c => `'${c}'`).join(', ')}],`);
311
- }
312
- if (pipelineSteps.length > 0) {
313
- lines.push(' pipeline: [');
314
- lines.push(pipelineSteps.join(',\n') + ',');
315
- lines.push(' ],');
316
- }
317
- lines.push('});');
318
- lines.push('');
319
- return lines.join('\n');
320
- }
321
- async function registerVerifiedAdapter(candidate, metadata) {
322
- const siteDir = path.join(USER_CLIS_DIR, candidate.site);
323
- const adapterPath = path.join(siteDir, `${candidate.name}.js`);
324
- const metadataPath = path.join(siteDir, `${candidate.name}.meta.json`);
325
- await fs.promises.mkdir(siteDir, { recursive: true });
326
- await fs.promises.writeFile(adapterPath, candidateToJs(candidate));
327
- await fs.promises.writeFile(metadataPath, JSON.stringify(metadata, null, 2));
328
- registerCommand(candidateToCommand(candidate, adapterPath));
329
- return { adapterPath, metadataPath };
330
- }
331
- async function writeVerifiedArtifact(candidate, exploreDir, metadata) {
332
- const outDir = path.join(exploreDir, 'verified');
333
- const adapterPath = path.join(outDir, `${candidate.name}.verified.js`);
334
- const metadataPath = path.join(outDir, `${candidate.name}.verified.meta.json`);
335
- await fs.promises.mkdir(outDir, { recursive: true });
336
- await fs.promises.writeFile(adapterPath, candidateToJs(candidate));
337
- await fs.promises.writeFile(metadataPath, JSON.stringify(metadata, null, 2));
338
- return { adapterPath, metadataPath };
339
- }
340
- // ── Session error classification ──────────────────────────────────────────────
341
- function classifySessionError(error, summary, stats, site) {
342
- if (error instanceof BrowserConnectError) {
343
- return {
344
- status: 'blocked',
345
- reason: 'execution-environment-unavailable',
346
- stage: 'verify',
347
- confidence: 'high',
348
- message: getErrorMessage(error),
349
- stats,
350
- };
351
- }
352
- if (error instanceof AuthRequiredError) {
353
- return {
354
- status: 'blocked',
355
- reason: 'auth-too-complex',
356
- stage: 'verify',
357
- confidence: 'high',
358
- message: getErrorMessage(error),
359
- stats,
360
- };
361
- }
362
- return {
363
- status: 'needs-human-check',
364
- escalation: buildEscalation('verify', 'verify-inconclusive', summary, site, {
365
- reusability: 'unverified-candidate',
366
- confidence: 'low',
367
- }),
368
- reusability: 'unverified-candidate',
369
- message: getErrorMessage(error),
370
- stats,
371
- };
372
- }
373
- // ── Main orchestrator ─────────────────────────────────────────────────────────
374
- export async function generateVerifiedFromUrl(opts) {
375
- const normalizedGoal = normalizeGoal(opts.goal) ?? opts.goal ?? undefined;
376
- const exploreResult = await exploreUrl(opts.url, {
377
- BrowserFactory: opts.BrowserFactory,
378
- site: opts.site,
379
- goal: normalizedGoal,
380
- waitSeconds: opts.waitSeconds ?? 3,
381
- workspace: opts.workspace,
382
- });
383
- const bundle = loadExploreBundle(exploreResult.out_dir);
384
- const synthesizeResult = synthesizeFromExplore(exploreResult.out_dir, { top: opts.top ?? 3 });
385
- const selected = selectCandidate(synthesizeResult.candidates ?? [], opts.goal);
386
- const baseStats = buildStats({
387
- endpointCount: exploreResult.endpoint_count,
388
- apiEndpointCount: exploreResult.api_endpoint_count,
389
- candidateCount: synthesizeResult.candidate_count,
390
- exploreDir: exploreResult.out_dir,
391
- });
392
- // ── Early hint: explore result ──────────────────────────────────────────
393
- if (exploreResult.api_endpoint_count === 0) {
394
- opts.onEarlyHint?.({
395
- stage: 'explore',
396
- continue: false,
397
- reason: 'no-viable-api-surface',
398
- confidence: 'high',
399
- });
400
- return {
401
- status: 'blocked',
402
- reason: 'no-viable-api-surface',
403
- stage: 'explore',
404
- confidence: 'high',
405
- message: 'No JSON API endpoints discovered on this site.',
406
- stats: baseStats,
407
- };
408
- }
409
- opts.onEarlyHint?.({
410
- stage: 'explore',
411
- continue: true,
412
- reason: 'api-surface-looks-viable',
413
- confidence: 'medium',
414
- });
415
- // ── Early hint: synthesize result ───────────────────────────────────────
416
- if (!selected || synthesizeResult.candidate_count === 0) {
417
- opts.onEarlyHint?.({
418
- stage: 'synthesize',
419
- continue: false,
420
- reason: 'no-viable-candidate',
421
- confidence: 'high',
422
- });
423
- return {
424
- status: 'blocked',
425
- reason: 'no-viable-candidate',
426
- stage: 'synthesize',
427
- confidence: 'high',
428
- message: 'No candidate met the quality threshold for verification.',
429
- stats: baseStats,
430
- };
431
- }
432
- const context = {
433
- capability: bundle.capabilities.find((capability) => capability.name === selected.name),
434
- endpoint: chooseEndpoint(bundle.capabilities.find((capability) => capability.name === selected.name), bundle.endpoints),
435
- };
436
- if (!context.endpoint) {
437
- opts.onEarlyHint?.({
438
- stage: 'synthesize',
439
- continue: false,
440
- reason: 'no-viable-candidate',
441
- confidence: 'medium',
442
- });
443
- return {
444
- status: 'blocked',
445
- reason: 'no-viable-candidate',
446
- stage: 'synthesize',
447
- confidence: 'medium',
448
- message: 'No endpoint could be matched to the selected candidate.',
449
- stats: baseStats,
450
- };
451
- }
452
- const expectedFields = Object.keys(context.endpoint.detectedFields ?? {});
453
- const originalCandidate = readCandidateJson(selected.path);
454
- const unsupportedArgs = getUnsupportedVerificationArgs(originalCandidate);
455
- // ── Escalation: unsupported required args ───────────────────────────────
456
- // Note: unsupported-required-args goes directly to P1 terminal.
457
- // No P2 hint is emitted — this is a P1-only decision per design guardrail.
458
- if (unsupportedArgs.length > 0) {
459
- return {
460
- status: 'needs-human-check',
461
- escalation: buildEscalation('synthesize', 'unsupported-required-args', selected, bundle.manifest.site, {
462
- reusability: 'unverified-candidate',
463
- confidence: 'high',
464
- }),
465
- reusability: 'unverified-candidate',
466
- message: `Auto-verification does not support required args: ${unsupportedArgs.join(', ')}`,
467
- stats: baseStats,
468
- };
469
- }
470
- opts.onEarlyHint?.({
471
- stage: 'synthesize',
472
- continue: true,
473
- reason: 'candidate-ready-for-verify',
474
- confidence: 'high',
475
- candidate: {
476
- name: selected.name,
477
- command: commandName(bundle.manifest.site, selected.name),
478
- path: selected.path ?? null,
479
- reusability: 'unverified-candidate',
480
- },
481
- });
482
- // ── Phase 3: single browser session (probe + verify + repair) ───────────
483
- try {
484
- return await browserSession(opts.BrowserFactory, async (page) => {
485
- await page.goto(bundle.manifest.final_url ?? bundle.manifest.target_url);
486
- // ── Probe: candidate-bound strategy ─────────────────────────────────
487
- const bestStrategy = await probeCandidateStrategy(page, context.endpoint.url);
488
- if (!bestStrategy) {
489
- opts.onEarlyHint?.({
490
- stage: 'cascade',
491
- continue: false,
492
- reason: 'auth-too-complex',
493
- confidence: 'high',
494
- });
495
- return {
496
- status: 'blocked',
497
- reason: 'auth-too-complex',
498
- stage: 'cascade',
499
- confidence: 'high',
500
- message: 'No PUBLIC or COOKIE strategy succeeded for this endpoint.',
501
- stats: baseStats,
502
- };
503
- }
504
- opts.onEarlyHint?.({
505
- stage: 'cascade',
506
- continue: true,
507
- reason: 'candidate-ready-for-verify',
508
- confidence: 'high',
509
- candidate: {
510
- name: selected.name,
511
- command: commandName(bundle.manifest.site, selected.name),
512
- path: selected.path ?? null,
513
- reusability: 'unverified-candidate',
514
- },
515
- });
516
- const candidate = applyStrategy(originalCandidate, bestStrategy);
517
- const goalStr = normalizedGoal ?? opts.goal ?? null;
518
- const buildMetadata = () => ({
519
- artifact_kind: 'verified',
520
- schema_version: 1,
521
- source_url: opts.url,
522
- goal: goalStr,
523
- strategy: bestStrategy,
524
- verified: true,
525
- reusable: true,
526
- reusability_reason: 'verified-artifact',
527
- });
528
- // ── First verify attempt ────────────────────────────────────────────
529
- const firstAttempt = await verifyCandidate(page, candidate, expectedFields);
530
- if (firstAttempt.ok) {
531
- const artifact = opts.noRegister
532
- ? await writeVerifiedArtifact(candidate, exploreResult.out_dir, buildMetadata())
533
- : await registerVerifiedAdapter(candidate, buildMetadata());
534
- return {
535
- status: 'success',
536
- adapter: {
537
- site: candidate.site,
538
- name: candidate.name,
539
- command: commandName(candidate.site, candidate.name),
540
- strategy: bestStrategy,
541
- path: artifact.adapterPath,
542
- metadata_path: artifact.metadataPath,
543
- reusability: 'verified-artifact',
544
- },
545
- reusability: 'verified-artifact',
546
- stats: buildStats({
547
- endpointCount: exploreResult.endpoint_count,
548
- apiEndpointCount: exploreResult.api_endpoint_count,
549
- candidateCount: synthesizeResult.candidate_count,
550
- verified: true,
551
- repairAttempted: false,
552
- exploreDir: exploreResult.out_dir,
553
- }),
554
- };
555
- }
556
- // ── Terminal from first attempt ─────────────────────────────────────
557
- if ('terminal' in firstAttempt) {
558
- if (firstAttempt.terminal === 'blocked') {
559
- return {
560
- status: 'blocked',
561
- reason: firstAttempt.reason ?? 'execution-environment-unavailable',
562
- stage: 'verify',
563
- confidence: 'high',
564
- message: firstAttempt.issue,
565
- stats: baseStats,
566
- };
567
- }
568
- return {
569
- status: 'needs-human-check',
570
- escalation: buildEscalation('verify', firstAttempt.escalationReason ?? 'verify-inconclusive', selected, bundle.manifest.site, { reusability: 'unverified-candidate', confidence: 'medium' }),
571
- reusability: 'unverified-candidate',
572
- message: firstAttempt.issue,
573
- stats: baseStats,
574
- };
575
- }
576
- // ── Bounded repair: itemPath relocation ─────────────────────────────
577
- const repaired = firstAttempt.reason === 'empty-result'
578
- ? withItemPath(candidate, context.endpoint?.itemPath ?? null)
579
- : null;
580
- if (!repaired) {
581
- const escalationReason = mapVerifyFailureToEscalation(firstAttempt.reason);
582
- return {
583
- status: 'needs-human-check',
584
- escalation: buildEscalation('verify', escalationReason, selected, bundle.manifest.site, {
585
- reusability: 'unverified-candidate',
586
- confidence: 'medium',
587
- }),
588
- reusability: 'unverified-candidate',
589
- message: `Verification failed: ${firstAttempt.reason}`,
590
- stats: buildStats({
591
- endpointCount: exploreResult.endpoint_count,
592
- apiEndpointCount: exploreResult.api_endpoint_count,
593
- candidateCount: synthesizeResult.candidate_count,
594
- repairAttempted: firstAttempt.reason === 'empty-result',
595
- exploreDir: exploreResult.out_dir,
596
- }),
597
- };
598
- }
599
- // ── Second verify attempt (after repair) ───────────────────────────
600
- const secondAttempt = await verifyCandidate(page, repaired, expectedFields);
601
- const repairedStats = buildStats({
602
- endpointCount: exploreResult.endpoint_count,
603
- apiEndpointCount: exploreResult.api_endpoint_count,
604
- candidateCount: synthesizeResult.candidate_count,
605
- repairAttempted: true,
606
- exploreDir: exploreResult.out_dir,
607
- });
608
- if (secondAttempt.ok) {
609
- const artifact = opts.noRegister
610
- ? await writeVerifiedArtifact(repaired, exploreResult.out_dir, buildMetadata())
611
- : await registerVerifiedAdapter(repaired, buildMetadata());
612
- return {
613
- status: 'success',
614
- adapter: {
615
- site: repaired.site,
616
- name: repaired.name,
617
- command: commandName(repaired.site, repaired.name),
618
- strategy: bestStrategy,
619
- path: artifact.adapterPath,
620
- metadata_path: artifact.metadataPath,
621
- reusability: 'verified-artifact',
622
- },
623
- reusability: 'verified-artifact',
624
- stats: { ...repairedStats, verified: true },
625
- };
626
- }
627
- if ('terminal' in secondAttempt) {
628
- if (secondAttempt.terminal === 'blocked') {
629
- return {
630
- status: 'blocked',
631
- reason: secondAttempt.reason ?? 'execution-environment-unavailable',
632
- stage: 'fallback',
633
- confidence: 'high',
634
- message: secondAttempt.issue,
635
- stats: repairedStats,
636
- };
637
- }
638
- return {
639
- status: 'needs-human-check',
640
- escalation: buildEscalation('fallback', secondAttempt.escalationReason ?? 'verify-inconclusive', selected, bundle.manifest.site, { reusability: 'unverified-candidate', confidence: 'low' }),
641
- reusability: 'unverified-candidate',
642
- message: secondAttempt.issue,
643
- stats: repairedStats,
644
- };
645
- }
646
- // ── Repair exhausted ────────────────────────────────────────────────
647
- const escalationReason = mapVerifyFailureToEscalation(secondAttempt.reason);
648
- return {
649
- status: 'needs-human-check',
650
- escalation: buildEscalation('fallback', escalationReason, selected, bundle.manifest.site, {
651
- reusability: 'unverified-candidate',
652
- confidence: 'low',
653
- }),
654
- reusability: 'unverified-candidate',
655
- message: `Repair exhausted: ${secondAttempt.reason}`,
656
- stats: repairedStats,
657
- };
658
- }, { workspace: opts.workspace });
659
- }
660
- catch (error) {
661
- return classifySessionError(error, selected, baseStats, bundle.manifest.site);
662
- }
663
- }
664
- // ── Render ────────────────────────────────────────────────────────────────────
665
- export function renderGenerateVerifiedSummary(result) {
666
- const lines = [
667
- `opencli generate: ${result.status.toUpperCase()}`,
668
- ];
669
- if (result.status === 'success' && result.adapter) {
670
- lines.push(`Command: ${result.adapter.command}`);
671
- lines.push(`Strategy: ${result.adapter.strategy}`);
672
- lines.push(`Path: ${result.adapter.path}`);
673
- }
674
- else if (result.status === 'blocked') {
675
- lines.push(`Reason: ${result.reason}`);
676
- lines.push(`Stage: ${result.stage}`);
677
- lines.push(`Confidence: ${result.confidence}`);
678
- if (result.message)
679
- lines.push(`Message: ${result.message}`);
680
- }
681
- else if (result.status === 'needs-human-check' && result.escalation) {
682
- lines.push(`Stage: ${result.escalation.stage}`);
683
- lines.push(`Reason: ${result.escalation.reason}`);
684
- lines.push(`Suggested action: ${result.escalation.suggested_action}`);
685
- lines.push(`Candidate: ${result.escalation.candidate.command}`);
686
- lines.push(`Reusability: ${result.escalation.candidate.reusability}`);
687
- if (result.message)
688
- lines.push(`Message: ${result.message}`);
689
- }
690
- lines.push('');
691
- lines.push(`Explore: ${result.stats.endpoint_count} endpoints, ${result.stats.api_endpoint_count} API`);
692
- lines.push(`Candidates: ${result.stats.candidate_count}`);
693
- lines.push(`Verified: ${result.stats.verified ? 'yes' : 'no'}`);
694
- lines.push(`Repair attempted: ${result.stats.repair_attempted ? 'yes' : 'no'}`);
695
- return lines.join('\n');
696
- }