@jackwener/opencli 0.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (94) hide show
  1. package/.github/workflows/ci.yml +26 -0
  2. package/.github/workflows/release.yml +40 -0
  3. package/README.md +67 -0
  4. package/SKILL.md +230 -0
  5. package/dist/bilibili.d.ts +13 -0
  6. package/dist/bilibili.js +93 -0
  7. package/dist/browser.d.ts +48 -0
  8. package/dist/browser.js +261 -0
  9. package/dist/clis/bilibili/favorite.d.ts +1 -0
  10. package/dist/clis/bilibili/favorite.js +39 -0
  11. package/dist/clis/bilibili/feed.d.ts +1 -0
  12. package/dist/clis/bilibili/feed.js +64 -0
  13. package/dist/clis/bilibili/history.d.ts +1 -0
  14. package/dist/clis/bilibili/history.js +44 -0
  15. package/dist/clis/bilibili/me.d.ts +1 -0
  16. package/dist/clis/bilibili/me.js +13 -0
  17. package/dist/clis/bilibili/search.d.ts +1 -0
  18. package/dist/clis/bilibili/search.js +24 -0
  19. package/dist/clis/bilibili/user-videos.d.ts +1 -0
  20. package/dist/clis/bilibili/user-videos.js +38 -0
  21. package/dist/clis/github/search.d.ts +1 -0
  22. package/dist/clis/github/search.js +20 -0
  23. package/dist/clis/index.d.ts +13 -0
  24. package/dist/clis/index.js +16 -0
  25. package/dist/clis/zhihu/search.d.ts +1 -0
  26. package/dist/clis/zhihu/search.js +58 -0
  27. package/dist/engine.d.ts +6 -0
  28. package/dist/engine.js +77 -0
  29. package/dist/explore.d.ts +17 -0
  30. package/dist/explore.js +603 -0
  31. package/dist/generate.d.ts +11 -0
  32. package/dist/generate.js +134 -0
  33. package/dist/main.d.ts +5 -0
  34. package/dist/main.js +117 -0
  35. package/dist/output.d.ts +11 -0
  36. package/dist/output.js +98 -0
  37. package/dist/pipeline.d.ts +9 -0
  38. package/dist/pipeline.js +315 -0
  39. package/dist/promote.d.ts +1 -0
  40. package/dist/promote.js +3 -0
  41. package/dist/register.d.ts +2 -0
  42. package/dist/register.js +2 -0
  43. package/dist/registry.d.ts +50 -0
  44. package/dist/registry.js +42 -0
  45. package/dist/runtime.d.ts +12 -0
  46. package/dist/runtime.js +27 -0
  47. package/dist/scaffold.d.ts +2 -0
  48. package/dist/scaffold.js +2 -0
  49. package/dist/smoke.d.ts +2 -0
  50. package/dist/smoke.js +2 -0
  51. package/dist/snapshotFormatter.d.ts +9 -0
  52. package/dist/snapshotFormatter.js +41 -0
  53. package/dist/synthesize.d.ts +10 -0
  54. package/dist/synthesize.js +191 -0
  55. package/dist/validate.d.ts +2 -0
  56. package/dist/validate.js +73 -0
  57. package/dist/verify.d.ts +2 -0
  58. package/dist/verify.js +9 -0
  59. package/package.json +47 -0
  60. package/src/bilibili.ts +111 -0
  61. package/src/browser.ts +260 -0
  62. package/src/clis/bilibili/favorite.ts +42 -0
  63. package/src/clis/bilibili/feed.ts +71 -0
  64. package/src/clis/bilibili/history.ts +48 -0
  65. package/src/clis/bilibili/hot.yaml +38 -0
  66. package/src/clis/bilibili/me.ts +14 -0
  67. package/src/clis/bilibili/search.ts +25 -0
  68. package/src/clis/bilibili/user-videos.ts +42 -0
  69. package/src/clis/github/search.ts +21 -0
  70. package/src/clis/github/trending.yaml +58 -0
  71. package/src/clis/hackernews/top.yaml +36 -0
  72. package/src/clis/index.ts +19 -0
  73. package/src/clis/twitter/trending.yaml +40 -0
  74. package/src/clis/v2ex/hot.yaml +29 -0
  75. package/src/clis/v2ex/latest.yaml +28 -0
  76. package/src/clis/zhihu/hot.yaml +28 -0
  77. package/src/clis/zhihu/search.ts +65 -0
  78. package/src/engine.ts +86 -0
  79. package/src/explore.ts +648 -0
  80. package/src/generate.ts +145 -0
  81. package/src/main.ts +103 -0
  82. package/src/output.ts +96 -0
  83. package/src/pipeline.ts +295 -0
  84. package/src/promote.ts +3 -0
  85. package/src/register.ts +2 -0
  86. package/src/registry.ts +87 -0
  87. package/src/runtime.ts +36 -0
  88. package/src/scaffold.ts +2 -0
  89. package/src/smoke.ts +2 -0
  90. package/src/snapshotFormatter.ts +51 -0
  91. package/src/synthesize.ts +210 -0
  92. package/src/validate.ts +55 -0
  93. package/src/verify.ts +9 -0
  94. package/tsconfig.json +17 -0
@@ -0,0 +1,145 @@
1
+ /**
2
+ * Generate: one-shot CLI creation from URL.
3
+ *
4
+ * Orchestrates the full pipeline:
5
+ * explore (Deep Explore) → synthesize (YAML generation) → register → verify
6
+ *
7
+ * Includes Strategy Cascade: if the initial strategy fails,
8
+ * automatically downgrades and retries.
9
+ */
10
+
11
+ import { exploreUrl } from './explore.js';
12
+ import { synthesizeFromExplore } from './synthesize.js';
13
+ import { registerCandidates } from './register.js';
14
+
15
+ const CAPABILITY_ALIASES: Record<string, string[]> = {
16
+ search: ['search', '搜索', '查找', 'query', 'keyword'],
17
+ hot: ['hot', '热门', '热榜', '热搜', 'popular', 'top', 'ranking'],
18
+ trending: ['trending', '趋势', '流行', 'discover'],
19
+ feed: ['feed', '动态', '关注', '时间线', 'timeline', 'following'],
20
+ me: ['profile', 'me', '个人信息', 'myinfo', '账号'],
21
+ detail: ['detail', '详情', 'video', 'article', 'view'],
22
+ comments: ['comments', '评论', '回复', 'reply'],
23
+ history: ['history', '历史', '记录'],
24
+ favorite: ['favorite', '收藏', 'bookmark', 'collect'],
25
+ };
26
+
27
+ /**
28
+ * Normalize a goal string to a standard capability name.
29
+ */
30
+ function normalizeGoal(goal?: string | null): string | null {
31
+ if (!goal) return null;
32
+ const lower = goal.trim().toLowerCase();
33
+ for (const [cap, aliases] of Object.entries(CAPABILITY_ALIASES)) {
34
+ if (lower === cap || aliases.some(a => lower.includes(a.toLowerCase()))) return cap;
35
+ }
36
+ return null;
37
+ }
38
+
39
+ /**
40
+ * Select the best candidate matching the user's goal.
41
+ */
42
+ function selectCandidate(candidates: any[], goal?: string | null): any {
43
+ if (!candidates.length) return null;
44
+ if (!goal) return candidates[0]; // highest confidence first
45
+
46
+ const normalized = normalizeGoal(goal);
47
+ if (normalized) {
48
+ const exact = candidates.find(c => c.name === normalized);
49
+ if (exact) return exact;
50
+ }
51
+
52
+ const lower = (goal ?? '').trim().toLowerCase();
53
+ const partial = candidates.find(c =>
54
+ c.name?.toLowerCase().includes(lower) || lower.includes(c.name?.toLowerCase())
55
+ );
56
+ return partial ?? candidates[0];
57
+ }
58
+
59
+ export async function generateCliFromUrl(opts: any): Promise<any> {
60
+ // Step 1: Deep Explore
61
+ const exploreResult = await exploreUrl(opts.url, {
62
+ BrowserFactory: opts.BrowserFactory,
63
+ site: opts.site,
64
+ goal: normalizeGoal(opts.goal) ?? opts.goal,
65
+ waitSeconds: opts.waitSeconds ?? 3,
66
+ });
67
+
68
+ // Step 2: Synthesize candidates
69
+ const synthesizeResult = synthesizeFromExplore(exploreResult.out_dir, {
70
+ top: opts.top ?? 5,
71
+ });
72
+
73
+ // Step 3: Select best candidate for goal
74
+ const selected = selectCandidate(synthesizeResult.candidates ?? [], opts.goal);
75
+ const selectedSite = selected?.site ?? synthesizeResult.site ?? exploreResult.site;
76
+
77
+ // Step 4: Register (if requested)
78
+ let registerResult: any = null;
79
+ if (opts.register !== false && synthesizeResult.candidate_count > 0) {
80
+ try {
81
+ registerResult = registerCandidates({
82
+ target: synthesizeResult.out_dir,
83
+ builtinClis: opts.builtinClis,
84
+ userClis: opts.userClis,
85
+ name: selected?.name,
86
+ });
87
+ } catch {}
88
+ }
89
+
90
+ const ok = exploreResult.endpoint_count > 0 && synthesizeResult.candidate_count > 0;
91
+
92
+ return {
93
+ ok,
94
+ goal: opts.goal,
95
+ normalized_goal: normalizeGoal(opts.goal),
96
+ site: selectedSite,
97
+ selected_candidate: selected,
98
+ selected_command: selected ? `${selectedSite}/${selected.name}` : '(none)',
99
+ explore: {
100
+ endpoint_count: exploreResult.endpoint_count,
101
+ api_endpoint_count: exploreResult.api_endpoint_count,
102
+ capability_count: exploreResult.capabilities?.length ?? 0,
103
+ top_strategy: exploreResult.top_strategy,
104
+ framework: exploreResult.framework,
105
+ },
106
+ synthesize: {
107
+ candidate_count: synthesizeResult.candidate_count,
108
+ candidates: (synthesizeResult.candidates ?? []).map((c: any) => ({
109
+ name: c.name,
110
+ strategy: c.strategy,
111
+ confidence: c.confidence,
112
+ })),
113
+ },
114
+ register: registerResult,
115
+ };
116
+ }
117
+
118
+ export function renderGenerateSummary(r: any): string {
119
+ const lines = [
120
+ `opencli generate: ${r.ok ? 'OK' : 'FAIL'}`,
121
+ `Site: ${r.site}`,
122
+ `Goal: ${r.goal ?? '(auto)'}`,
123
+ `Selected: ${r.selected_command}`,
124
+ '',
125
+ `Explore:`,
126
+ ` Endpoints: ${r.explore?.endpoint_count ?? 0} total, ${r.explore?.api_endpoint_count ?? 0} API`,
127
+ ` Capabilities: ${r.explore?.capability_count ?? 0}`,
128
+ ` Strategy: ${r.explore?.top_strategy ?? 'unknown'}`,
129
+ '',
130
+ `Synthesize:`,
131
+ ` Candidates: ${r.synthesize?.candidate_count ?? 0}`,
132
+ ];
133
+
134
+ for (const c of r.synthesize?.candidates ?? []) {
135
+ lines.push(` • ${c.name} (${c.strategy}, ${((c.confidence ?? 0) * 100).toFixed(0)}%)`);
136
+ }
137
+
138
+ if (r.register) lines.push(`\nRegistered: ${r.register.count ?? 0}`);
139
+
140
+ const fw = r.explore?.framework ?? {};
141
+ const fwNames = Object.entries(fw).filter(([, v]) => v).map(([k]) => k);
142
+ if (fwNames.length) lines.push(`Framework: ${fwNames.join(', ')}`);
143
+
144
+ return lines.join('\n');
145
+ }
package/src/main.ts ADDED
@@ -0,0 +1,103 @@
1
+ #!/usr/bin/env node
2
+ /**
3
+ * opencli — Make any website your CLI. AI-powered.
4
+ */
5
+
6
+ import * as os from 'node:os';
7
+ import * as path from 'node:path';
8
+ import { fileURLToPath } from 'node:url';
9
+ import { Command } from 'commander';
10
+ import chalk from 'chalk';
11
+ import { discoverClis, executeCommand } from './engine.js';
12
+ import { type CliCommand, fullName, getRegistry, strategyLabel } from './registry.js';
13
+ import { render as renderOutput } from './output.js';
14
+ import './clis/index.js';
15
+ import { PlaywrightMCP } from './browser.js';
16
+ import { browserSession, DEFAULT_BROWSER_COMMAND_TIMEOUT, runWithTimeout } from './runtime.js';
17
+
18
+ const __filename = fileURLToPath(import.meta.url);
19
+ const __dirname = path.dirname(__filename);
20
+ const BUILTIN_CLIS = path.resolve(__dirname, 'clis');
21
+ const USER_CLIS = path.join(os.homedir(), '.opencli', 'clis');
22
+
23
+ discoverClis(BUILTIN_CLIS, USER_CLIS);
24
+
25
+ const program = new Command();
26
+ program.name('opencli').description('Make any website your CLI. Zero setup. AI-powered.').version('0.1.0');
27
+
28
+ // ── Built-in commands ──────────────────────────────────────────────────────
29
+
30
+ program.command('list').description('List all available CLI commands').option('--json', 'JSON output')
31
+ .action((opts) => {
32
+ const registry = getRegistry();
33
+ const commands = [...registry.values()].sort((a, b) => fullName(a).localeCompare(fullName(b)));
34
+ if (opts.json) { console.log(JSON.stringify(commands.map(c => ({ command: fullName(c), site: c.site, name: c.name, description: c.description, strategy: strategyLabel(c), browser: c.browser, args: c.args.map(a => a.name) })), null, 2)); return; }
35
+ const sites = new Map<string, CliCommand[]>();
36
+ for (const cmd of commands) { const g = sites.get(cmd.site) ?? []; g.push(cmd); sites.set(cmd.site, g); }
37
+ console.log(); console.log(chalk.bold(' opencli') + chalk.dim(' — available commands')); console.log();
38
+ for (const [site, cmds] of sites) {
39
+ console.log(chalk.bold.cyan(` ${site}`));
40
+ for (const cmd of cmds) { const tag = strategyLabel(cmd) === 'public' ? chalk.green('[public]') : chalk.yellow(`[${strategyLabel(cmd)}]`); console.log(` ${cmd.name} ${tag}${cmd.description ? chalk.dim(` — ${cmd.description}`) : ''}`); }
41
+ console.log();
42
+ }
43
+ console.log(chalk.dim(` ${commands.length} commands across ${sites.size} sites`)); console.log();
44
+ });
45
+
46
+ program.command('validate').description('Validate CLI definitions').argument('[target]', 'site or site/name')
47
+ .action(async (target) => { const { validateClisWithTarget, renderValidationReport } = await import('./validate.js'); console.log(renderValidationReport(validateClisWithTarget([BUILTIN_CLIS, USER_CLIS], target))); });
48
+
49
+ program.command('verify').description('Validate + smoke test').argument('[target]').option('--smoke', 'Run smoke tests', false)
50
+ .action(async (target, opts) => { const { verifyClis, renderVerifyReport } = await import('./verify.js'); const r = await verifyClis({ builtinClis: BUILTIN_CLIS, userClis: USER_CLIS, target, smoke: opts.smoke }); console.log(renderVerifyReport(r)); process.exitCode = r.ok ? 0 : 1; });
51
+
52
+ program.command('explore').description('Explore a website').argument('<url>').option('--site <name>').option('--goal <text>').option('--wait <s>', '', '3')
53
+ .action(async (url, opts) => { const { exploreUrl, renderExploreSummary } = await import('./explore.js'); console.log(renderExploreSummary(await exploreUrl(url, { BrowserFactory: PlaywrightMCP, site: opts.site, goal: opts.goal, waitSeconds: parseFloat(opts.wait) }))); });
54
+
55
+ program.command('synthesize').description('Synthesize CLIs from explore').argument('<target>').option('--top <n>', '', '3')
56
+ .action(async (target, opts) => { const { synthesizeFromExplore, renderSynthesizeSummary } = await import('./synthesize.js'); console.log(renderSynthesizeSummary(synthesizeFromExplore(target, { top: parseInt(opts.top) }))); });
57
+
58
+ program.command('generate').description('One-shot: explore → synthesize → register').argument('<url>').option('--goal <text>').option('--site <name>')
59
+ .action(async (url, opts) => { const { generateCliFromUrl, renderGenerateSummary } = await import('./generate.js'); const r = await generateCliFromUrl({ url, BrowserFactory: PlaywrightMCP, builtinClis: BUILTIN_CLIS, userClis: USER_CLIS, goal: opts.goal, site: opts.site }); console.log(renderGenerateSummary(r)); process.exitCode = r.ok ? 0 : 1; });
60
+
61
+ // ── Dynamic site commands ──────────────────────────────────────────────────
62
+
63
+ const registry = getRegistry();
64
+ const siteGroups = new Map<string, Command>();
65
+
66
+ for (const [, cmd] of registry) {
67
+ let siteCmd = siteGroups.get(cmd.site);
68
+ if (!siteCmd) { siteCmd = program.command(cmd.site).description(`${cmd.site} commands`); siteGroups.set(cmd.site, siteCmd); }
69
+ const subCmd = siteCmd.command(cmd.name).description(cmd.description);
70
+
71
+ for (const arg of cmd.args) {
72
+ const flag = arg.required ? `--${arg.name} <value>` : `--${arg.name} [value]`;
73
+ if (arg.required) subCmd.requiredOption(flag, arg.help ?? '');
74
+ else if (arg.default != null) subCmd.option(flag, arg.help ?? '', String(arg.default));
75
+ else subCmd.option(flag, arg.help ?? '');
76
+ }
77
+ subCmd.option('-f, --format <fmt>', 'Output format: table, json, md, csv', 'table').option('-v, --verbose', 'Debug output', false);
78
+
79
+ subCmd.action(async (actionOpts) => {
80
+ const startTime = Date.now();
81
+ const kwargs: Record<string, any> = {};
82
+ for (const arg of cmd.args) {
83
+ const v = actionOpts[arg.name]; if (v !== undefined) kwargs[arg.name] = coerce(v, arg.type ?? 'str');
84
+ else if (arg.default != null) kwargs[arg.name] = arg.default;
85
+ }
86
+ try {
87
+ let result: any;
88
+ if (cmd.browser) {
89
+ result = await browserSession(PlaywrightMCP, async (page) => runWithTimeout(executeCommand(cmd, page, kwargs, actionOpts.verbose), { timeout: cmd.timeoutSeconds ?? DEFAULT_BROWSER_COMMAND_TIMEOUT, label: fullName(cmd) }));
90
+ } else { result = await executeCommand(cmd, null, kwargs, actionOpts.verbose); }
91
+ renderOutput(result, { fmt: actionOpts.format, columns: cmd.columns, title: `${cmd.site}/${cmd.name}`, elapsed: (Date.now() - startTime) / 1000, source: fullName(cmd) });
92
+ } catch (err: any) { console.error(chalk.red(`Error: ${err.message ?? err}`)); process.exitCode = 1; }
93
+ });
94
+ }
95
+
96
+ function coerce(v: any, t: string): any {
97
+ if (t === 'bool') return ['1', 'true', 'yes', 'on'].includes(String(v).toLowerCase());
98
+ if (t === 'int') return parseInt(String(v), 10);
99
+ if (t === 'float') return parseFloat(String(v));
100
+ return String(v);
101
+ }
102
+
103
+ program.parse();
package/src/output.ts ADDED
@@ -0,0 +1,96 @@
1
+ /**
2
+ * Output formatting: table, JSON, Markdown, CSV.
3
+ */
4
+
5
+ import chalk from 'chalk';
6
+ import Table from 'cli-table3';
7
+
8
+ export interface RenderOptions {
9
+ fmt?: string;
10
+ columns?: string[];
11
+ title?: string;
12
+ elapsed?: number;
13
+ source?: string;
14
+ }
15
+
16
+ export function render(data: any, opts: RenderOptions = {}): void {
17
+ const fmt = opts.fmt ?? 'table';
18
+ if (data === null || data === undefined) {
19
+ console.log(data);
20
+ return;
21
+ }
22
+ switch (fmt) {
23
+ case 'json': renderJson(data); break;
24
+ case 'md': case 'markdown': renderMarkdown(data, opts); break;
25
+ case 'csv': renderCsv(data, opts); break;
26
+ default: renderTable(data, opts); break;
27
+ }
28
+ }
29
+
30
+ function renderTable(data: any, opts: RenderOptions): void {
31
+ const rows: any[] = Array.isArray(data) ? data : [data];
32
+ if (!rows.length) { console.log(chalk.dim('(no data)')); return; }
33
+ const columns = opts.columns ?? Object.keys(rows[0]);
34
+
35
+ const header = columns.map((c, i) => i === 0 ? '#' : capitalize(c));
36
+ const table = new Table({
37
+ head: header.map(h => chalk.bold(h)),
38
+ style: { head: [], border: [] },
39
+ wordWrap: true,
40
+ wrapOnWordBoundary: true,
41
+ colWidths: columns.map((c, i) => {
42
+ if (i === 0) return 4;
43
+ if (c === 'url' || c === 'description') return null as any;
44
+ if (c === 'title' || c === 'name' || c === 'repo') return null as any;
45
+ return null as any;
46
+ }).filter(() => true),
47
+ });
48
+
49
+ for (const row of rows) {
50
+ table.push(columns.map(c => {
51
+ const v = row[c];
52
+ return v === null || v === undefined ? '' : String(v);
53
+ }));
54
+ }
55
+
56
+ console.log();
57
+ if (opts.title) console.log(chalk.dim(` ${opts.title}`));
58
+ console.log(table.toString());
59
+ const footer: string[] = [];
60
+ footer.push(`${rows.length} items`);
61
+ if (opts.elapsed) footer.push(`${opts.elapsed.toFixed(1)}s`);
62
+ if (opts.source) footer.push(opts.source);
63
+ console.log(chalk.dim(footer.join(' · ')));
64
+ }
65
+
66
+ function renderJson(data: any): void {
67
+ console.log(JSON.stringify(data, null, 2));
68
+ }
69
+
70
+ function renderMarkdown(data: any, opts: RenderOptions): void {
71
+ const rows: any[] = Array.isArray(data) ? data : [data];
72
+ if (!rows.length) return;
73
+ const columns = opts.columns ?? Object.keys(rows[0]);
74
+ console.log('| ' + columns.join(' | ') + ' |');
75
+ console.log('| ' + columns.map(() => '---').join(' | ') + ' |');
76
+ for (const row of rows) {
77
+ console.log('| ' + columns.map(c => String(row[c] ?? '')).join(' | ') + ' |');
78
+ }
79
+ }
80
+
81
+ function renderCsv(data: any, opts: RenderOptions): void {
82
+ const rows: any[] = Array.isArray(data) ? data : [data];
83
+ if (!rows.length) return;
84
+ const columns = opts.columns ?? Object.keys(rows[0]);
85
+ console.log(columns.join(','));
86
+ for (const row of rows) {
87
+ console.log(columns.map(c => {
88
+ const v = String(row[c] ?? '');
89
+ return v.includes(',') || v.includes('"') ? `"${v.replace(/"/g, '""')}"` : v;
90
+ }).join(','));
91
+ }
92
+ }
93
+
94
+ function capitalize(s: string): string {
95
+ return s.charAt(0).toUpperCase() + s.slice(1);
96
+ }
@@ -0,0 +1,295 @@
1
+ /**
2
+ * YAML pipeline executor.
3
+ * Steps: fetch, navigate, evaluate, map, filter, sort, limit, select, snapshot, click, type, wait, press, intercept.
4
+ */
5
+
6
+ import chalk from 'chalk';
7
+
8
+ export interface PipelineContext {
9
+ args?: Record<string, any>;
10
+ debug?: boolean;
11
+ }
12
+
13
+ export async function executePipeline(
14
+ page: any,
15
+ pipeline: any[],
16
+ ctx: PipelineContext = {},
17
+ ): Promise<any> {
18
+ const args = ctx.args ?? {};
19
+ const debug = ctx.debug ?? false;
20
+ let data: any = null;
21
+ const total = pipeline.length;
22
+
23
+ for (let i = 0; i < pipeline.length; i++) {
24
+ const step = pipeline[i];
25
+ if (!step || typeof step !== 'object') continue;
26
+ for (const [op, params] of Object.entries(step)) {
27
+ if (debug) debugStepStart(i + 1, total, op, params);
28
+ data = await executeStep(page, op, params, data, args);
29
+ if (debug) debugStepResult(op, data);
30
+ }
31
+ }
32
+ return data;
33
+ }
34
+
35
+ function normalizeEvaluateSource(source: string): string {
36
+ const stripped = source.trim();
37
+ if (!stripped) return '() => undefined';
38
+ if (stripped.startsWith('(') && stripped.endsWith(')()')) return `() => (${stripped})`;
39
+ if (/^(async\s+)?\([^)]*\)\s*=>/.test(stripped)) return stripped;
40
+ if (/^(async\s+)?[A-Za-z_][A-Za-z0-9_]*\s*=>/.test(stripped)) return stripped;
41
+ if (stripped.startsWith('function ') || stripped.startsWith('async function ')) return stripped;
42
+ return `() => (${stripped})`;
43
+ }
44
+
45
+ function debugStepStart(stepNum: number, total: number, op: string, params: any): void {
46
+ let preview = '';
47
+ if (typeof params === 'string') {
48
+ preview = params.length <= 80 ? ` → ${params}` : ` → ${params.slice(0, 77)}...`;
49
+ } else if (params && typeof params === 'object' && !Array.isArray(params)) {
50
+ preview = ` (${Object.keys(params).join(', ')})`;
51
+ }
52
+ process.stderr.write(` ${chalk.dim(`[${stepNum}/${total}]`)} ${chalk.bold.cyan(op)}${preview}\n`);
53
+ }
54
+
55
+ function debugStepResult(op: string, data: any): void {
56
+ if (data === null || data === undefined) {
57
+ process.stderr.write(` ${chalk.dim('→ (no data)')}\n`);
58
+ } else if (Array.isArray(data)) {
59
+ process.stderr.write(` ${chalk.dim(`→ ${data.length} items`)}\n`);
60
+ } else if (typeof data === 'object') {
61
+ const keys = Object.keys(data).slice(0, 5);
62
+ process.stderr.write(` ${chalk.dim(`→ dict (${keys.join(', ')}${Object.keys(data).length > 5 ? '...' : ''})`)}\n`);
63
+ } else if (typeof data === 'string') {
64
+ const p = data.slice(0, 60).replace(/\n/g, '\\n');
65
+ process.stderr.write(` ${chalk.dim(`→ "${p}${data.length > 60 ? '...' : ''}"`)}\n`);
66
+ } else {
67
+ process.stderr.write(` ${chalk.dim(`→ ${typeof data}`)}\n`);
68
+ }
69
+ }
70
+
71
+ // Single URL fetch helper
72
+ async function fetchSingle(
73
+ page: any, url: string, method: string,
74
+ queryParams: Record<string, any>, headers: Record<string, any>,
75
+ args: Record<string, any>, data: any,
76
+ ): Promise<any> {
77
+ const renderedParams: Record<string, string> = {};
78
+ for (const [k, v] of Object.entries(queryParams)) renderedParams[k] = String(render(v, { args, data }));
79
+ const renderedHeaders: Record<string, string> = {};
80
+ for (const [k, v] of Object.entries(headers)) renderedHeaders[k] = String(render(v, { args, data }));
81
+
82
+ let finalUrl = url;
83
+ if (Object.keys(renderedParams).length > 0) {
84
+ const qs = new URLSearchParams(renderedParams).toString();
85
+ finalUrl = `${finalUrl}${finalUrl.includes('?') ? '&' : '?'}${qs}`;
86
+ }
87
+
88
+ if (page === null) {
89
+ const resp = await fetch(finalUrl, { method: method.toUpperCase(), headers: renderedHeaders });
90
+ return resp.json();
91
+ }
92
+
93
+ const headersJs = JSON.stringify(renderedHeaders);
94
+ const escapedUrl = finalUrl.replace(/"/g, '\\"');
95
+ return page.evaluate(`
96
+ async () => {
97
+ const resp = await fetch("${escapedUrl}", {
98
+ method: "${method}", headers: ${headersJs}, credentials: "include"
99
+ });
100
+ return await resp.json();
101
+ }
102
+ `);
103
+ }
104
+
105
+ async function executeStep(page: any, op: string, params: any, data: any, args: Record<string, any>): Promise<any> {
106
+ switch (op) {
107
+ case 'navigate': {
108
+ const url = render(params, { args, data });
109
+ await page.goto(String(url));
110
+ return data;
111
+ }
112
+ case 'fetch': {
113
+ const urlOrObj = typeof params === 'string' ? params : (params?.url ?? '');
114
+ const method = params?.method ?? 'GET';
115
+ const queryParams: Record<string, any> = params?.params ?? {};
116
+ const headers: Record<string, any> = params?.headers ?? {};
117
+ const urlTemplate = String(urlOrObj);
118
+
119
+ // Per-item fetch when data is array and URL references item
120
+ if (Array.isArray(data) && urlTemplate.includes('item')) {
121
+ const results: any[] = [];
122
+ for (let i = 0; i < data.length; i++) {
123
+ const itemUrl = String(render(urlTemplate, { args, data, item: data[i], index: i }));
124
+ results.push(await fetchSingle(page, itemUrl, method, queryParams, headers, args, data));
125
+ }
126
+ return results;
127
+ }
128
+ const url = render(urlOrObj, { args, data });
129
+ return fetchSingle(page, String(url), method, queryParams, headers, args, data);
130
+ }
131
+ case 'select': {
132
+ const pathStr = String(render(params, { args, data }));
133
+ if (data && typeof data === 'object') {
134
+ let current = data;
135
+ for (const part of pathStr.split('.')) {
136
+ if (current && typeof current === 'object' && !Array.isArray(current)) current = (current as any)[part];
137
+ else if (Array.isArray(current) && /^\d+$/.test(part)) current = current[parseInt(part, 10)];
138
+ else return null;
139
+ }
140
+ return current;
141
+ }
142
+ return data;
143
+ }
144
+ case 'evaluate': {
145
+ const js = String(render(params, { args, data }));
146
+ return page.evaluate(normalizeEvaluateSource(js));
147
+ }
148
+ case 'snapshot': {
149
+ const opts = (typeof params === 'object' && params) ? params : {};
150
+ return page.snapshot({ interactive: opts.interactive ?? false, compact: opts.compact ?? false, maxDepth: opts.max_depth, raw: opts.raw ?? false });
151
+ }
152
+ case 'click': {
153
+ await page.click(String(render(params, { args, data })).replace(/^@/, ''));
154
+ return data;
155
+ }
156
+ case 'type': {
157
+ if (typeof params === 'object' && params) {
158
+ const ref = String(render(params.ref ?? '', { args, data })).replace(/^@/, '');
159
+ const text = String(render(params.text ?? '', { args, data }));
160
+ await page.typeText(ref, text);
161
+ if (params.submit) await page.pressKey('Enter');
162
+ }
163
+ return data;
164
+ }
165
+ case 'wait': {
166
+ if (typeof params === 'number') await page.wait(params);
167
+ else if (typeof params === 'object' && params) {
168
+ if ('text' in params) {
169
+ const timeout = params.timeout ?? 10;
170
+ const start = Date.now();
171
+ while ((Date.now() - start) / 1000 < timeout) {
172
+ const snap = await page.snapshot({ raw: true });
173
+ if (typeof snap === 'string' && snap.includes(params.text)) break;
174
+ await page.wait(0.5);
175
+ }
176
+ } else if ('time' in params) await page.wait(Number(params.time));
177
+ } else if (typeof params === 'string') await page.wait(Number(render(params, { args, data })));
178
+ return data;
179
+ }
180
+ case 'press': {
181
+ await page.pressKey(String(render(params, { args, data })));
182
+ return data;
183
+ }
184
+ case 'map': {
185
+ if (!data || typeof data !== 'object') return data;
186
+ let items: any[] = Array.isArray(data) ? data : [data];
187
+ if (!Array.isArray(data) && typeof data === 'object' && 'data' in data) items = data.data;
188
+ const result: any[] = [];
189
+ for (let i = 0; i < items.length; i++) {
190
+ const item = items[i];
191
+ const row: Record<string, any> = {};
192
+ for (const [key, template] of Object.entries(params)) row[key] = render(template, { args, data, item, index: i });
193
+ result.push(row);
194
+ }
195
+ return result;
196
+ }
197
+ case 'filter': {
198
+ if (!Array.isArray(data)) return data;
199
+ return data.filter((item, i) => evalExpr(String(params), { args, item, index: i }));
200
+ }
201
+ case 'sort': {
202
+ if (!Array.isArray(data)) return data;
203
+ const key = typeof params === 'object' ? (params.by ?? '') : String(params);
204
+ const reverse = typeof params === 'object' ? params.order === 'desc' : false;
205
+ return [...data].sort((a, b) => { const va = a[key] ?? ''; const vb = b[key] ?? ''; const cmp = va < vb ? -1 : va > vb ? 1 : 0; return reverse ? -cmp : cmp; });
206
+ }
207
+ case 'limit': {
208
+ if (!Array.isArray(data)) return data;
209
+ return data.slice(0, Number(render(params, { args, data })));
210
+ }
211
+ case 'intercept': return data;
212
+ default: return data;
213
+ }
214
+ }
215
+
216
+ // Template engine: ${{ ... }}
217
+ interface RenderContext { args?: Record<string, any>; data?: any; item?: any; index?: number; }
218
+
219
+ function render(template: any, ctx: RenderContext): any {
220
+ if (typeof template !== 'string') return template;
221
+ const fullMatch = template.match(/^\$\{\{\s*(.*?)\s*\}\}$/);
222
+ if (fullMatch) return evalExpr(fullMatch[1].trim(), ctx);
223
+ return template.replace(/\$\{\{\s*(.*?)\s*\}\}/g, (_m, expr) => String(evalExpr(expr.trim(), ctx)));
224
+ }
225
+
226
+ function evalExpr(expr: string, ctx: RenderContext): any {
227
+ const args = ctx.args ?? {};
228
+ const item = ctx.item ?? {};
229
+ const data = ctx.data;
230
+ const index = ctx.index ?? 0;
231
+
232
+ // Default filter: args.limit | default(20)
233
+ if (expr.includes('|') && expr.includes('default(')) {
234
+ const [mainExpr, rest] = expr.split('|', 2);
235
+ const defaultMatch = rest.match(/default\((.+?)\)/);
236
+ const defaultVal = defaultMatch ? defaultMatch[1] : null;
237
+ const result = resolvePath(mainExpr.trim(), { args, item, data, index });
238
+ if (result === null || result === undefined) {
239
+ if (defaultVal !== null) {
240
+ const intVal = parseInt(defaultVal!, 10);
241
+ if (!isNaN(intVal) && String(intVal) === defaultVal!.trim()) return intVal;
242
+ return defaultVal!.replace(/^['"]|['"]$/g, '');
243
+ }
244
+ }
245
+ return result;
246
+ }
247
+
248
+ // Arithmetic: index + 1
249
+ const arithMatch = expr.match(/^([\w][\w.]*)\s*([+\-*/])\s*(\d+)$/);
250
+ if (arithMatch) {
251
+ const [, varName, op, numStr] = arithMatch;
252
+ const val = resolvePath(varName, { args, item, data, index });
253
+ if (val !== null && val !== undefined) {
254
+ const numVal = Number(val); const num = Number(numStr);
255
+ if (!isNaN(numVal)) {
256
+ switch (op) {
257
+ case '+': return numVal + num; case '-': return numVal - num;
258
+ case '*': return numVal * num; case '/': return num !== 0 ? numVal / num : 0;
259
+ }
260
+ }
261
+ }
262
+ }
263
+
264
+ // JS-like fallback expression: item.tweetCount || 'N/A'
265
+ const orMatch = expr.match(/^(.+?)\s*\|\|\s*(.+)$/);
266
+ if (orMatch) {
267
+ const left = evalExpr(orMatch[1].trim(), ctx);
268
+ if (left) return left;
269
+ const right = orMatch[2].trim();
270
+ return right.replace(/^['"]|['"]$/g, '');
271
+ }
272
+
273
+ return resolvePath(expr, { args, item, data, index });
274
+ }
275
+
276
+ function resolvePath(pathStr: string, ctx: RenderContext): any {
277
+ const args = ctx.args ?? {};
278
+ const item = ctx.item ?? {};
279
+ const data = ctx.data;
280
+ const index = ctx.index ?? 0;
281
+ const parts = pathStr.split('.');
282
+ const rootName = parts[0];
283
+ let obj: any; let rest: string[];
284
+ if (rootName === 'args') { obj = args; rest = parts.slice(1); }
285
+ else if (rootName === 'item') { obj = item; rest = parts.slice(1); }
286
+ else if (rootName === 'data') { obj = data; rest = parts.slice(1); }
287
+ else if (rootName === 'index') return index;
288
+ else { obj = item; rest = parts; }
289
+ for (const part of rest) {
290
+ if (obj && typeof obj === 'object' && !Array.isArray(obj)) obj = obj[part];
291
+ else if (Array.isArray(obj) && /^\d+$/.test(part)) obj = obj[parseInt(part, 10)];
292
+ else return null;
293
+ }
294
+ return obj;
295
+ }
package/src/promote.ts ADDED
@@ -0,0 +1,3 @@
1
+ /** Promote verified drafts. */
2
+ import { registerCandidates } from './register.js';
3
+ export function promoteCandidate(opts: any): any { return registerCandidates(opts); }
@@ -0,0 +1,2 @@
1
+ /** Register candidates, promote, scaffold, generate — stubs with exports. */
2
+ export function registerCandidates(opts: any): any { return { ok: true, count: 0 }; }