@jackwener/opencli 0.4.0 → 0.4.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CLI-CREATOR.md +128 -79
- package/README.md +3 -3
- package/README.zh-CN.md +3 -3
- package/SKILL.md +4 -2
- package/dist/build-manifest.d.ts +11 -0
- package/dist/build-manifest.js +161 -0
- package/dist/cli-manifest.json +1793 -0
- package/dist/clis/bilibili/following.d.ts +1 -0
- package/dist/clis/bilibili/following.js +41 -0
- package/dist/clis/xiaohongshu/search.d.ts +5 -2
- package/dist/clis/xiaohongshu/search.js +35 -41
- package/dist/engine.d.ts +13 -0
- package/dist/engine.js +122 -17
- package/dist/pipeline/steps/fetch.js +57 -1
- package/dist/registry.d.ts +3 -0
- package/package.json +3 -2
- package/src/build-manifest.ts +194 -0
- package/src/clis/bilibili/following.ts +50 -0
- package/src/clis/xiaohongshu/search.ts +41 -44
- package/src/engine.ts +123 -17
- package/src/pipeline/steps/fetch.ts +63 -1
- package/src/registry.ts +3 -0
|
@@ -0,0 +1 @@
|
|
|
1
|
+
export {};
|
|
@@ -0,0 +1,41 @@
|
|
|
1
|
+
import { cli, Strategy } from '../../registry.js';
|
|
2
|
+
import { fetchJson, getSelfUid, resolveUid } from '../../bilibili.js';
|
|
3
|
+
cli({
|
|
4
|
+
site: 'bilibili',
|
|
5
|
+
name: 'following',
|
|
6
|
+
description: '获取 Bilibili 用户的关注列表',
|
|
7
|
+
strategy: Strategy.COOKIE,
|
|
8
|
+
args: [
|
|
9
|
+
{ name: 'uid', required: false, help: '目标用户 ID(默认为当前登录用户)' },
|
|
10
|
+
{ name: 'page', type: 'int', required: false, default: 1, help: '页码' },
|
|
11
|
+
{ name: 'limit', type: 'int', required: false, default: 50, help: '每页数量 (最大 50)' },
|
|
12
|
+
],
|
|
13
|
+
columns: ['mid', 'name', 'sign', 'following', 'fans'],
|
|
14
|
+
func: async (page, kwargs) => {
|
|
15
|
+
if (!page)
|
|
16
|
+
throw new Error('Requires browser');
|
|
17
|
+
// 1. Resolve UID (default to self)
|
|
18
|
+
const uid = kwargs.uid
|
|
19
|
+
? await resolveUid(page, kwargs.uid)
|
|
20
|
+
: await getSelfUid(page);
|
|
21
|
+
const pn = kwargs.page ?? 1;
|
|
22
|
+
const ps = Math.min(kwargs.limit ?? 50, 50);
|
|
23
|
+
// 2. Fetch following list (standard Cookie API, no Wbi signing needed)
|
|
24
|
+
const payload = await fetchJson(page, `https://api.bilibili.com/x/relation/followings?vmid=${uid}&pn=${pn}&ps=${ps}&order=desc`);
|
|
25
|
+
if (payload.code !== 0) {
|
|
26
|
+
throw new Error(`获取关注列表失败: ${payload.message} (${payload.code})`);
|
|
27
|
+
}
|
|
28
|
+
const list = payload.data?.list || [];
|
|
29
|
+
if (list.length === 0) {
|
|
30
|
+
return [{ mid: '-', name: `共 ${payload.data?.total ?? 0} 人关注,当前页无数据`, sign: '', following: '', fans: '' }];
|
|
31
|
+
}
|
|
32
|
+
// 3. Map to output
|
|
33
|
+
return list.map((u) => ({
|
|
34
|
+
mid: u.mid,
|
|
35
|
+
name: u.uname,
|
|
36
|
+
sign: (u.sign || '').slice(0, 40),
|
|
37
|
+
following: u.attribute === 6 ? '互相关注' : '已关注',
|
|
38
|
+
fans: u.official_verify?.desc || '',
|
|
39
|
+
}));
|
|
40
|
+
},
|
|
41
|
+
});
|
|
@@ -1,5 +1,8 @@
|
|
|
1
1
|
/**
|
|
2
|
-
* Xiaohongshu search —
|
|
3
|
-
*
|
|
2
|
+
* Xiaohongshu search — DOM-based extraction from search results page.
|
|
3
|
+
* The previous Pinia store + XHR interception approach broke because
|
|
4
|
+
* the API now returns empty items. This version navigates directly to
|
|
5
|
+
* the search results page and extracts data from rendered DOM elements.
|
|
6
|
+
* Ref: https://github.com/jackwener/opencli/issues/10
|
|
4
7
|
*/
|
|
5
8
|
export {};
|
|
@@ -1,6 +1,9 @@
|
|
|
1
1
|
/**
|
|
2
|
-
* Xiaohongshu search —
|
|
3
|
-
*
|
|
2
|
+
* Xiaohongshu search — DOM-based extraction from search results page.
|
|
3
|
+
* The previous Pinia store + XHR interception approach broke because
|
|
4
|
+
* the API now returns empty items. This version navigates directly to
|
|
5
|
+
* the search results page and extracts data from rendered DOM elements.
|
|
6
|
+
* Ref: https://github.com/jackwener/opencli/issues/10
|
|
4
7
|
*/
|
|
5
8
|
import { cli, Strategy } from '../../registry.js';
|
|
6
9
|
cli({
|
|
@@ -13,54 +16,45 @@ cli({
|
|
|
13
16
|
{ name: 'keyword', required: true, help: 'Search keyword' },
|
|
14
17
|
{ name: 'limit', type: 'int', default: 20, help: 'Number of results' },
|
|
15
18
|
],
|
|
16
|
-
columns: ['rank', 'title', 'author', 'likes'
|
|
19
|
+
columns: ['rank', 'title', 'author', 'likes'],
|
|
17
20
|
func: async (page, kwargs) => {
|
|
18
|
-
|
|
19
|
-
await page.
|
|
21
|
+
const keyword = encodeURIComponent(kwargs.keyword);
|
|
22
|
+
await page.goto(`https://www.xiaohongshu.com/search_result?keyword=${keyword}&source=web_search_result_notes`);
|
|
23
|
+
await page.wait(3);
|
|
24
|
+
// Scroll a couple of times to load more results
|
|
25
|
+
await page.autoScroll({ times: 2 });
|
|
20
26
|
const data = await page.evaluate(`
|
|
21
|
-
(
|
|
22
|
-
const
|
|
23
|
-
const
|
|
24
|
-
|
|
27
|
+
(() => {
|
|
28
|
+
const notes = document.querySelectorAll('section.note-item');
|
|
29
|
+
const results = [];
|
|
30
|
+
notes.forEach(el => {
|
|
31
|
+
// Skip "related searches" sections
|
|
32
|
+
if (el.classList.contains('query-note-item')) return;
|
|
25
33
|
|
|
26
|
-
|
|
27
|
-
|
|
34
|
+
const titleEl = el.querySelector('.title, .note-title, a.title');
|
|
35
|
+
const nameEl = el.querySelector('.name, .author-name, .nick-name');
|
|
36
|
+
const likesEl = el.querySelector('.count, .like-count, .like-wrapper .count');
|
|
37
|
+
const linkEl = el.querySelector('a[href*="/explore/"], a[href*="/search_result/"], a[href*="/note/"]');
|
|
28
38
|
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
const origSend = XMLHttpRequest.prototype.send;
|
|
32
|
-
XMLHttpRequest.prototype.open = function(m, u) { this.__url = u; return origOpen.apply(this, arguments); };
|
|
33
|
-
XMLHttpRequest.prototype.send = function(b) {
|
|
34
|
-
if (this.__url?.includes('search/notes')) {
|
|
35
|
-
const x = this;
|
|
36
|
-
const orig = x.onreadystatechange;
|
|
37
|
-
x.onreadystatechange = function() { if (x.readyState === 4 && !captured) { try { captured = JSON.parse(x.responseText); } catch {} } if (orig) orig.apply(this, arguments); };
|
|
38
|
-
}
|
|
39
|
-
return origSend.apply(this, arguments);
|
|
40
|
-
};
|
|
39
|
+
const href = linkEl?.getAttribute('href') || '';
|
|
40
|
+
const noteId = href.match(/\\/(?:explore|note)\\/([a-f0-9]+)/)?.[1] || '';
|
|
41
41
|
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
if (!captured?.success) return {error: captured?.msg || 'Search failed'};
|
|
52
|
-
return (captured.data?.items || []).map(i => ({
|
|
53
|
-
title: i.note_card?.display_title || '',
|
|
54
|
-
type: i.note_card?.type || '',
|
|
55
|
-
url: 'https://www.xiaohongshu.com/explore/' + i.id,
|
|
56
|
-
author: i.note_card?.user?.nickname || '',
|
|
57
|
-
likes: i.note_card?.interact_info?.liked_count || '0',
|
|
58
|
-
}));
|
|
42
|
+
results.push({
|
|
43
|
+
title: (titleEl?.textContent || '').trim(),
|
|
44
|
+
author: (nameEl?.textContent || '').trim(),
|
|
45
|
+
likes: (likesEl?.textContent || '0').trim(),
|
|
46
|
+
url: noteId ? 'https://www.xiaohongshu.com/explore/' + noteId : '',
|
|
47
|
+
});
|
|
48
|
+
});
|
|
49
|
+
return results;
|
|
59
50
|
})()
|
|
60
51
|
`);
|
|
61
52
|
if (!Array.isArray(data))
|
|
62
53
|
return [];
|
|
63
|
-
return data
|
|
54
|
+
return data
|
|
55
|
+
.filter((item) => item.title)
|
|
56
|
+
.slice(0, kwargs.limit)
|
|
57
|
+
.map((item, i) => ({
|
|
64
58
|
rank: i + 1,
|
|
65
59
|
...item,
|
|
66
60
|
}));
|
package/dist/engine.d.ts
CHANGED
|
@@ -1,7 +1,20 @@
|
|
|
1
1
|
/**
|
|
2
2
|
* CLI discovery: finds YAML/TS CLI definitions and registers them.
|
|
3
|
+
*
|
|
4
|
+
* Supports two modes:
|
|
5
|
+
* 1. FAST PATH (manifest): If a pre-compiled cli-manifest.json exists,
|
|
6
|
+
* registers all YAML commands instantly without runtime YAML parsing.
|
|
7
|
+
* TS modules are loaded lazily only when their command is executed.
|
|
8
|
+
* 2. FALLBACK (filesystem scan): Traditional runtime discovery for development.
|
|
3
9
|
*/
|
|
4
10
|
import { type CliCommand } from './registry.js';
|
|
5
11
|
import type { IPage } from './types.js';
|
|
12
|
+
/**
|
|
13
|
+
* Discover and register CLI commands.
|
|
14
|
+
* Uses pre-compiled manifest when available for instant startup.
|
|
15
|
+
*/
|
|
6
16
|
export declare function discoverClis(...dirs: string[]): Promise<void>;
|
|
17
|
+
/**
|
|
18
|
+
* Execute a CLI command. Handles lazy-loading of TS modules.
|
|
19
|
+
*/
|
|
7
20
|
export declare function executeCommand(cmd: CliCommand, page: IPage | null, kwargs: Record<string, any>, debug?: boolean): Promise<any>;
|
package/dist/engine.js
CHANGED
|
@@ -1,31 +1,110 @@
|
|
|
1
1
|
/**
|
|
2
2
|
* CLI discovery: finds YAML/TS CLI definitions and registers them.
|
|
3
|
+
*
|
|
4
|
+
* Supports two modes:
|
|
5
|
+
* 1. FAST PATH (manifest): If a pre-compiled cli-manifest.json exists,
|
|
6
|
+
* registers all YAML commands instantly without runtime YAML parsing.
|
|
7
|
+
* TS modules are loaded lazily only when their command is executed.
|
|
8
|
+
* 2. FALLBACK (filesystem scan): Traditional runtime discovery for development.
|
|
3
9
|
*/
|
|
4
10
|
import * as fs from 'node:fs';
|
|
5
11
|
import * as path from 'node:path';
|
|
6
12
|
import yaml from 'js-yaml';
|
|
7
13
|
import { Strategy, registerCommand } from './registry.js';
|
|
8
14
|
import { executePipeline } from './pipeline.js';
|
|
15
|
+
/** Set of TS module paths that have been loaded */
|
|
16
|
+
const _loadedModules = new Set();
|
|
17
|
+
/**
|
|
18
|
+
* Discover and register CLI commands.
|
|
19
|
+
* Uses pre-compiled manifest when available for instant startup.
|
|
20
|
+
*/
|
|
9
21
|
export async function discoverClis(...dirs) {
|
|
10
|
-
|
|
22
|
+
// Fast path: try manifest first (production / post-build)
|
|
11
23
|
for (const dir of dirs) {
|
|
12
|
-
|
|
24
|
+
const manifestPath = path.resolve(dir, '..', 'cli-manifest.json');
|
|
25
|
+
if (fs.existsSync(manifestPath)) {
|
|
26
|
+
loadFromManifest(manifestPath, dir);
|
|
27
|
+
continue; // Skip filesystem scan for this directory
|
|
28
|
+
}
|
|
29
|
+
// Fallback: runtime filesystem scan (development)
|
|
30
|
+
await discoverClisFromFs(dir);
|
|
31
|
+
}
|
|
32
|
+
}
|
|
33
|
+
/**
|
|
34
|
+
* Fast-path: register commands from pre-compiled manifest.
|
|
35
|
+
* YAML pipelines are inlined — zero YAML parsing at runtime.
|
|
36
|
+
* TS modules are deferred — loaded lazily on first execution.
|
|
37
|
+
*/
|
|
38
|
+
function loadFromManifest(manifestPath, clisDir) {
|
|
39
|
+
try {
|
|
40
|
+
const manifest = JSON.parse(fs.readFileSync(manifestPath, 'utf-8'));
|
|
41
|
+
for (const entry of manifest) {
|
|
42
|
+
if (entry.type === 'yaml') {
|
|
43
|
+
// YAML pipelines fully inlined in manifest — register directly
|
|
44
|
+
const strategy = Strategy[entry.strategy.toUpperCase()] ?? Strategy.COOKIE;
|
|
45
|
+
const cmd = {
|
|
46
|
+
site: entry.site,
|
|
47
|
+
name: entry.name,
|
|
48
|
+
description: entry.description ?? '',
|
|
49
|
+
domain: entry.domain,
|
|
50
|
+
strategy,
|
|
51
|
+
browser: entry.browser,
|
|
52
|
+
args: entry.args ?? [],
|
|
53
|
+
columns: entry.columns,
|
|
54
|
+
pipeline: entry.pipeline,
|
|
55
|
+
timeoutSeconds: entry.timeout,
|
|
56
|
+
source: `manifest:${entry.site}/${entry.name}`,
|
|
57
|
+
};
|
|
58
|
+
registerCommand(cmd);
|
|
59
|
+
}
|
|
60
|
+
else if (entry.type === 'ts' && entry.modulePath) {
|
|
61
|
+
// TS adapters: register a lightweight stub.
|
|
62
|
+
// The actual module is loaded lazily on first executeCommand().
|
|
63
|
+
const strategy = Strategy[(entry.strategy ?? 'cookie').toUpperCase()] ?? Strategy.COOKIE;
|
|
64
|
+
const modulePath = path.resolve(clisDir, entry.modulePath);
|
|
65
|
+
const cmd = {
|
|
66
|
+
site: entry.site,
|
|
67
|
+
name: entry.name,
|
|
68
|
+
description: entry.description ?? '',
|
|
69
|
+
domain: entry.domain,
|
|
70
|
+
strategy,
|
|
71
|
+
browser: entry.browser ?? true,
|
|
72
|
+
args: entry.args ?? [],
|
|
73
|
+
columns: entry.columns,
|
|
74
|
+
timeoutSeconds: entry.timeout,
|
|
75
|
+
source: modulePath,
|
|
76
|
+
// Mark as lazy — executeCommand will load the module before running
|
|
77
|
+
_lazy: true,
|
|
78
|
+
_modulePath: modulePath,
|
|
79
|
+
};
|
|
80
|
+
registerCommand(cmd);
|
|
81
|
+
}
|
|
82
|
+
}
|
|
83
|
+
}
|
|
84
|
+
catch (err) {
|
|
85
|
+
process.stderr.write(`Warning: failed to load manifest ${manifestPath}: ${err.message}\n`);
|
|
86
|
+
}
|
|
87
|
+
}
|
|
88
|
+
/**
|
|
89
|
+
* Fallback: traditional filesystem scan (used during development with tsx).
|
|
90
|
+
*/
|
|
91
|
+
async function discoverClisFromFs(dir) {
|
|
92
|
+
if (!fs.existsSync(dir))
|
|
93
|
+
return;
|
|
94
|
+
const promises = [];
|
|
95
|
+
for (const site of fs.readdirSync(dir)) {
|
|
96
|
+
const siteDir = path.join(dir, site);
|
|
97
|
+
if (!fs.statSync(siteDir).isDirectory())
|
|
13
98
|
continue;
|
|
14
|
-
for (const
|
|
15
|
-
const
|
|
16
|
-
if (
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
}
|
|
23
|
-
else if (file.endsWith('.js')) {
|
|
24
|
-
// Dynamic import of compiled adapter modules
|
|
25
|
-
promises.push(import(`file://${filePath}`).catch((err) => {
|
|
26
|
-
process.stderr.write(`Warning: failed to load module ${filePath}: ${err.message}\n`);
|
|
27
|
-
}));
|
|
28
|
-
}
|
|
99
|
+
for (const file of fs.readdirSync(siteDir)) {
|
|
100
|
+
const filePath = path.join(siteDir, file);
|
|
101
|
+
if (file.endsWith('.yaml') || file.endsWith('.yml')) {
|
|
102
|
+
registerYamlCli(filePath, site);
|
|
103
|
+
}
|
|
104
|
+
else if (file.endsWith('.js') && !file.endsWith('.d.js')) {
|
|
105
|
+
promises.push(import(`file://${filePath}`).catch((err) => {
|
|
106
|
+
process.stderr.write(`Warning: failed to load module ${filePath}: ${err.message}\n`);
|
|
107
|
+
}));
|
|
29
108
|
}
|
|
30
109
|
}
|
|
31
110
|
}
|
|
@@ -74,7 +153,33 @@ function registerYamlCli(filePath, defaultSite) {
|
|
|
74
153
|
process.stderr.write(`Warning: failed to load ${filePath}: ${err.message}\n`);
|
|
75
154
|
}
|
|
76
155
|
}
|
|
156
|
+
/**
|
|
157
|
+
* Execute a CLI command. Handles lazy-loading of TS modules.
|
|
158
|
+
*/
|
|
77
159
|
export async function executeCommand(cmd, page, kwargs, debug = false) {
|
|
160
|
+
// Lazy-load TS module on first execution
|
|
161
|
+
if (cmd._lazy && cmd._modulePath) {
|
|
162
|
+
const modulePath = cmd._modulePath;
|
|
163
|
+
if (!_loadedModules.has(modulePath)) {
|
|
164
|
+
try {
|
|
165
|
+
await import(`file://${modulePath}`);
|
|
166
|
+
_loadedModules.add(modulePath);
|
|
167
|
+
}
|
|
168
|
+
catch (err) {
|
|
169
|
+
throw new Error(`Failed to load adapter module ${modulePath}: ${err.message}`);
|
|
170
|
+
}
|
|
171
|
+
}
|
|
172
|
+
// After loading, the module's cli() call will have updated the registry
|
|
173
|
+
// with the real func/pipeline. Re-fetch the command.
|
|
174
|
+
const { getRegistry, fullName } = await import('./registry.js');
|
|
175
|
+
const updated = getRegistry().get(fullName(cmd));
|
|
176
|
+
if (updated && updated.func) {
|
|
177
|
+
return updated.func(page, kwargs, debug);
|
|
178
|
+
}
|
|
179
|
+
if (updated && updated.pipeline) {
|
|
180
|
+
return executePipeline(page, updated.pipeline, { args: kwargs, debug });
|
|
181
|
+
}
|
|
182
|
+
}
|
|
78
183
|
if (cmd.func) {
|
|
79
184
|
return cmd.func(page, kwargs, debug);
|
|
80
185
|
}
|
|
@@ -44,6 +44,42 @@ async function fetchSingle(page, url, method, queryParams, headers, args, data)
|
|
|
44
44
|
}
|
|
45
45
|
`);
|
|
46
46
|
}
|
|
47
|
+
/**
|
|
48
|
+
* Batch fetch: send all URLs into the browser as a single evaluate() call.
|
|
49
|
+
* This eliminates N-1 cross-process IPC round trips, performing all fetches
|
|
50
|
+
* inside the V8 engine and returning results as one JSON array.
|
|
51
|
+
*/
|
|
52
|
+
async function fetchBatchInBrowser(page, urls, method, headers, concurrency) {
|
|
53
|
+
const headersJs = JSON.stringify(headers);
|
|
54
|
+
const urlsJs = JSON.stringify(urls);
|
|
55
|
+
return page.evaluate(`
|
|
56
|
+
async () => {
|
|
57
|
+
const urls = ${urlsJs};
|
|
58
|
+
const method = "${method}";
|
|
59
|
+
const headers = ${headersJs};
|
|
60
|
+
const concurrency = ${concurrency};
|
|
61
|
+
|
|
62
|
+
const results = new Array(urls.length);
|
|
63
|
+
let idx = 0;
|
|
64
|
+
|
|
65
|
+
async function worker() {
|
|
66
|
+
while (idx < urls.length) {
|
|
67
|
+
const i = idx++;
|
|
68
|
+
try {
|
|
69
|
+
const resp = await fetch(urls[i], { method, headers, credentials: "include" });
|
|
70
|
+
results[i] = await resp.json();
|
|
71
|
+
} catch (e) {
|
|
72
|
+
results[i] = { error: e.message };
|
|
73
|
+
}
|
|
74
|
+
}
|
|
75
|
+
}
|
|
76
|
+
|
|
77
|
+
const workers = Array.from({ length: Math.min(concurrency, urls.length) }, () => worker());
|
|
78
|
+
await Promise.all(workers);
|
|
79
|
+
return results;
|
|
80
|
+
}
|
|
81
|
+
`);
|
|
82
|
+
}
|
|
47
83
|
export async function stepFetch(page, params, data, args) {
|
|
48
84
|
const urlOrObj = typeof params === 'string' ? params : (params?.url ?? '');
|
|
49
85
|
const method = params?.method ?? 'GET';
|
|
@@ -53,9 +89,29 @@ export async function stepFetch(page, params, data, args) {
|
|
|
53
89
|
// Per-item fetch when data is array and URL references item
|
|
54
90
|
if (Array.isArray(data) && urlTemplate.includes('item')) {
|
|
55
91
|
const concurrency = typeof params?.concurrency === 'number' ? params.concurrency : 5;
|
|
92
|
+
// Render all URLs upfront
|
|
93
|
+
const renderedHeaders = {};
|
|
94
|
+
for (const [k, v] of Object.entries(headers))
|
|
95
|
+
renderedHeaders[k] = String(render(v, { args, data }));
|
|
96
|
+
const renderedParams = {};
|
|
97
|
+
for (const [k, v] of Object.entries(queryParams))
|
|
98
|
+
renderedParams[k] = String(render(v, { args, data }));
|
|
99
|
+
const urls = data.map((item, index) => {
|
|
100
|
+
let url = String(render(urlTemplate, { args, data, item, index }));
|
|
101
|
+
if (Object.keys(renderedParams).length > 0) {
|
|
102
|
+
const qs = new URLSearchParams(renderedParams).toString();
|
|
103
|
+
url = `${url}${url.includes('?') ? '&' : '?'}${qs}`;
|
|
104
|
+
}
|
|
105
|
+
return url;
|
|
106
|
+
});
|
|
107
|
+
// BATCH IPC: if browser is available, batch all fetches into a single evaluate() call
|
|
108
|
+
if (page !== null) {
|
|
109
|
+
return fetchBatchInBrowser(page, urls, method.toUpperCase(), renderedHeaders, concurrency);
|
|
110
|
+
}
|
|
111
|
+
// Non-browser: use concurrent pool (already optimized)
|
|
56
112
|
return mapConcurrent(data, concurrency, async (item, index) => {
|
|
57
113
|
const itemUrl = String(render(urlTemplate, { args, data, item, index }));
|
|
58
|
-
return fetchSingle(
|
|
114
|
+
return fetchSingle(null, itemUrl, method, queryParams, headers, args, data);
|
|
59
115
|
});
|
|
60
116
|
}
|
|
61
117
|
const url = render(urlOrObj, { args, data });
|
package/dist/registry.d.ts
CHANGED
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@jackwener/opencli",
|
|
3
|
-
"version": "0.4.
|
|
3
|
+
"version": "0.4.2",
|
|
4
4
|
"publishConfig": {
|
|
5
5
|
"access": "public"
|
|
6
6
|
},
|
|
@@ -12,7 +12,8 @@
|
|
|
12
12
|
},
|
|
13
13
|
"scripts": {
|
|
14
14
|
"dev": "tsx src/main.ts",
|
|
15
|
-
"build": "tsc && npm run clean-yaml && npm run copy-yaml",
|
|
15
|
+
"build": "tsc && npm run clean-yaml && npm run copy-yaml && npm run build-manifest",
|
|
16
|
+
"build-manifest": "node dist/build-manifest.js || true",
|
|
16
17
|
"clean-yaml": "find dist/clis -name '*.yaml' -o -name '*.yml' 2>/dev/null | xargs rm -f",
|
|
17
18
|
"copy-yaml": "find src/clis -name '*.yaml' -o -name '*.yml' | while read f; do d=\"dist/${f#src/}\"; mkdir -p \"$(dirname \"$d\")\"; cp \"$f\" \"$d\"; done",
|
|
18
19
|
"start": "node dist/main.js",
|
|
@@ -0,0 +1,194 @@
|
|
|
1
|
+
#!/usr/bin/env node
|
|
2
|
+
/**
|
|
3
|
+
* Build-time CLI manifest compiler.
|
|
4
|
+
*
|
|
5
|
+
* Scans all YAML/TS CLI definitions and pre-compiles them into a single
|
|
6
|
+
* manifest.json for instant cold-start registration (no runtime YAML parsing).
|
|
7
|
+
*
|
|
8
|
+
* Usage: npx tsx src/build-manifest.ts
|
|
9
|
+
* Output: dist/cli-manifest.json
|
|
10
|
+
*/
|
|
11
|
+
|
|
12
|
+
import * as fs from 'node:fs';
|
|
13
|
+
import * as path from 'node:path';
|
|
14
|
+
import { fileURLToPath } from 'node:url';
|
|
15
|
+
import yaml from 'js-yaml';
|
|
16
|
+
|
|
17
|
+
const __dirname = path.dirname(fileURLToPath(import.meta.url));
|
|
18
|
+
const CLIS_DIR = path.resolve(__dirname, 'clis');
|
|
19
|
+
const OUTPUT = path.resolve(__dirname, '..', 'dist', 'cli-manifest.json');
|
|
20
|
+
|
|
21
|
+
interface ManifestEntry {
|
|
22
|
+
site: string;
|
|
23
|
+
name: string;
|
|
24
|
+
description: string;
|
|
25
|
+
domain?: string;
|
|
26
|
+
strategy: string;
|
|
27
|
+
browser: boolean;
|
|
28
|
+
args: Array<{
|
|
29
|
+
name: string;
|
|
30
|
+
type?: string;
|
|
31
|
+
default?: any;
|
|
32
|
+
required?: boolean;
|
|
33
|
+
help?: string;
|
|
34
|
+
choices?: string[];
|
|
35
|
+
}>;
|
|
36
|
+
columns?: string[];
|
|
37
|
+
pipeline?: any[];
|
|
38
|
+
timeout?: number;
|
|
39
|
+
/** 'yaml' or 'ts' — determines how executeCommand loads the handler */
|
|
40
|
+
type: 'yaml' | 'ts';
|
|
41
|
+
/** Relative path from clis/ dir, e.g. 'bilibili/hot.yaml' or 'bilibili/search.js' */
|
|
42
|
+
modulePath?: string;
|
|
43
|
+
}
|
|
44
|
+
|
|
45
|
+
function scanYaml(filePath: string, site: string): ManifestEntry | null {
|
|
46
|
+
try {
|
|
47
|
+
const raw = fs.readFileSync(filePath, 'utf-8');
|
|
48
|
+
const def = yaml.load(raw) as any;
|
|
49
|
+
if (!def || typeof def !== 'object') return null;
|
|
50
|
+
|
|
51
|
+
const strategyStr = def.strategy ?? (def.browser === false ? 'public' : 'cookie');
|
|
52
|
+
const strategy = strategyStr.toUpperCase();
|
|
53
|
+
const browser = def.browser ?? (strategy !== 'PUBLIC');
|
|
54
|
+
|
|
55
|
+
const args: ManifestEntry['args'] = [];
|
|
56
|
+
if (def.args && typeof def.args === 'object') {
|
|
57
|
+
for (const [argName, argDef] of Object.entries(def.args as Record<string, any>)) {
|
|
58
|
+
args.push({
|
|
59
|
+
name: argName,
|
|
60
|
+
type: argDef?.type ?? 'str',
|
|
61
|
+
default: argDef?.default,
|
|
62
|
+
required: argDef?.required ?? false,
|
|
63
|
+
help: argDef?.description ?? argDef?.help ?? '',
|
|
64
|
+
choices: argDef?.choices,
|
|
65
|
+
});
|
|
66
|
+
}
|
|
67
|
+
}
|
|
68
|
+
|
|
69
|
+
return {
|
|
70
|
+
site: def.site ?? site,
|
|
71
|
+
name: def.name ?? path.basename(filePath, path.extname(filePath)),
|
|
72
|
+
description: def.description ?? '',
|
|
73
|
+
domain: def.domain,
|
|
74
|
+
strategy: strategy.toLowerCase(),
|
|
75
|
+
browser,
|
|
76
|
+
args,
|
|
77
|
+
columns: def.columns,
|
|
78
|
+
pipeline: def.pipeline,
|
|
79
|
+
timeout: def.timeout,
|
|
80
|
+
type: 'yaml',
|
|
81
|
+
};
|
|
82
|
+
} catch (err: any) {
|
|
83
|
+
process.stderr.write(`Warning: failed to parse ${filePath}: ${err.message}\n`);
|
|
84
|
+
return null;
|
|
85
|
+
}
|
|
86
|
+
}
|
|
87
|
+
|
|
88
|
+
function scanTs(filePath: string, site: string): ManifestEntry {
|
|
89
|
+
// TS adapters self-register via cli() at import time.
|
|
90
|
+
// We statically parse the source to extract metadata for the manifest stub.
|
|
91
|
+
const baseName = path.basename(filePath, path.extname(filePath));
|
|
92
|
+
const relativePath = `${site}/${baseName}.js`;
|
|
93
|
+
|
|
94
|
+
const entry: ManifestEntry = {
|
|
95
|
+
site,
|
|
96
|
+
name: baseName,
|
|
97
|
+
description: '',
|
|
98
|
+
strategy: 'cookie',
|
|
99
|
+
browser: true,
|
|
100
|
+
args: [],
|
|
101
|
+
type: 'ts',
|
|
102
|
+
modulePath: relativePath,
|
|
103
|
+
};
|
|
104
|
+
|
|
105
|
+
try {
|
|
106
|
+
const src = fs.readFileSync(filePath, 'utf-8');
|
|
107
|
+
|
|
108
|
+
// Extract description
|
|
109
|
+
const descMatch = src.match(/description\s*:\s*['"`]([^'"`]*)['"`]/);
|
|
110
|
+
if (descMatch) entry.description = descMatch[1];
|
|
111
|
+
|
|
112
|
+
// Extract domain
|
|
113
|
+
const domainMatch = src.match(/domain\s*:\s*['"`]([^'"`]*)['"`]/);
|
|
114
|
+
if (domainMatch) entry.domain = domainMatch[1];
|
|
115
|
+
|
|
116
|
+
// Extract strategy
|
|
117
|
+
const stratMatch = src.match(/strategy\s*:\s*Strategy\.(\w+)/);
|
|
118
|
+
if (stratMatch) entry.strategy = stratMatch[1].toLowerCase();
|
|
119
|
+
|
|
120
|
+
// Extract columns
|
|
121
|
+
const colMatch = src.match(/columns\s*:\s*\[([^\]]*)\]/);
|
|
122
|
+
if (colMatch) {
|
|
123
|
+
entry.columns = colMatch[1].split(',').map(s => s.trim().replace(/^['"`]|['"`]$/g, '')).filter(Boolean);
|
|
124
|
+
}
|
|
125
|
+
|
|
126
|
+
// Extract args array items: { name: '...', ... }
|
|
127
|
+
const argsBlockMatch = src.match(/args\s*:\s*\[([\s\S]*?)\]\s*,/);
|
|
128
|
+
if (argsBlockMatch) {
|
|
129
|
+
const argsBlock = argsBlockMatch[1];
|
|
130
|
+
const argRegex = /\{\s*name\s*:\s*['"`](\w+)['"`]([^}]*)\}/g;
|
|
131
|
+
let m;
|
|
132
|
+
while ((m = argRegex.exec(argsBlock)) !== null) {
|
|
133
|
+
const argName = m[1];
|
|
134
|
+
const body = m[2];
|
|
135
|
+
const typeMatch = body.match(/type\s*:\s*['"`](\w+)['"`]/);
|
|
136
|
+
const defaultMatch = body.match(/default\s*:\s*([^,}]+)/);
|
|
137
|
+
const requiredMatch = body.match(/required\s*:\s*(true|false)/);
|
|
138
|
+
const helpMatch = body.match(/help\s*:\s*['"`]([^'"`]*)['"`]/);
|
|
139
|
+
|
|
140
|
+
let defaultVal: any = undefined;
|
|
141
|
+
if (defaultMatch) {
|
|
142
|
+
const raw = defaultMatch[1].trim();
|
|
143
|
+
if (raw === 'true') defaultVal = true;
|
|
144
|
+
else if (raw === 'false') defaultVal = false;
|
|
145
|
+
else if (/^\d+$/.test(raw)) defaultVal = parseInt(raw, 10);
|
|
146
|
+
else if (/^\d+\.\d+$/.test(raw)) defaultVal = parseFloat(raw);
|
|
147
|
+
else defaultVal = raw.replace(/^['"`]|['"`]$/g, '');
|
|
148
|
+
}
|
|
149
|
+
|
|
150
|
+
entry.args.push({
|
|
151
|
+
name: argName,
|
|
152
|
+
type: typeMatch?.[1] ?? 'str',
|
|
153
|
+
default: defaultVal,
|
|
154
|
+
required: requiredMatch?.[1] === 'true',
|
|
155
|
+
help: helpMatch?.[1] ?? '',
|
|
156
|
+
});
|
|
157
|
+
}
|
|
158
|
+
}
|
|
159
|
+
} catch {
|
|
160
|
+
// If parsing fails, fall back to empty metadata — module will self-register at runtime
|
|
161
|
+
}
|
|
162
|
+
|
|
163
|
+
return entry;
|
|
164
|
+
}
|
|
165
|
+
|
|
166
|
+
// Main
|
|
167
|
+
const manifest: ManifestEntry[] = [];
|
|
168
|
+
|
|
169
|
+
if (fs.existsSync(CLIS_DIR)) {
|
|
170
|
+
for (const site of fs.readdirSync(CLIS_DIR)) {
|
|
171
|
+
const siteDir = path.join(CLIS_DIR, site);
|
|
172
|
+
if (!fs.statSync(siteDir).isDirectory()) continue;
|
|
173
|
+
for (const file of fs.readdirSync(siteDir)) {
|
|
174
|
+
const filePath = path.join(siteDir, file);
|
|
175
|
+
if (file.endsWith('.yaml') || file.endsWith('.yml')) {
|
|
176
|
+
const entry = scanYaml(filePath, site);
|
|
177
|
+
if (entry) manifest.push(entry);
|
|
178
|
+
} else if (
|
|
179
|
+
(file.endsWith('.ts') && !file.endsWith('.d.ts') && file !== 'index.ts') ||
|
|
180
|
+
(file.endsWith('.js') && !file.endsWith('.d.js') && file !== 'index.js')
|
|
181
|
+
) {
|
|
182
|
+
manifest.push(scanTs(filePath, site));
|
|
183
|
+
}
|
|
184
|
+
}
|
|
185
|
+
}
|
|
186
|
+
}
|
|
187
|
+
|
|
188
|
+
// Ensure output directory exists
|
|
189
|
+
fs.mkdirSync(path.dirname(OUTPUT), { recursive: true });
|
|
190
|
+
fs.writeFileSync(OUTPUT, JSON.stringify(manifest, null, 2));
|
|
191
|
+
|
|
192
|
+
const yamlCount = manifest.filter(e => e.type === 'yaml').length;
|
|
193
|
+
const tsCount = manifest.filter(e => e.type === 'ts').length;
|
|
194
|
+
console.log(`✅ Manifest compiled: ${manifest.length} entries (${yamlCount} YAML, ${tsCount} TS) → ${OUTPUT}`);
|