@jackwener/opencli 0.2.0 → 0.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CLI-CREATOR.md +51 -72
- package/README.md +8 -5
- package/README.zh-CN.md +8 -5
- package/SKILL.md +27 -14
- package/dist/browser.d.ts +6 -0
- package/dist/browser.js +65 -1
- package/dist/clis/bilibili/dynamic.d.ts +1 -0
- package/dist/clis/bilibili/dynamic.js +33 -0
- package/dist/clis/bilibili/ranking.d.ts +1 -0
- package/dist/clis/bilibili/ranking.js +24 -0
- package/dist/clis/reddit/frontpage.yaml +30 -0
- package/dist/clis/reddit/hot.yaml +3 -2
- package/dist/clis/reddit/search.yaml +34 -0
- package/dist/clis/reddit/subreddit.yaml +39 -0
- package/dist/clis/twitter/bookmarks.yaml +85 -0
- package/dist/clis/twitter/profile.d.ts +1 -0
- package/dist/clis/twitter/profile.js +56 -0
- package/dist/clis/twitter/search.d.ts +1 -0
- package/dist/clis/twitter/search.js +60 -0
- package/dist/clis/twitter/timeline.d.ts +1 -0
- package/dist/clis/twitter/timeline.js +47 -0
- package/dist/clis/xiaohongshu/user.d.ts +1 -0
- package/dist/clis/xiaohongshu/user.js +40 -0
- package/dist/clis/xueqiu/feed.yaml +53 -0
- package/dist/clis/xueqiu/hot-stock.yaml +49 -0
- package/dist/clis/xueqiu/hot.yaml +46 -0
- package/dist/clis/xueqiu/search.yaml +53 -0
- package/dist/clis/xueqiu/stock.yaml +67 -0
- package/dist/clis/xueqiu/watchlist.yaml +46 -0
- package/dist/clis/zhihu/hot.yaml +6 -2
- package/dist/clis/zhihu/search.yaml +3 -1
- package/dist/engine.d.ts +1 -1
- package/dist/engine.js +9 -1
- package/dist/main.d.ts +1 -1
- package/dist/main.js +10 -3
- package/dist/pipeline/steps/intercept.js +56 -29
- package/dist/pipeline/template.js +3 -1
- package/dist/pipeline/template.test.js +6 -0
- package/dist/types.d.ts +6 -0
- package/package.json +1 -1
- package/src/browser.ts +72 -4
- package/src/clis/bilibili/dynamic.ts +34 -0
- package/src/clis/bilibili/ranking.ts +25 -0
- package/src/clis/reddit/frontpage.yaml +30 -0
- package/src/clis/reddit/hot.yaml +3 -2
- package/src/clis/reddit/search.yaml +34 -0
- package/src/clis/reddit/subreddit.yaml +39 -0
- package/src/clis/twitter/bookmarks.yaml +85 -0
- package/src/clis/twitter/profile.ts +61 -0
- package/src/clis/twitter/search.ts +65 -0
- package/src/clis/twitter/timeline.ts +50 -0
- package/src/clis/xiaohongshu/user.ts +45 -0
- package/src/clis/xueqiu/feed.yaml +53 -0
- package/src/clis/xueqiu/hot-stock.yaml +49 -0
- package/src/clis/xueqiu/hot.yaml +46 -0
- package/src/clis/xueqiu/search.yaml +53 -0
- package/src/clis/xueqiu/stock.yaml +67 -0
- package/src/clis/xueqiu/watchlist.yaml +46 -0
- package/src/clis/zhihu/hot.yaml +6 -2
- package/src/clis/zhihu/search.yaml +3 -1
- package/src/engine.ts +10 -1
- package/src/main.ts +9 -3
- package/src/pipeline/steps/intercept.ts +58 -28
- package/src/pipeline/template.test.ts +6 -0
- package/src/pipeline/template.ts +3 -1
- package/src/types.ts +3 -0
- package/dist/clis/index.d.ts +0 -22
- package/dist/clis/index.js +0 -34
- package/src/clis/index.ts +0 -46
package/dist/engine.js
CHANGED
|
@@ -6,7 +6,8 @@ import * as path from 'node:path';
|
|
|
6
6
|
import yaml from 'js-yaml';
|
|
7
7
|
import { Strategy, registerCommand } from './registry.js';
|
|
8
8
|
import { executePipeline } from './pipeline.js';
|
|
9
|
-
export function discoverClis(...dirs) {
|
|
9
|
+
export async function discoverClis(...dirs) {
|
|
10
|
+
const promises = [];
|
|
10
11
|
for (const dir of dirs) {
|
|
11
12
|
if (!fs.existsSync(dir))
|
|
12
13
|
continue;
|
|
@@ -19,9 +20,16 @@ export function discoverClis(...dirs) {
|
|
|
19
20
|
if (file.endsWith('.yaml') || file.endsWith('.yml')) {
|
|
20
21
|
registerYamlCli(filePath, site);
|
|
21
22
|
}
|
|
23
|
+
else if (file.endsWith('.js')) {
|
|
24
|
+
// Dynamic import of compiled adapter modules
|
|
25
|
+
promises.push(import(`file://${filePath}`).catch((err) => {
|
|
26
|
+
process.stderr.write(`Warning: failed to load module ${filePath}: ${err.message}\n`);
|
|
27
|
+
}));
|
|
28
|
+
}
|
|
22
29
|
}
|
|
23
30
|
}
|
|
24
31
|
}
|
|
32
|
+
await Promise.all(promises);
|
|
25
33
|
}
|
|
26
34
|
function registerYamlCli(filePath, defaultSite) {
|
|
27
35
|
try {
|
package/dist/main.d.ts
CHANGED
package/dist/main.js
CHANGED
|
@@ -11,7 +11,6 @@ import chalk from 'chalk';
|
|
|
11
11
|
import { discoverClis, executeCommand } from './engine.js';
|
|
12
12
|
import { fullName, getRegistry, strategyLabel } from './registry.js';
|
|
13
13
|
import { render as renderOutput } from './output.js';
|
|
14
|
-
import './clis/index.js';
|
|
15
14
|
import { PlaywrightMCP } from './browser.js';
|
|
16
15
|
import { browserSession, DEFAULT_BROWSER_COMMAND_TIMEOUT, runWithTimeout } from './runtime.js';
|
|
17
16
|
const __filename = fileURLToPath(import.meta.url);
|
|
@@ -21,7 +20,7 @@ const USER_CLIS = path.join(os.homedir(), '.opencli', 'clis');
|
|
|
21
20
|
// Read version from package.json (single source of truth)
|
|
22
21
|
const pkgJsonPath = path.resolve(__dirname, '..', 'package.json');
|
|
23
22
|
const PKG_VERSION = JSON.parse(fs.readFileSync(pkgJsonPath, 'utf-8')).version ?? '0.0.0';
|
|
24
|
-
discoverClis(BUILTIN_CLIS, USER_CLIS);
|
|
23
|
+
await discoverClis(BUILTIN_CLIS, USER_CLIS);
|
|
25
24
|
const program = new Command();
|
|
26
25
|
program.name('opencli').description('Make any website your CLI. Zero setup. AI-powered.').version(PKG_VERSION);
|
|
27
26
|
// ── Built-in commands ──────────────────────────────────────────────────────
|
|
@@ -116,10 +115,18 @@ for (const [, cmd] of registry) {
|
|
|
116
115
|
else {
|
|
117
116
|
result = await executeCommand(cmd, null, kwargs, actionOpts.verbose);
|
|
118
117
|
}
|
|
118
|
+
if (actionOpts.verbose && (!result || (Array.isArray(result) && result.length === 0))) {
|
|
119
|
+
console.error(chalk.yellow(`[Verbose] Warning: Command returned an empty result. If the website structural API changed or requires authentication, check the network or update the adapter.`));
|
|
120
|
+
}
|
|
119
121
|
renderOutput(result, { fmt: actionOpts.format, columns: cmd.columns, title: `${cmd.site}/${cmd.name}`, elapsed: (Date.now() - startTime) / 1000, source: fullName(cmd) });
|
|
120
122
|
}
|
|
121
123
|
catch (err) {
|
|
122
|
-
|
|
124
|
+
if (actionOpts.verbose && err.stack) {
|
|
125
|
+
console.error(chalk.red(err.stack));
|
|
126
|
+
}
|
|
127
|
+
else {
|
|
128
|
+
console.error(chalk.red(`Error: ${err.message ?? err}`));
|
|
129
|
+
}
|
|
123
130
|
process.exitCode = 1;
|
|
124
131
|
}
|
|
125
132
|
});
|
|
@@ -10,7 +10,54 @@ export async function stepIntercept(page, params, data, args) {
|
|
|
10
10
|
const selectPath = cfg.select ?? null;
|
|
11
11
|
if (!capturePattern)
|
|
12
12
|
return data;
|
|
13
|
-
// Step 1:
|
|
13
|
+
// Step 1: Inject fetch/XHR interceptor BEFORE trigger
|
|
14
|
+
await page.evaluate(`
|
|
15
|
+
() => {
|
|
16
|
+
window.__opencli_intercepted = window.__opencli_intercepted || [];
|
|
17
|
+
const pattern = ${JSON.stringify(capturePattern)};
|
|
18
|
+
|
|
19
|
+
if (!window.__opencli_fetch_patched) {
|
|
20
|
+
const origFetch = window.fetch;
|
|
21
|
+
window.fetch = async function(...args) {
|
|
22
|
+
const reqUrl = typeof args[0] === 'string' ? args[0] : (args[0] && args[0].url) || '';
|
|
23
|
+
const response = await origFetch.apply(this, args);
|
|
24
|
+
setTimeout(async () => {
|
|
25
|
+
try {
|
|
26
|
+
if (reqUrl.includes(pattern)) {
|
|
27
|
+
const clone = response.clone();
|
|
28
|
+
const json = await clone.json();
|
|
29
|
+
window.__opencli_intercepted.push(json);
|
|
30
|
+
}
|
|
31
|
+
} catch(e) {}
|
|
32
|
+
}, 0);
|
|
33
|
+
return response;
|
|
34
|
+
};
|
|
35
|
+
window.__opencli_fetch_patched = true;
|
|
36
|
+
}
|
|
37
|
+
|
|
38
|
+
if (!window.__opencli_xhr_patched) {
|
|
39
|
+
const XHR = XMLHttpRequest.prototype;
|
|
40
|
+
const open = XHR.open;
|
|
41
|
+
const send = XHR.send;
|
|
42
|
+
XHR.open = function(method, url, ...args) {
|
|
43
|
+
this._reqUrl = url;
|
|
44
|
+
return open.call(this, method, url, ...args);
|
|
45
|
+
};
|
|
46
|
+
XHR.send = function(...args) {
|
|
47
|
+
this.addEventListener('load', function() {
|
|
48
|
+
try {
|
|
49
|
+
if (this._reqUrl && this._reqUrl.includes(pattern)) {
|
|
50
|
+
window.__opencli_intercepted.push(JSON.parse(this.responseText));
|
|
51
|
+
}
|
|
52
|
+
} catch(e) {}
|
|
53
|
+
});
|
|
54
|
+
return send.apply(this, args);
|
|
55
|
+
};
|
|
56
|
+
window.__opencli_xhr_patched = true;
|
|
57
|
+
}
|
|
58
|
+
}
|
|
59
|
+
`);
|
|
60
|
+
// Step 2: Execute the trigger action
|
|
14
61
|
if (trigger.startsWith('navigate:')) {
|
|
15
62
|
const url = render(trigger.slice('navigate:'.length), { args, data });
|
|
16
63
|
await page.goto(String(url));
|
|
@@ -27,36 +74,16 @@ export async function stepIntercept(page, params, data, args) {
|
|
|
27
74
|
else if (trigger === 'scroll') {
|
|
28
75
|
await page.scroll('down');
|
|
29
76
|
}
|
|
30
|
-
// Step
|
|
77
|
+
// Step 3: Wait a bit for network requests to fire
|
|
31
78
|
await page.wait(Math.min(timeout, 3));
|
|
32
|
-
// Step
|
|
33
|
-
const
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
const match = line.match(/\[?(GET|POST)\]?\s+(\S+)\s*(?:=>|→)\s*\[?(\d+)\]?/i);
|
|
39
|
-
if (match) {
|
|
40
|
-
const [, , url, status] = match;
|
|
41
|
-
if (url.includes(capturePattern) && status === '200') {
|
|
42
|
-
try {
|
|
43
|
-
const body = await page.evaluate(`
|
|
44
|
-
async () => {
|
|
45
|
-
try {
|
|
46
|
-
const resp = await fetch(${JSON.stringify(url)}, { credentials: 'include' });
|
|
47
|
-
if (!resp.ok) return null;
|
|
48
|
-
return await resp.json();
|
|
49
|
-
} catch { return null; }
|
|
50
|
-
}
|
|
51
|
-
`);
|
|
52
|
-
if (body)
|
|
53
|
-
matchingResponses.push(body);
|
|
54
|
-
}
|
|
55
|
-
catch { }
|
|
56
|
-
}
|
|
57
|
-
}
|
|
58
|
-
}
|
|
79
|
+
// Step 4: Retrieve captured data
|
|
80
|
+
const matchingResponses = await page.evaluate(`
|
|
81
|
+
() => {
|
|
82
|
+
const data = window.__opencli_intercepted || [];
|
|
83
|
+
window.__opencli_intercepted = []; // clear after reading
|
|
84
|
+
return data;
|
|
59
85
|
}
|
|
86
|
+
`);
|
|
60
87
|
// Step 4: Select from response if specified
|
|
61
88
|
let result = matchingResponses.length === 1 ? matchingResponses[0] :
|
|
62
89
|
matchingResponses.length > 1 ? matchingResponses : data;
|
|
@@ -68,7 +68,7 @@ export function evalExpr(expr, ctx) {
|
|
|
68
68
|
* Apply a named filter to a value.
|
|
69
69
|
* Supported filters:
|
|
70
70
|
* default(val), join(sep), upper, lower, truncate(n), trim,
|
|
71
|
-
* replace(old,new), keys, length, first, last
|
|
71
|
+
* replace(old,new), keys, length, first, last, json
|
|
72
72
|
*/
|
|
73
73
|
function applyFilter(filterExpr, value) {
|
|
74
74
|
const match = filterExpr.match(/^(\w+)(?:\((.+)\))?$/);
|
|
@@ -112,6 +112,8 @@ function applyFilter(filterExpr, value) {
|
|
|
112
112
|
return Array.isArray(value) ? value[0] : value;
|
|
113
113
|
case 'last':
|
|
114
114
|
return Array.isArray(value) ? value[value.length - 1] : value;
|
|
115
|
+
case 'json':
|
|
116
|
+
return JSON.stringify(value ?? null);
|
|
115
117
|
default:
|
|
116
118
|
return value;
|
|
117
119
|
}
|
|
@@ -72,6 +72,12 @@ describe('evalExpr', () => {
|
|
|
72
72
|
it('applies length filter', () => {
|
|
73
73
|
expect(evalExpr('item.items | length', { item: { items: [1, 2, 3] } })).toBe(3);
|
|
74
74
|
});
|
|
75
|
+
it('applies json filter to strings with quotes', () => {
|
|
76
|
+
expect(evalExpr('args.keyword | json', { args: { keyword: "O'Reilly" } })).toBe('"O\'Reilly"');
|
|
77
|
+
});
|
|
78
|
+
it('applies json filter to nullish values', () => {
|
|
79
|
+
expect(evalExpr('args.keyword | json', { args: {} })).toBe('null');
|
|
80
|
+
});
|
|
75
81
|
});
|
|
76
82
|
describe('render', () => {
|
|
77
83
|
it('renders full expression', () => {
|
package/dist/types.d.ts
CHANGED
|
@@ -24,4 +24,10 @@ export interface IPage {
|
|
|
24
24
|
networkRequests(includeStatic?: boolean): Promise<any>;
|
|
25
25
|
consoleMessages(level?: string): Promise<any>;
|
|
26
26
|
scroll(direction?: string, amount?: number): Promise<void>;
|
|
27
|
+
autoScroll(options?: {
|
|
28
|
+
times?: number;
|
|
29
|
+
delayMs?: number;
|
|
30
|
+
}): Promise<void>;
|
|
31
|
+
installInterceptor(pattern: string): Promise<void>;
|
|
32
|
+
getInterceptedRequests(): Promise<any[]>;
|
|
27
33
|
}
|
package/package.json
CHANGED
package/src/browser.ts
CHANGED
|
@@ -135,6 +135,69 @@ export class Page implements IPage {
|
|
|
135
135
|
async scroll(direction: string = 'down', amount: number = 500): Promise<void> {
|
|
136
136
|
await this.call('tools/call', { name: 'browser_press_key', arguments: { key: direction === 'down' ? 'PageDown' : 'PageUp' } });
|
|
137
137
|
}
|
|
138
|
+
|
|
139
|
+
async autoScroll(options: { times?: number; delayMs?: number } = {}): Promise<void> {
|
|
140
|
+
const times = options.times ?? 3;
|
|
141
|
+
const delayMs = options.delayMs ?? 2000;
|
|
142
|
+
for (let i = 0; i < times; i++) {
|
|
143
|
+
await this.evaluate('() => window.scrollTo(0, document.body.scrollHeight)');
|
|
144
|
+
await this.wait(delayMs / 1000);
|
|
145
|
+
}
|
|
146
|
+
}
|
|
147
|
+
|
|
148
|
+
async installInterceptor(pattern: string): Promise<void> {
|
|
149
|
+
const js = `
|
|
150
|
+
() => {
|
|
151
|
+
window.__opencli_xhr = window.__opencli_xhr || [];
|
|
152
|
+
window.__opencli_patterns = window.__opencli_patterns || [];
|
|
153
|
+
if (!window.__opencli_patterns.includes('${pattern}')) {
|
|
154
|
+
window.__opencli_patterns.push('${pattern}');
|
|
155
|
+
}
|
|
156
|
+
|
|
157
|
+
if (!window.__patched_xhr) {
|
|
158
|
+
const checkMatch = (url) => window.__opencli_patterns.some(p => url.includes(p));
|
|
159
|
+
|
|
160
|
+
const XHR = XMLHttpRequest.prototype;
|
|
161
|
+
const open = XHR.open;
|
|
162
|
+
const send = XHR.send;
|
|
163
|
+
XHR.open = function(method, url) {
|
|
164
|
+
this._url = url;
|
|
165
|
+
return open.call(this, method, url, ...Array.prototype.slice.call(arguments, 2));
|
|
166
|
+
};
|
|
167
|
+
XHR.send = function() {
|
|
168
|
+
this.addEventListener('load', function() {
|
|
169
|
+
if (checkMatch(this._url)) {
|
|
170
|
+
try { window.__opencli_xhr.push({url: this._url, data: JSON.parse(this.responseText)}); } catch(e){}
|
|
171
|
+
}
|
|
172
|
+
});
|
|
173
|
+
return send.apply(this, arguments);
|
|
174
|
+
};
|
|
175
|
+
|
|
176
|
+
const origFetch = window.fetch;
|
|
177
|
+
window.fetch = async function(...args) {
|
|
178
|
+
let u = typeof args[0] === 'string' ? args[0] : (args[0] && args[0].url) || '';
|
|
179
|
+
const res = await origFetch.apply(this, args);
|
|
180
|
+
setTimeout(async () => {
|
|
181
|
+
try {
|
|
182
|
+
if (checkMatch(u)) {
|
|
183
|
+
const clone = res.clone();
|
|
184
|
+
const j = await clone.json();
|
|
185
|
+
window.__opencli_xhr.push({url: u, data: j});
|
|
186
|
+
}
|
|
187
|
+
} catch(e) {}
|
|
188
|
+
}, 0);
|
|
189
|
+
return res;
|
|
190
|
+
};
|
|
191
|
+
window.__patched_xhr = true;
|
|
192
|
+
}
|
|
193
|
+
}
|
|
194
|
+
`;
|
|
195
|
+
await this.evaluate(js);
|
|
196
|
+
}
|
|
197
|
+
|
|
198
|
+
async getInterceptedRequests(): Promise<any[]> {
|
|
199
|
+
return (await this.evaluate('() => window.__opencli_xhr')) || [];
|
|
200
|
+
}
|
|
138
201
|
}
|
|
139
202
|
|
|
140
203
|
/**
|
|
@@ -158,10 +221,15 @@ export class PlaywrightMCP {
|
|
|
158
221
|
return new Promise<Page>((resolve, reject) => {
|
|
159
222
|
const timer = setTimeout(() => reject(new Error(`Timed out connecting to browser (${timeout}s)`)), timeout * 1000);
|
|
160
223
|
|
|
161
|
-
|
|
162
|
-
|
|
163
|
-
|
|
164
|
-
|
|
224
|
+
const mcpArgs = [mcpPath, '--extension'];
|
|
225
|
+
if (process.env.OPENCLI_BROWSER_EXECUTABLE_PATH) {
|
|
226
|
+
mcpArgs.push('--executablePath', process.env.OPENCLI_BROWSER_EXECUTABLE_PATH);
|
|
227
|
+
}
|
|
228
|
+
|
|
229
|
+
this._proc = spawn('node', mcpArgs, {
|
|
230
|
+
stdio: ['pipe', 'pipe', 'pipe'],
|
|
231
|
+
env: { ...process.env, ...(process.env.PLAYWRIGHT_MCP_EXTENSION_TOKEN ? { PLAYWRIGHT_MCP_EXTENSION_TOKEN: process.env.PLAYWRIGHT_MCP_EXTENSION_TOKEN } : {}) },
|
|
232
|
+
});
|
|
165
233
|
|
|
166
234
|
// Increase max listeners to avoid warnings
|
|
167
235
|
this._proc.setMaxListeners(20);
|
|
@@ -0,0 +1,34 @@
|
|
|
1
|
+
import { cli, Strategy } from '../../registry.js';
|
|
2
|
+
import { apiGet } from '../../bilibili.js';
|
|
3
|
+
|
|
4
|
+
cli({
|
|
5
|
+
site: 'bilibili',
|
|
6
|
+
name: 'dynamic',
|
|
7
|
+
description: 'Get Bilibili user dynamic feed',
|
|
8
|
+
domain: 'www.bilibili.com',
|
|
9
|
+
strategy: Strategy.COOKIE,
|
|
10
|
+
args: [
|
|
11
|
+
{ name: 'limit', type: 'int', default: 15 },
|
|
12
|
+
],
|
|
13
|
+
columns: ['id', 'author', 'text', 'likes', 'url'],
|
|
14
|
+
func: async (page, kwargs) => {
|
|
15
|
+
const payload = await apiGet(page, '/x/polymer/web-dynamic/v1/feed/all', { params: {}, signed: false });
|
|
16
|
+
const results: any[] = payload?.data?.items ?? [];
|
|
17
|
+
return results.slice(0, Number(kwargs.limit)).map((item: any) => {
|
|
18
|
+
let text = '';
|
|
19
|
+
if (item.modules?.module_dynamic?.desc?.text) {
|
|
20
|
+
text = item.modules.module_dynamic.desc.text;
|
|
21
|
+
} else if (item.modules?.module_dynamic?.major?.archive?.title) {
|
|
22
|
+
text = item.modules.module_dynamic.major.archive.title;
|
|
23
|
+
}
|
|
24
|
+
|
|
25
|
+
return {
|
|
26
|
+
id: item.id_str ?? '',
|
|
27
|
+
author: item.modules?.module_author?.name ?? '',
|
|
28
|
+
text: text,
|
|
29
|
+
likes: item.modules?.module_stat?.like?.count ?? 0,
|
|
30
|
+
url: item.id_str ? `https://t.bilibili.com/${item.id_str}` : ''
|
|
31
|
+
};
|
|
32
|
+
});
|
|
33
|
+
},
|
|
34
|
+
});
|
|
@@ -0,0 +1,25 @@
|
|
|
1
|
+
import { cli, Strategy } from '../../registry.js';
|
|
2
|
+
import { apiGet } from '../../bilibili.js';
|
|
3
|
+
|
|
4
|
+
cli({
|
|
5
|
+
site: 'bilibili',
|
|
6
|
+
name: 'ranking',
|
|
7
|
+
description: 'Get Bilibili video ranking board',
|
|
8
|
+
domain: 'www.bilibili.com',
|
|
9
|
+
strategy: Strategy.COOKIE,
|
|
10
|
+
args: [
|
|
11
|
+
{ name: 'limit', type: 'int', default: 20 },
|
|
12
|
+
],
|
|
13
|
+
columns: ['rank', 'title', 'author', 'score', 'url'],
|
|
14
|
+
func: async (page, kwargs) => {
|
|
15
|
+
const payload = await apiGet(page, '/x/web-interface/ranking/v2', { params: { rid: 0, type: 'all' }, signed: false });
|
|
16
|
+
const results: any[] = payload?.data?.list ?? [];
|
|
17
|
+
return results.slice(0, Number(kwargs.limit)).map((item: any, i: number) => ({
|
|
18
|
+
rank: i + 1,
|
|
19
|
+
title: item.title ?? '',
|
|
20
|
+
author: item.owner?.name ?? '',
|
|
21
|
+
score: item.stat?.view ?? 0,
|
|
22
|
+
url: item.bvid ? `https://www.bilibili.com/video/${item.bvid}` : ''
|
|
23
|
+
}));
|
|
24
|
+
},
|
|
25
|
+
});
|
|
@@ -0,0 +1,30 @@
|
|
|
1
|
+
site: reddit
|
|
2
|
+
name: frontpage
|
|
3
|
+
description: Reddit Frontpage / r/all
|
|
4
|
+
domain: reddit.com
|
|
5
|
+
strategy: cookie
|
|
6
|
+
browser: true
|
|
7
|
+
|
|
8
|
+
args:
|
|
9
|
+
limit:
|
|
10
|
+
type: int
|
|
11
|
+
default: 15
|
|
12
|
+
|
|
13
|
+
columns: [title, subreddit, author, upvotes, comments, url]
|
|
14
|
+
|
|
15
|
+
pipeline:
|
|
16
|
+
- navigate: https://www.reddit.com
|
|
17
|
+
- evaluate: |
|
|
18
|
+
(async () => {
|
|
19
|
+
const res = await fetch('/r/all.json?limit=${{ args.limit }}', { credentials: 'include' });
|
|
20
|
+
const j = await res.json();
|
|
21
|
+
return j?.data?.children || [];
|
|
22
|
+
})()
|
|
23
|
+
- map:
|
|
24
|
+
title: ${{ item.data.title }}
|
|
25
|
+
subreddit: ${{ item.data.subreddit_name_prefixed }}
|
|
26
|
+
author: ${{ item.data.author }}
|
|
27
|
+
upvotes: ${{ item.data.score }}
|
|
28
|
+
comments: ${{ item.data.num_comments }}
|
|
29
|
+
url: https://www.reddit.com${{ item.data.permalink }}
|
|
30
|
+
- limit: ${{ args.limit }}
|
package/src/clis/reddit/hot.yaml
CHANGED
|
@@ -18,9 +18,10 @@ pipeline:
|
|
|
18
18
|
|
|
19
19
|
- evaluate: |
|
|
20
20
|
(async () => {
|
|
21
|
-
const sub =
|
|
21
|
+
const sub = ${{ args.subreddit | json }};
|
|
22
22
|
const path = sub ? '/r/' + sub + '/hot.json' : '/hot.json';
|
|
23
|
-
const
|
|
23
|
+
const limit = ${{ args.limit }};
|
|
24
|
+
const res = await fetch(path + '?limit=' + limit + '&raw_json=1', {
|
|
24
25
|
credentials: 'include'
|
|
25
26
|
});
|
|
26
27
|
const d = await res.json();
|
|
@@ -0,0 +1,34 @@
|
|
|
1
|
+
site: reddit
|
|
2
|
+
name: search
|
|
3
|
+
description: Search Reddit Posts
|
|
4
|
+
domain: reddit.com
|
|
5
|
+
strategy: cookie
|
|
6
|
+
browser: true
|
|
7
|
+
|
|
8
|
+
args:
|
|
9
|
+
query:
|
|
10
|
+
type: string
|
|
11
|
+
required: true
|
|
12
|
+
limit:
|
|
13
|
+
type: int
|
|
14
|
+
default: 15
|
|
15
|
+
|
|
16
|
+
columns: [title, subreddit, author, upvotes, comments, url]
|
|
17
|
+
|
|
18
|
+
pipeline:
|
|
19
|
+
- navigate: https://www.reddit.com
|
|
20
|
+
- evaluate: |
|
|
21
|
+
(async () => {
|
|
22
|
+
const q = encodeURIComponent('${{ args.query }}');
|
|
23
|
+
const res = await fetch('/search.json?q=' + q + '&limit=${{ args.limit }}', { credentials: 'include' });
|
|
24
|
+
const j = await res.json();
|
|
25
|
+
return j?.data?.children || [];
|
|
26
|
+
})()
|
|
27
|
+
- map:
|
|
28
|
+
title: ${{ item.data.title }}
|
|
29
|
+
subreddit: ${{ item.data.subreddit_name_prefixed }}
|
|
30
|
+
author: ${{ item.data.author }}
|
|
31
|
+
upvotes: ${{ item.data.score }}
|
|
32
|
+
comments: ${{ item.data.num_comments }}
|
|
33
|
+
url: https://www.reddit.com${{ item.data.permalink }}
|
|
34
|
+
- limit: ${{ args.limit }}
|
|
@@ -0,0 +1,39 @@
|
|
|
1
|
+
site: reddit
|
|
2
|
+
name: subreddit
|
|
3
|
+
description: Get posts from a specific Subreddit
|
|
4
|
+
domain: reddit.com
|
|
5
|
+
strategy: cookie
|
|
6
|
+
browser: true
|
|
7
|
+
|
|
8
|
+
args:
|
|
9
|
+
name:
|
|
10
|
+
type: string
|
|
11
|
+
required: true
|
|
12
|
+
sort:
|
|
13
|
+
type: string
|
|
14
|
+
default: hot
|
|
15
|
+
description: "Sorting method: hot, new, top, rising"
|
|
16
|
+
limit:
|
|
17
|
+
type: int
|
|
18
|
+
default: 15
|
|
19
|
+
|
|
20
|
+
columns: [title, author, upvotes, comments, url]
|
|
21
|
+
|
|
22
|
+
pipeline:
|
|
23
|
+
- navigate: https://www.reddit.com
|
|
24
|
+
- evaluate: |
|
|
25
|
+
(async () => {
|
|
26
|
+
let sub = '${{ args.name }}';
|
|
27
|
+
if (sub.startsWith('r/')) sub = sub.slice(2);
|
|
28
|
+
const sort = '${{ args.sort }}';
|
|
29
|
+
const res = await fetch('/r/' + sub + '/' + sort + '.json?limit=${{ args.limit }}', { credentials: 'include' });
|
|
30
|
+
const j = await res.json();
|
|
31
|
+
return j?.data?.children || [];
|
|
32
|
+
})()
|
|
33
|
+
- map:
|
|
34
|
+
title: ${{ item.data.title }}
|
|
35
|
+
author: ${{ item.data.author }}
|
|
36
|
+
upvotes: ${{ item.data.score }}
|
|
37
|
+
comments: ${{ item.data.num_comments }}
|
|
38
|
+
url: https://www.reddit.com${{ item.data.permalink }}
|
|
39
|
+
- limit: ${{ args.limit }}
|
|
@@ -0,0 +1,85 @@
|
|
|
1
|
+
site: twitter
|
|
2
|
+
name: bookmarks
|
|
3
|
+
description: 获取 Twitter 书签列表
|
|
4
|
+
domain: x.com
|
|
5
|
+
browser: true
|
|
6
|
+
|
|
7
|
+
args:
|
|
8
|
+
limit:
|
|
9
|
+
type: int
|
|
10
|
+
default: 20
|
|
11
|
+
description: Number of bookmarks to return (default 20)
|
|
12
|
+
|
|
13
|
+
pipeline:
|
|
14
|
+
- navigate: https://x.com/i/bookmarks
|
|
15
|
+
- wait: 2
|
|
16
|
+
- evaluate: |
|
|
17
|
+
(async () => {
|
|
18
|
+
const ct0 = document.cookie.split(';').map(c=>c.trim()).find(c=>c.startsWith('ct0='))?.split('=')[1];
|
|
19
|
+
if (!ct0) throw new Error('No ct0 cookie. Hint: Not logged into x.com.');
|
|
20
|
+
const bearer = decodeURIComponent('AAAAAAAAAAAAAAAAAAAAANRILgAAAAAAnNwIzUejRCOuH5E6I8xnZz4puTs%3D1Zv7ttfk8LF81IUq16cHjhLTvJu4FA33AGWWjCpTnA');
|
|
21
|
+
const _h = {'Authorization':'Bearer '+bearer, 'X-Csrf-Token':ct0, 'X-Twitter-Auth-Type':'OAuth2Session', 'X-Twitter-Active-User':'yes'};
|
|
22
|
+
|
|
23
|
+
const count = Math.min(${{ args.limit }}, 100);
|
|
24
|
+
const variables = JSON.stringify({count, includePromotedContent: false});
|
|
25
|
+
const features = JSON.stringify({
|
|
26
|
+
rweb_video_screen_enabled: false, profile_label_improvements_pcf_label_in_post_enabled: true,
|
|
27
|
+
responsive_web_profile_redirect_enabled: false, rweb_tipjar_consumption_enabled: false,
|
|
28
|
+
verified_phone_label_enabled: false, creator_subscriptions_tweet_preview_api_enabled: true,
|
|
29
|
+
responsive_web_graphql_timeline_navigation_enabled: true,
|
|
30
|
+
responsive_web_graphql_skip_user_profile_image_extensions_enabled: false,
|
|
31
|
+
premium_content_api_read_enabled: false, communities_web_enable_tweet_community_results_fetch: true,
|
|
32
|
+
c9s_tweet_anatomy_moderator_badge_enabled: true,
|
|
33
|
+
articles_preview_enabled: true, responsive_web_edit_tweet_api_enabled: true,
|
|
34
|
+
graphql_is_translatable_rweb_tweet_is_translatable_enabled: true,
|
|
35
|
+
view_counts_everywhere_api_enabled: true, longform_notetweets_consumption_enabled: true,
|
|
36
|
+
responsive_web_twitter_article_tweet_consumption_enabled: true,
|
|
37
|
+
tweet_awards_web_tipping_enabled: false,
|
|
38
|
+
content_disclosure_indicator_enabled: true, content_disclosure_ai_generated_indicator_enabled: true,
|
|
39
|
+
freedom_of_speech_not_reach_fetch_enabled: true, standardized_nudges_misinfo: true,
|
|
40
|
+
tweet_with_visibility_results_prefer_gql_limited_actions_policy_enabled: true,
|
|
41
|
+
longform_notetweets_rich_text_read_enabled: true, longform_notetweets_inline_media_enabled: false,
|
|
42
|
+
responsive_web_enhance_cards_enabled: false
|
|
43
|
+
});
|
|
44
|
+
const url = '/i/api/graphql/Fy0QMy4q_aZCpkO0PnyLYw/Bookmarks?variables=' + encodeURIComponent(variables) + '&features=' + encodeURIComponent(features);
|
|
45
|
+
const resp = await fetch(url, {headers: _h, credentials: 'include'});
|
|
46
|
+
if (!resp.ok) throw new Error('HTTP ' + resp.status + '. Hint: queryId may have changed.');
|
|
47
|
+
const d = await resp.json();
|
|
48
|
+
|
|
49
|
+
const instructions = d.data?.bookmark_timeline_v2?.timeline?.instructions || d.data?.bookmark_timeline?.timeline?.instructions || [];
|
|
50
|
+
let tweets = [], seen = new Set();
|
|
51
|
+
for (const inst of instructions) {
|
|
52
|
+
for (const entry of (inst.entries || [])) {
|
|
53
|
+
const r = entry.content?.itemContent?.tweet_results?.result;
|
|
54
|
+
if (!r) continue;
|
|
55
|
+
const tw = r.tweet || r;
|
|
56
|
+
const l = tw.legacy || {};
|
|
57
|
+
if (!tw.rest_id || seen.has(tw.rest_id)) continue;
|
|
58
|
+
seen.add(tw.rest_id);
|
|
59
|
+
const u = tw.core?.user_results?.result;
|
|
60
|
+
const nt = tw.note_tweet?.note_tweet_results?.result?.text;
|
|
61
|
+
const screenName = u?.legacy?.screen_name || u?.core?.screen_name;
|
|
62
|
+
tweets.push({
|
|
63
|
+
id: tw.rest_id,
|
|
64
|
+
author: screenName,
|
|
65
|
+
name: u?.legacy?.name || u?.core?.name,
|
|
66
|
+
url: 'https://x.com/' + (screenName || '_') + '/status/' + tw.rest_id,
|
|
67
|
+
text: nt || l.full_text || '',
|
|
68
|
+
likes: l.favorite_count,
|
|
69
|
+
retweets: l.retweet_count,
|
|
70
|
+
created_at: l.created_at
|
|
71
|
+
});
|
|
72
|
+
}
|
|
73
|
+
}
|
|
74
|
+
return tweets;
|
|
75
|
+
})()
|
|
76
|
+
|
|
77
|
+
- map:
|
|
78
|
+
author: ${{ item.author }}
|
|
79
|
+
text: ${{ item.text }}
|
|
80
|
+
likes: ${{ item.likes }}
|
|
81
|
+
url: ${{ item.url }}
|
|
82
|
+
|
|
83
|
+
- limit: ${{ args.limit }}
|
|
84
|
+
|
|
85
|
+
columns: [author, text, likes, url]
|
|
@@ -0,0 +1,61 @@
|
|
|
1
|
+
import { cli, Strategy } from '../../registry.js';
|
|
2
|
+
|
|
3
|
+
cli({
|
|
4
|
+
site: 'twitter',
|
|
5
|
+
name: 'profile',
|
|
6
|
+
description: 'Fetch tweets from a user profile',
|
|
7
|
+
domain: 'x.com',
|
|
8
|
+
strategy: Strategy.INTERCEPT,
|
|
9
|
+
browser: true,
|
|
10
|
+
args: [
|
|
11
|
+
{ name: 'username', type: 'string', required: true },
|
|
12
|
+
{ name: 'limit', type: 'int', default: 15 },
|
|
13
|
+
],
|
|
14
|
+
columns: ['id', 'text', 'likes', 'views', 'url'],
|
|
15
|
+
func: async (page, kwargs) => {
|
|
16
|
+
// Navigate to user profile via search for reliability
|
|
17
|
+
await page.goto(`https://x.com/search?q=from:${kwargs.username}&f=live`);
|
|
18
|
+
await page.wait(5);
|
|
19
|
+
|
|
20
|
+
// Inject XHR interceptor
|
|
21
|
+
await page.installInterceptor('SearchTimeline');
|
|
22
|
+
|
|
23
|
+
// Trigger API by scrolling
|
|
24
|
+
await page.autoScroll({ times: 3, delayMs: 2000 });
|
|
25
|
+
|
|
26
|
+
// Retrieve data
|
|
27
|
+
const requests = await page.getInterceptedRequests();
|
|
28
|
+
if (!requests || requests.length === 0) return [];
|
|
29
|
+
|
|
30
|
+
let results: any[] = [];
|
|
31
|
+
for (const req of requests) {
|
|
32
|
+
try {
|
|
33
|
+
const insts = req.data.data.search_by_raw_query.search_timeline.timeline.instructions;
|
|
34
|
+
const addEntries = insts.find((i: any) => i.type === 'TimelineAddEntries');
|
|
35
|
+
if (!addEntries) continue;
|
|
36
|
+
|
|
37
|
+
for (const entry of addEntries.entries) {
|
|
38
|
+
if (!entry.entryId.startsWith('tweet-')) continue;
|
|
39
|
+
|
|
40
|
+
let tweet = entry.content?.itemContent?.tweet_results?.result;
|
|
41
|
+
if (!tweet) continue;
|
|
42
|
+
|
|
43
|
+
if (tweet.__typename === 'TweetWithVisibilityResults' && tweet.tweet) {
|
|
44
|
+
tweet = tweet.tweet;
|
|
45
|
+
}
|
|
46
|
+
|
|
47
|
+
results.push({
|
|
48
|
+
id: tweet.rest_id,
|
|
49
|
+
text: tweet.legacy?.full_text || '',
|
|
50
|
+
likes: tweet.legacy?.favorite_count || 0,
|
|
51
|
+
views: tweet.views?.count || '0',
|
|
52
|
+
url: `https://x.com/i/status/${tweet.rest_id}`
|
|
53
|
+
});
|
|
54
|
+
}
|
|
55
|
+
} catch (e) {
|
|
56
|
+
}
|
|
57
|
+
}
|
|
58
|
+
|
|
59
|
+
return results.slice(0, kwargs.limit);
|
|
60
|
+
}
|
|
61
|
+
});
|