@jackwener/opencli 0.2.0 → 0.3.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (69) hide show
  1. package/CLI-CREATOR.md +51 -72
  2. package/README.md +8 -5
  3. package/README.zh-CN.md +8 -5
  4. package/SKILL.md +27 -14
  5. package/dist/browser.d.ts +6 -0
  6. package/dist/browser.js +65 -1
  7. package/dist/clis/bilibili/dynamic.d.ts +1 -0
  8. package/dist/clis/bilibili/dynamic.js +33 -0
  9. package/dist/clis/bilibili/ranking.d.ts +1 -0
  10. package/dist/clis/bilibili/ranking.js +24 -0
  11. package/dist/clis/reddit/frontpage.yaml +30 -0
  12. package/dist/clis/reddit/hot.yaml +3 -2
  13. package/dist/clis/reddit/search.yaml +34 -0
  14. package/dist/clis/reddit/subreddit.yaml +39 -0
  15. package/dist/clis/twitter/bookmarks.yaml +85 -0
  16. package/dist/clis/twitter/profile.d.ts +1 -0
  17. package/dist/clis/twitter/profile.js +56 -0
  18. package/dist/clis/twitter/search.d.ts +1 -0
  19. package/dist/clis/twitter/search.js +60 -0
  20. package/dist/clis/twitter/timeline.d.ts +1 -0
  21. package/dist/clis/twitter/timeline.js +47 -0
  22. package/dist/clis/xiaohongshu/user.d.ts +1 -0
  23. package/dist/clis/xiaohongshu/user.js +40 -0
  24. package/dist/clis/xueqiu/feed.yaml +53 -0
  25. package/dist/clis/xueqiu/hot-stock.yaml +49 -0
  26. package/dist/clis/xueqiu/hot.yaml +46 -0
  27. package/dist/clis/xueqiu/search.yaml +53 -0
  28. package/dist/clis/xueqiu/stock.yaml +67 -0
  29. package/dist/clis/xueqiu/watchlist.yaml +46 -0
  30. package/dist/clis/zhihu/hot.yaml +6 -2
  31. package/dist/clis/zhihu/search.yaml +3 -1
  32. package/dist/engine.d.ts +1 -1
  33. package/dist/engine.js +9 -1
  34. package/dist/main.d.ts +1 -1
  35. package/dist/main.js +10 -3
  36. package/dist/pipeline/steps/intercept.js +56 -29
  37. package/dist/pipeline/template.js +3 -1
  38. package/dist/pipeline/template.test.js +6 -0
  39. package/dist/types.d.ts +6 -0
  40. package/package.json +1 -1
  41. package/src/browser.ts +72 -4
  42. package/src/clis/bilibili/dynamic.ts +34 -0
  43. package/src/clis/bilibili/ranking.ts +25 -0
  44. package/src/clis/reddit/frontpage.yaml +30 -0
  45. package/src/clis/reddit/hot.yaml +3 -2
  46. package/src/clis/reddit/search.yaml +34 -0
  47. package/src/clis/reddit/subreddit.yaml +39 -0
  48. package/src/clis/twitter/bookmarks.yaml +85 -0
  49. package/src/clis/twitter/profile.ts +61 -0
  50. package/src/clis/twitter/search.ts +65 -0
  51. package/src/clis/twitter/timeline.ts +50 -0
  52. package/src/clis/xiaohongshu/user.ts +45 -0
  53. package/src/clis/xueqiu/feed.yaml +53 -0
  54. package/src/clis/xueqiu/hot-stock.yaml +49 -0
  55. package/src/clis/xueqiu/hot.yaml +46 -0
  56. package/src/clis/xueqiu/search.yaml +53 -0
  57. package/src/clis/xueqiu/stock.yaml +67 -0
  58. package/src/clis/xueqiu/watchlist.yaml +46 -0
  59. package/src/clis/zhihu/hot.yaml +6 -2
  60. package/src/clis/zhihu/search.yaml +3 -1
  61. package/src/engine.ts +10 -1
  62. package/src/main.ts +9 -3
  63. package/src/pipeline/steps/intercept.ts +58 -28
  64. package/src/pipeline/template.test.ts +6 -0
  65. package/src/pipeline/template.ts +3 -1
  66. package/src/types.ts +3 -0
  67. package/dist/clis/index.d.ts +0 -22
  68. package/dist/clis/index.js +0 -34
  69. package/src/clis/index.ts +0 -46
package/dist/engine.js CHANGED
@@ -6,7 +6,8 @@ import * as path from 'node:path';
6
6
  import yaml from 'js-yaml';
7
7
  import { Strategy, registerCommand } from './registry.js';
8
8
  import { executePipeline } from './pipeline.js';
9
- export function discoverClis(...dirs) {
9
+ export async function discoverClis(...dirs) {
10
+ const promises = [];
10
11
  for (const dir of dirs) {
11
12
  if (!fs.existsSync(dir))
12
13
  continue;
@@ -19,9 +20,16 @@ export function discoverClis(...dirs) {
19
20
  if (file.endsWith('.yaml') || file.endsWith('.yml')) {
20
21
  registerYamlCli(filePath, site);
21
22
  }
23
+ else if (file.endsWith('.js')) {
24
+ // Dynamic import of compiled adapter modules
25
+ promises.push(import(`file://${filePath}`).catch((err) => {
26
+ process.stderr.write(`Warning: failed to load module ${filePath}: ${err.message}\n`);
27
+ }));
28
+ }
22
29
  }
23
30
  }
24
31
  }
32
+ await Promise.all(promises);
25
33
  }
26
34
  function registerYamlCli(filePath, defaultSite) {
27
35
  try {
package/dist/main.d.ts CHANGED
@@ -2,4 +2,4 @@
2
2
  /**
3
3
  * opencli — Make any website your CLI. AI-powered.
4
4
  */
5
- import './clis/index.js';
5
+ export {};
package/dist/main.js CHANGED
@@ -11,7 +11,6 @@ import chalk from 'chalk';
11
11
  import { discoverClis, executeCommand } from './engine.js';
12
12
  import { fullName, getRegistry, strategyLabel } from './registry.js';
13
13
  import { render as renderOutput } from './output.js';
14
- import './clis/index.js';
15
14
  import { PlaywrightMCP } from './browser.js';
16
15
  import { browserSession, DEFAULT_BROWSER_COMMAND_TIMEOUT, runWithTimeout } from './runtime.js';
17
16
  const __filename = fileURLToPath(import.meta.url);
@@ -21,7 +20,7 @@ const USER_CLIS = path.join(os.homedir(), '.opencli', 'clis');
21
20
  // Read version from package.json (single source of truth)
22
21
  const pkgJsonPath = path.resolve(__dirname, '..', 'package.json');
23
22
  const PKG_VERSION = JSON.parse(fs.readFileSync(pkgJsonPath, 'utf-8')).version ?? '0.0.0';
24
- discoverClis(BUILTIN_CLIS, USER_CLIS);
23
+ await discoverClis(BUILTIN_CLIS, USER_CLIS);
25
24
  const program = new Command();
26
25
  program.name('opencli').description('Make any website your CLI. Zero setup. AI-powered.').version(PKG_VERSION);
27
26
  // ── Built-in commands ──────────────────────────────────────────────────────
@@ -116,10 +115,18 @@ for (const [, cmd] of registry) {
116
115
  else {
117
116
  result = await executeCommand(cmd, null, kwargs, actionOpts.verbose);
118
117
  }
118
+ if (actionOpts.verbose && (!result || (Array.isArray(result) && result.length === 0))) {
119
+ console.error(chalk.yellow(`[Verbose] Warning: Command returned an empty result. If the website structural API changed or requires authentication, check the network or update the adapter.`));
120
+ }
119
121
  renderOutput(result, { fmt: actionOpts.format, columns: cmd.columns, title: `${cmd.site}/${cmd.name}`, elapsed: (Date.now() - startTime) / 1000, source: fullName(cmd) });
120
122
  }
121
123
  catch (err) {
122
- console.error(chalk.red(`Error: ${err.message ?? err}`));
124
+ if (actionOpts.verbose && err.stack) {
125
+ console.error(chalk.red(err.stack));
126
+ }
127
+ else {
128
+ console.error(chalk.red(`Error: ${err.message ?? err}`));
129
+ }
123
130
  process.exitCode = 1;
124
131
  }
125
132
  });
@@ -10,7 +10,54 @@ export async function stepIntercept(page, params, data, args) {
10
10
  const selectPath = cfg.select ?? null;
11
11
  if (!capturePattern)
12
12
  return data;
13
- // Step 1: Execute the trigger action
13
+ // Step 1: Inject fetch/XHR interceptor BEFORE trigger
14
+ await page.evaluate(`
15
+ () => {
16
+ window.__opencli_intercepted = window.__opencli_intercepted || [];
17
+ const pattern = ${JSON.stringify(capturePattern)};
18
+
19
+ if (!window.__opencli_fetch_patched) {
20
+ const origFetch = window.fetch;
21
+ window.fetch = async function(...args) {
22
+ const reqUrl = typeof args[0] === 'string' ? args[0] : (args[0] && args[0].url) || '';
23
+ const response = await origFetch.apply(this, args);
24
+ setTimeout(async () => {
25
+ try {
26
+ if (reqUrl.includes(pattern)) {
27
+ const clone = response.clone();
28
+ const json = await clone.json();
29
+ window.__opencli_intercepted.push(json);
30
+ }
31
+ } catch(e) {}
32
+ }, 0);
33
+ return response;
34
+ };
35
+ window.__opencli_fetch_patched = true;
36
+ }
37
+
38
+ if (!window.__opencli_xhr_patched) {
39
+ const XHR = XMLHttpRequest.prototype;
40
+ const open = XHR.open;
41
+ const send = XHR.send;
42
+ XHR.open = function(method, url, ...args) {
43
+ this._reqUrl = url;
44
+ return open.call(this, method, url, ...args);
45
+ };
46
+ XHR.send = function(...args) {
47
+ this.addEventListener('load', function() {
48
+ try {
49
+ if (this._reqUrl && this._reqUrl.includes(pattern)) {
50
+ window.__opencli_intercepted.push(JSON.parse(this.responseText));
51
+ }
52
+ } catch(e) {}
53
+ });
54
+ return send.apply(this, args);
55
+ };
56
+ window.__opencli_xhr_patched = true;
57
+ }
58
+ }
59
+ `);
60
+ // Step 2: Execute the trigger action
14
61
  if (trigger.startsWith('navigate:')) {
15
62
  const url = render(trigger.slice('navigate:'.length), { args, data });
16
63
  await page.goto(String(url));
@@ -27,36 +74,16 @@ export async function stepIntercept(page, params, data, args) {
27
74
  else if (trigger === 'scroll') {
28
75
  await page.scroll('down');
29
76
  }
30
- // Step 2: Wait a bit for network requests to fire
77
+ // Step 3: Wait a bit for network requests to fire
31
78
  await page.wait(Math.min(timeout, 3));
32
- // Step 3: Get network requests and find matching ones
33
- const rawNetwork = await page.networkRequests(false);
34
- const matchingResponses = [];
35
- if (typeof rawNetwork === 'string') {
36
- const lines = rawNetwork.split('\n');
37
- for (const line of lines) {
38
- const match = line.match(/\[?(GET|POST)\]?\s+(\S+)\s*(?:=>|→)\s*\[?(\d+)\]?/i);
39
- if (match) {
40
- const [, , url, status] = match;
41
- if (url.includes(capturePattern) && status === '200') {
42
- try {
43
- const body = await page.evaluate(`
44
- async () => {
45
- try {
46
- const resp = await fetch(${JSON.stringify(url)}, { credentials: 'include' });
47
- if (!resp.ok) return null;
48
- return await resp.json();
49
- } catch { return null; }
50
- }
51
- `);
52
- if (body)
53
- matchingResponses.push(body);
54
- }
55
- catch { }
56
- }
57
- }
58
- }
79
+ // Step 4: Retrieve captured data
80
+ const matchingResponses = await page.evaluate(`
81
+ () => {
82
+ const data = window.__opencli_intercepted || [];
83
+ window.__opencli_intercepted = []; // clear after reading
84
+ return data;
59
85
  }
86
+ `);
60
87
  // Step 4: Select from response if specified
61
88
  let result = matchingResponses.length === 1 ? matchingResponses[0] :
62
89
  matchingResponses.length > 1 ? matchingResponses : data;
@@ -68,7 +68,7 @@ export function evalExpr(expr, ctx) {
68
68
  * Apply a named filter to a value.
69
69
  * Supported filters:
70
70
  * default(val), join(sep), upper, lower, truncate(n), trim,
71
- * replace(old,new), keys, length, first, last
71
+ * replace(old,new), keys, length, first, last, json
72
72
  */
73
73
  function applyFilter(filterExpr, value) {
74
74
  const match = filterExpr.match(/^(\w+)(?:\((.+)\))?$/);
@@ -112,6 +112,8 @@ function applyFilter(filterExpr, value) {
112
112
  return Array.isArray(value) ? value[0] : value;
113
113
  case 'last':
114
114
  return Array.isArray(value) ? value[value.length - 1] : value;
115
+ case 'json':
116
+ return JSON.stringify(value ?? null);
115
117
  default:
116
118
  return value;
117
119
  }
@@ -72,6 +72,12 @@ describe('evalExpr', () => {
72
72
  it('applies length filter', () => {
73
73
  expect(evalExpr('item.items | length', { item: { items: [1, 2, 3] } })).toBe(3);
74
74
  });
75
+ it('applies json filter to strings with quotes', () => {
76
+ expect(evalExpr('args.keyword | json', { args: { keyword: "O'Reilly" } })).toBe('"O\'Reilly"');
77
+ });
78
+ it('applies json filter to nullish values', () => {
79
+ expect(evalExpr('args.keyword | json', { args: {} })).toBe('null');
80
+ });
75
81
  });
76
82
  describe('render', () => {
77
83
  it('renders full expression', () => {
package/dist/types.d.ts CHANGED
@@ -24,4 +24,10 @@ export interface IPage {
24
24
  networkRequests(includeStatic?: boolean): Promise<any>;
25
25
  consoleMessages(level?: string): Promise<any>;
26
26
  scroll(direction?: string, amount?: number): Promise<void>;
27
+ autoScroll(options?: {
28
+ times?: number;
29
+ delayMs?: number;
30
+ }): Promise<void>;
31
+ installInterceptor(pattern: string): Promise<void>;
32
+ getInterceptedRequests(): Promise<any[]>;
27
33
  }
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@jackwener/opencli",
3
- "version": "0.2.0",
3
+ "version": "0.3.0",
4
4
  "publishConfig": {
5
5
  "access": "public"
6
6
  },
package/src/browser.ts CHANGED
@@ -135,6 +135,69 @@ export class Page implements IPage {
135
135
  async scroll(direction: string = 'down', amount: number = 500): Promise<void> {
136
136
  await this.call('tools/call', { name: 'browser_press_key', arguments: { key: direction === 'down' ? 'PageDown' : 'PageUp' } });
137
137
  }
138
+
139
+ async autoScroll(options: { times?: number; delayMs?: number } = {}): Promise<void> {
140
+ const times = options.times ?? 3;
141
+ const delayMs = options.delayMs ?? 2000;
142
+ for (let i = 0; i < times; i++) {
143
+ await this.evaluate('() => window.scrollTo(0, document.body.scrollHeight)');
144
+ await this.wait(delayMs / 1000);
145
+ }
146
+ }
147
+
148
+ async installInterceptor(pattern: string): Promise<void> {
149
+ const js = `
150
+ () => {
151
+ window.__opencli_xhr = window.__opencli_xhr || [];
152
+ window.__opencli_patterns = window.__opencli_patterns || [];
153
+ if (!window.__opencli_patterns.includes('${pattern}')) {
154
+ window.__opencli_patterns.push('${pattern}');
155
+ }
156
+
157
+ if (!window.__patched_xhr) {
158
+ const checkMatch = (url) => window.__opencli_patterns.some(p => url.includes(p));
159
+
160
+ const XHR = XMLHttpRequest.prototype;
161
+ const open = XHR.open;
162
+ const send = XHR.send;
163
+ XHR.open = function(method, url) {
164
+ this._url = url;
165
+ return open.call(this, method, url, ...Array.prototype.slice.call(arguments, 2));
166
+ };
167
+ XHR.send = function() {
168
+ this.addEventListener('load', function() {
169
+ if (checkMatch(this._url)) {
170
+ try { window.__opencli_xhr.push({url: this._url, data: JSON.parse(this.responseText)}); } catch(e){}
171
+ }
172
+ });
173
+ return send.apply(this, arguments);
174
+ };
175
+
176
+ const origFetch = window.fetch;
177
+ window.fetch = async function(...args) {
178
+ let u = typeof args[0] === 'string' ? args[0] : (args[0] && args[0].url) || '';
179
+ const res = await origFetch.apply(this, args);
180
+ setTimeout(async () => {
181
+ try {
182
+ if (checkMatch(u)) {
183
+ const clone = res.clone();
184
+ const j = await clone.json();
185
+ window.__opencli_xhr.push({url: u, data: j});
186
+ }
187
+ } catch(e) {}
188
+ }, 0);
189
+ return res;
190
+ };
191
+ window.__patched_xhr = true;
192
+ }
193
+ }
194
+ `;
195
+ await this.evaluate(js);
196
+ }
197
+
198
+ async getInterceptedRequests(): Promise<any[]> {
199
+ return (await this.evaluate('() => window.__opencli_xhr')) || [];
200
+ }
138
201
  }
139
202
 
140
203
  /**
@@ -158,10 +221,15 @@ export class PlaywrightMCP {
158
221
  return new Promise<Page>((resolve, reject) => {
159
222
  const timer = setTimeout(() => reject(new Error(`Timed out connecting to browser (${timeout}s)`)), timeout * 1000);
160
223
 
161
- this._proc = spawn('node', [mcpPath, '--extension'], {
162
- stdio: ['pipe', 'pipe', 'pipe'],
163
- env: { ...process.env, ...(process.env.PLAYWRIGHT_MCP_EXTENSION_TOKEN ? { PLAYWRIGHT_MCP_EXTENSION_TOKEN: process.env.PLAYWRIGHT_MCP_EXTENSION_TOKEN } : {}) },
164
- });
224
+ const mcpArgs = [mcpPath, '--extension'];
225
+ if (process.env.OPENCLI_BROWSER_EXECUTABLE_PATH) {
226
+ mcpArgs.push('--executablePath', process.env.OPENCLI_BROWSER_EXECUTABLE_PATH);
227
+ }
228
+
229
+ this._proc = spawn('node', mcpArgs, {
230
+ stdio: ['pipe', 'pipe', 'pipe'],
231
+ env: { ...process.env, ...(process.env.PLAYWRIGHT_MCP_EXTENSION_TOKEN ? { PLAYWRIGHT_MCP_EXTENSION_TOKEN: process.env.PLAYWRIGHT_MCP_EXTENSION_TOKEN } : {}) },
232
+ });
165
233
 
166
234
  // Increase max listeners to avoid warnings
167
235
  this._proc.setMaxListeners(20);
@@ -0,0 +1,34 @@
1
+ import { cli, Strategy } from '../../registry.js';
2
+ import { apiGet } from '../../bilibili.js';
3
+
4
+ cli({
5
+ site: 'bilibili',
6
+ name: 'dynamic',
7
+ description: 'Get Bilibili user dynamic feed',
8
+ domain: 'www.bilibili.com',
9
+ strategy: Strategy.COOKIE,
10
+ args: [
11
+ { name: 'limit', type: 'int', default: 15 },
12
+ ],
13
+ columns: ['id', 'author', 'text', 'likes', 'url'],
14
+ func: async (page, kwargs) => {
15
+ const payload = await apiGet(page, '/x/polymer/web-dynamic/v1/feed/all', { params: {}, signed: false });
16
+ const results: any[] = payload?.data?.items ?? [];
17
+ return results.slice(0, Number(kwargs.limit)).map((item: any) => {
18
+ let text = '';
19
+ if (item.modules?.module_dynamic?.desc?.text) {
20
+ text = item.modules.module_dynamic.desc.text;
21
+ } else if (item.modules?.module_dynamic?.major?.archive?.title) {
22
+ text = item.modules.module_dynamic.major.archive.title;
23
+ }
24
+
25
+ return {
26
+ id: item.id_str ?? '',
27
+ author: item.modules?.module_author?.name ?? '',
28
+ text: text,
29
+ likes: item.modules?.module_stat?.like?.count ?? 0,
30
+ url: item.id_str ? `https://t.bilibili.com/${item.id_str}` : ''
31
+ };
32
+ });
33
+ },
34
+ });
@@ -0,0 +1,25 @@
1
+ import { cli, Strategy } from '../../registry.js';
2
+ import { apiGet } from '../../bilibili.js';
3
+
4
+ cli({
5
+ site: 'bilibili',
6
+ name: 'ranking',
7
+ description: 'Get Bilibili video ranking board',
8
+ domain: 'www.bilibili.com',
9
+ strategy: Strategy.COOKIE,
10
+ args: [
11
+ { name: 'limit', type: 'int', default: 20 },
12
+ ],
13
+ columns: ['rank', 'title', 'author', 'score', 'url'],
14
+ func: async (page, kwargs) => {
15
+ const payload = await apiGet(page, '/x/web-interface/ranking/v2', { params: { rid: 0, type: 'all' }, signed: false });
16
+ const results: any[] = payload?.data?.list ?? [];
17
+ return results.slice(0, Number(kwargs.limit)).map((item: any, i: number) => ({
18
+ rank: i + 1,
19
+ title: item.title ?? '',
20
+ author: item.owner?.name ?? '',
21
+ score: item.stat?.view ?? 0,
22
+ url: item.bvid ? `https://www.bilibili.com/video/${item.bvid}` : ''
23
+ }));
24
+ },
25
+ });
@@ -0,0 +1,30 @@
1
+ site: reddit
2
+ name: frontpage
3
+ description: Reddit Frontpage / r/all
4
+ domain: reddit.com
5
+ strategy: cookie
6
+ browser: true
7
+
8
+ args:
9
+ limit:
10
+ type: int
11
+ default: 15
12
+
13
+ columns: [title, subreddit, author, upvotes, comments, url]
14
+
15
+ pipeline:
16
+ - navigate: https://www.reddit.com
17
+ - evaluate: |
18
+ (async () => {
19
+ const res = await fetch('/r/all.json?limit=${{ args.limit }}', { credentials: 'include' });
20
+ const j = await res.json();
21
+ return j?.data?.children || [];
22
+ })()
23
+ - map:
24
+ title: ${{ item.data.title }}
25
+ subreddit: ${{ item.data.subreddit_name_prefixed }}
26
+ author: ${{ item.data.author }}
27
+ upvotes: ${{ item.data.score }}
28
+ comments: ${{ item.data.num_comments }}
29
+ url: https://www.reddit.com${{ item.data.permalink }}
30
+ - limit: ${{ args.limit }}
@@ -18,9 +18,10 @@ pipeline:
18
18
 
19
19
  - evaluate: |
20
20
  (async () => {
21
- const sub = '${{ args.subreddit }}';
21
+ const sub = ${{ args.subreddit | json }};
22
22
  const path = sub ? '/r/' + sub + '/hot.json' : '/hot.json';
23
- const res = await fetch(path + '?limit=${{ args.limit }}&raw_json=1', {
23
+ const limit = ${{ args.limit }};
24
+ const res = await fetch(path + '?limit=' + limit + '&raw_json=1', {
24
25
  credentials: 'include'
25
26
  });
26
27
  const d = await res.json();
@@ -0,0 +1,34 @@
1
+ site: reddit
2
+ name: search
3
+ description: Search Reddit Posts
4
+ domain: reddit.com
5
+ strategy: cookie
6
+ browser: true
7
+
8
+ args:
9
+ query:
10
+ type: string
11
+ required: true
12
+ limit:
13
+ type: int
14
+ default: 15
15
+
16
+ columns: [title, subreddit, author, upvotes, comments, url]
17
+
18
+ pipeline:
19
+ - navigate: https://www.reddit.com
20
+ - evaluate: |
21
+ (async () => {
22
+ const q = encodeURIComponent('${{ args.query }}');
23
+ const res = await fetch('/search.json?q=' + q + '&limit=${{ args.limit }}', { credentials: 'include' });
24
+ const j = await res.json();
25
+ return j?.data?.children || [];
26
+ })()
27
+ - map:
28
+ title: ${{ item.data.title }}
29
+ subreddit: ${{ item.data.subreddit_name_prefixed }}
30
+ author: ${{ item.data.author }}
31
+ upvotes: ${{ item.data.score }}
32
+ comments: ${{ item.data.num_comments }}
33
+ url: https://www.reddit.com${{ item.data.permalink }}
34
+ - limit: ${{ args.limit }}
@@ -0,0 +1,39 @@
1
+ site: reddit
2
+ name: subreddit
3
+ description: Get posts from a specific Subreddit
4
+ domain: reddit.com
5
+ strategy: cookie
6
+ browser: true
7
+
8
+ args:
9
+ name:
10
+ type: string
11
+ required: true
12
+ sort:
13
+ type: string
14
+ default: hot
15
+ description: "Sorting method: hot, new, top, rising"
16
+ limit:
17
+ type: int
18
+ default: 15
19
+
20
+ columns: [title, author, upvotes, comments, url]
21
+
22
+ pipeline:
23
+ - navigate: https://www.reddit.com
24
+ - evaluate: |
25
+ (async () => {
26
+ let sub = '${{ args.name }}';
27
+ if (sub.startsWith('r/')) sub = sub.slice(2);
28
+ const sort = '${{ args.sort }}';
29
+ const res = await fetch('/r/' + sub + '/' + sort + '.json?limit=${{ args.limit }}', { credentials: 'include' });
30
+ const j = await res.json();
31
+ return j?.data?.children || [];
32
+ })()
33
+ - map:
34
+ title: ${{ item.data.title }}
35
+ author: ${{ item.data.author }}
36
+ upvotes: ${{ item.data.score }}
37
+ comments: ${{ item.data.num_comments }}
38
+ url: https://www.reddit.com${{ item.data.permalink }}
39
+ - limit: ${{ args.limit }}
@@ -0,0 +1,85 @@
1
+ site: twitter
2
+ name: bookmarks
3
+ description: 获取 Twitter 书签列表
4
+ domain: x.com
5
+ browser: true
6
+
7
+ args:
8
+ limit:
9
+ type: int
10
+ default: 20
11
+ description: Number of bookmarks to return (default 20)
12
+
13
+ pipeline:
14
+ - navigate: https://x.com/i/bookmarks
15
+ - wait: 2
16
+ - evaluate: |
17
+ (async () => {
18
+ const ct0 = document.cookie.split(';').map(c=>c.trim()).find(c=>c.startsWith('ct0='))?.split('=')[1];
19
+ if (!ct0) throw new Error('No ct0 cookie. Hint: Not logged into x.com.');
20
+ const bearer = decodeURIComponent('AAAAAAAAAAAAAAAAAAAAANRILgAAAAAAnNwIzUejRCOuH5E6I8xnZz4puTs%3D1Zv7ttfk8LF81IUq16cHjhLTvJu4FA33AGWWjCpTnA');
21
+ const _h = {'Authorization':'Bearer '+bearer, 'X-Csrf-Token':ct0, 'X-Twitter-Auth-Type':'OAuth2Session', 'X-Twitter-Active-User':'yes'};
22
+
23
+ const count = Math.min(${{ args.limit }}, 100);
24
+ const variables = JSON.stringify({count, includePromotedContent: false});
25
+ const features = JSON.stringify({
26
+ rweb_video_screen_enabled: false, profile_label_improvements_pcf_label_in_post_enabled: true,
27
+ responsive_web_profile_redirect_enabled: false, rweb_tipjar_consumption_enabled: false,
28
+ verified_phone_label_enabled: false, creator_subscriptions_tweet_preview_api_enabled: true,
29
+ responsive_web_graphql_timeline_navigation_enabled: true,
30
+ responsive_web_graphql_skip_user_profile_image_extensions_enabled: false,
31
+ premium_content_api_read_enabled: false, communities_web_enable_tweet_community_results_fetch: true,
32
+ c9s_tweet_anatomy_moderator_badge_enabled: true,
33
+ articles_preview_enabled: true, responsive_web_edit_tweet_api_enabled: true,
34
+ graphql_is_translatable_rweb_tweet_is_translatable_enabled: true,
35
+ view_counts_everywhere_api_enabled: true, longform_notetweets_consumption_enabled: true,
36
+ responsive_web_twitter_article_tweet_consumption_enabled: true,
37
+ tweet_awards_web_tipping_enabled: false,
38
+ content_disclosure_indicator_enabled: true, content_disclosure_ai_generated_indicator_enabled: true,
39
+ freedom_of_speech_not_reach_fetch_enabled: true, standardized_nudges_misinfo: true,
40
+ tweet_with_visibility_results_prefer_gql_limited_actions_policy_enabled: true,
41
+ longform_notetweets_rich_text_read_enabled: true, longform_notetweets_inline_media_enabled: false,
42
+ responsive_web_enhance_cards_enabled: false
43
+ });
44
+ const url = '/i/api/graphql/Fy0QMy4q_aZCpkO0PnyLYw/Bookmarks?variables=' + encodeURIComponent(variables) + '&features=' + encodeURIComponent(features);
45
+ const resp = await fetch(url, {headers: _h, credentials: 'include'});
46
+ if (!resp.ok) throw new Error('HTTP ' + resp.status + '. Hint: queryId may have changed.');
47
+ const d = await resp.json();
48
+
49
+ const instructions = d.data?.bookmark_timeline_v2?.timeline?.instructions || d.data?.bookmark_timeline?.timeline?.instructions || [];
50
+ let tweets = [], seen = new Set();
51
+ for (const inst of instructions) {
52
+ for (const entry of (inst.entries || [])) {
53
+ const r = entry.content?.itemContent?.tweet_results?.result;
54
+ if (!r) continue;
55
+ const tw = r.tweet || r;
56
+ const l = tw.legacy || {};
57
+ if (!tw.rest_id || seen.has(tw.rest_id)) continue;
58
+ seen.add(tw.rest_id);
59
+ const u = tw.core?.user_results?.result;
60
+ const nt = tw.note_tweet?.note_tweet_results?.result?.text;
61
+ const screenName = u?.legacy?.screen_name || u?.core?.screen_name;
62
+ tweets.push({
63
+ id: tw.rest_id,
64
+ author: screenName,
65
+ name: u?.legacy?.name || u?.core?.name,
66
+ url: 'https://x.com/' + (screenName || '_') + '/status/' + tw.rest_id,
67
+ text: nt || l.full_text || '',
68
+ likes: l.favorite_count,
69
+ retweets: l.retweet_count,
70
+ created_at: l.created_at
71
+ });
72
+ }
73
+ }
74
+ return tweets;
75
+ })()
76
+
77
+ - map:
78
+ author: ${{ item.author }}
79
+ text: ${{ item.text }}
80
+ likes: ${{ item.likes }}
81
+ url: ${{ item.url }}
82
+
83
+ - limit: ${{ args.limit }}
84
+
85
+ columns: [author, text, likes, url]
@@ -0,0 +1,61 @@
1
+ import { cli, Strategy } from '../../registry.js';
2
+
3
+ cli({
4
+ site: 'twitter',
5
+ name: 'profile',
6
+ description: 'Fetch tweets from a user profile',
7
+ domain: 'x.com',
8
+ strategy: Strategy.INTERCEPT,
9
+ browser: true,
10
+ args: [
11
+ { name: 'username', type: 'string', required: true },
12
+ { name: 'limit', type: 'int', default: 15 },
13
+ ],
14
+ columns: ['id', 'text', 'likes', 'views', 'url'],
15
+ func: async (page, kwargs) => {
16
+ // Navigate to user profile via search for reliability
17
+ await page.goto(`https://x.com/search?q=from:${kwargs.username}&f=live`);
18
+ await page.wait(5);
19
+
20
+ // Inject XHR interceptor
21
+ await page.installInterceptor('SearchTimeline');
22
+
23
+ // Trigger API by scrolling
24
+ await page.autoScroll({ times: 3, delayMs: 2000 });
25
+
26
+ // Retrieve data
27
+ const requests = await page.getInterceptedRequests();
28
+ if (!requests || requests.length === 0) return [];
29
+
30
+ let results: any[] = [];
31
+ for (const req of requests) {
32
+ try {
33
+ const insts = req.data.data.search_by_raw_query.search_timeline.timeline.instructions;
34
+ const addEntries = insts.find((i: any) => i.type === 'TimelineAddEntries');
35
+ if (!addEntries) continue;
36
+
37
+ for (const entry of addEntries.entries) {
38
+ if (!entry.entryId.startsWith('tweet-')) continue;
39
+
40
+ let tweet = entry.content?.itemContent?.tweet_results?.result;
41
+ if (!tweet) continue;
42
+
43
+ if (tweet.__typename === 'TweetWithVisibilityResults' && tweet.tweet) {
44
+ tweet = tweet.tweet;
45
+ }
46
+
47
+ results.push({
48
+ id: tweet.rest_id,
49
+ text: tweet.legacy?.full_text || '',
50
+ likes: tweet.legacy?.favorite_count || 0,
51
+ views: tweet.views?.count || '0',
52
+ url: `https://x.com/i/status/${tweet.rest_id}`
53
+ });
54
+ }
55
+ } catch (e) {
56
+ }
57
+ }
58
+
59
+ return results.slice(0, kwargs.limit);
60
+ }
61
+ });