@jackwener/opencli 1.7.5 → 1.7.7

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (121) hide show
  1. package/README.md +22 -10
  2. package/README.zh-CN.md +18 -9
  3. package/cli-manifest.json +401 -11
  4. package/clis/51job/company.js +125 -0
  5. package/clis/51job/detail.js +108 -0
  6. package/clis/51job/hot.js +55 -0
  7. package/clis/51job/search.js +79 -0
  8. package/clis/51job/utils.js +302 -0
  9. package/clis/51job/utils.test.js +69 -0
  10. package/clis/bilibili/video.js +68 -0
  11. package/clis/bilibili/video.test.js +132 -0
  12. package/clis/chatgpt/image.js +1 -1
  13. package/clis/deepseek/ask.js +37 -11
  14. package/clis/deepseek/ask.test.js +165 -0
  15. package/clis/deepseek/utils.js +192 -24
  16. package/clis/deepseek/utils.test.js +145 -0
  17. package/clis/gemini/image.js +1 -1
  18. package/clis/instagram/download.js +1 -1
  19. package/clis/jianyu/search.js +139 -3
  20. package/clis/jianyu/search.test.js +25 -0
  21. package/clis/jianyu/shared/procurement-detail.js +15 -0
  22. package/clis/jianyu/shared/procurement-detail.test.js +12 -0
  23. package/clis/twitter/likes.js +3 -2
  24. package/clis/twitter/search.js +4 -2
  25. package/clis/twitter/search.test.js +4 -0
  26. package/clis/twitter/shared.js +35 -2
  27. package/clis/twitter/shared.test.js +96 -0
  28. package/clis/twitter/thread.js +3 -1
  29. package/clis/twitter/timeline.js +3 -2
  30. package/clis/twitter/tweets.js +219 -0
  31. package/clis/twitter/tweets.test.js +125 -0
  32. package/clis/web/read.js +25 -5
  33. package/clis/web/read.test.js +76 -0
  34. package/clis/weread/ai-outline.js +170 -0
  35. package/clis/weread/ai-outline.test.js +83 -0
  36. package/clis/weread/book.js +57 -44
  37. package/clis/weread/commands.test.js +24 -0
  38. package/clis/xiaoyuzhou/podcast-episodes.js +2 -2
  39. package/clis/xiaoyuzhou/podcast-episodes.test.js +78 -0
  40. package/clis/youtube/channel.js +35 -0
  41. package/dist/src/browser/analyze.d.ts +103 -0
  42. package/dist/src/browser/analyze.js +230 -0
  43. package/dist/src/browser/analyze.test.d.ts +1 -0
  44. package/dist/src/browser/analyze.test.js +164 -0
  45. package/dist/src/browser/article-extract.d.ts +57 -0
  46. package/dist/src/browser/article-extract.e2e.test.d.ts +1 -0
  47. package/dist/src/browser/article-extract.e2e.test.js +105 -0
  48. package/dist/src/browser/article-extract.js +169 -0
  49. package/dist/src/browser/article-extract.test.d.ts +1 -0
  50. package/dist/src/browser/article-extract.test.js +94 -0
  51. package/dist/src/browser/base-page.d.ts +13 -3
  52. package/dist/src/browser/base-page.js +35 -25
  53. package/dist/src/browser/cdp.d.ts +1 -0
  54. package/dist/src/browser/cdp.js +23 -5
  55. package/dist/src/browser/compound.d.ts +59 -0
  56. package/dist/src/browser/compound.js +112 -0
  57. package/dist/src/browser/compound.test.d.ts +1 -0
  58. package/dist/src/browser/compound.test.js +175 -0
  59. package/dist/src/browser/dom-snapshot.d.ts +7 -0
  60. package/dist/src/browser/dom-snapshot.js +76 -3
  61. package/dist/src/browser/dom-snapshot.test.js +65 -0
  62. package/dist/src/browser/extract.d.ts +69 -0
  63. package/dist/src/browser/extract.js +132 -0
  64. package/dist/src/browser/extract.test.d.ts +1 -0
  65. package/dist/src/browser/extract.test.js +129 -0
  66. package/dist/src/browser/find.d.ts +76 -0
  67. package/dist/src/browser/find.js +179 -0
  68. package/dist/src/browser/find.test.d.ts +1 -0
  69. package/dist/src/browser/find.test.js +120 -0
  70. package/dist/src/browser/html-tree.d.ts +75 -0
  71. package/dist/src/browser/html-tree.js +112 -0
  72. package/dist/src/browser/html-tree.test.d.ts +1 -0
  73. package/dist/src/browser/html-tree.test.js +181 -0
  74. package/dist/src/browser/network-cache.d.ts +48 -0
  75. package/dist/src/browser/network-cache.js +66 -0
  76. package/dist/src/browser/network-cache.test.d.ts +1 -0
  77. package/dist/src/browser/network-cache.test.js +58 -0
  78. package/dist/src/browser/network-key.d.ts +22 -0
  79. package/dist/src/browser/network-key.js +66 -0
  80. package/dist/src/browser/network-key.test.d.ts +1 -0
  81. package/dist/src/browser/network-key.test.js +49 -0
  82. package/dist/src/browser/shape-filter.d.ts +52 -0
  83. package/dist/src/browser/shape-filter.js +101 -0
  84. package/dist/src/browser/shape-filter.test.d.ts +1 -0
  85. package/dist/src/browser/shape-filter.test.js +101 -0
  86. package/dist/src/browser/shape.d.ts +23 -0
  87. package/dist/src/browser/shape.js +95 -0
  88. package/dist/src/browser/shape.test.d.ts +1 -0
  89. package/dist/src/browser/shape.test.js +82 -0
  90. package/dist/src/browser/target-errors.d.ts +14 -1
  91. package/dist/src/browser/target-errors.js +13 -0
  92. package/dist/src/browser/target-errors.test.js +39 -6
  93. package/dist/src/browser/target-resolver.d.ts +57 -10
  94. package/dist/src/browser/target-resolver.js +195 -75
  95. package/dist/src/browser/target-resolver.test.js +80 -5
  96. package/dist/src/browser/verify-fixture.d.ts +59 -0
  97. package/dist/src/browser/verify-fixture.js +213 -0
  98. package/dist/src/browser/verify-fixture.test.d.ts +1 -0
  99. package/dist/src/browser/verify-fixture.test.js +161 -0
  100. package/dist/src/cli.d.ts +32 -0
  101. package/dist/src/cli.js +936 -141
  102. package/dist/src/cli.test.js +1051 -1
  103. package/dist/src/daemon.d.ts +3 -2
  104. package/dist/src/daemon.js +16 -4
  105. package/dist/src/daemon.test.d.ts +1 -0
  106. package/dist/src/daemon.test.js +19 -0
  107. package/dist/src/download/article-download.d.ts +12 -0
  108. package/dist/src/download/article-download.js +141 -17
  109. package/dist/src/download/article-download.test.js +196 -0
  110. package/dist/src/download/index.js +73 -86
  111. package/dist/src/errors.js +4 -2
  112. package/dist/src/errors.test.js +13 -0
  113. package/dist/src/execution.js +7 -2
  114. package/dist/src/execution.test.js +54 -0
  115. package/dist/src/launcher.d.ts +1 -1
  116. package/dist/src/launcher.js +3 -3
  117. package/dist/src/main.js +16 -0
  118. package/dist/src/output.js +1 -1
  119. package/dist/src/output.test.js +6 -0
  120. package/dist/src/types.d.ts +18 -3
  121. package/package.json +5 -1
@@ -7,6 +7,13 @@ const RETRYABLE_DETAIL_ERROR_PATTERNS = [
7
7
  /cannot find context with specified id/i,
8
8
  /\[taxonomy=empty_result\]/i,
9
9
  ];
10
+ const DETAIL_AUTH_CHALLENGE_PATTERNS = [
11
+ /请在下图依次点击/i,
12
+ /验证码/i,
13
+ /请完成验证/i,
14
+ /验证登录/i,
15
+ /登录即可获得更多浏览权限/i,
16
+ ];
10
17
  function isRetryableDetailError(error) {
11
18
  const message = error instanceof Error
12
19
  ? cleanText(error.message)
@@ -61,6 +68,14 @@ export async function runProcurementDetail(page, { url, site, query = '', }) {
61
68
  const title = cleanText(row.title);
62
69
  const detailText = cleanText(row.detailText);
63
70
  const publishTime = cleanText(row.publishTime);
71
+ const authGateText = cleanText(`${title} ${detailText}`);
72
+ if (DETAIL_AUTH_CHALLENGE_PATTERNS.some((pattern) => pattern.test(authGateText))) {
73
+ throw taxonomyError('selector_drift', {
74
+ site,
75
+ command: 'detail',
76
+ detail: `detail page blocked by verification challenge: ${targetUrl}`,
77
+ });
78
+ }
64
79
  if (!title && !detailText) {
65
80
  throw taxonomyError('empty_result', {
66
81
  site,
@@ -69,4 +69,16 @@ describe('procurement detail runner', () => {
69
69
  })).rejects.toThrow('[taxonomy=extraction_drift]');
70
70
  expect(attempts).toBe(1);
71
71
  });
72
+ it('rejects captcha/verification pages as selector_drift', async () => {
73
+ const page = createPage(async () => ({
74
+ title: '验证码',
75
+ detailText: '请在下图依次点击:槨畽黛',
76
+ publishTime: '',
77
+ }));
78
+ await expect(runProcurementDetail(page, {
79
+ url: 'https://www.jianyu360.cn/nologin/content/ABC.html',
80
+ site: 'jianyu',
81
+ query: '电梯',
82
+ })).rejects.toThrow('[taxonomy=selector_drift]');
83
+ });
72
84
  });
@@ -1,6 +1,6 @@
1
1
  import { cli, Strategy } from '@jackwener/opencli/registry';
2
2
  import { AuthRequiredError, CommandExecutionError } from '@jackwener/opencli/errors';
3
- import { resolveTwitterQueryId, sanitizeQueryId } from './shared.js';
3
+ import { resolveTwitterQueryId, sanitizeQueryId, extractMedia } from './shared.js';
4
4
  const BEARER_TOKEN = 'AAAAAAAAAAAAAAAAAAAAANRILgAAAAAAnNwIzUejRCOuH5E6I8xnZz4puTs%3D1Zv7ttfk8LF81IUq16cHjhLTvJu4FA33AGWWjCpTnA';
5
5
  const LIKES_QUERY_ID = 'RozQdCp4CilQzrcuU0NY5w';
6
6
  const USER_BY_SCREEN_NAME_QUERY_ID = 'qRednkZG-rn1P6b48NINmQ';
@@ -99,6 +99,7 @@ function extractLikedTweet(result, seen) {
99
99
  retweets: legacy.retweet_count || 0,
100
100
  created_at: legacy.created_at || '',
101
101
  url: `https://x.com/${screenName}/status/${tw.rest_id}`,
102
+ ...extractMedia(legacy),
102
103
  };
103
104
  }
104
105
  function parseLikes(data, seen) {
@@ -144,7 +145,7 @@ cli({
144
145
  { name: 'username', type: 'string', positional: true, help: 'Twitter screen name (without @). Defaults to logged-in user.' },
145
146
  { name: 'limit', type: 'int', default: 20 },
146
147
  ],
147
- columns: ['author', 'name', 'text', 'likes', 'url'],
148
+ columns: ['author', 'name', 'text', 'likes', 'url', 'has_media', 'media_urls'],
148
149
  func: async (page, kwargs) => {
149
150
  const limit = kwargs.limit || 20;
150
151
  let username = (kwargs.username || '').replace(/^@/, '');
@@ -1,5 +1,6 @@
1
1
  import { CommandExecutionError } from '@jackwener/opencli/errors';
2
2
  import { cli, Strategy } from '@jackwener/opencli/registry';
3
+ import { extractMedia } from './shared.js';
3
4
  /**
4
5
  * Trigger Twitter search SPA navigation with fallback strategies.
5
6
  *
@@ -102,7 +103,7 @@ cli({
102
103
  { name: 'filter', type: 'string', default: 'top', choices: ['top', 'live'] },
103
104
  { name: 'limit', type: 'int', default: 15 },
104
105
  ],
105
- columns: ['id', 'author', 'text', 'created_at', 'likes', 'views', 'url'],
106
+ columns: ['id', 'author', 'text', 'created_at', 'likes', 'views', 'url', 'has_media', 'media_urls'],
106
107
  func: async (page, kwargs) => {
107
108
  const query = kwargs.query;
108
109
  const filter = kwargs.filter === 'live' ? 'live' : 'top';
@@ -156,7 +157,8 @@ cli({
156
157
  created_at: tweet.legacy?.created_at || '',
157
158
  likes: tweet.legacy?.favorite_count || 0,
158
159
  views: tweet.views?.count || '0',
159
- url: `https://x.com/i/status/${tweet.rest_id}`
160
+ url: `https://x.com/i/status/${tweet.rest_id}`,
161
+ ...extractMedia(tweet.legacy),
160
162
  });
161
163
  }
162
164
  }
@@ -75,6 +75,8 @@ describe('twitter search command', () => {
75
75
  likes: 7,
76
76
  views: '12',
77
77
  url: 'https://x.com/i/status/1',
78
+ has_media: false,
79
+ media_urls: [],
78
80
  },
79
81
  ]);
80
82
  expect(page.installInterceptor).toHaveBeenCalledWith('SearchTimeline');
@@ -203,6 +205,8 @@ describe('twitter search command', () => {
203
205
  likes: 3,
204
206
  views: '5',
205
207
  url: 'https://x.com/i/status/99',
208
+ has_media: false,
209
+ media_urls: [],
206
210
  },
207
211
  ]);
208
212
  // 6 evaluate calls: 2x pushState + 2x pathname check + 1x fallback + 1x pathname check
@@ -5,14 +5,19 @@ export function sanitizeQueryId(resolved, fallbackId) {
5
5
  export async function resolveTwitterQueryId(page, operationName, fallbackId) {
6
6
  const resolved = await page.evaluate(`async () => {
7
7
  const operationName = ${JSON.stringify(operationName)};
8
+ const controller = new AbortController();
9
+ const timeout = setTimeout(() => controller.abort(), 5000);
8
10
  try {
9
- const ghResp = await fetch('https://raw.githubusercontent.com/fa0311/twitter-openapi/refs/heads/main/src/config/placeholder.json');
11
+ const ghResp = await fetch('https://raw.githubusercontent.com/fa0311/twitter-openapi/refs/heads/main/src/config/placeholder.json', { signal: controller.signal });
12
+ clearTimeout(timeout);
10
13
  if (ghResp.ok) {
11
14
  const data = await ghResp.json();
12
15
  const entry = data?.[operationName];
13
16
  if (entry && entry.queryId) return entry.queryId;
14
17
  }
15
- } catch {}
18
+ } catch {
19
+ clearTimeout(timeout);
20
+ }
16
21
  try {
17
22
  const scripts = performance.getEntriesByType('resource')
18
23
  .filter(r => r.name.includes('client-web') && r.name.endsWith('.js'))
@@ -30,6 +35,34 @@ export async function resolveTwitterQueryId(page, operationName, fallbackId) {
30
35
  }`);
31
36
  return sanitizeQueryId(resolved, fallbackId);
32
37
  }
38
+ /**
39
+ * Extract media flags and URLs from a tweet's `legacy` object.
40
+ *
41
+ * Prefers `extended_entities.media` (superset with full video_info) and falls
42
+ * back to `entities.media` when the extended form is missing. For videos and
43
+ * animated GIFs, returns the mp4 variant URL; for photos, returns
44
+ * `media_url_https`.
45
+ */
46
+ export function extractMedia(legacy) {
47
+ const media = legacy?.extended_entities?.media || legacy?.entities?.media;
48
+ if (!Array.isArray(media) || media.length === 0) {
49
+ return { has_media: false, media_urls: [] };
50
+ }
51
+ const urls = [];
52
+ for (const m of media) {
53
+ if (!m) continue;
54
+ if (m.type === 'video' || m.type === 'animated_gif') {
55
+ const variants = m.video_info?.variants || [];
56
+ const mp4 = variants.find((v) => v?.content_type === 'video/mp4');
57
+ const url = mp4?.url || m.media_url_https;
58
+ if (url) urls.push(url);
59
+ } else {
60
+ if (m.media_url_https) urls.push(m.media_url_https);
61
+ }
62
+ }
63
+ return { has_media: urls.length > 0, media_urls: urls };
64
+ }
33
65
  export const __test__ = {
34
66
  sanitizeQueryId,
67
+ extractMedia,
35
68
  };
@@ -0,0 +1,96 @@
1
+ import { describe, expect, it } from 'vitest';
2
+ import { __test__ } from './shared.js';
3
+
4
+ const { extractMedia } = __test__;
5
+
6
+ describe('twitter extractMedia', () => {
7
+ it('returns false + empty list when legacy has no media', () => {
8
+ expect(extractMedia({})).toEqual({ has_media: false, media_urls: [] });
9
+ expect(extractMedia(undefined)).toEqual({ has_media: false, media_urls: [] });
10
+ expect(extractMedia({ extended_entities: { media: [] } })).toEqual({
11
+ has_media: false,
12
+ media_urls: [],
13
+ });
14
+ });
15
+
16
+ it('extracts photo urls from extended_entities', () => {
17
+ const result = extractMedia({
18
+ extended_entities: {
19
+ media: [
20
+ { type: 'photo', media_url_https: 'https://pbs.twimg.com/media/a.jpg' },
21
+ { type: 'photo', media_url_https: 'https://pbs.twimg.com/media/b.jpg' },
22
+ ],
23
+ },
24
+ });
25
+ expect(result.has_media).toBe(true);
26
+ expect(result.media_urls).toEqual([
27
+ 'https://pbs.twimg.com/media/a.jpg',
28
+ 'https://pbs.twimg.com/media/b.jpg',
29
+ ]);
30
+ });
31
+
32
+ it('prefers mp4 variant for video and animated_gif', () => {
33
+ const result = extractMedia({
34
+ extended_entities: {
35
+ media: [
36
+ {
37
+ type: 'video',
38
+ media_url_https: 'https://pbs.twimg.com/media/thumb.jpg',
39
+ video_info: {
40
+ variants: [
41
+ { content_type: 'application/x-mpegURL', url: 'https://video.twimg.com/x.m3u8' },
42
+ { content_type: 'video/mp4', url: 'https://video.twimg.com/x.mp4' },
43
+ ],
44
+ },
45
+ },
46
+ {
47
+ type: 'animated_gif',
48
+ media_url_https: 'https://pbs.twimg.com/tweet_video_thumb/g.jpg',
49
+ video_info: {
50
+ variants: [
51
+ { content_type: 'video/mp4', url: 'https://video.twimg.com/g.mp4' },
52
+ ],
53
+ },
54
+ },
55
+ ],
56
+ },
57
+ });
58
+ expect(result.has_media).toBe(true);
59
+ expect(result.media_urls).toEqual([
60
+ 'https://video.twimg.com/x.mp4',
61
+ 'https://video.twimg.com/g.mp4',
62
+ ]);
63
+ });
64
+
65
+ it('falls back to media_url_https when no mp4 variant is available', () => {
66
+ const result = extractMedia({
67
+ extended_entities: {
68
+ media: [
69
+ {
70
+ type: 'video',
71
+ media_url_https: 'https://pbs.twimg.com/media/thumb.jpg',
72
+ video_info: { variants: [] },
73
+ },
74
+ ],
75
+ },
76
+ });
77
+ expect(result).toEqual({
78
+ has_media: true,
79
+ media_urls: ['https://pbs.twimg.com/media/thumb.jpg'],
80
+ });
81
+ });
82
+
83
+ it('falls back to entities.media when extended_entities is missing', () => {
84
+ const result = extractMedia({
85
+ entities: {
86
+ media: [
87
+ { type: 'photo', media_url_https: 'https://pbs.twimg.com/media/c.jpg' },
88
+ ],
89
+ },
90
+ });
91
+ expect(result).toEqual({
92
+ has_media: true,
93
+ media_urls: ['https://pbs.twimg.com/media/c.jpg'],
94
+ });
95
+ });
96
+ });
@@ -1,5 +1,6 @@
1
1
  import { cli, Strategy } from '@jackwener/opencli/registry';
2
2
  import { AuthRequiredError, CommandExecutionError } from '@jackwener/opencli/errors';
3
+ import { extractMedia } from './shared.js';
3
4
  // ── Twitter GraphQL constants ──────────────────────────────────────────
4
5
  const BEARER_TOKEN = 'AAAAAAAAAAAAAAAAAAAAANRILgAAAAAAnNwIzUejRCOuH5E6I8xnZz4puTs%3D1Zv7ttfk8LF81IUq16cHjhLTvJu4FA33AGWWjCpTnA';
5
6
  const TWEET_DETAIL_QUERY_ID = 'nBS-WpgA6ZG0CyNHD517JQ';
@@ -54,6 +55,7 @@ function extractTweet(r, seen) {
54
55
  in_reply_to: l.in_reply_to_status_id_str || undefined,
55
56
  created_at: l.created_at,
56
57
  url: `https://x.com/${screenName}/status/${tw.rest_id}`,
58
+ ...extractMedia(l),
57
59
  };
58
60
  }
59
61
  function parseTweetDetail(data, seen) {
@@ -101,7 +103,7 @@ cli({
101
103
  { name: 'tweet-id', positional: true, type: 'string', required: true },
102
104
  { name: 'limit', type: 'int', default: 50 },
103
105
  ],
104
- columns: ['id', 'author', 'text', 'likes', 'retweets', 'url'],
106
+ columns: ['id', 'author', 'text', 'likes', 'retweets', 'url', 'has_media', 'media_urls'],
105
107
  func: async (page, kwargs) => {
106
108
  let tweetId = kwargs['tweet-id'];
107
109
  const urlMatch = tweetId.match(/\/status\/(\d+)/);
@@ -1,6 +1,6 @@
1
1
  import { AuthRequiredError, CommandExecutionError } from '@jackwener/opencli/errors';
2
2
  import { cli, Strategy } from '@jackwener/opencli/registry';
3
- import { resolveTwitterQueryId } from './shared.js';
3
+ import { resolveTwitterQueryId, extractMedia } from './shared.js';
4
4
  // ── Twitter GraphQL constants ──────────────────────────────────────────
5
5
  const BEARER_TOKEN = 'AAAAAAAAAAAAAAAAAAAAANRILgAAAAAAnNwIzUejRCOuH5E6I8xnZz4puTs%3D1Zv7ttfk8LF81IUq16cHjhLTvJu4FA33AGWWjCpTnA';
6
6
  const HOME_TIMELINE_QUERY_ID = 'c-CzHF1LboFilMpsx4ZCrQ';
@@ -85,6 +85,7 @@ function extractTweet(result, seen) {
85
85
  views,
86
86
  created_at: l.created_at || '',
87
87
  url: `https://x.com/${screenName}/status/${tw.rest_id}`,
88
+ ...extractMedia(l),
88
89
  };
89
90
  }
90
91
  function parseHomeTimeline(data, seen) {
@@ -148,7 +149,7 @@ cli({
148
149
  },
149
150
  { name: 'limit', type: 'int', default: 20 },
150
151
  ],
151
- columns: ['id', 'author', 'text', 'likes', 'retweets', 'replies', 'views', 'created_at', 'url'],
152
+ columns: ['id', 'author', 'text', 'likes', 'retweets', 'replies', 'views', 'created_at', 'url', 'has_media', 'media_urls'],
152
153
  func: async (page, kwargs) => {
153
154
  const limit = kwargs.limit || 20;
154
155
  const timelineType = kwargs.type === 'following' ? 'following' : 'for-you';
@@ -0,0 +1,219 @@
1
+ import { cli, Strategy } from '@jackwener/opencli/registry';
2
+ import { AuthRequiredError, CommandExecutionError, EmptyResultError } from '@jackwener/opencli/errors';
3
+ import { resolveTwitterQueryId, sanitizeQueryId, extractMedia } from './shared.js';
4
+
5
+ const BEARER_TOKEN = 'AAAAAAAAAAAAAAAAAAAAANRILgAAAAAAnNwIzUejRCOuH5E6I8xnZz4puTs%3D1Zv7ttfk8LF81IUq16cHjhLTvJu4FA33AGWWjCpTnA';
6
+ const USER_TWEETS_QUERY_ID = '6fWQaBPK51aGyC_VC7t9GQ';
7
+ const USER_BY_SCREEN_NAME_QUERY_ID = 'IGgvgiOx4QZndDHuD3x9TQ';
8
+
9
+ const USER_TWEETS_FEATURES = {
10
+ rweb_video_screen_enabled: false,
11
+ payments_enabled: false,
12
+ profile_label_improvements_pcf_label_in_post_enabled: true,
13
+ rweb_tipjar_consumption_enabled: true,
14
+ verified_phone_label_enabled: false,
15
+ creator_subscriptions_tweet_preview_api_enabled: true,
16
+ responsive_web_graphql_timeline_navigation_enabled: true,
17
+ responsive_web_graphql_skip_user_profile_image_extensions_enabled: false,
18
+ premium_content_api_read_enabled: false,
19
+ communities_web_enable_tweet_community_results_fetch: true,
20
+ c9s_tweet_anatomy_moderator_badge_enabled: true,
21
+ responsive_web_grok_analyze_button_fetch_trends_enabled: false,
22
+ responsive_web_grok_analyze_post_followups_enabled: true,
23
+ responsive_web_jetfuel_frame: true,
24
+ responsive_web_grok_share_attachment_enabled: true,
25
+ responsive_web_grok_annotations_enabled: true,
26
+ articles_preview_enabled: true,
27
+ responsive_web_edit_tweet_api_enabled: true,
28
+ graphql_is_translatable_rweb_tweet_is_translatable_enabled: true,
29
+ view_counts_everywhere_api_enabled: true,
30
+ longform_notetweets_consumption_enabled: true,
31
+ responsive_web_twitter_article_tweet_consumption_enabled: true,
32
+ tweet_awards_web_tipping_enabled: false,
33
+ content_disclosure_indicator_enabled: true,
34
+ content_disclosure_ai_generated_indicator_enabled: true,
35
+ responsive_web_grok_show_grok_translated_post: false,
36
+ responsive_web_grok_analysis_button_from_backend: true,
37
+ post_ctas_fetch_enabled: false,
38
+ freedom_of_speech_not_reach_fetch_enabled: true,
39
+ standardized_nudges_misinfo: true,
40
+ tweet_with_visibility_results_prefer_gql_limited_actions_policy_enabled: true,
41
+ longform_notetweets_rich_text_read_enabled: true,
42
+ longform_notetweets_inline_media_enabled: true,
43
+ responsive_web_grok_image_annotation_enabled: true,
44
+ responsive_web_grok_imagine_annotation_enabled: true,
45
+ responsive_web_grok_community_note_auto_translation_is_enabled: false,
46
+ responsive_web_enhance_cards_enabled: false,
47
+ };
48
+
49
+ const USER_BY_SCREEN_NAME_FEATURES = {
50
+ hidden_profile_subscriptions_enabled: true,
51
+ rweb_tipjar_consumption_enabled: true,
52
+ responsive_web_graphql_exclude_directive_enabled: true,
53
+ verified_phone_label_enabled: false,
54
+ subscriptions_verification_info_is_identity_verified_enabled: true,
55
+ subscriptions_verification_info_verified_since_enabled: true,
56
+ highlights_tweets_tab_ui_enabled: true,
57
+ responsive_web_twitter_article_notes_tab_enabled: true,
58
+ subscriptions_feature_can_gift_premium: true,
59
+ creator_subscriptions_tweet_preview_api_enabled: true,
60
+ responsive_web_graphql_skip_user_profile_image_extensions_enabled: false,
61
+ responsive_web_graphql_timeline_navigation_enabled: true,
62
+ };
63
+
64
+ function buildUserTweetsUrl(queryId, userId, count, cursor) {
65
+ const vars = {
66
+ userId,
67
+ count,
68
+ includePromotedContent: false,
69
+ withQuickPromoteEligibilityTweetFields: true,
70
+ withVoice: true,
71
+ };
72
+ if (cursor) vars.cursor = cursor;
73
+ return `/i/api/graphql/${queryId}/UserTweets`
74
+ + `?variables=${encodeURIComponent(JSON.stringify(vars))}`
75
+ + `&features=${encodeURIComponent(JSON.stringify(USER_TWEETS_FEATURES))}`;
76
+ }
77
+
78
+ function buildUserByScreenNameUrl(queryId, screenName) {
79
+ const vars = { screen_name: screenName, withSafetyModeUserFields: true };
80
+ return `/i/api/graphql/${queryId}/UserByScreenName`
81
+ + `?variables=${encodeURIComponent(JSON.stringify(vars))}`
82
+ + `&features=${encodeURIComponent(JSON.stringify(USER_BY_SCREEN_NAME_FEATURES))}`;
83
+ }
84
+
85
+ function extractTweet(result, seen) {
86
+ if (!result) return null;
87
+ const tw = result.tweet || result;
88
+ const legacy = tw.legacy || {};
89
+ if (!tw.rest_id || seen.has(tw.rest_id)) return null;
90
+ seen.add(tw.rest_id);
91
+ const user = tw.core?.user_results?.result;
92
+ const screenName = user?.legacy?.screen_name || user?.core?.screen_name || 'unknown';
93
+ const displayName = user?.legacy?.name || user?.core?.name || '';
94
+ const noteText = tw.note_tweet?.note_tweet_results?.result?.text;
95
+ const isRetweet = Boolean(legacy.retweeted_status_result || legacy.full_text?.startsWith('RT @'));
96
+ return {
97
+ id: tw.rest_id,
98
+ author: screenName,
99
+ name: displayName,
100
+ text: noteText || legacy.full_text || '',
101
+ likes: legacy.favorite_count || 0,
102
+ retweets: legacy.retweet_count || 0,
103
+ replies: legacy.reply_count || 0,
104
+ views: Number(tw.views?.count) || 0,
105
+ is_retweet: isRetweet,
106
+ created_at: legacy.created_at || '',
107
+ url: `https://x.com/${screenName}/status/${tw.rest_id}`,
108
+ ...extractMedia(legacy),
109
+ };
110
+ }
111
+
112
+ function parseUserTweets(data, seen) {
113
+ const tweets = [];
114
+ let nextCursor = null;
115
+ const instructions = data?.data?.user?.result?.timeline_v2?.timeline?.instructions
116
+ || data?.data?.user?.result?.timeline?.timeline?.instructions
117
+ || [];
118
+ for (const inst of instructions) {
119
+ if (inst.type === 'TimelinePinEntry') continue;
120
+ for (const entry of inst.entries || []) {
121
+ const content = entry.content;
122
+ if (content?.entryType === 'TimelineTimelineCursor' || content?.__typename === 'TimelineTimelineCursor') {
123
+ if (content.cursorType === 'Bottom' || content.cursorType === 'ShowMore') nextCursor = content.value;
124
+ continue;
125
+ }
126
+ if (entry.entryId?.startsWith('cursor-bottom-') || entry.entryId?.startsWith('cursor-showMore-')) {
127
+ nextCursor = content?.value || content?.itemContent?.value || nextCursor;
128
+ continue;
129
+ }
130
+ const direct = extractTweet(content?.itemContent?.tweet_results?.result, seen);
131
+ if (direct) {
132
+ tweets.push(direct);
133
+ continue;
134
+ }
135
+ for (const item of content?.items || []) {
136
+ const nested = extractTweet(item.item?.itemContent?.tweet_results?.result, seen);
137
+ if (nested) tweets.push(nested);
138
+ }
139
+ }
140
+ }
141
+ return { tweets, nextCursor };
142
+ }
143
+
144
+ cli({
145
+ site: 'twitter',
146
+ name: 'tweets',
147
+ description: "Fetch a Twitter user's most recent tweets (chronological, excludes pinned)",
148
+ domain: 'x.com',
149
+ strategy: Strategy.COOKIE,
150
+ browser: true,
151
+ args: [
152
+ { name: 'username', type: 'string', positional: true, required: true, help: 'Twitter screen name (with or without @)' },
153
+ { name: 'limit', type: 'int', default: 20, help: 'Max tweets to return' },
154
+ ],
155
+ columns: ['author', 'created_at', 'is_retweet', 'text', 'likes', 'retweets', 'replies', 'views', 'url', 'has_media', 'media_urls'],
156
+ func: async (page, kwargs) => {
157
+ const limit = Math.max(1, Math.min(200, kwargs.limit || 20));
158
+ const username = String(kwargs.username || '').replace(/^@/, '').trim();
159
+ if (!username) throw new CommandExecutionError('username is required');
160
+
161
+ await page.goto('https://x.com');
162
+ await page.wait(3);
163
+
164
+ const ct0 = await page.evaluate(`() => {
165
+ return document.cookie.split(';').map(c => c.trim()).find(c => c.startsWith('ct0='))?.split('=')[1] || null;
166
+ }`);
167
+ if (!ct0) throw new AuthRequiredError('x.com', 'Not logged into x.com (no ct0 cookie)');
168
+
169
+ const userTweetsQueryId = await resolveTwitterQueryId(page, 'UserTweets', USER_TWEETS_QUERY_ID);
170
+ const userByScreenNameQueryId = await resolveTwitterQueryId(page, 'UserByScreenName', USER_BY_SCREEN_NAME_QUERY_ID);
171
+
172
+ const headers = JSON.stringify({
173
+ 'Authorization': `Bearer ${decodeURIComponent(BEARER_TOKEN)}`,
174
+ 'X-Csrf-Token': ct0,
175
+ 'X-Twitter-Auth-Type': 'OAuth2Session',
176
+ 'X-Twitter-Active-User': 'yes',
177
+ });
178
+
179
+ const ubsUrl = buildUserByScreenNameUrl(userByScreenNameQueryId, username);
180
+ const userId = await page.evaluate(`async () => {
181
+ const resp = await fetch("${ubsUrl}", { headers: ${headers}, credentials: 'include' });
182
+ if (!resp.ok) return null;
183
+ const d = await resp.json();
184
+ return d?.data?.user?.result?.rest_id || null;
185
+ }`);
186
+ if (!userId) throw new CommandExecutionError(`Could not resolve @${username}`);
187
+
188
+ const seen = new Set();
189
+ const all = [];
190
+ let cursor = null;
191
+ for (let i = 0; i < 5 && all.length < limit; i++) {
192
+ const fetchCount = Math.min(100, limit - all.length + 10);
193
+ const url = buildUserTweetsUrl(userTweetsQueryId, userId, fetchCount, cursor);
194
+ const data = await page.evaluate(`async () => {
195
+ const r = await fetch("${url}", { headers: ${headers}, credentials: 'include' });
196
+ return r.ok ? await r.json() : { error: r.status };
197
+ }`);
198
+ if (data?.error) {
199
+ if (all.length === 0) throw new CommandExecutionError(`HTTP ${data.error}: UserTweets fetch failed — queryId may have expired`);
200
+ break;
201
+ }
202
+ const { tweets, nextCursor } = parseUserTweets(data, seen);
203
+ all.push(...tweets);
204
+ if (!nextCursor || nextCursor === cursor) break;
205
+ cursor = nextCursor;
206
+ }
207
+
208
+ if (all.length === 0) throw new EmptyResultError(`@${username} has no recent tweets`, 'Account may be private or suspended');
209
+ return all.slice(0, limit);
210
+ },
211
+ });
212
+
213
+ export const __test__ = {
214
+ sanitizeQueryId,
215
+ buildUserTweetsUrl,
216
+ buildUserByScreenNameUrl,
217
+ extractTweet,
218
+ parseUserTweets,
219
+ };
@@ -0,0 +1,125 @@
1
+ import { describe, expect, it } from 'vitest';
2
+ import { getRegistry } from '@jackwener/opencli/registry';
3
+ import { __test__ } from './tweets.js';
4
+
5
+ describe('twitter tweets helpers', () => {
6
+ it('registers is_retweet in the default columns', () => {
7
+ const cmd = getRegistry().get('twitter/tweets');
8
+ expect(cmd?.columns).toEqual(['author', 'created_at', 'is_retweet', 'text', 'likes', 'retweets', 'replies', 'views', 'url', 'has_media', 'media_urls']);
9
+ });
10
+
11
+ it('falls back when queryId contains unsafe characters', () => {
12
+ expect(__test__.sanitizeQueryId('safe_Query-123', 'fallback')).toBe('safe_Query-123');
13
+ expect(__test__.sanitizeQueryId('bad"id', 'fallback')).toBe('fallback');
14
+ expect(__test__.sanitizeQueryId('bad/id', 'fallback')).toBe('fallback');
15
+ expect(__test__.sanitizeQueryId(null, 'fallback')).toBe('fallback');
16
+ });
17
+
18
+ it('builds UserTweets url with cursor and features', () => {
19
+ const url = __test__.buildUserTweetsUrl('query123', '42', 20, 'cursor-1');
20
+ expect(url).toContain('/i/api/graphql/query123/UserTweets');
21
+ const decoded = decodeURIComponent(url);
22
+ expect(decoded).toContain('"userId":"42"');
23
+ expect(decoded).toContain('"count":20');
24
+ expect(decoded).toContain('"cursor":"cursor-1"');
25
+ expect(decoded).toContain('longform_notetweets_consumption_enabled');
26
+ });
27
+
28
+ it('builds UserByScreenName url for the given handle', () => {
29
+ const url = __test__.buildUserByScreenNameUrl('uquery', 'jakevin7');
30
+ expect(url).toContain('/i/api/graphql/uquery/UserByScreenName');
31
+ expect(decodeURIComponent(url)).toContain('"screen_name":"jakevin7"');
32
+ });
33
+
34
+ it('prefers note_tweet text over legacy.full_text for long posts', () => {
35
+ const seen = new Set();
36
+ const tweet = __test__.extractTweet({
37
+ rest_id: '99',
38
+ legacy: { full_text: 'short truncated…', favorite_count: 1, retweet_count: 0, reply_count: 0, created_at: 'now' },
39
+ note_tweet: { note_tweet_results: { result: { text: 'full long-form body' } } },
40
+ core: { user_results: { result: { legacy: { screen_name: 'bob', name: 'Bob' } } } },
41
+ views: { count: '42' },
42
+ }, seen);
43
+ expect(tweet.text).toBe('full long-form body');
44
+ expect(tweet.views).toBe(42);
45
+ });
46
+
47
+ it('flags retweets via RT prefix or retweeted_status_result', () => {
48
+ const a = __test__.extractTweet({
49
+ rest_id: '1',
50
+ legacy: { full_text: 'RT @foo: hi', favorite_count: 0, retweet_count: 0, reply_count: 0, created_at: '' },
51
+ core: { user_results: { result: { legacy: { screen_name: 'u', name: 'U' } } } },
52
+ }, new Set());
53
+ expect(a.is_retweet).toBe(true);
54
+
55
+ const b = __test__.extractTweet({
56
+ rest_id: '2',
57
+ legacy: { full_text: 'hello', favorite_count: 0, retweet_count: 0, reply_count: 0, created_at: '', retweeted_status_result: { result: {} } },
58
+ core: { user_results: { result: { legacy: { screen_name: 'u', name: 'U' } } } },
59
+ }, new Set());
60
+ expect(b.is_retweet).toBe(true);
61
+ });
62
+
63
+ it('parses chronological tweets and skips pinned instruction', () => {
64
+ const chronEntry = {
65
+ entryId: 'tweet-1',
66
+ content: {
67
+ itemContent: {
68
+ tweet_results: {
69
+ result: {
70
+ rest_id: '1',
71
+ legacy: { full_text: 'chronological post', favorite_count: 5, retweet_count: 1, reply_count: 2, created_at: 'now' },
72
+ core: { user_results: { result: { legacy: { screen_name: 'alice', name: 'Alice' } } } },
73
+ views: { count: '100' },
74
+ },
75
+ },
76
+ },
77
+ },
78
+ };
79
+ const cursorEntry = {
80
+ entryId: 'cursor-bottom-1',
81
+ content: { entryType: 'TimelineTimelineCursor', cursorType: 'Bottom', value: 'cursor-next' },
82
+ };
83
+ const pinnedEntry = {
84
+ entryId: 'tweet-pinned-999',
85
+ content: {
86
+ itemContent: {
87
+ tweet_results: {
88
+ result: {
89
+ rest_id: '999',
90
+ legacy: { full_text: 'pinned post', favorite_count: 0, retweet_count: 0, reply_count: 0, created_at: 'old' },
91
+ core: { user_results: { result: { legacy: { screen_name: 'alice', name: 'Alice' } } } },
92
+ },
93
+ },
94
+ },
95
+ },
96
+ };
97
+ const payload = {
98
+ data: {
99
+ user: {
100
+ result: {
101
+ timeline_v2: {
102
+ timeline: {
103
+ instructions: [
104
+ { type: 'TimelinePinEntry', entries: [pinnedEntry] },
105
+ { entries: [chronEntry, cursorEntry] },
106
+ ],
107
+ },
108
+ },
109
+ },
110
+ },
111
+ },
112
+ };
113
+ const result = __test__.parseUserTweets(payload, new Set());
114
+ expect(result.nextCursor).toBe('cursor-next');
115
+ expect(result.tweets).toHaveLength(1);
116
+ expect(result.tweets[0]).toMatchObject({
117
+ id: '1',
118
+ author: 'alice',
119
+ text: 'chronological post',
120
+ likes: 5,
121
+ views: 100,
122
+ url: 'https://x.com/alice/status/1',
123
+ });
124
+ });
125
+ });