@jackwener/opencli 1.7.18 → 1.7.19
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +7 -8
- package/README.zh-CN.md +7 -8
- package/cli-manifest.json +305 -9
- package/clis/ctrip/ctrip.test.js +486 -1
- package/clis/ctrip/flight.js +136 -0
- package/clis/ctrip/hotel-search.js +132 -0
- package/clis/ctrip/utils.js +298 -0
- package/clis/google/search.js +16 -6
- package/clis/google-scholar/search.js +20 -5
- package/clis/google-scholar/search.test.js +35 -2
- package/clis/reddit/home.js +117 -0
- package/clis/reddit/home.test.js +127 -0
- package/clis/reddit/read.js +400 -54
- package/clis/reddit/read.test.js +315 -12
- package/clis/reddit/subreddit-info.js +117 -0
- package/clis/reddit/subreddit-info.test.js +163 -0
- package/clis/reddit/whoami.js +84 -0
- package/clis/reddit/whoami.test.js +105 -0
- package/clis/rednote/search.js +6 -2
- package/clis/twitter/bookmark-folder.js +3 -1
- package/clis/twitter/bookmarks.js +3 -1
- package/clis/twitter/followers.js +20 -5
- package/clis/twitter/followers.test.js +44 -0
- package/clis/twitter/following.js +36 -20
- package/clis/twitter/following.test.js +60 -8
- package/clis/twitter/likes.js +28 -13
- package/clis/twitter/likes.test.js +111 -1
- package/clis/twitter/list-add.js +128 -204
- package/clis/twitter/list-add.test.js +97 -1
- package/clis/twitter/list-tweets.js +13 -4
- package/clis/twitter/list-tweets.test.js +48 -0
- package/clis/twitter/lists.js +5 -2
- package/clis/twitter/post.js +23 -4
- package/clis/twitter/post.test.js +30 -0
- package/clis/twitter/profile.js +16 -8
- package/clis/twitter/profile.test.js +39 -0
- package/clis/twitter/reply.js +133 -10
- package/clis/twitter/reply.test.js +55 -0
- package/clis/twitter/search.js +188 -170
- package/clis/twitter/search.test.js +96 -258
- package/clis/twitter/shared.js +167 -16
- package/clis/twitter/shared.test.js +102 -1
- package/clis/twitter/timeline.js +3 -1
- package/clis/twitter/tweets.js +147 -51
- package/clis/twitter/tweets.test.js +238 -1
- package/clis/xiaohongshu/comments.js +23 -2
- package/clis/xiaohongshu/comments.test.js +63 -1
- package/clis/xiaohongshu/search.js +168 -13
- package/clis/xiaohongshu/search.test.js +82 -8
- package/clis/xueqiu/earnings-date.js +2 -2
- package/clis/xueqiu/kline.js +2 -2
- package/clis/xueqiu/utils.js +19 -0
- package/clis/xueqiu/utils.test.js +26 -0
- package/clis/zhihu/answer-detail.js +233 -0
- package/clis/zhihu/answer-detail.test.js +330 -0
- package/clis/zhihu/question.js +44 -10
- package/clis/zhihu/question.test.js +78 -1
- package/clis/zhihu/recommend.js +103 -0
- package/clis/zhihu/recommend.test.js +143 -0
- package/dist/src/browser/base-page.d.ts +3 -2
- package/dist/src/browser/base-page.test.js +2 -2
- package/dist/src/browser/cdp.js +3 -3
- package/dist/src/browser/page.d.ts +3 -2
- package/dist/src/browser/page.js +4 -4
- package/dist/src/browser/page.test.js +31 -0
- package/dist/src/browser/utils.d.ts +10 -0
- package/dist/src/browser/utils.js +37 -0
- package/dist/src/browser/utils.test.d.ts +1 -0
- package/dist/src/browser/utils.test.js +29 -0
- package/dist/src/cli-argv-preprocess.d.ts +37 -0
- package/dist/src/cli-argv-preprocess.js +131 -0
- package/dist/src/cli-argv-preprocess.test.d.ts +1 -0
- package/dist/src/cli-argv-preprocess.test.js +130 -0
- package/dist/src/cli.js +123 -86
- package/dist/src/cli.test.js +33 -28
- package/dist/src/commands/daemon.js +6 -7
- package/dist/src/doctor.js +15 -16
- package/dist/src/download/progress.js +15 -11
- package/dist/src/download/progress.test.d.ts +1 -0
- package/dist/src/download/progress.test.js +25 -0
- package/dist/src/execution.js +1 -3
- package/dist/src/execution.test.js +4 -16
- package/dist/src/help.d.ts +11 -0
- package/dist/src/help.js +46 -5
- package/dist/src/logger.js +8 -9
- package/dist/src/main.js +16 -0
- package/dist/src/output.js +4 -5
- package/dist/src/runtime-detect.d.ts +1 -1
- package/dist/src/runtime-detect.js +1 -1
- package/dist/src/runtime-detect.test.js +3 -2
- package/dist/src/tui.d.ts +0 -1
- package/dist/src/tui.js +9 -22
- package/dist/src/types.d.ts +3 -1
- package/dist/src/update-check.js +4 -5
- package/package.json +5 -4
package/clis/twitter/shared.js
CHANGED
|
@@ -1,8 +1,28 @@
|
|
|
1
1
|
import { ArgumentError } from '@jackwener/opencli/errors';
|
|
2
2
|
|
|
3
3
|
const QUERY_ID_PATTERN = /^[A-Za-z0-9_-]+$/;
|
|
4
|
+
const SCREEN_NAME_PATTERN = /^[A-Za-z0-9_]{1,15}$/;
|
|
4
5
|
const TWEET_PATH_PATTERN = /^\/(?:[^/]+|i)\/status\/(\d+)\/?$/;
|
|
5
6
|
const TWEET_HOSTS = new Set(['x.com', 'twitter.com']);
|
|
7
|
+
const SCREEN_NAME_HOSTS = new Set(['x.com', 'twitter.com', 'mobile.twitter.com']);
|
|
8
|
+
const RESERVED_SCREEN_NAME_PATHS = new Set([
|
|
9
|
+
'compose',
|
|
10
|
+
'explore',
|
|
11
|
+
'help',
|
|
12
|
+
'home',
|
|
13
|
+
'i',
|
|
14
|
+
'intent',
|
|
15
|
+
'jobs',
|
|
16
|
+
'login',
|
|
17
|
+
'logout',
|
|
18
|
+
'messages',
|
|
19
|
+
'notifications',
|
|
20
|
+
'privacy',
|
|
21
|
+
'search',
|
|
22
|
+
'settings',
|
|
23
|
+
'signup',
|
|
24
|
+
'tos',
|
|
25
|
+
]);
|
|
6
26
|
|
|
7
27
|
function isTwitterHost(hostname) {
|
|
8
28
|
return TWEET_HOSTS.has(hostname)
|
|
@@ -81,9 +101,138 @@ export function buildTwitterArticleScopeSource(tweetId) {
|
|
|
81
101
|
export function sanitizeQueryId(resolved, fallbackId) {
|
|
82
102
|
return typeof resolved === 'string' && QUERY_ID_PATTERN.test(resolved) ? resolved : fallbackId;
|
|
83
103
|
}
|
|
84
|
-
|
|
104
|
+
|
|
105
|
+
export function normalizeTwitterScreenName(value) {
|
|
106
|
+
const raw = String(value ?? '').trim();
|
|
107
|
+
if (!raw) return '';
|
|
108
|
+
let candidate = '';
|
|
109
|
+
try {
|
|
110
|
+
const url = raw.startsWith('/') ? new URL(raw, 'https://x.com') : new URL(raw);
|
|
111
|
+
if (
|
|
112
|
+
url.protocol !== 'https:' ||
|
|
113
|
+
url.username ||
|
|
114
|
+
url.password ||
|
|
115
|
+
url.port ||
|
|
116
|
+
!SCREEN_NAME_HOSTS.has(url.hostname)
|
|
117
|
+
) {
|
|
118
|
+
return '';
|
|
119
|
+
}
|
|
120
|
+
const segments = url.pathname.split('/').filter(Boolean);
|
|
121
|
+
if (segments.length !== 1) return '';
|
|
122
|
+
candidate = segments[0];
|
|
123
|
+
} catch {
|
|
124
|
+
if (raw.includes('/') || raw.includes('?') || raw.includes('#')) return '';
|
|
125
|
+
candidate = raw.replace(/^@+/, '');
|
|
126
|
+
}
|
|
127
|
+
if (!SCREEN_NAME_PATTERN.test(candidate)) return '';
|
|
128
|
+
if (RESERVED_SCREEN_NAME_PATHS.has(candidate.toLowerCase())) return '';
|
|
129
|
+
return candidate;
|
|
130
|
+
}
|
|
131
|
+
|
|
132
|
+
function keysToFlags(keys) {
|
|
133
|
+
if (!Array.isArray(keys)) return {};
|
|
134
|
+
return Object.fromEntries(keys.filter((key) => typeof key === 'string' && key).map((key) => [key, true]));
|
|
135
|
+
}
|
|
136
|
+
|
|
137
|
+
function normalizeOperationFallback(fallback) {
|
|
138
|
+
if (typeof fallback === 'string') return { queryId: fallback, features: {}, fieldToggles: {} };
|
|
139
|
+
return {
|
|
140
|
+
queryId: fallback?.queryId || null,
|
|
141
|
+
features: fallback?.features || {},
|
|
142
|
+
fieldToggles: fallback?.fieldToggles || {},
|
|
143
|
+
};
|
|
144
|
+
}
|
|
145
|
+
|
|
146
|
+
export function unwrapBrowserResult(value) {
|
|
147
|
+
if (
|
|
148
|
+
value
|
|
149
|
+
&& typeof value === 'object'
|
|
150
|
+
&& typeof value.session === 'string'
|
|
151
|
+
&& Object.prototype.hasOwnProperty.call(value, 'data')
|
|
152
|
+
) {
|
|
153
|
+
return value.data;
|
|
154
|
+
}
|
|
155
|
+
return value;
|
|
156
|
+
}
|
|
157
|
+
|
|
158
|
+
export function normalizeTwitterGraphqlPayload(value) {
|
|
159
|
+
const unwrapped = unwrapBrowserResult(value);
|
|
160
|
+
if (unwrapped?.data && typeof unwrapped.data === 'object') return unwrapped;
|
|
161
|
+
if (
|
|
162
|
+
unwrapped
|
|
163
|
+
&& typeof unwrapped === 'object'
|
|
164
|
+
&& (
|
|
165
|
+
Object.prototype.hasOwnProperty.call(unwrapped, 'user')
|
|
166
|
+
|| Object.prototype.hasOwnProperty.call(unwrapped, 'search_by_raw_query')
|
|
167
|
+
)
|
|
168
|
+
) {
|
|
169
|
+
return { data: unwrapped };
|
|
170
|
+
}
|
|
171
|
+
return unwrapped;
|
|
172
|
+
}
|
|
173
|
+
|
|
174
|
+
export function sanitizeTwitterOperationMetadata(resolved, fallback) {
|
|
175
|
+
const value = unwrapBrowserResult(resolved);
|
|
176
|
+
const normalizedFallback = normalizeOperationFallback(fallback);
|
|
177
|
+
// Empty resolved features / fieldToggles must defer to the baked fallback.
|
|
178
|
+
// The bundle parser can find a queryId but miss `featureSwitches:[...]` (e.g.
|
|
179
|
+
// a minification change, or the 2500-char snippet window truncating before
|
|
180
|
+
// the array). When that happens, keysToFlags(undefined) returns {}; if we
|
|
181
|
+
// kept it, Twitter would receive an empty `features` map and respond 400,
|
|
182
|
+
// surfacing a misleading "queryId expired" error.
|
|
183
|
+
return {
|
|
184
|
+
queryId: sanitizeQueryId(value?.queryId, normalizedFallback.queryId),
|
|
185
|
+
features: value?.features
|
|
186
|
+
&& typeof value.features === 'object'
|
|
187
|
+
&& Object.keys(value.features).length > 0
|
|
188
|
+
? value.features
|
|
189
|
+
: normalizedFallback.features,
|
|
190
|
+
fieldToggles: value?.fieldToggles
|
|
191
|
+
&& typeof value.fieldToggles === 'object'
|
|
192
|
+
&& Object.keys(value.fieldToggles).length > 0
|
|
193
|
+
? value.fieldToggles
|
|
194
|
+
: normalizedFallback.fieldToggles,
|
|
195
|
+
};
|
|
196
|
+
}
|
|
197
|
+
|
|
198
|
+
export async function resolveTwitterOperationMetadata(page, operationName, fallback) {
|
|
85
199
|
const resolved = await page.evaluate(`async () => {
|
|
86
200
|
const operationName = ${JSON.stringify(operationName)};
|
|
201
|
+
const keysToFlags = (keys) => Object.fromEntries((keys || []).map((key) => [key, true]));
|
|
202
|
+
const quotedKeys = (source) => source
|
|
203
|
+
? Array.from(source.matchAll(/"([^"]+)"/g)).map((match) => match[1])
|
|
204
|
+
: [];
|
|
205
|
+
const parseOperation = (text) => {
|
|
206
|
+
const marker = 'operationName:"' + operationName + '"';
|
|
207
|
+
const index = text.indexOf(marker);
|
|
208
|
+
if (index < 0) return null;
|
|
209
|
+
const start = Math.max(0, text.lastIndexOf('e.exports=', index));
|
|
210
|
+
const endMarker = text.indexOf('}}}', index);
|
|
211
|
+
const snippet = text.slice(start, endMarker > index ? endMarker + 3 : index + 2500);
|
|
212
|
+
const queryId = snippet.match(/queryId:"([A-Za-z0-9_-]+)"/)?.[1] || null;
|
|
213
|
+
if (!queryId) return null;
|
|
214
|
+
return {
|
|
215
|
+
queryId,
|
|
216
|
+
features: keysToFlags(quotedKeys(snippet.match(/featureSwitches:\\[([^\\]]*)\\]/)?.[1])),
|
|
217
|
+
fieldToggles: keysToFlags(quotedKeys(snippet.match(/fieldToggles:\\[([^\\]]*)\\]/)?.[1])),
|
|
218
|
+
};
|
|
219
|
+
};
|
|
220
|
+
try {
|
|
221
|
+
const scripts = Array.from(document.scripts)
|
|
222
|
+
.map(s => s.src)
|
|
223
|
+
.filter(Boolean)
|
|
224
|
+
.concat(performance.getEntriesByType('resource')
|
|
225
|
+
.map(r => r.name)
|
|
226
|
+
.filter(r => r.includes('client-web') && r.endsWith('.js')));
|
|
227
|
+
const uniqueScripts = Array.from(new Set(scripts));
|
|
228
|
+
for (const scriptUrl of uniqueScripts.slice(-30)) {
|
|
229
|
+
try {
|
|
230
|
+
const text = await (await fetch(scriptUrl)).text();
|
|
231
|
+
const operation = parseOperation(text);
|
|
232
|
+
if (operation) return operation;
|
|
233
|
+
} catch {}
|
|
234
|
+
}
|
|
235
|
+
} catch {}
|
|
87
236
|
const controller = new AbortController();
|
|
88
237
|
const timeout = setTimeout(() => controller.abort(), 5000);
|
|
89
238
|
try {
|
|
@@ -92,27 +241,25 @@ export async function resolveTwitterQueryId(page, operationName, fallbackId) {
|
|
|
92
241
|
if (ghResp.ok) {
|
|
93
242
|
const data = await ghResp.json();
|
|
94
243
|
const entry = data?.[operationName];
|
|
95
|
-
if (entry && entry.queryId)
|
|
244
|
+
if (entry && entry.queryId) {
|
|
245
|
+
return {
|
|
246
|
+
queryId: entry.queryId,
|
|
247
|
+
features: keysToFlags(entry.featureSwitches),
|
|
248
|
+
fieldToggles: keysToFlags(entry.fieldToggles),
|
|
249
|
+
};
|
|
250
|
+
}
|
|
96
251
|
}
|
|
97
252
|
} catch {
|
|
98
253
|
clearTimeout(timeout);
|
|
99
254
|
}
|
|
100
|
-
try {
|
|
101
|
-
const scripts = performance.getEntriesByType('resource')
|
|
102
|
-
.filter(r => r.name.includes('client-web') && r.name.endsWith('.js'))
|
|
103
|
-
.map(r => r.name);
|
|
104
|
-
for (const scriptUrl of scripts.slice(0, 15)) {
|
|
105
|
-
try {
|
|
106
|
-
const text = await (await fetch(scriptUrl)).text();
|
|
107
|
-
const re = new RegExp('queryId:"([A-Za-z0-9_-]+)"[^}]{0,200}operationName:"' + operationName + '"');
|
|
108
|
-
const match = text.match(re);
|
|
109
|
-
if (match) return match[1];
|
|
110
|
-
} catch {}
|
|
111
|
-
}
|
|
112
|
-
} catch {}
|
|
113
255
|
return null;
|
|
114
256
|
}`);
|
|
115
|
-
return
|
|
257
|
+
return sanitizeTwitterOperationMetadata(resolved, fallback);
|
|
258
|
+
}
|
|
259
|
+
|
|
260
|
+
export async function resolveTwitterQueryId(page, operationName, fallbackId) {
|
|
261
|
+
const operation = await resolveTwitterOperationMetadata(page, operationName, fallbackId);
|
|
262
|
+
return operation.queryId;
|
|
116
263
|
}
|
|
117
264
|
/**
|
|
118
265
|
* Extract media flags and URLs from a tweet's `legacy` object.
|
|
@@ -143,6 +290,10 @@ export function extractMedia(legacy) {
|
|
|
143
290
|
}
|
|
144
291
|
export const __test__ = {
|
|
145
292
|
sanitizeQueryId,
|
|
293
|
+
sanitizeTwitterOperationMetadata,
|
|
294
|
+
unwrapBrowserResult,
|
|
295
|
+
normalizeTwitterGraphqlPayload,
|
|
296
|
+
normalizeTwitterScreenName,
|
|
146
297
|
extractMedia,
|
|
147
298
|
parseTweetUrl,
|
|
148
299
|
buildTwitterArticleScopeSource,
|
|
@@ -3,7 +3,108 @@ import { JSDOM } from 'jsdom';
|
|
|
3
3
|
import { __test__ } from './shared.js';
|
|
4
4
|
import { ArgumentError } from '@jackwener/opencli/errors';
|
|
5
5
|
|
|
6
|
-
const { extractMedia, parseTweetUrl, buildTwitterArticleScopeSource } = __test__;
|
|
6
|
+
const { extractMedia, parseTweetUrl, buildTwitterArticleScopeSource, unwrapBrowserResult, normalizeTwitterGraphqlPayload, normalizeTwitterScreenName, sanitizeTwitterOperationMetadata } = __test__;
|
|
7
|
+
|
|
8
|
+
describe('twitter browser result helpers', () => {
|
|
9
|
+
it('unwraps Browser Bridge exec envelopes', () => {
|
|
10
|
+
expect(unwrapBrowserResult({ session: 'site:twitter', data: '123' })).toBe('123');
|
|
11
|
+
expect(unwrapBrowserResult({ data: { user: true } })).toEqual({ data: { user: true } });
|
|
12
|
+
});
|
|
13
|
+
|
|
14
|
+
it('sanitizes operation metadata after unwrapping Browser Bridge envelopes', () => {
|
|
15
|
+
const result = sanitizeTwitterOperationMetadata({
|
|
16
|
+
session: 'site:twitter',
|
|
17
|
+
data: {
|
|
18
|
+
queryId: 'abc_123',
|
|
19
|
+
features: { feature: true },
|
|
20
|
+
fieldToggles: { field: true },
|
|
21
|
+
},
|
|
22
|
+
}, { queryId: 'fallback', features: {}, fieldToggles: {} });
|
|
23
|
+
expect(result).toEqual({
|
|
24
|
+
queryId: 'abc_123',
|
|
25
|
+
features: { feature: true },
|
|
26
|
+
fieldToggles: { field: true },
|
|
27
|
+
});
|
|
28
|
+
});
|
|
29
|
+
|
|
30
|
+
it('falls back to baked features / fieldToggles when the bundle parser returns empty maps', () => {
|
|
31
|
+
// Regression guard: resolveTwitterOperationMetadata's bundle parser can
|
|
32
|
+
// find a queryId but miss `featureSwitches:[...]` (e.g. minification
|
|
33
|
+
// change, or the 2500-char snippet window truncating before the array).
|
|
34
|
+
// In that case keysToFlags(undefined) returns {}; if sanitize kept the
|
|
35
|
+
// empty map, Twitter would receive a request with no features and reply
|
|
36
|
+
// 400, surfacing a misleading "queryId expired" error.
|
|
37
|
+
const result = sanitizeTwitterOperationMetadata({
|
|
38
|
+
queryId: 'newQueryId',
|
|
39
|
+
features: {},
|
|
40
|
+
fieldToggles: {},
|
|
41
|
+
}, {
|
|
42
|
+
queryId: 'fallback',
|
|
43
|
+
features: { fallback_feature: true },
|
|
44
|
+
fieldToggles: { fallback_field: true },
|
|
45
|
+
});
|
|
46
|
+
expect(result).toEqual({
|
|
47
|
+
queryId: 'newQueryId',
|
|
48
|
+
features: { fallback_feature: true },
|
|
49
|
+
fieldToggles: { fallback_field: true },
|
|
50
|
+
});
|
|
51
|
+
});
|
|
52
|
+
|
|
53
|
+
it('falls back when resolved features are non-object falsy values', () => {
|
|
54
|
+
const result = sanitizeTwitterOperationMetadata({
|
|
55
|
+
queryId: 'newQueryId',
|
|
56
|
+
features: null,
|
|
57
|
+
fieldToggles: undefined,
|
|
58
|
+
}, {
|
|
59
|
+
queryId: 'fallback',
|
|
60
|
+
features: { fallback_feature: true },
|
|
61
|
+
fieldToggles: { fallback_field: true },
|
|
62
|
+
});
|
|
63
|
+
expect(result.features).toEqual({ fallback_feature: true });
|
|
64
|
+
expect(result.fieldToggles).toEqual({ fallback_field: true });
|
|
65
|
+
});
|
|
66
|
+
|
|
67
|
+
it('normalizes GraphQL payloads when the bridge strips the top-level data key', () => {
|
|
68
|
+
expect(normalizeTwitterGraphqlPayload({ user: { result: {} } })).toEqual({
|
|
69
|
+
data: { user: { result: {} } },
|
|
70
|
+
});
|
|
71
|
+
expect(normalizeTwitterGraphqlPayload({ search_by_raw_query: { search_timeline: {} } })).toEqual({
|
|
72
|
+
data: { search_by_raw_query: { search_timeline: {} } },
|
|
73
|
+
});
|
|
74
|
+
expect(normalizeTwitterGraphqlPayload({ data: { user: {} } })).toEqual({ data: { user: {} } });
|
|
75
|
+
});
|
|
76
|
+
});
|
|
77
|
+
|
|
78
|
+
describe('twitter normalizeTwitterScreenName', () => {
|
|
79
|
+
it('accepts exact handles and exact Twitter/X profile URLs', () => {
|
|
80
|
+
expect(normalizeTwitterScreenName('@viewer')).toBe('viewer');
|
|
81
|
+
expect(normalizeTwitterScreenName('/viewer')).toBe('viewer');
|
|
82
|
+
expect(normalizeTwitterScreenName('https://x.com/viewer')).toBe('viewer');
|
|
83
|
+
expect(normalizeTwitterScreenName('https://twitter.com/viewer?lang=en')).toBe('viewer');
|
|
84
|
+
expect(normalizeTwitterScreenName('https://mobile.twitter.com/viewer')).toBe('viewer');
|
|
85
|
+
});
|
|
86
|
+
|
|
87
|
+
it('rejects route collisions, malformed handles, and non-exact profile URLs', () => {
|
|
88
|
+
const invalid = [
|
|
89
|
+
'/home',
|
|
90
|
+
'/viewer/extra',
|
|
91
|
+
'viewer/extra',
|
|
92
|
+
'viewer?tab=posts',
|
|
93
|
+
'https://x.com/home',
|
|
94
|
+
'https://x.com/viewer/status/1',
|
|
95
|
+
'http://x.com/viewer',
|
|
96
|
+
'https://evil.com/viewer',
|
|
97
|
+
'https://x.com.evil.com/viewer',
|
|
98
|
+
'https://x.com:444/viewer',
|
|
99
|
+
'https://user:pass@x.com/viewer',
|
|
100
|
+
'bad-handle',
|
|
101
|
+
'abcdefghijklmnop',
|
|
102
|
+
];
|
|
103
|
+
for (const value of invalid) {
|
|
104
|
+
expect(normalizeTwitterScreenName(value)).toBe('');
|
|
105
|
+
}
|
|
106
|
+
});
|
|
107
|
+
});
|
|
7
108
|
|
|
8
109
|
describe('twitter parseTweetUrl', () => {
|
|
9
110
|
it('accepts exact Twitter/X tweet URLs and preserves query parameters', () => {
|
package/clis/twitter/timeline.js
CHANGED
|
@@ -5,6 +5,7 @@ import { TWITTER_BEARER_TOKEN, applyTopByEngagement } from './utils.js';
|
|
|
5
5
|
// ── Twitter GraphQL constants ──────────────────────────────────────────
|
|
6
6
|
const HOME_TIMELINE_QUERY_ID = 'c-CzHF1LboFilMpsx4ZCrQ';
|
|
7
7
|
const HOME_LATEST_TIMELINE_QUERY_ID = 'BKB7oi212Fi7kQtCBGE4zA';
|
|
8
|
+
const MAX_PAGINATION_PAGES = 100;
|
|
8
9
|
// Endpoint config: for-you uses GET HomeTimeline, following uses POST HomeLatestTimeline
|
|
9
10
|
const TIMELINE_ENDPOINTS = {
|
|
10
11
|
'for-you': { endpoint: 'HomeTimeline', method: 'GET', fallbackQueryId: HOME_TIMELINE_QUERY_ID },
|
|
@@ -176,7 +177,8 @@ cli({
|
|
|
176
177
|
const allTweets = [];
|
|
177
178
|
const seen = new Set();
|
|
178
179
|
let cursor = null;
|
|
179
|
-
|
|
180
|
+
// Runaway guard only; --limit and cursor exhaustion control normal pagination.
|
|
181
|
+
for (let i = 0; i < MAX_PAGINATION_PAGES && allTweets.length < limit; i++) {
|
|
180
182
|
const fetchCount = Math.min(40, limit - allTweets.length + 5); // over-fetch slightly for promoted filtering
|
|
181
183
|
const variables = buildTimelineVariables(timelineType, fetchCount, cursor);
|
|
182
184
|
const apiUrl = buildHomeTimelineUrl(queryId, endpoint, variables);
|
package/clis/twitter/tweets.js
CHANGED
|
@@ -1,15 +1,19 @@
|
|
|
1
1
|
import { cli, Strategy } from '@jackwener/opencli/registry';
|
|
2
|
-
import { AuthRequiredError, CommandExecutionError, EmptyResultError } from '@jackwener/opencli/errors';
|
|
3
|
-
import {
|
|
2
|
+
import { ArgumentError, AuthRequiredError, CommandExecutionError, EmptyResultError } from '@jackwener/opencli/errors';
|
|
3
|
+
import { resolveTwitterOperationMetadata, sanitizeQueryId, extractMedia, normalizeTwitterGraphqlPayload, unwrapBrowserResult } from './shared.js';
|
|
4
|
+
import { normalizeTwitterScreenName } from './shared.js';
|
|
4
5
|
import { TWITTER_BEARER_TOKEN, applyTopByEngagement } from './utils.js';
|
|
5
6
|
|
|
6
|
-
const USER_TWEETS_QUERY_ID = '
|
|
7
|
+
const USER_TWEETS_QUERY_ID = 'lrMzG9qPQHpqJdP3AbM-bQ';
|
|
7
8
|
const USER_BY_SCREEN_NAME_QUERY_ID = 'IGgvgiOx4QZndDHuD3x9TQ';
|
|
9
|
+
const MAX_PAGINATION_PAGES = 100;
|
|
8
10
|
|
|
9
11
|
const USER_TWEETS_FEATURES = {
|
|
10
|
-
rweb_video_screen_enabled:
|
|
12
|
+
rweb_video_screen_enabled: true,
|
|
13
|
+
rweb_cashtags_enabled: true,
|
|
11
14
|
payments_enabled: false,
|
|
12
15
|
profile_label_improvements_pcf_label_in_post_enabled: true,
|
|
16
|
+
responsive_web_profile_redirect_enabled: true,
|
|
13
17
|
rweb_tipjar_consumption_enabled: true,
|
|
14
18
|
verified_phone_label_enabled: false,
|
|
15
19
|
creator_subscriptions_tweet_preview_api_enabled: true,
|
|
@@ -20,6 +24,7 @@ const USER_TWEETS_FEATURES = {
|
|
|
20
24
|
c9s_tweet_anatomy_moderator_badge_enabled: true,
|
|
21
25
|
responsive_web_grok_analyze_button_fetch_trends_enabled: false,
|
|
22
26
|
responsive_web_grok_analyze_post_followups_enabled: true,
|
|
27
|
+
rweb_cashtags_composer_attachment_enabled: true,
|
|
23
28
|
responsive_web_jetfuel_frame: true,
|
|
24
29
|
responsive_web_grok_share_attachment_enabled: true,
|
|
25
30
|
responsive_web_grok_annotations_enabled: true,
|
|
@@ -46,8 +51,21 @@ const USER_TWEETS_FEATURES = {
|
|
|
46
51
|
responsive_web_enhance_cards_enabled: false,
|
|
47
52
|
};
|
|
48
53
|
|
|
54
|
+
const USER_TWEETS_FIELD_TOGGLES = {
|
|
55
|
+
withPayments: true,
|
|
56
|
+
withAuxiliaryUserLabels: true,
|
|
57
|
+
withArticleRichContentState: true,
|
|
58
|
+
withArticlePlainText: true,
|
|
59
|
+
withArticleSummaryText: true,
|
|
60
|
+
withArticleVoiceOver: true,
|
|
61
|
+
withGrokAnalyze: true,
|
|
62
|
+
withDisallowedReplyControls: true,
|
|
63
|
+
};
|
|
64
|
+
|
|
49
65
|
const USER_BY_SCREEN_NAME_FEATURES = {
|
|
50
66
|
hidden_profile_subscriptions_enabled: true,
|
|
67
|
+
profile_label_improvements_pcf_label_in_post_enabled: true,
|
|
68
|
+
responsive_web_profile_redirect_enabled: true,
|
|
51
69
|
rweb_tipjar_consumption_enabled: true,
|
|
52
70
|
responsive_web_graphql_exclude_directive_enabled: true,
|
|
53
71
|
verified_phone_label_enabled: false,
|
|
@@ -61,7 +79,59 @@ const USER_BY_SCREEN_NAME_FEATURES = {
|
|
|
61
79
|
responsive_web_graphql_timeline_navigation_enabled: true,
|
|
62
80
|
};
|
|
63
81
|
|
|
64
|
-
|
|
82
|
+
const USER_BY_SCREEN_NAME_FIELD_TOGGLES = {
|
|
83
|
+
withPayments: true,
|
|
84
|
+
withAuxiliaryUserLabels: true,
|
|
85
|
+
};
|
|
86
|
+
|
|
87
|
+
const USER_TWEETS_OPERATION = {
|
|
88
|
+
queryId: USER_TWEETS_QUERY_ID,
|
|
89
|
+
features: USER_TWEETS_FEATURES,
|
|
90
|
+
fieldToggles: USER_TWEETS_FIELD_TOGGLES,
|
|
91
|
+
};
|
|
92
|
+
|
|
93
|
+
const USER_BY_SCREEN_NAME_OPERATION = {
|
|
94
|
+
queryId: USER_BY_SCREEN_NAME_QUERY_ID,
|
|
95
|
+
features: USER_BY_SCREEN_NAME_FEATURES,
|
|
96
|
+
fieldToggles: USER_BY_SCREEN_NAME_FIELD_TOGGLES,
|
|
97
|
+
};
|
|
98
|
+
|
|
99
|
+
function normalizeUserTweetsOperation(operation) {
|
|
100
|
+
if (typeof operation === 'string') {
|
|
101
|
+
return { queryId: operation, features: USER_TWEETS_FEATURES, fieldToggles: USER_TWEETS_FIELD_TOGGLES };
|
|
102
|
+
}
|
|
103
|
+
return {
|
|
104
|
+
queryId: operation?.queryId || USER_TWEETS_QUERY_ID,
|
|
105
|
+
features: operation?.features || USER_TWEETS_FEATURES,
|
|
106
|
+
fieldToggles: operation?.fieldToggles || USER_TWEETS_FIELD_TOGGLES,
|
|
107
|
+
};
|
|
108
|
+
}
|
|
109
|
+
|
|
110
|
+
function normalizeUserByScreenNameOperation(operation) {
|
|
111
|
+
if (typeof operation === 'string') {
|
|
112
|
+
return { queryId: operation, features: USER_BY_SCREEN_NAME_FEATURES, fieldToggles: USER_BY_SCREEN_NAME_FIELD_TOGGLES };
|
|
113
|
+
}
|
|
114
|
+
return {
|
|
115
|
+
queryId: operation?.queryId || USER_BY_SCREEN_NAME_QUERY_ID,
|
|
116
|
+
features: operation?.features || USER_BY_SCREEN_NAME_FEATURES,
|
|
117
|
+
fieldToggles: operation?.fieldToggles || USER_BY_SCREEN_NAME_FIELD_TOGGLES,
|
|
118
|
+
};
|
|
119
|
+
}
|
|
120
|
+
|
|
121
|
+
function appendGraphqlParams(path, variables, operation) {
|
|
122
|
+
const fieldToggles = operation.fieldToggles || {};
|
|
123
|
+
const params = [
|
|
124
|
+
`variables=${encodeURIComponent(JSON.stringify(variables))}`,
|
|
125
|
+
`features=${encodeURIComponent(JSON.stringify(operation.features || {}))}`,
|
|
126
|
+
];
|
|
127
|
+
if (Object.keys(fieldToggles).length > 0) {
|
|
128
|
+
params.push(`fieldToggles=${encodeURIComponent(JSON.stringify(fieldToggles))}`);
|
|
129
|
+
}
|
|
130
|
+
return `${path}?${params.join('&')}`;
|
|
131
|
+
}
|
|
132
|
+
|
|
133
|
+
function buildUserTweetsUrl(operation, userId, count, cursor) {
|
|
134
|
+
const normalized = normalizeUserTweetsOperation(operation);
|
|
65
135
|
const vars = {
|
|
66
136
|
userId,
|
|
67
137
|
count,
|
|
@@ -70,21 +140,20 @@ function buildUserTweetsUrl(queryId, userId, count, cursor) {
|
|
|
70
140
|
withVoice: true,
|
|
71
141
|
};
|
|
72
142
|
if (cursor) vars.cursor = cursor;
|
|
73
|
-
return `/i/api/graphql/${queryId}/UserTweets
|
|
74
|
-
+ `?variables=${encodeURIComponent(JSON.stringify(vars))}`
|
|
75
|
-
+ `&features=${encodeURIComponent(JSON.stringify(USER_TWEETS_FEATURES))}`;
|
|
143
|
+
return appendGraphqlParams(`/i/api/graphql/${normalized.queryId}/UserTweets`, vars, normalized);
|
|
76
144
|
}
|
|
77
145
|
|
|
78
|
-
function buildUserByScreenNameUrl(
|
|
146
|
+
function buildUserByScreenNameUrl(operation, screenName) {
|
|
147
|
+
const normalized = normalizeUserByScreenNameOperation(operation);
|
|
79
148
|
const vars = { screen_name: screenName, withSafetyModeUserFields: true };
|
|
80
|
-
return `/i/api/graphql/${queryId}/UserByScreenName
|
|
81
|
-
+ `?variables=${encodeURIComponent(JSON.stringify(vars))}`
|
|
82
|
-
+ `&features=${encodeURIComponent(JSON.stringify(USER_BY_SCREEN_NAME_FEATURES))}`;
|
|
149
|
+
return appendGraphqlParams(`/i/api/graphql/${normalized.queryId}/UserByScreenName`, vars, normalized);
|
|
83
150
|
}
|
|
84
151
|
|
|
85
152
|
function extractTweet(result, seen) {
|
|
86
153
|
if (!result) return null;
|
|
87
|
-
const tw = result.
|
|
154
|
+
const tw = result.__typename === 'TweetWithVisibilityResults' && result.tweet
|
|
155
|
+
? result.tweet
|
|
156
|
+
: (result.tweet || result);
|
|
88
157
|
const legacy = tw.legacy || {};
|
|
89
158
|
if (!tw.rest_id || seen.has(tw.rest_id)) return null;
|
|
90
159
|
seen.add(tw.rest_id);
|
|
@@ -112,32 +181,35 @@ function extractTweet(result, seen) {
|
|
|
112
181
|
function parseUserTweets(data, seen) {
|
|
113
182
|
const tweets = [];
|
|
114
183
|
let nextCursor = null;
|
|
115
|
-
const
|
|
116
|
-
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
|
|
120
|
-
|
|
121
|
-
|
|
122
|
-
|
|
123
|
-
|
|
124
|
-
|
|
125
|
-
|
|
126
|
-
if (
|
|
127
|
-
nextCursor = content?.value || content?.itemContent?.value || nextCursor;
|
|
128
|
-
continue;
|
|
129
|
-
}
|
|
130
|
-
const direct = extractTweet(content?.itemContent?.tweet_results?.result, seen);
|
|
131
|
-
if (direct) {
|
|
132
|
-
tweets.push(direct);
|
|
133
|
-
continue;
|
|
134
|
-
}
|
|
135
|
-
for (const item of content?.items || []) {
|
|
136
|
-
const nested = extractTweet(item.item?.itemContent?.tweet_results?.result, seen);
|
|
137
|
-
if (nested) tweets.push(nested);
|
|
138
|
-
}
|
|
184
|
+
const result = data?.data?.user?.result || {};
|
|
185
|
+
const instructionSets = [
|
|
186
|
+
result.timeline_v2?.timeline?.instructions,
|
|
187
|
+
result.timeline?.timeline?.instructions,
|
|
188
|
+
].filter(Array.isArray);
|
|
189
|
+
const instructions = instructionSets.flat();
|
|
190
|
+
const visit = (value) => {
|
|
191
|
+
if (!value || typeof value !== 'object') return;
|
|
192
|
+
if (value.type === 'TimelinePinEntry') return;
|
|
193
|
+
if (value.tweet_results?.result) {
|
|
194
|
+
const tweet = extractTweet(value.tweet_results.result, seen);
|
|
195
|
+
if (tweet) tweets.push(tweet);
|
|
139
196
|
}
|
|
140
|
-
|
|
197
|
+
if (
|
|
198
|
+
(value.entryType === 'TimelineTimelineCursor' || value.__typename === 'TimelineTimelineCursor')
|
|
199
|
+
&& (value.cursorType === 'Bottom' || value.cursorType === 'ShowMore')
|
|
200
|
+
&& value.value
|
|
201
|
+
) {
|
|
202
|
+
nextCursor = value.value;
|
|
203
|
+
}
|
|
204
|
+
if (Array.isArray(value)) {
|
|
205
|
+
for (const item of value) visit(item);
|
|
206
|
+
return;
|
|
207
|
+
}
|
|
208
|
+
for (const child of Object.values(value)) {
|
|
209
|
+
if (child && typeof child === 'object') visit(child);
|
|
210
|
+
}
|
|
211
|
+
};
|
|
212
|
+
visit(instructions);
|
|
141
213
|
return { tweets, nextCursor };
|
|
142
214
|
}
|
|
143
215
|
|
|
@@ -145,28 +217,51 @@ cli({
|
|
|
145
217
|
site: 'twitter',
|
|
146
218
|
name: 'tweets',
|
|
147
219
|
access: 'read',
|
|
148
|
-
description: "Fetch a Twitter user's most recent tweets (chronological, excludes pinned)",
|
|
220
|
+
description: "Fetch a Twitter user's most recent tweets (chronological, excludes pinned; defaults to the logged-in user when no username is given)",
|
|
149
221
|
domain: 'x.com',
|
|
150
222
|
strategy: Strategy.COOKIE,
|
|
151
223
|
browser: true,
|
|
152
224
|
siteSession: 'persistent',
|
|
153
225
|
args: [
|
|
154
|
-
{ name: 'username', type: 'string', positional: true,
|
|
226
|
+
{ name: 'username', type: 'string', positional: true, help: 'Twitter screen name (with or without @). Defaults to the logged-in user when omitted.' },
|
|
155
227
|
{ name: 'limit', type: 'int', default: 20, help: 'Max tweets to return' },
|
|
156
228
|
{ name: 'top-by-engagement', type: 'int', default: 0, help: 'When set to N>0, re-rank the tweets by weighted engagement (likes×1 + retweets×3 + replies×2 + bookmarks×5 + log10(views+1)×0.5) and return the top N. Default 0 keeps the chronological ordering.' },
|
|
157
229
|
],
|
|
158
230
|
columns: ['id', 'author', 'created_at', 'is_retweet', 'text', 'likes', 'retweets', 'replies', 'views', 'url', 'has_media', 'media_urls'],
|
|
159
231
|
func: async (page, kwargs) => {
|
|
160
232
|
const limit = Math.max(1, Math.min(200, kwargs.limit || 20));
|
|
161
|
-
const
|
|
162
|
-
|
|
233
|
+
const rawUsername = String(kwargs.username ?? '').trim();
|
|
234
|
+
let username = normalizeTwitterScreenName(rawUsername);
|
|
235
|
+
if (rawUsername && !username) {
|
|
236
|
+
throw new ArgumentError('twitter tweets username must be a valid Twitter/X handle', 'Example: opencli twitter tweets @jack --limit 20');
|
|
237
|
+
}
|
|
238
|
+
// When no username is given, detect the logged-in user (own tweets).
|
|
239
|
+
// Mirrors the self-detection pattern used by twitter/profile and
|
|
240
|
+
// twitter/likes so agents can pull own-account data without having
|
|
241
|
+
// to know their own screen name up front.
|
|
242
|
+
if (!username) {
|
|
243
|
+
await page.goto('https://x.com/home');
|
|
244
|
+
await page.wait({ selector: '[data-testid="primaryColumn"]' });
|
|
245
|
+
// Bridge wraps primitive page.evaluate returns as { session, data:<value> }.
|
|
246
|
+
// unwrapBrowserResult drops that envelope so the href string is usable.
|
|
247
|
+
const href = unwrapBrowserResult(await page.evaluate(`() => {
|
|
248
|
+
const link = document.querySelector('a[data-testid="AppTabBar_Profile_Link"]');
|
|
249
|
+
return link ? link.getAttribute('href') : null;
|
|
250
|
+
}`));
|
|
251
|
+
if (!href || typeof href !== 'string')
|
|
252
|
+
throw new AuthRequiredError('x.com', 'Could not detect logged-in user. Are you logged in?');
|
|
253
|
+
username = normalizeTwitterScreenName(href);
|
|
254
|
+
if (!username) {
|
|
255
|
+
throw new AuthRequiredError('x.com', 'Could not detect logged-in user. Are you logged in?');
|
|
256
|
+
}
|
|
257
|
+
}
|
|
163
258
|
|
|
164
259
|
const cookies = await page.getCookies({ url: 'https://x.com' });
|
|
165
260
|
const ct0 = cookies.find((c) => c.name === 'ct0')?.value || null;
|
|
166
261
|
if (!ct0) throw new AuthRequiredError('x.com', 'Not logged into x.com (no ct0 cookie)');
|
|
167
262
|
|
|
168
|
-
const
|
|
169
|
-
const
|
|
263
|
+
const userTweetsOperation = await resolveTwitterOperationMetadata(page, 'UserTweets', USER_TWEETS_OPERATION);
|
|
264
|
+
const userByScreenNameOperation = await resolveTwitterOperationMetadata(page, 'UserByScreenName', USER_BY_SCREEN_NAME_OPERATION);
|
|
170
265
|
|
|
171
266
|
const headers = JSON.stringify({
|
|
172
267
|
'Authorization': `Bearer ${decodeURIComponent(TWITTER_BEARER_TOKEN)}`,
|
|
@@ -175,25 +270,26 @@ cli({
|
|
|
175
270
|
'X-Twitter-Active-User': 'yes',
|
|
176
271
|
});
|
|
177
272
|
|
|
178
|
-
const ubsUrl = buildUserByScreenNameUrl(
|
|
179
|
-
const userId = await page.evaluate(`async () => {
|
|
273
|
+
const ubsUrl = buildUserByScreenNameUrl(userByScreenNameOperation, username);
|
|
274
|
+
const userId = unwrapBrowserResult(await page.evaluate(`async () => {
|
|
180
275
|
const resp = await fetch("${ubsUrl}", { headers: ${headers}, credentials: 'include' });
|
|
181
276
|
if (!resp.ok) return null;
|
|
182
277
|
const d = await resp.json();
|
|
183
278
|
return d?.data?.user?.result?.rest_id || null;
|
|
184
|
-
}`);
|
|
279
|
+
}`));
|
|
185
280
|
if (!userId) throw new CommandExecutionError(`Could not resolve @${username}`);
|
|
186
281
|
|
|
187
282
|
const seen = new Set();
|
|
188
283
|
const all = [];
|
|
189
284
|
let cursor = null;
|
|
190
|
-
|
|
285
|
+
// Runaway guard only; --limit and cursor exhaustion control normal pagination.
|
|
286
|
+
for (let i = 0; i < MAX_PAGINATION_PAGES && all.length < limit; i++) {
|
|
191
287
|
const fetchCount = Math.min(100, limit - all.length + 10);
|
|
192
|
-
const url = buildUserTweetsUrl(
|
|
193
|
-
const data = await page.evaluate(`async () => {
|
|
288
|
+
const url = buildUserTweetsUrl(userTweetsOperation, userId, fetchCount, cursor);
|
|
289
|
+
const data = normalizeTwitterGraphqlPayload(await page.evaluate(`async () => {
|
|
194
290
|
const r = await fetch("${url}", { headers: ${headers}, credentials: 'include' });
|
|
195
291
|
return r.ok ? await r.json() : { error: r.status };
|
|
196
|
-
}`);
|
|
292
|
+
}`));
|
|
197
293
|
if (data?.error) {
|
|
198
294
|
if (all.length === 0) throw new CommandExecutionError(`HTTP ${data.error}: UserTweets fetch failed — queryId may have expired`);
|
|
199
295
|
break;
|