@jackwener/opencli 1.7.19 → 1.7.21
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +11 -9
- package/README.zh-CN.md +9 -10
- package/cli-manifest.json +239 -249
- package/clis/_shared/search-adapter.js +70 -0
- package/clis/boss/chatlist.js +96 -14
- package/clis/boss/chatlist.test.js +211 -0
- package/clis/boss/chatmsg.js +98 -24
- package/clis/boss/chatmsg.test.js +230 -0
- package/clis/boss/utils.js +240 -11
- package/clis/brave/search.js +80 -0
- package/clis/brave/search.test.js +76 -0
- package/clis/duckduckgo/search.js +131 -0
- package/clis/duckduckgo/search.test.js +128 -0
- package/clis/duckduckgo/suggest.js +45 -0
- package/clis/duckduckgo/suggest.test.js +66 -0
- package/clis/facebook/feed.js +301 -56
- package/clis/facebook/feed.test.js +169 -0
- package/clis/reddit/comment.js +0 -1
- package/clis/reddit/frontpage.js +0 -1
- package/clis/reddit/home.js +0 -1
- package/clis/reddit/popular.js +0 -1
- package/clis/reddit/read.js +0 -1
- package/clis/reddit/read.test.js +2 -2
- package/clis/reddit/save.js +0 -1
- package/clis/reddit/saved.js +0 -1
- package/clis/reddit/search.js +0 -1
- package/clis/reddit/subreddit-info.js +0 -1
- package/clis/reddit/subreddit.js +0 -1
- package/clis/reddit/subscribe.js +0 -1
- package/clis/reddit/upvote.js +0 -1
- package/clis/reddit/upvoted.js +0 -1
- package/clis/reddit/user-comments.js +0 -1
- package/clis/reddit/user-posts.js +0 -1
- package/clis/reddit/user.js +0 -1
- package/clis/reddit/whoami.js +0 -1
- package/clis/rednote/rednote.test.js +65 -0
- package/clis/rednote/search.js +11 -5
- package/clis/twitter/article.js +0 -1
- package/clis/twitter/bookmark-folder.js +5 -4
- package/clis/twitter/bookmark-folder.test.js +59 -1
- package/clis/twitter/bookmark-folders.js +0 -1
- package/clis/twitter/bookmarks.js +9 -4
- package/clis/twitter/bookmarks.test.js +205 -0
- package/clis/twitter/download.js +0 -1
- package/clis/twitter/followers.js +0 -1
- package/clis/twitter/following.js +0 -1
- package/clis/twitter/likes.js +0 -1
- package/clis/twitter/list-tweets.js +0 -1
- package/clis/twitter/lists.js +0 -1
- package/clis/twitter/notifications.js +0 -1
- package/clis/twitter/profile.js +0 -1
- package/clis/twitter/search.js +0 -1
- package/clis/twitter/thread.js +0 -1
- package/clis/twitter/timeline.js +0 -1
- package/clis/twitter/trending.js +0 -1
- package/clis/twitter/tweets.js +0 -1
- package/clis/xiaohongshu/search.js +34 -16
- package/clis/xiaohongshu/search.test.js +66 -11
- package/clis/yahoo/search.js +92 -0
- package/clis/yahoo/search.test.js +94 -0
- package/dist/src/browser/daemon-client.d.ts +1 -0
- package/dist/src/browser/daemon-client.js +3 -0
- package/dist/src/browser/daemon-client.test.js +20 -0
- package/dist/src/cli.js +8 -3
- package/dist/src/cli.test.js +1 -0
- package/dist/src/daemon-utils.d.ts +18 -0
- package/dist/src/daemon-utils.js +37 -0
- package/dist/src/daemon.d.ts +1 -1
- package/dist/src/daemon.js +44 -13
- package/dist/src/daemon.test.js +42 -1
- package/dist/src/electron-apps.js +0 -1
- package/dist/src/electron-apps.test.js +1 -0
- package/dist/src/external-clis.yaml +12 -3
- package/dist/src/external.d.ts +4 -0
- package/dist/src/external.js +3 -0
- package/dist/src/external.test.js +24 -1
- package/dist/src/help.d.ts +5 -1
- package/dist/src/help.js +4 -3
- package/dist/src/help.test.js +5 -1
- package/package.json +1 -1
- package/clis/notion/export.js +0 -32
- package/clis/notion/favorites.js +0 -85
- package/clis/notion/new.js +0 -35
- package/clis/notion/read.js +0 -31
- package/clis/notion/search.js +0 -47
- package/clis/notion/sidebar.js +0 -42
- package/clis/notion/status.js +0 -17
- package/clis/notion/write.js +0 -41
|
@@ -7,7 +7,6 @@ cli({
|
|
|
7
7
|
domain: 'reddit.com',
|
|
8
8
|
strategy: Strategy.COOKIE,
|
|
9
9
|
browser: true,
|
|
10
|
-
siteSession: 'persistent',
|
|
11
10
|
args: [
|
|
12
11
|
{ name: 'username', type: 'string', required: true, positional: true, help: 'Reddit username (no `u/` prefix needed)' },
|
|
13
12
|
{ name: 'limit', type: 'int', default: 15 },
|
|
@@ -7,7 +7,6 @@ cli({
|
|
|
7
7
|
domain: 'reddit.com',
|
|
8
8
|
strategy: Strategy.COOKIE,
|
|
9
9
|
browser: true,
|
|
10
|
-
siteSession: 'persistent',
|
|
11
10
|
args: [
|
|
12
11
|
{ name: 'username', type: 'string', required: true, positional: true, help: 'Reddit username (no `u/` prefix needed)' },
|
|
13
12
|
{ name: 'limit', type: 'int', default: 15 },
|
package/clis/reddit/user.js
CHANGED
package/clis/reddit/whoami.js
CHANGED
|
@@ -31,6 +31,14 @@ function createPageMock(evaluateResult) {
|
|
|
31
31
|
getCookies: vi.fn().mockResolvedValue([{ name: 'sid', value: 'secret', domain: 'www.rednote.com' }]),
|
|
32
32
|
};
|
|
33
33
|
}
|
|
34
|
+
function createSearchPageMock(evaluateResults) {
|
|
35
|
+
const page = createPageMock(undefined);
|
|
36
|
+
page.evaluate = vi.fn();
|
|
37
|
+
for (const result of evaluateResults) {
|
|
38
|
+
page.evaluate.mockResolvedValueOnce(result);
|
|
39
|
+
}
|
|
40
|
+
return page;
|
|
41
|
+
}
|
|
34
42
|
|
|
35
43
|
describe('rednote note URL identity', () => {
|
|
36
44
|
const download = getRegistry().get('rednote/download');
|
|
@@ -130,6 +138,63 @@ describe('rednote argument validation', () => {
|
|
|
130
138
|
});
|
|
131
139
|
});
|
|
132
140
|
|
|
141
|
+
describe('rednote search browser-bridge envelopes', () => {
|
|
142
|
+
const search = getRegistry().get('rednote/search');
|
|
143
|
+
|
|
144
|
+
it('unwraps login-wall wait result envelopes before auth handling', async () => {
|
|
145
|
+
const page = createSearchPageMock([
|
|
146
|
+
{ session: 'site:rednote', data: 'login_wall' },
|
|
147
|
+
]);
|
|
148
|
+
|
|
149
|
+
await expect(search.func(page, { query: 'tesla', limit: 5 })).rejects.toMatchObject({
|
|
150
|
+
code: 'AUTH_REQUIRED',
|
|
151
|
+
message: expect.stringContaining('blocked behind a login wall'),
|
|
152
|
+
});
|
|
153
|
+
expect(page.evaluate).toHaveBeenCalledTimes(1);
|
|
154
|
+
});
|
|
155
|
+
|
|
156
|
+
it('unwraps search extraction envelopes and preserves rednote row shape', async () => {
|
|
157
|
+
const url = 'https://www.rednote.com/search_result/68e90be80000000004022e66?xsec_token=test-token';
|
|
158
|
+
const page = createSearchPageMock([
|
|
159
|
+
'content',
|
|
160
|
+
1,
|
|
161
|
+
{
|
|
162
|
+
session: 'site:rednote',
|
|
163
|
+
data: [{
|
|
164
|
+
title: 'rednote result',
|
|
165
|
+
author: 'author',
|
|
166
|
+
likes: '12',
|
|
167
|
+
url,
|
|
168
|
+
author_url: 'https://www.rednote.com/user/profile/u1',
|
|
169
|
+
}],
|
|
170
|
+
},
|
|
171
|
+
]);
|
|
172
|
+
|
|
173
|
+
await expect(search.func(page, { query: 'tesla', limit: 1 })).resolves.toEqual([{
|
|
174
|
+
rank: 1,
|
|
175
|
+
title: 'rednote result',
|
|
176
|
+
author: 'author',
|
|
177
|
+
likes: '12',
|
|
178
|
+
published_at: '2025-10-10',
|
|
179
|
+
url,
|
|
180
|
+
author_url: 'https://www.rednote.com/user/profile/u1',
|
|
181
|
+
}]);
|
|
182
|
+
});
|
|
183
|
+
|
|
184
|
+
it('fails typed instead of silently returning [] for malformed extraction payloads', async () => {
|
|
185
|
+
const page = createSearchPageMock([
|
|
186
|
+
'content',
|
|
187
|
+
1,
|
|
188
|
+
{ session: 'site:rednote', data: { rows: [] } },
|
|
189
|
+
]);
|
|
190
|
+
|
|
191
|
+
await expect(search.func(page, { query: 'tesla', limit: 1 })).rejects.toMatchObject({
|
|
192
|
+
code: 'COMMAND_EXEC',
|
|
193
|
+
message: expect.stringContaining('payload shape'),
|
|
194
|
+
});
|
|
195
|
+
});
|
|
196
|
+
});
|
|
197
|
+
|
|
133
198
|
describe('rednote Pinia store failures', () => {
|
|
134
199
|
it('maps feed store read failure to CommandExecutionError', async () => {
|
|
135
200
|
const command = getRegistry().get('rednote/feed');
|
package/clis/rednote/search.js
CHANGED
|
@@ -6,8 +6,8 @@
|
|
|
6
6
|
* 1:1 comparison between the two frontends.
|
|
7
7
|
*/
|
|
8
8
|
import { cli, Strategy } from '@jackwener/opencli/registry';
|
|
9
|
-
import { ArgumentError, AuthRequiredError } from '@jackwener/opencli/errors';
|
|
10
|
-
import { buildScrollUntilJs, buildSearchExtractJs, noteIdToDate } from '../xiaohongshu/search.js';
|
|
9
|
+
import { ArgumentError, AuthRequiredError, CommandExecutionError } from '@jackwener/opencli/errors';
|
|
10
|
+
import { buildScrollUntilJs, buildSearchExtractJs, noteIdToDate, unwrapEvaluateResult } from '../xiaohongshu/search.js';
|
|
11
11
|
|
|
12
12
|
function parseLimit(raw) {
|
|
13
13
|
const parsed = Number(raw);
|
|
@@ -19,6 +19,13 @@ function parseLimit(raw) {
|
|
|
19
19
|
}
|
|
20
20
|
return parsed;
|
|
21
21
|
}
|
|
22
|
+
function requireSearchRows(payload) {
|
|
23
|
+
const rows = unwrapEvaluateResult(payload);
|
|
24
|
+
if (!Array.isArray(rows)) {
|
|
25
|
+
throw new CommandExecutionError('Unexpected Rednote search extraction payload shape; expected an array of rows.');
|
|
26
|
+
}
|
|
27
|
+
return rows;
|
|
28
|
+
}
|
|
22
29
|
|
|
23
30
|
/**
|
|
24
31
|
* Wait for search results or login wall using MutationObserver (max 5s).
|
|
@@ -78,7 +85,7 @@ cli({
|
|
|
78
85
|
const limit = parseLimit(kwargs.limit ?? 20);
|
|
79
86
|
const keyword = encodeURIComponent(kwargs.query);
|
|
80
87
|
await page.goto(`https://www.rednote.com/search_result?keyword=${keyword}&source=web_search_result_notes`);
|
|
81
|
-
const waitResult = await page.evaluate(WAIT_FOR_CONTENT_JS);
|
|
88
|
+
const waitResult = unwrapEvaluateResult(await page.evaluate(WAIT_FOR_CONTENT_JS));
|
|
82
89
|
if (waitResult === 'login_wall') {
|
|
83
90
|
throw new AuthRequiredError('www.rednote.com', 'Rednote search results are blocked behind a login wall');
|
|
84
91
|
}
|
|
@@ -87,8 +94,7 @@ cli({
|
|
|
87
94
|
// `autoScroll({ times: 2 })` capped extraction at ~13 notes regardless
|
|
88
95
|
// of `--limit`.
|
|
89
96
|
await page.evaluate(buildScrollUntilJs(limit));
|
|
90
|
-
const
|
|
91
|
-
const data = Array.isArray(payload) ? payload : [];
|
|
97
|
+
const data = requireSearchRows(await page.evaluate(buildSearchExtractJs('www.rednote.com')));
|
|
92
98
|
return data
|
|
93
99
|
.filter((item) => item.title)
|
|
94
100
|
.slice(0, limit)
|
package/clis/twitter/article.js
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
import { cli, Strategy } from '@jackwener/opencli/registry';
|
|
2
2
|
import { ArgumentError, AuthRequiredError, CommandExecutionError } from '@jackwener/opencli/errors';
|
|
3
3
|
import { TWITTER_BEARER_TOKEN, applyTopByEngagement } from './utils.js';
|
|
4
|
-
import { resolveTwitterQueryId } from './shared.js';
|
|
4
|
+
import { extractMedia, resolveTwitterQueryId } from './shared.js';
|
|
5
5
|
|
|
6
6
|
// Companion to bookmark-folders.js: reads tweets inside a single folder.
|
|
7
7
|
// X exposes folder contents through a separate timeline operation
|
|
@@ -54,7 +54,7 @@ function buildFolderTimelineUrl(queryId, folderId, count, cursor) {
|
|
|
54
54
|
+ `&features=${encodeURIComponent(JSON.stringify(FEATURES))}`;
|
|
55
55
|
}
|
|
56
56
|
|
|
57
|
-
function extractFolderTweet(result, seen) {
|
|
57
|
+
export function extractFolderTweet(result, seen) {
|
|
58
58
|
if (!result) return null;
|
|
59
59
|
const tw = result.tweet || result;
|
|
60
60
|
const legacy = tw.legacy || {};
|
|
@@ -72,6 +72,7 @@ function extractFolderTweet(result, seen) {
|
|
|
72
72
|
bookmarks: legacy.bookmark_count || 0,
|
|
73
73
|
created_at: legacy.created_at || '',
|
|
74
74
|
url: screenName ? `https://x.com/${screenName}/status/${tw.rest_id}` : `https://x.com/i/status/${tw.rest_id}`,
|
|
75
|
+
...extractMedia(legacy),
|
|
75
76
|
};
|
|
76
77
|
}
|
|
77
78
|
|
|
@@ -123,13 +124,12 @@ cli({
|
|
|
123
124
|
domain: 'x.com',
|
|
124
125
|
strategy: Strategy.COOKIE,
|
|
125
126
|
browser: true,
|
|
126
|
-
siteSession: 'persistent',
|
|
127
127
|
args: [
|
|
128
128
|
{ name: 'folder-id', positional: true, type: 'string', required: true, help: 'Folder id from `opencli twitter bookmark-folders`.' },
|
|
129
129
|
{ name: 'limit', type: 'int', default: 20, help: 'Maximum number of bookmarks to return (default 20).' },
|
|
130
130
|
{ name: 'top-by-engagement', type: 'int', default: 0, help: 'When set to N>0, re-rank the folder by weighted engagement (likes×1 + retweets×3 + replies×2 + bookmarks×5 + log10(views+1)×0.5) and return the top N. Default 0 keeps the API\'s native (saved-time) ordering.' },
|
|
131
131
|
],
|
|
132
|
-
columns: ['id', 'author', 'text', 'likes', 'retweets', 'bookmarks', 'created_at', 'url'],
|
|
132
|
+
columns: ['id', 'author', 'text', 'likes', 'retweets', 'bookmarks', 'created_at', 'url', 'has_media', 'media_urls'],
|
|
133
133
|
func: async (page, kwargs) => {
|
|
134
134
|
const folderId = String(kwargs['folder-id'] || '').trim();
|
|
135
135
|
if (!folderId || !FOLDER_ID_PATTERN.test(folderId)) {
|
|
@@ -184,6 +184,7 @@ cli({
|
|
|
184
184
|
|
|
185
185
|
export const __test__ = {
|
|
186
186
|
parseBookmarkFolderTimeline,
|
|
187
|
+
extractFolderTweet,
|
|
187
188
|
buildFolderTimelineUrl,
|
|
188
189
|
FOLDER_ID_PATTERN,
|
|
189
190
|
};
|
|
@@ -2,7 +2,7 @@ import { describe, expect, it, vi } from 'vitest';
|
|
|
2
2
|
import { getRegistry } from '@jackwener/opencli/registry';
|
|
3
3
|
import { __test__ } from './bookmark-folder.js';
|
|
4
4
|
|
|
5
|
-
const { parseBookmarkFolderTimeline, buildFolderTimelineUrl, FOLDER_ID_PATTERN } = __test__;
|
|
5
|
+
const { parseBookmarkFolderTimeline, extractFolderTweet, buildFolderTimelineUrl, FOLDER_ID_PATTERN } = __test__;
|
|
6
6
|
|
|
7
7
|
describe('twitter bookmark-folder URL builder', () => {
|
|
8
8
|
it('embeds the folder id and count in the variables payload', () => {
|
|
@@ -97,6 +97,8 @@ describe('twitter bookmark-folder timeline parser', () => {
|
|
|
97
97
|
bookmarks: 3,
|
|
98
98
|
created_at: 'Tue Mar 17 09:00:00 +0000 2026',
|
|
99
99
|
url: 'https://x.com/alice/status/1',
|
|
100
|
+
has_media: false,
|
|
101
|
+
media_urls: [],
|
|
100
102
|
},
|
|
101
103
|
]);
|
|
102
104
|
expect(nextCursor).toBe('NEXT_CURSOR');
|
|
@@ -247,6 +249,62 @@ describe('twitter bookmark-folder timeline parser', () => {
|
|
|
247
249
|
it('returns empty array + null cursor for unknown envelope', () => {
|
|
248
250
|
expect(parseBookmarkFolderTimeline({}, new Set())).toEqual({ tweets: [], nextCursor: null });
|
|
249
251
|
});
|
|
252
|
+
|
|
253
|
+
it('includes photo media URLs from extended_entities', () => {
|
|
254
|
+
const tweet = extractFolderTweet({
|
|
255
|
+
rest_id: '101',
|
|
256
|
+
legacy: {
|
|
257
|
+
full_text: 'pic folder tweet',
|
|
258
|
+
extended_entities: {
|
|
259
|
+
media: [
|
|
260
|
+
{ type: 'photo', media_url_https: 'https://pbs.twimg.com/media/abc.jpg' },
|
|
261
|
+
{ type: 'photo', media_url_https: 'https://pbs.twimg.com/media/def.jpg' },
|
|
262
|
+
],
|
|
263
|
+
},
|
|
264
|
+
},
|
|
265
|
+
core: { user_results: { result: { legacy: { screen_name: 'eve' } } } },
|
|
266
|
+
}, new Set());
|
|
267
|
+
expect(tweet?.has_media).toBe(true);
|
|
268
|
+
expect(tweet?.media_urls).toEqual([
|
|
269
|
+
'https://pbs.twimg.com/media/abc.jpg',
|
|
270
|
+
'https://pbs.twimg.com/media/def.jpg',
|
|
271
|
+
]);
|
|
272
|
+
});
|
|
273
|
+
|
|
274
|
+
it('extracts mp4 variant URL for video media', () => {
|
|
275
|
+
const tweet = extractFolderTweet({
|
|
276
|
+
rest_id: '102',
|
|
277
|
+
legacy: {
|
|
278
|
+
full_text: 'video folder tweet',
|
|
279
|
+
extended_entities: {
|
|
280
|
+
media: [{
|
|
281
|
+
type: 'video',
|
|
282
|
+
media_url_https: 'https://pbs.twimg.com/amplify_video_thumb/thumb.jpg',
|
|
283
|
+
video_info: {
|
|
284
|
+
variants: [
|
|
285
|
+
{ content_type: 'application/x-mpegURL', url: 'https://video.twimg.com/playlist.m3u8' },
|
|
286
|
+
{ content_type: 'video/mp4', bitrate: 832000, url: 'https://video.twimg.com/low.mp4' },
|
|
287
|
+
{ content_type: 'video/mp4', bitrate: 2176000, url: 'https://video.twimg.com/high.mp4' },
|
|
288
|
+
],
|
|
289
|
+
},
|
|
290
|
+
}],
|
|
291
|
+
},
|
|
292
|
+
},
|
|
293
|
+
core: { user_results: { result: { legacy: { screen_name: 'frank' } } } },
|
|
294
|
+
}, new Set());
|
|
295
|
+
expect(tweet?.has_media).toBe(true);
|
|
296
|
+
expect(tweet?.media_urls?.[0]).toMatch(/\.mp4$/);
|
|
297
|
+
});
|
|
298
|
+
|
|
299
|
+
it('returns has_media false / media_urls empty when no media present', () => {
|
|
300
|
+
const tweet = extractFolderTweet({
|
|
301
|
+
rest_id: '103',
|
|
302
|
+
legacy: { full_text: 'text only', favorite_count: 0, retweet_count: 0, bookmark_count: 0 },
|
|
303
|
+
core: { user_results: { result: { legacy: { screen_name: 'gail' } } } },
|
|
304
|
+
}, new Set());
|
|
305
|
+
expect(tweet?.has_media).toBe(false);
|
|
306
|
+
expect(tweet?.media_urls).toEqual([]);
|
|
307
|
+
});
|
|
250
308
|
});
|
|
251
309
|
|
|
252
310
|
describe('twitter bookmark-folder id validation', () => {
|
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
import { cli, Strategy } from '@jackwener/opencli/registry';
|
|
2
2
|
import { AuthRequiredError, CommandExecutionError } from '@jackwener/opencli/errors';
|
|
3
|
+
import { extractMedia } from './shared.js';
|
|
3
4
|
import { TWITTER_BEARER_TOKEN, applyTopByEngagement } from './utils.js';
|
|
4
5
|
const BOOKMARKS_QUERY_ID = 'Fy0QMy4q_aZCpkO0PnyLYw';
|
|
5
6
|
const MAX_PAGINATION_PAGES = 100;
|
|
@@ -42,7 +43,7 @@ function buildBookmarksUrl(count, cursor) {
|
|
|
42
43
|
+ `?variables=${encodeURIComponent(JSON.stringify(vars))}`
|
|
43
44
|
+ `&features=${encodeURIComponent(JSON.stringify(FEATURES))}`;
|
|
44
45
|
}
|
|
45
|
-
function extractBookmarkTweet(result, seen) {
|
|
46
|
+
export function extractBookmarkTweet(result, seen) {
|
|
46
47
|
if (!result)
|
|
47
48
|
return null;
|
|
48
49
|
const tw = result.tweet || result;
|
|
@@ -64,9 +65,10 @@ function extractBookmarkTweet(result, seen) {
|
|
|
64
65
|
bookmarks: legacy.bookmark_count || 0,
|
|
65
66
|
created_at: legacy.created_at || '',
|
|
66
67
|
url: `https://x.com/${screenName}/status/${tw.rest_id}`,
|
|
68
|
+
...extractMedia(legacy),
|
|
67
69
|
};
|
|
68
70
|
}
|
|
69
|
-
function parseBookmarks(data, seen) {
|
|
71
|
+
export function parseBookmarks(data, seen) {
|
|
70
72
|
const tweets = [];
|
|
71
73
|
let nextCursor = null;
|
|
72
74
|
const instructions = data?.data?.bookmark_timeline_v2?.timeline?.instructions
|
|
@@ -106,12 +108,11 @@ cli({
|
|
|
106
108
|
domain: 'x.com',
|
|
107
109
|
strategy: Strategy.COOKIE,
|
|
108
110
|
browser: true,
|
|
109
|
-
siteSession: 'persistent',
|
|
110
111
|
args: [
|
|
111
112
|
{ name: 'limit', type: 'int', default: 20, help: 'Maximum number of bookmarks to return (default 20).' },
|
|
112
113
|
{ name: 'top-by-engagement', type: 'int', default: 0, help: 'When set to N>0, re-rank the bookmarks by weighted engagement (likes×1 + retweets×3 + replies×2 + bookmarks×5 + log10(views+1)×0.5) and return the top N. Default 0 keeps the API\'s native (saved-time) ordering.' },
|
|
113
114
|
],
|
|
114
|
-
columns: ['id', 'author', 'text', 'likes', 'retweets', 'bookmarks', 'created_at', 'url'],
|
|
115
|
+
columns: ['id', 'author', 'text', 'likes', 'retweets', 'bookmarks', 'created_at', 'url', 'has_media', 'media_urls'],
|
|
115
116
|
func: async (page, kwargs) => {
|
|
116
117
|
const limit = kwargs.limit || 20;
|
|
117
118
|
const cookies = await page.getCookies({ url: 'https://x.com' });
|
|
@@ -174,3 +175,7 @@ cli({
|
|
|
174
175
|
return applyTopByEngagement(trimmed, kwargs['top-by-engagement']);
|
|
175
176
|
},
|
|
176
177
|
});
|
|
178
|
+
export const __test__ = {
|
|
179
|
+
parseBookmarks,
|
|
180
|
+
extractBookmarkTweet,
|
|
181
|
+
};
|
|
@@ -0,0 +1,205 @@
|
|
|
1
|
+
import { describe, expect, it } from 'vitest';
|
|
2
|
+
import { __test__ } from './bookmarks.js';
|
|
3
|
+
|
|
4
|
+
const { parseBookmarks, extractBookmarkTweet } = __test__;
|
|
5
|
+
|
|
6
|
+
describe('twitter bookmarks parser', () => {
|
|
7
|
+
it('extracts a baseline tweet with no media (has_media false, media_urls empty)', () => {
|
|
8
|
+
const tweet = extractBookmarkTweet({
|
|
9
|
+
rest_id: '1',
|
|
10
|
+
legacy: {
|
|
11
|
+
full_text: 'plain bookmark',
|
|
12
|
+
favorite_count: 5,
|
|
13
|
+
retweet_count: 1,
|
|
14
|
+
bookmark_count: 2,
|
|
15
|
+
created_at: 'Wed Apr 16 10:00:00 +0000 2026',
|
|
16
|
+
},
|
|
17
|
+
core: { user_results: { result: { legacy: { screen_name: 'alice', name: 'Alice' } } } },
|
|
18
|
+
}, new Set());
|
|
19
|
+
expect(tweet).toEqual({
|
|
20
|
+
id: '1',
|
|
21
|
+
author: 'alice',
|
|
22
|
+
name: 'Alice',
|
|
23
|
+
text: 'plain bookmark',
|
|
24
|
+
likes: 5,
|
|
25
|
+
retweets: 1,
|
|
26
|
+
bookmarks: 2,
|
|
27
|
+
created_at: 'Wed Apr 16 10:00:00 +0000 2026',
|
|
28
|
+
url: 'https://x.com/alice/status/1',
|
|
29
|
+
has_media: false,
|
|
30
|
+
media_urls: [],
|
|
31
|
+
});
|
|
32
|
+
});
|
|
33
|
+
|
|
34
|
+
it('includes photo media URLs from extended_entities', () => {
|
|
35
|
+
const tweet = extractBookmarkTweet({
|
|
36
|
+
rest_id: '101',
|
|
37
|
+
legacy: {
|
|
38
|
+
full_text: 'pic bookmark',
|
|
39
|
+
extended_entities: {
|
|
40
|
+
media: [
|
|
41
|
+
{ type: 'photo', media_url_https: 'https://pbs.twimg.com/media/abc.jpg' },
|
|
42
|
+
{ type: 'photo', media_url_https: 'https://pbs.twimg.com/media/def.jpg' },
|
|
43
|
+
],
|
|
44
|
+
},
|
|
45
|
+
},
|
|
46
|
+
core: { user_results: { result: { legacy: { screen_name: 'bob' } } } },
|
|
47
|
+
}, new Set());
|
|
48
|
+
expect(tweet?.has_media).toBe(true);
|
|
49
|
+
expect(tweet?.media_urls).toEqual([
|
|
50
|
+
'https://pbs.twimg.com/media/abc.jpg',
|
|
51
|
+
'https://pbs.twimg.com/media/def.jpg',
|
|
52
|
+
]);
|
|
53
|
+
});
|
|
54
|
+
|
|
55
|
+
it('extracts mp4 variant URL for video media', () => {
|
|
56
|
+
const tweet = extractBookmarkTweet({
|
|
57
|
+
rest_id: '102',
|
|
58
|
+
legacy: {
|
|
59
|
+
full_text: 'video bookmark',
|
|
60
|
+
extended_entities: {
|
|
61
|
+
media: [{
|
|
62
|
+
type: 'video',
|
|
63
|
+
media_url_https: 'https://pbs.twimg.com/amplify_video_thumb/thumb.jpg',
|
|
64
|
+
video_info: {
|
|
65
|
+
variants: [
|
|
66
|
+
{ content_type: 'application/x-mpegURL', url: 'https://video.twimg.com/playlist.m3u8' },
|
|
67
|
+
{ content_type: 'video/mp4', bitrate: 832000, url: 'https://video.twimg.com/low.mp4' },
|
|
68
|
+
{ content_type: 'video/mp4', bitrate: 2176000, url: 'https://video.twimg.com/high.mp4' },
|
|
69
|
+
],
|
|
70
|
+
},
|
|
71
|
+
}],
|
|
72
|
+
},
|
|
73
|
+
},
|
|
74
|
+
core: { user_results: { result: { legacy: { screen_name: 'carol' } } } },
|
|
75
|
+
}, new Set());
|
|
76
|
+
expect(tweet?.has_media).toBe(true);
|
|
77
|
+
expect(tweet?.media_urls?.[0]).toMatch(/\.mp4$/);
|
|
78
|
+
});
|
|
79
|
+
|
|
80
|
+
it('falls back to entities.media when extended_entities is absent', () => {
|
|
81
|
+
const tweet = extractBookmarkTweet({
|
|
82
|
+
rest_id: '103',
|
|
83
|
+
legacy: {
|
|
84
|
+
full_text: 'entities-only media',
|
|
85
|
+
entities: {
|
|
86
|
+
media: [{ type: 'photo', media_url_https: 'https://pbs.twimg.com/media/legacy.jpg' }],
|
|
87
|
+
},
|
|
88
|
+
},
|
|
89
|
+
core: { user_results: { result: { legacy: { screen_name: 'dave' } } } },
|
|
90
|
+
}, new Set());
|
|
91
|
+
expect(tweet?.has_media).toBe(true);
|
|
92
|
+
expect(tweet?.media_urls).toEqual(['https://pbs.twimg.com/media/legacy.jpg']);
|
|
93
|
+
});
|
|
94
|
+
|
|
95
|
+
it('prefers note_tweet text over truncated full_text', () => {
|
|
96
|
+
const tweet = extractBookmarkTweet({
|
|
97
|
+
rest_id: '2',
|
|
98
|
+
legacy: { full_text: 'short text…', favorite_count: 0, retweet_count: 0, bookmark_count: 0 },
|
|
99
|
+
note_tweet: { note_tweet_results: { result: { text: 'full long-form text body' } } },
|
|
100
|
+
core: { user_results: { result: { core: { screen_name: 'erin' } } } },
|
|
101
|
+
}, new Set());
|
|
102
|
+
expect(tweet?.text).toBe('full long-form text body');
|
|
103
|
+
});
|
|
104
|
+
|
|
105
|
+
it('deduplicates tweets across the seen Set', () => {
|
|
106
|
+
const data = {
|
|
107
|
+
data: {
|
|
108
|
+
bookmark_timeline_v2: {
|
|
109
|
+
timeline: {
|
|
110
|
+
instructions: [{
|
|
111
|
+
entries: [
|
|
112
|
+
{
|
|
113
|
+
entryId: 'tweet-3',
|
|
114
|
+
content: {
|
|
115
|
+
itemContent: {
|
|
116
|
+
tweet_results: {
|
|
117
|
+
result: {
|
|
118
|
+
rest_id: '3',
|
|
119
|
+
legacy: { full_text: 'first', favorite_count: 0, retweet_count: 0, bookmark_count: 0 },
|
|
120
|
+
core: { user_results: { result: { legacy: { screen_name: 'frank' } } } },
|
|
121
|
+
},
|
|
122
|
+
},
|
|
123
|
+
},
|
|
124
|
+
},
|
|
125
|
+
},
|
|
126
|
+
{
|
|
127
|
+
entryId: 'tweet-3-dup',
|
|
128
|
+
content: {
|
|
129
|
+
itemContent: {
|
|
130
|
+
tweet_results: {
|
|
131
|
+
result: {
|
|
132
|
+
rest_id: '3',
|
|
133
|
+
legacy: { full_text: 'duplicate' },
|
|
134
|
+
core: { user_results: { result: { legacy: { screen_name: 'frank' } } } },
|
|
135
|
+
},
|
|
136
|
+
},
|
|
137
|
+
},
|
|
138
|
+
},
|
|
139
|
+
},
|
|
140
|
+
],
|
|
141
|
+
}],
|
|
142
|
+
},
|
|
143
|
+
},
|
|
144
|
+
},
|
|
145
|
+
};
|
|
146
|
+
const seen = new Set();
|
|
147
|
+
const { tweets } = parseBookmarks(data, seen);
|
|
148
|
+
expect(tweets).toHaveLength(1);
|
|
149
|
+
expect(tweets[0].text).toBe('first');
|
|
150
|
+
});
|
|
151
|
+
|
|
152
|
+
it('extracts cursor + tweets from the bookmark_timeline_v2 envelope', () => {
|
|
153
|
+
const data = {
|
|
154
|
+
data: {
|
|
155
|
+
bookmark_timeline_v2: {
|
|
156
|
+
timeline: {
|
|
157
|
+
instructions: [
|
|
158
|
+
{
|
|
159
|
+
type: 'TimelineAddEntries',
|
|
160
|
+
entries: [
|
|
161
|
+
{
|
|
162
|
+
entryId: 'tweet-4',
|
|
163
|
+
content: {
|
|
164
|
+
itemContent: {
|
|
165
|
+
tweet_results: {
|
|
166
|
+
result: {
|
|
167
|
+
rest_id: '4',
|
|
168
|
+
legacy: {
|
|
169
|
+
full_text: 'envelope tweet',
|
|
170
|
+
favorite_count: 1,
|
|
171
|
+
retweet_count: 0,
|
|
172
|
+
bookmark_count: 0,
|
|
173
|
+
extended_entities: {
|
|
174
|
+
media: [{ type: 'photo', media_url_https: 'https://pbs.twimg.com/media/x.jpg' }],
|
|
175
|
+
},
|
|
176
|
+
},
|
|
177
|
+
core: { user_results: { result: { legacy: { screen_name: 'gina' } } } },
|
|
178
|
+
},
|
|
179
|
+
},
|
|
180
|
+
},
|
|
181
|
+
},
|
|
182
|
+
},
|
|
183
|
+
{
|
|
184
|
+
entryId: 'cursor-bottom-Y',
|
|
185
|
+
content: { __typename: 'TimelineTimelineCursor', cursorType: 'Bottom', value: 'NEXT' },
|
|
186
|
+
},
|
|
187
|
+
],
|
|
188
|
+
},
|
|
189
|
+
],
|
|
190
|
+
},
|
|
191
|
+
},
|
|
192
|
+
},
|
|
193
|
+
};
|
|
194
|
+
const { tweets, nextCursor } = parseBookmarks(data, new Set());
|
|
195
|
+
expect(tweets).toHaveLength(1);
|
|
196
|
+
expect(tweets[0].id).toBe('4');
|
|
197
|
+
expect(tweets[0].has_media).toBe(true);
|
|
198
|
+
expect(tweets[0].media_urls).toEqual(['https://pbs.twimg.com/media/x.jpg']);
|
|
199
|
+
expect(nextCursor).toBe('NEXT');
|
|
200
|
+
});
|
|
201
|
+
|
|
202
|
+
it('returns empty tweets + null cursor for unknown envelope', () => {
|
|
203
|
+
expect(parseBookmarks({}, new Set())).toEqual({ tweets: [], nextCursor: null });
|
|
204
|
+
});
|
|
205
|
+
});
|
package/clis/twitter/download.js
CHANGED
|
@@ -15,7 +15,6 @@ cli({
|
|
|
15
15
|
description: 'Download Twitter/X media (images and videos). Provide either <username> to scan a profile\'s media tab, or --tweet-url to download a single tweet.',
|
|
16
16
|
domain: 'x.com',
|
|
17
17
|
strategy: Strategy.COOKIE,
|
|
18
|
-
siteSession: 'persistent',
|
|
19
18
|
args: [
|
|
20
19
|
{ name: 'username', positional: true, help: 'Twitter username (with or without @) to scan their /media tab. Either <username> or --tweet-url is required.' },
|
|
21
20
|
{ name: 'tweet-url', help: 'Single tweet URL to download. Use this OR <username>, not both required at once.' },
|
package/clis/twitter/likes.js
CHANGED
|
@@ -143,7 +143,6 @@ cli({
|
|
|
143
143
|
domain: 'x.com',
|
|
144
144
|
strategy: Strategy.COOKIE,
|
|
145
145
|
browser: true,
|
|
146
|
-
siteSession: 'persistent',
|
|
147
146
|
args: [
|
|
148
147
|
{ name: 'username', type: 'string', positional: true, help: 'Twitter screen name (with or without @). Defaults to the logged-in user when omitted.' },
|
|
149
148
|
{ name: 'limit', type: 'int', default: 20, help: 'Maximum number of liked tweets to return (default 20).' },
|
|
@@ -115,7 +115,6 @@ cli({
|
|
|
115
115
|
domain: 'x.com',
|
|
116
116
|
strategy: Strategy.COOKIE,
|
|
117
117
|
browser: true,
|
|
118
|
-
siteSession: 'persistent',
|
|
119
118
|
args: [
|
|
120
119
|
{ name: 'listId', positional: true, type: 'string', required: true, help: 'Numeric ID of a Twitter/X list (e.g. from `opencli twitter lists`)' },
|
|
121
120
|
{ name: 'limit', type: 'int', default: 50 },
|
package/clis/twitter/lists.js
CHANGED
package/clis/twitter/profile.js
CHANGED
|
@@ -11,7 +11,6 @@ cli({
|
|
|
11
11
|
domain: 'x.com',
|
|
12
12
|
strategy: Strategy.COOKIE,
|
|
13
13
|
browser: true,
|
|
14
|
-
siteSession: 'persistent',
|
|
15
14
|
args: [
|
|
16
15
|
{ name: 'username', type: 'string', positional: true, help: 'Twitter screen name (with or without @). Defaults to the logged-in user when omitted.' },
|
|
17
16
|
],
|
package/clis/twitter/search.js
CHANGED
|
@@ -261,7 +261,6 @@ cli({
|
|
|
261
261
|
domain: 'x.com',
|
|
262
262
|
strategy: Strategy.COOKIE,
|
|
263
263
|
browser: true,
|
|
264
|
-
siteSession: 'persistent',
|
|
265
264
|
args: [
|
|
266
265
|
{ name: 'query', type: 'string', required: true, positional: true, help: 'Search query. Raw X operators (e.g. "exact phrase", #tag, OR, lang:en, since:YYYY-MM-DD, from:, since:) are passed through unchanged.' },
|
|
267
266
|
{ name: 'filter', type: 'string', default: 'top', choices: ['top', 'live'], help: 'Legacy alias for --product. Kept for backwards compatibility; if --product is set it wins.' },
|
package/clis/twitter/thread.js
CHANGED
|
@@ -100,7 +100,6 @@ cli({
|
|
|
100
100
|
domain: 'x.com',
|
|
101
101
|
strategy: Strategy.COOKIE,
|
|
102
102
|
browser: true,
|
|
103
|
-
siteSession: 'persistent',
|
|
104
103
|
args: [
|
|
105
104
|
{ name: 'tweet-id', positional: true, type: 'string', required: true, help: 'Tweet numeric ID (e.g. 1234567890) or full status URL' },
|
|
106
105
|
{ name: 'limit', type: 'int', default: 50 },
|