@jackwener/opencli 1.7.22 → 1.8.0

package/clis/twitter/download.test.js

@@ -0,0 +1,457 @@
+import { beforeEach, describe, expect, it, vi } from 'vitest';
+const { mockDownloadMedia, mockFormatCookieHeader } = vi.hoisted(() => ({
+    mockDownloadMedia: vi.fn(),
+    mockFormatCookieHeader: vi.fn(() => 'ct0=token'),
+}));
+vi.mock('@jackwener/opencli/download/media-download', () => ({
+    downloadMedia: mockDownloadMedia,
+}));
+vi.mock('@jackwener/opencli/download', () => ({
+    formatCookieHeader: mockFormatCookieHeader,
+}));
+import { getRegistry } from '@jackwener/opencli/registry';
+import { ArgumentError, AuthRequiredError, CommandExecutionError, EmptyResultError } from '@jackwener/opencli/errors';
+import { __test__ } from './download.js';
+
+const {
+    buildUserMediaUrl,
+    buildUserByScreenNameUrl,
+    parseUserMedia,
+    classifyMediaUrl,
+    requireLimit,
+    nextUserMediaFetchCount,
+} = __test__;
+
+function createPageMock(evaluateResults = []) {
+    const evaluate = vi.fn();
+    for (const result of evaluateResults) evaluate.mockResolvedValueOnce(result);
+    evaluate.mockResolvedValue(undefined);
+    return {
+        goto: vi.fn().mockResolvedValue(undefined),
+        wait: vi.fn().mockResolvedValue(undefined),
+        evaluate,
+        getCookies: vi.fn().mockResolvedValue([{ name: 'ct0', value: 'token', domain: '.x.com' }]),
+    };
+}
+
+function userLookupPayload(userId = '42') {
+    return {
+        ok: true,
+        payload: {
+            data: {
+                user: {
+                    result: { rest_id: userId },
+                },
+            },
+        },
+    };
+}
+
+function userMediaPayload(entries) {
+    return {
+        ok: true,
+        payload: {
+            data: {
+                user: {
+                    result: {
+                        timeline_v2: {
+                            timeline: {
+                                instructions: [{ entries }],
+                            },
+                        },
+                    },
+                },
+            },
+        },
+    };
+}
+
+function tweetEntry(id, url = `https://pbs.twimg.com/media/${id}.jpg`) {
+    return {
+        content: {
+            itemContent: {
+                tweet_results: {
+                    result: {
+                        rest_id: id,
+                        legacy: {
+                            extended_entities: {
+                                media: [{ type: 'photo', media_url_https: url }],
+                            },
+                        },
+                    },
+                },
+            },
+        },
+    };
+}
+
+describe('twitter download helpers', () => {
+    beforeEach(() => {
+        mockDownloadMedia.mockReset();
+        mockDownloadMedia.mockResolvedValue([{ index: 1, type: 'image', status: 'success', size: '1 KB' }]);
+        mockFormatCookieHeader.mockClear();
+    });
+
+    it('registers the canonical download columns', () => {
+        const cmd = getRegistry().get('twitter/download');
+        expect(cmd?.columns).toEqual(['index', 'tweet_id', 'url', 'type', 'status', 'size']);
+    });
+
+    it('makes username positional and tweet-url a flag', () => {
+        const cmd = getRegistry().get('twitter/download');
+        const usernameArg = cmd?.args?.find((a) => a.name === 'username');
+        const tweetUrlArg = cmd?.args?.find((a) => a.name === 'tweet-url');
+        expect(usernameArg?.positional).toBe(true);
+        expect(tweetUrlArg?.positional).not.toBe(true);
+    });
+
+    it('builds a UserMedia URL with userId, count and cursor', () => {
+        const url = buildUserMediaUrl(
+            { queryId: 'QID', features: { fa: true }, fieldToggles: { fb: true } },
+            '42',
+            50,
+            'cursor-xyz',
+        );
+        expect(url.startsWith('/i/api/graphql/QID/UserMedia?')).toBe(true);
+        const vars = JSON.parse(decodeURIComponent(url.match(/variables=([^&]+)/)[1]));
+        expect(vars.userId).toBe('42');
+        expect(vars.count).toBe(50);
+        expect(vars.cursor).toBe('cursor-xyz');
+        expect(vars.includePromotedContent).toBe(false);
+    });
+
+    it('omits cursor variable when not paging', () => {
+        const url = buildUserMediaUrl({ queryId: 'QID', features: {}, fieldToggles: {} }, '42', 10, null);
+        const vars = JSON.parse(decodeURIComponent(url.match(/variables=([^&]+)/)[1]));
+        expect(vars.cursor).toBeUndefined();
+    });
+
+    it('builds a UserByScreenName URL with the screen_name variable', () => {
+        const url = buildUserByScreenNameUrl(
+            { queryId: 'UBSN', features: {}, fieldToggles: {} },
+            'jack',
+        );
+        expect(url.startsWith('/i/api/graphql/UBSN/UserByScreenName?')).toBe(true);
+        expect(decodeURIComponent(url)).toContain('"screen_name":"jack"');
+    });
+
+    it('classifies twimg video URLs as video and pbs URLs as image', () => {
+        expect(classifyMediaUrl('https://video.twimg.com/amplify_video/123/vid/avc1/720x1280/abc.mp4?tag=27')).toBe('video');
+        expect(classifyMediaUrl('https://pbs.twimg.com/media/AbCdEf.jpg')).toBe('image');
+        expect(classifyMediaUrl('https://example.com/clip.m3u8')).toBe('video');
+        expect(classifyMediaUrl(null)).toBe('unknown');
+    });
+
+    it('strictly validates profile download limit', () => {
+        expect(requireLimit(undefined)).toBe(10);
+        expect(requireLimit(1)).toBe(1);
+        for (const value of [0, -1, 1.5, 'abc', 1001]) {
+            expect(() => requireLimit(value)).toThrow(ArgumentError);
+        }
+    });
+
+    it('calculates profile media page sizes without silently clamping user input', () => {
+        expect(nextUserMediaFetchCount(1, 0)).toBe(11);
+        expect(nextUserMediaFetchCount(1000, 0)).toBe(100);
+        expect(nextUserMediaFetchCount(1000, 950)).toBe(60);
+        expect(nextUserMediaFetchCount(10, 10)).toBe(0);
+    });
+
+    it('extracts media urls and the bottom cursor from a UserMedia payload', () => {
+        const payload = {
+            data: {
+                user: {
+                    result: {
+                        timeline_v2: {
+                            timeline: {
+                                instructions: [
+                                    {
+                                        entries: [
+                                            {
+                                                content: {
+                                                    itemContent: {
+                                                        tweet_results: {
+                                                            result: {
+                                                                rest_id: 'tweet-1',
+                                                                legacy: {
+                                                                    extended_entities: {
+                                                                        media: [
+                                                                            { type: 'photo', media_url_https: 'https://pbs.twimg.com/media/IMG1.jpg' },
+                                                                            { type: 'video', video_info: { variants: [{ content_type: 'video/mp4', url: 'https://video.twimg.com/v/1.mp4' }] } },
+                                                                        ],
+                                                                    },
+                                                                },
+                                                            },
+                                                        },
+                                                    },
+                                                },
+                                            },
+                                            {
+                                                content: {
+                                                    entryType: 'TimelineTimelineCursor',
+                                                    cursorType: 'Bottom',
+                                                    value: 'next-cursor-abc',
+                                                },
+                                            },
+                                        ],
+                                    },
+                                ],
+                            },
+                        },
+                    },
+                },
+            },
+        };
+        const seen = new Set();
+        const { items, nextCursor } = parseUserMedia(payload, seen);
+        expect(nextCursor).toBe('next-cursor-abc');
+        expect(items).toHaveLength(2);
+        expect(items[0]).toMatchObject({ tweet_id: 'tweet-1', url: 'https://pbs.twimg.com/media/IMG1.jpg', type: 'image' });
+        expect(items[1]).toMatchObject({ tweet_id: 'tweet-1', url: 'https://video.twimg.com/v/1.mp4', type: 'video' });
+        expect(seen.has('tweet-1')).toBe(true);
+    });
+
+    it('skips already-seen tweets across pages', () => {
+        const tweetEntry = (id) => ({
+            content: {
+                itemContent: {
+                    tweet_results: {
+                        result: {
+                            rest_id: id,
+                            legacy: {
+                                extended_entities: {
+                                    media: [{ type: 'photo', media_url_https: `https://pbs.twimg.com/media/${id}.jpg` }],
+                                },
+                            },
+                        },
+                    },
+                },
+            },
+        });
+        const payload = {
+            data: {
+                user: {
+                    result: {
+                        timeline_v2: {
+                            timeline: {
+                                instructions: [{ entries: [tweetEntry('A'), tweetEntry('A'), tweetEntry('B')] }],
+                            },
+                        },
+                    },
+                },
+            },
+        };
+        const seen = new Set();
+        const { items } = parseUserMedia(payload, seen);
+        expect(items.map((item) => item.tweet_id)).toEqual(['A', 'B']);
+    });
+
+    it('treats TweetWithVisibilityResults wrappers as tweets', () => {
+        const payload = {
+            data: {
+                user: {
+                    result: {
+                        timeline_v2: {
+                            timeline: {
+                                instructions: [
+                                    {
+                                        entries: [
+                                            {
+                                                content: {
+                                                    itemContent: {
+                                                        tweet_results: {
+                                                            result: {
+                                                                __typename: 'TweetWithVisibilityResults',
+                                                                tweet: {
+                                                                    rest_id: 'wrapped-1',
+                                                                    legacy: {
+                                                                        extended_entities: {
+                                                                            media: [{ type: 'photo', media_url_https: 'https://pbs.twimg.com/media/W.jpg' }],
+                                                                        },
+                                                                    },
+                                                                },
+                                                            },
+                                                        },
+                                                    },
+                                                },
+                                            },
+                                        ],
+                                    },
+                                ],
+                            },
+                        },
+                    },
+                },
+            },
+        };
+        const { items } = parseUserMedia(payload, new Set());
+        expect(items).toHaveLength(1);
+        expect(items[0].tweet_id).toBe('wrapped-1');
+    });
+
+    it('fails typed when UserMedia payload has no timeline instructions', () => {
+        expect(() => parseUserMedia({ data: { user: { result: {} } } }, new Set()))
+            .toThrow(CommandExecutionError);
+    });
+
+    it('rejects missing, mixed, invalid username and invalid limit before navigation', async () => {
+        const cmd = getRegistry().get('twitter/download');
+        for (const args of [
+            {},
+            { username: 'jack', 'tweet-url': 'https://x.com/jack/status/123' },
+            { username: 'bad/name' },
+            { username: 'jack', limit: 0 },
+        ]) {
+            const page = createPageMock();
+            await expect(cmd.func(page, args)).rejects.toBeInstanceOf(ArgumentError);
+            expect(page.goto).not.toHaveBeenCalled();
+        }
+    });
+
+    it('downloads profile media through UserByScreenName and UserMedia GraphQL payloads', async () => {
+        const cmd = getRegistry().get('twitter/download');
+        mockDownloadMedia.mockResolvedValueOnce([
+            { index: 1, type: 'image', status: 'success', size: '1 KB' },
+            { index: 2, type: 'image', status: 'success', size: '2 KB' },
+        ]);
+        const page = createPageMock([
+            { queryId: 'UM', features: { a: true }, fieldToggles: {} },
+            { queryId: 'UB', features: {}, fieldToggles: {} },
+            userLookupPayload('42'),
+            userMediaPayload([
+                tweetEntry('A'),
+                {
+                    content: {
+                        entryType: 'TimelineTimelineCursor',
+                        cursorType: 'Bottom',
+                        value: 'cursor-1',
+                    },
+                },
+            ]),
+            userMediaPayload([tweetEntry('B')]),
+        ]);
+        const rows = await cmd.func(page, { username: '@jack', limit: 2, output: './out' });
+        expect(page.goto).toHaveBeenCalledWith('https://x.com/jack');
+        expect(page.evaluate).toHaveBeenCalledTimes(5);
+        expect(mockDownloadMedia).toHaveBeenCalledWith([
+            { tweet_id: 'A', url: 'https://pbs.twimg.com/media/A.jpg', type: 'image' },
+            { tweet_id: 'B', url: 'https://pbs.twimg.com/media/B.jpg', type: 'image' },
+        ], expect.objectContaining({
+            output: './out',
+            subdir: 'jack',
+            filenamePrefix: 'jack',
+            cookies: 'ct0=token',
+        }));
+        expect(rows).toEqual([
+            {
+                index: 1,
+                tweet_id: 'A',
+                url: 'https://pbs.twimg.com/media/A.jpg',
+                type: 'image',
+                status: 'success',
+                size: '1 KB',
+            },
+            {
+                index: 2,
+                tweet_id: 'B',
+                url: 'https://pbs.twimg.com/media/B.jpg',
+                type: 'image',
+                status: 'success',
+                size: '2 KB',
+            },
+        ]);
+    });
+
+    it('maps missing ct0 and GraphQL auth failures to AuthRequiredError', async () => {
+        const cmd = getRegistry().get('twitter/download');
+        const noCt0Page = createPageMock();
+        noCt0Page.getCookies.mockResolvedValueOnce([]);
+        await expect(cmd.func(noCt0Page, { username: 'jack', limit: 1 }))
+            .rejects.toBeInstanceOf(AuthRequiredError);
+
+        const authPage = createPageMock([
+            { queryId: 'UM', features: {}, fieldToggles: {} },
+            { queryId: 'UB', features: {}, fieldToggles: {} },
+            { ok: false, status: 401 },
+        ]);
+        await expect(cmd.func(authPage, { username: 'jack', limit: 1 }))
+            .rejects.toBeInstanceOf(AuthRequiredError);
+    });
+
+    it('fails typed for malformed UserMedia and fetch failures instead of partial success', async () => {
+        const cmd = getRegistry().get('twitter/download');
+        const malformedPage = createPageMock([
+            { queryId: 'UM', features: {}, fieldToggles: {} },
+            { queryId: 'UB', features: {}, fieldToggles: {} },
+            userLookupPayload('42'),
+            { ok: true, payload: { data: { user: { result: {} } } } },
+        ]);
+        await expect(cmd.func(malformedPage, { username: 'jack', limit: 1 }))
+            .rejects.toBeInstanceOf(CommandExecutionError);
+
+        const partialPage = createPageMock([
+            { queryId: 'UM', features: {}, fieldToggles: {} },
+            { queryId: 'UB', features: {}, fieldToggles: {} },
+            userLookupPayload('42'),
+            userMediaPayload([
+                tweetEntry('A'),
+                {
+                    content: {
+                        entryType: 'TimelineTimelineCursor',
+                        cursorType: 'Bottom',
+                        value: 'cursor-1',
+                    },
+                },
+            ]),
+            { ok: false, status: 500 },
+        ]);
+        await expect(cmd.func(partialPage, { username: 'jack', limit: 2 }))
+            .rejects.toBeInstanceOf(CommandExecutionError);
+        expect(mockDownloadMedia).not.toHaveBeenCalled();
+
+        const repeatedCursorPage = createPageMock([
+            { queryId: 'UM', features: {}, fieldToggles: {} },
+            { queryId: 'UB', features: {}, fieldToggles: {} },
+            userLookupPayload('42'),
+            userMediaPayload([
+                tweetEntry('A'),
+                {
+                    content: {
+                        entryType: 'TimelineTimelineCursor',
+                        cursorType: 'Bottom',
+                        value: 'cursor-1',
+                    },
+                },
+            ]),
+            userMediaPayload([
+                tweetEntry('B'),
+                {
+                    content: {
+                        entryType: 'TimelineTimelineCursor',
+                        cursorType: 'Bottom',
+                        value: 'cursor-1',
+                    },
+                },
+            ]),
+        ]);
+        await expect(cmd.func(repeatedCursorPage, { username: 'jack', limit: 3 }))
+            .rejects.toThrowError(/same cursor twice/);
+        expect(mockDownloadMedia).not.toHaveBeenCalled();
+    });
+
+    it('uses typed empty result for profile or tweet media absence', async () => {
+        const cmd = getRegistry().get('twitter/download');
+        const profilePage = createPageMock([
+            { queryId: 'UM', features: {}, fieldToggles: {} },
+            { queryId: 'UB', features: {}, fieldToggles: {} },
+            userLookupPayload('42'),
+            userMediaPayload([]),
+        ]);
+        await expect(cmd.func(profilePage, { username: 'jack', limit: 1 }))
+            .rejects.toBeInstanceOf(EmptyResultError);
+
+        const tweetPage = createPageMock([[]]);
+        await expect(cmd.func(tweetPage, { 'tweet-url': 'https://x.com/jack/status/123' }))
+            .rejects.toBeInstanceOf(EmptyResultError);
+    });
+});

package/clis/twitter/list-create.js

@@ -0,0 +1,155 @@
+import { cli, Strategy } from '@jackwener/opencli/registry';
+import { ArgumentError, AuthRequiredError, CommandExecutionError } from '@jackwener/opencli/errors';
+import { unwrapBrowserResult } from './shared.js';
+import { TWITTER_BEARER_TOKEN } from './utils.js';
+
+const CREATE_LIST_QUERY_ID = 'UQRa0jJ9doxGEIQRea1Y0w';
+const NAME_MAX = 25;
+const DESCRIPTION_MAX = 100;
+
+// Minimal feature set as observed in the real CreateList web request payload.
+// Twitter rejects requests with extra/unknown features (DecodeException).
+const FEATURES = {
+    profile_label_improvements_pcf_label_in_post_enabled: true,
+    responsive_web_profile_redirect_enabled: false,
+    rweb_tipjar_consumption_enabled: false,
+    verified_phone_label_enabled: false,
+    responsive_web_graphql_skip_user_profile_image_extensions_enabled: false,
+    responsive_web_graphql_timeline_navigation_enabled: true,
+};
+
+export function parseListCreateArgs(kwargs) {
+    const name = String(kwargs.name || '').trim();
+    const description = String(kwargs.description || '').trim();
+    const modeRaw = String(kwargs.mode || 'public').trim().toLowerCase();
+    if (!name) {
+        throw new ArgumentError('List name is required', 'Example: opencli twitter list-create "My List"');
+    }
+    if (name.length > NAME_MAX) {
+        throw new ArgumentError(`List name too long: ${name.length} chars (max ${NAME_MAX})`);
+    }
+    if (description.length > DESCRIPTION_MAX) {
+        throw new ArgumentError(`Description too long: ${description.length} chars (max ${DESCRIPTION_MAX})`);
+    }
+    if (modeRaw !== 'public' && modeRaw !== 'private') {
+        throw new ArgumentError(`Invalid mode: ${JSON.stringify(kwargs.mode)}. Expected "public" or "private".`);
+    }
+    return { listName: name, listDescription: description, listMode: modeRaw, privateFlag: modeRaw === 'private' };
+}
+
+function requireCreateListResult(result, expectedName, expectedMode) {
+    if (!result || typeof result !== 'object') {
+        throw new CommandExecutionError(`Unexpected result from twitter list-create: ${JSON.stringify(result)}`);
+    }
+    if (result.httpStatus === 401 || result.httpStatus === 403) {
+        throw new AuthRequiredError('x.com', `Twitter CreateList returned HTTP ${result.httpStatus}`);
+    }
+    if (!result.ok) {
+        const snippet = String(result.bodyText || '').slice(0, 300);
+        throw new CommandExecutionError(`HTTP ${result.httpStatus} from CreateList: ${snippet}`);
+    }
+    if (!result.bodyJson || typeof result.bodyJson !== 'object') {
+        throw new CommandExecutionError(`CreateList returned malformed JSON payload. Body: ${String(result.bodyText || '').slice(0, 300)}`);
+    }
+    const list = result.bodyJson?.data?.list;
+    if (!list || typeof list !== 'object') {
+        const errors = result.bodyJson?.errors;
+        if (Array.isArray(errors) && errors.length > 0) {
+            throw new CommandExecutionError(`CreateList failed: ${errors[0].message || JSON.stringify(errors[0])}`);
+        }
+        throw new CommandExecutionError(`CreateList returned no list payload. Body: ${String(result.bodyText || '').slice(0, 300)}`);
+    }
+    const id = String(list.id_str || list.id || '');
+    if (!/^\d+$/.test(id)) {
+        throw new CommandExecutionError('CreateList returned a list payload without a numeric list id.');
+    }
+    if (typeof list.name !== 'string' || !list.name.trim()) {
+        throw new CommandExecutionError('CreateList returned a list payload without a list name.');
+    }
+    if (list.name.trim() !== expectedName) {
+        throw new CommandExecutionError(`CreateList returned name ${JSON.stringify(list.name)}, expected ${JSON.stringify(expectedName)}.`);
+    }
+    const modeValue = typeof list.mode === 'string' ? list.mode : '';
+    if (!modeValue) {
+        throw new CommandExecutionError('CreateList returned a list payload without list mode.');
+    }
+    const mode = /private/i.test(modeValue) ? 'private' : 'public';
+    if (mode !== expectedMode) {
+        throw new CommandExecutionError(`CreateList returned mode ${mode}, expected ${expectedMode}.`);
+    }
+    return { createdList: list, listId: id, listMode: mode };
+}
+
+export function buildListCreateRow({ result, name, description, mode }) {
+    const { createdList, listId, listMode } = requireCreateListResult(result, name, mode);
+    return {
+        id: listId,
+        name: createdList.name,
+        description: typeof createdList.description === 'string' ? createdList.description : description,
+        mode: listMode,
+        status: 'success',
+    };
+}
+
+cli({
+    site: 'twitter',
+    name: 'list-create',
+    description: 'Create a new Twitter/X list (returns the new list id)',
+    access: 'write',
+    domain: 'x.com',
+    strategy: Strategy.COOKIE,
+    browser: true,
+    args: [
+        { name: 'name', positional: true, type: 'string', required: true, help: `List name (max ${NAME_MAX} chars)` },
+        { name: 'description', type: 'string', default: '', help: `Optional list description (max ${DESCRIPTION_MAX} chars)` },
+        { name: 'mode', type: 'string', default: 'public', help: 'public | private' },
+    ],
+    columns: ['id', 'name', 'description', 'mode', 'status'],
+    func: async (page, kwargs) => {
+        const { listName: name, listDescription: description, listMode: mode, privateFlag: isPrivate } = parseListCreateArgs(kwargs);
+
+        await page.goto('https://x.com');
+        await page.wait(3);
+        const cookies = await page.getCookies({ url: 'https://x.com' });
+        const ct0 = cookies.find((c) => c.name === 'ct0')?.value || null;
+        if (!ct0) throw new AuthRequiredError('x.com', 'Not logged into x.com (no ct0 cookie)');
+
+        // Hardcode queryId: it must match the FEATURES schema below.
+        // Letting resolveTwitterQueryId() drift would pull a newer queryId
+        // whose schema would reject our simplified features payload.
+        const queryId = CREATE_LIST_QUERY_ID;
+
+        const headers = JSON.stringify({
+            'Authorization': `Bearer ${decodeURIComponent(TWITTER_BEARER_TOKEN)}`,
+            'X-Csrf-Token': ct0,
+            'X-Twitter-Auth-Type': 'OAuth2Session',
+            'X-Twitter-Active-User': 'yes',
+            'Content-Type': 'application/json',
+        });
+        const body = JSON.stringify({
+            variables: { isPrivate, name, description },
+            features: FEATURES,
+            queryId,
+        });
+        const apiUrl = `/i/api/graphql/${queryId}/CreateList`;
+
+        const result = unwrapBrowserResult(await page.evaluate(`async () => {
+            const r = await fetch(${JSON.stringify(apiUrl)}, {
+                method: 'POST',
+                headers: ${headers},
+                credentials: 'include',
+                body: ${JSON.stringify(body)},
+            });
+            const bodyText = await r.text();
+            let bodyJson = null;
+            try { bodyJson = JSON.parse(bodyText); } catch {}
+            return { ok: r.ok, httpStatus: r.status, bodyJson, bodyText };
+        }`));
+
+        // Note: Twitter sometimes returns a non-fatal `errors` array (e.g. a
+        // strato DecodeException from a side-effect serializer) WHILE STILL
+        // creating the list. So check for a valid list payload FIRST and
+        // only treat errors as fatal if no list came back.
+        return [buildListCreateRow({ result, name, description, mode })];
+    },
+});