@jackwener/opencli 1.6.6 → 1.6.8

package/README.md

@@ -133,7 +133,7 @@ git clone git@github.com:jackwener/opencli.git && cd opencli && npm install && n
 | **twitter** | `trending` `search` `timeline` `bookmarks` `post` `download` `profile` `article` `like` `likes` `notifications` `reply` `reply-dm` `thread` `follow` `unfollow` `followers` `following` `block` `unblock` `bookmark` `unbookmark` `delete` `hide-reply` `accept` |
 | **reddit** | `hot` `frontpage` `popular` `search` `subreddit` `read` `user` `user-posts` `user-comments` `upvote` `upvoted` `save` `saved` `comment` `subscribe` |
 | **amazon** | `bestsellers` `search` `product` `offer` `discussion` `movers-shakers` `new-releases` |
-| **1688** | `search` `item` `store` |
+| **1688** | `search` `item` `assets` `download` `store` |
 | **gemini** | `new` `ask` `image` `deep-research` `deep-research-result` |
 | **yuanbao** | `new` `ask` |
 | **notebooklm** | `status` `list` `open` `current` `get` `history` `summary` `note-list` `notes-get` `source-list` `source-get` `source-fulltext` `source-guide` |
@@ -191,6 +191,7 @@ OpenCLI supports downloading images, videos, and articles from supported platfor
 | **twitter** | Images, Videos | From user media tab or single tweet |
 | **douban** | Images | Poster / still image lists |
 | **pixiv** | Images | Original-quality illustrations, multi-page |
+| **1688** | Images, Videos | Downloads page-visible product media from item pages |
 | **zhihu** | Articles (Markdown) | Exports with optional image download |
 | **weixin** | Articles (Markdown) | WeChat Official Account articles |
 
@@ -200,6 +201,7 @@ For video downloads, install `yt-dlp` first: `brew install yt-dlp`
 opencli xiaohongshu download abc123 --output ./xhs
 opencli bilibili download BV1xxx --output ./bilibili
 opencli twitter download elonmusk --limit 20 --output ./twitter
+opencli 1688 download 841141931191 --output ./1688-downloads
 ```
 
 ## Output Formats

package/README.zh-CN.md

@@ -178,7 +178,7 @@ npx skills add jackwener/opencli --skill opencli-oneshot     # 快速命令参
 | **jike** | `feed` `search` `create` `like` `comment` `repost` `notifications` `post` `topic` `user` | 浏览器 |
 | **jimeng** | `generate` `history` | 浏览器 |
 | **yollomi** | `generate` `video` `edit` `upload` `models` `remove-bg` `upscale` `face-swap` `restore` `try-on` `background` `object-remover` | 浏览器 |
-| **linux-do** | `hot` `latest` `feed` `search` `categories` `category` `tags` `topic` `user-posts` `user-topics` | 浏览器 |
+| **linux-do** | `hot` `latest` `feed` `search` `categories` `category` `tags` `topic` `topic-content` `user-posts` `user-topics` | 浏览器 |
 | **stackoverflow** | `hot` `search` `bounties` `unanswered` | 公开 |
 | **steam** | `top-sellers` | 公开 |
 | **weread** | `shelf` `search` `book` `highlights` `notes` `notebooks` `ranking` | 浏览器 |
@@ -186,7 +186,7 @@ npx skills add jackwener/opencli --skill opencli-oneshot     # 快速命令参
 | **facebook** | `feed` `profile` `search` `friends` `groups` `events` `notifications` `memories` `add-friend` `join-group` | 浏览器 |
 | **google** | `news` `search` `suggest` `trends` | 公开 |
 | **amazon** | `bestsellers` `search` `product` `offer` `discussion` `movers-shakers` `new-releases` | 浏览器 |
-| **1688** | `search` `item` `store` | 浏览器 |
+| **1688** | `search` `item` `assets` `download` `store` | 浏览器 |
 | **gemini** | `new` `ask` `image` `deep-research` `deep-research-result` | 浏览器 |
 | **spotify** | `auth` `status` `play` `pause` `next` `prev` `volume` `search` `queue` `shuffle` `repeat` | OAuth API |
 | **notebooklm** | `status` `list` `open` `current` `get` `history` `summary` `note-list` `notes-get` `source-list` `source-get` `source-fulltext` `source-guide` | 浏览器 |
@@ -258,6 +258,7 @@ OpenCLI 支持从各平台下载图片、视频和文章。
 | **B站** | 视频 | 需要安装 `yt-dlp` |
 | **Twitter/X** | 图片、视频 | 从用户媒体页或单条推文下载 |
 | **Pixiv** | 图片 | 下载原始画质插画，支持多页作品 |
+| **1688** | 图片、视频 | 下载商品页中可见的商品素材 |
 | **知乎** | 文章（Markdown） | 导出文章，可选下载图片到本地 |
 | **微信公众号** | 文章（Markdown） | 导出微信公众号文章为 Markdown |
 | **豆瓣** | 图片 | 下载电影条目的海报 / 剧照图片 |
@@ -292,6 +293,9 @@ opencli twitter download --tweet-url "https://x.com/user/status/123" --output ./
 # 下载豆瓣电影海报 / 剧照
 opencli douban download 30382501 --output ./douban
 
+# 下载 1688 商品页中的图片 / 视频素材
+opencli 1688 download 841141931191 --output ./1688-downloads
+
 # 导出知乎文章为 Markdown
 opencli zhihu download "https://zhuanlan.zhihu.com/p/xxx" --output ./zhihu
 

package/dist/clis/1688/assets.d.ts

@@ -0,0 +1,42 @@
+import type { IPage } from '@jackwener/opencli/types';
+import { type MediaSource } from './shared.js';
+interface AssetBrowserPayload {
+    href?: string;
+    title?: string;
+    offerTitle?: string;
+    offerId?: string | number;
+    gallery?: {
+        mainImage?: string[];
+        offerImgList?: string[];
+        wlImageInfos?: Array<{
+            fullPathImageURI?: string;
+        }>;
+        [key: string]: unknown;
+    };
+    scannedAssets?: MediaSource[];
+}
+export interface Normalized1688Assets {
+    offer_id: string | null;
+    title: string | null;
+    item_url: string;
+    main_images: string[];
+    sku_images: string[];
+    detail_images: string[];
+    videos: string[];
+    other_images: string[];
+    raw_assets: MediaSource[];
+    source: string[];
+    main_count: number;
+    sku_count: number;
+    detail_count: number;
+    video_count: number;
+    source_url: string;
+    fetched_at: string;
+    strategy: string;
+}
+declare function normalizeAssets(payload: AssetBrowserPayload): Normalized1688Assets;
+export declare function extractAssetsForInput(page: IPage, input: string): Promise<Normalized1688Assets>;
+export declare const __test__: {
+    normalizeAssets: typeof normalizeAssets;
+};
+export {};

package/dist/clis/1688/assets.js

@@ -0,0 +1,204 @@
+import { cli, Strategy } from '@jackwener/opencli/registry';
+import { assertAuthenticatedState, buildDetailUrl, buildProvenance, cleanText, extractOfferId, gotoAndReadState, uniqueMediaSources, } from './shared.js';
+function scriptToReadAssets() {
+    return `
+    (() => {
+      const root = window.context ?? {};
+      const model = root.result?.global?.globalData?.model ?? null;
+      const gallery = root.result?.data?.gallery?.fields ?? null;
+      const defaultSrcProps = ['data-lazyload-src', 'data-src', 'data-ks-lazyload', 'currentSrc', 'src'];
+      const groups = [
+        { key: 'main', type: 'image', selectors: ['#dt-tab img', '.detail-gallery-turn img.detail-gallery-img', '.img-list-wrapper img.od-gallery-img', '.od-scroller-item span'] },
+        { key: 'video', type: 'video', selectors: ['.lib-video video', 'video[src]', 'video source[src]'] },
+        { key: 'sku', type: 'image', selectors: ['.pc-sku-wrapper .prop-item-inner-wrapper', '.sku-item-wrapper', '.specification-cell', '.sku-filter-button', '.expand-view-item', '.feature-item img'], srcProps: ['backgroundImage'] },
+        { key: 'detail', type: 'image', selectors: ['.de-description-detail img', '#detailContentContainer img', '.html-description img', '.html-description source', '.desc-lazyload-container img'] },
+      ];
+      const assets = [];
+      const seen = new Set();
+
+      const normalizeUrl = (value) => {
+        if (typeof value !== 'string') return '';
+        let next = value
+          .replace(/^url\\((.*)\\)$/i, '$1')
+          .replace(/^['"]|['"]$/g, '')
+          .replace(/\\\\u002F/g, '/')
+          .replace(/&/g, '&')
+          .trim();
+        if (!next || next.startsWith('blob:') || next.startsWith('data:')) return '';
+        if (next.startsWith('//')) next = 'https:' + next;
+        try {
+          return new URL(next, location.href).toString();
+        } catch {
+          return '';
+        }
+      };
+
+      const push = (type, group, url, source) => {
+        const normalized = normalizeUrl(url);
+        if (!normalized) return;
+        const key = type + ':' + normalized;
+        if (seen.has(key)) return;
+        seen.add(key);
+        assets.push({ type, group, url: normalized, source });
+      };
+
+      const queryAllDeep = (selector) => {
+        const results = [];
+        const visitedRoots = new Set();
+        const walkRoots = (root, fn) => {
+          if (!root || visitedRoots.has(root)) return;
+          visitedRoots.add(root);
+          fn(root);
+          const childElements = root.querySelectorAll ? Array.from(root.querySelectorAll('*')) : [];
+          for (const child of childElements) {
+            if (child && child.shadowRoot) {
+              walkRoots(child.shadowRoot, fn);
+            }
+          }
+        };
+        walkRoots(document, (root) => {
+          if (root.querySelectorAll) {
+            results.push(...Array.from(root.querySelectorAll(selector)));
+          }
+        });
+        return results;
+      };
+
+      const valuesFromElement = (element, srcProps) => {
+        const values = [];
+        const props = srcProps && srcProps.length ? srcProps : defaultSrcProps;
+        for (const prop of props) {
+          try {
+            if (prop === 'backgroundImage') {
+              const bg = getComputedStyle(element).backgroundImage || '';
+              const matches = bg.match(/url\\(([^)]+)\\)/g) || [];
+              for (const match of matches) {
+                const clean = match.replace(/^url\\(/, '').replace(/\\)$/, '');
+                values.push(clean);
+              }
+              continue;
+            }
+
+            const direct = element[prop];
+            if (typeof direct === 'string' && direct) values.push(direct);
+            const attr = element.getAttribute ? element.getAttribute(prop) : '';
+            if (attr) values.push(attr);
+          } catch {}
+        }
+
+        if (element.tagName === 'SOURCE' && element.parentElement?.tagName === 'VIDEO') {
+          values.push(element.src || element.getAttribute('src') || '');
+        }
+
+        if (element.tagName === 'VIDEO') {
+          values.push(element.currentSrc || '');
+          values.push(element.src || '');
+        }
+
+        return values;
+      };
+
+      for (const group of groups) {
+        for (const selector of group.selectors) {
+          for (const element of queryAllDeep(selector)) {
+            for (const value of valuesFromElement(element, group.srcProps)) {
+              push(group.type, group.key, value, 'dom:' + selector);
+            }
+          }
+        }
+      }
+
+      const scriptTexts = Array.from(document.scripts).map((script) => script.textContent || '');
+      const videoRegex = /https?:\\/\\/[^"'\\s]+\\.(?:mp4|m3u8)(?:\\?[^"'\\s]*)?/gi;
+      for (const scriptText of scriptTexts) {
+        const matches = scriptText.match(videoRegex) || [];
+        for (const match of matches) {
+          push('video', 'video', match, 'script');
+        }
+      }
+
+      const toJson = (value) => JSON.parse(JSON.stringify(value ?? null));
+      return {
+        href: window.location.href,
+        title: document.title || '',
+        offerTitle: model?.offerTitleModel?.subject ?? '',
+        offerId: model?.tradeModel?.offerId ?? '',
+        gallery: toJson(gallery),
+        scannedAssets: assets,
+      };
+    })()
+  `;
+}
+function normalizeAssets(payload) {
+    const offerId = cleanText(String(payload.offerId ?? '')) || extractOfferId(cleanText(payload.href)) || null;
+    const itemUrl = offerId ? buildDetailUrl(offerId) : cleanText(payload.href);
+    const seededAssets = [
+        ...((payload.gallery?.mainImage ?? []).map((url) => ({ type: 'image', group: 'main', url, source: 'page_state:mainImage' }))),
+        ...((payload.gallery?.offerImgList ?? []).map((url) => ({ type: 'image', group: 'main', url, source: 'page_state:offerImgList' }))),
+        ...((payload.gallery?.wlImageInfos ?? []).map((item) => ({
+            type: 'image',
+            group: 'main',
+            url: item?.fullPathImageURI ?? '',
+            source: 'page_state:wlImageInfos',
+        }))),
+    ];
+    const assets = uniqueMediaSources([...seededAssets, ...(payload.scannedAssets ?? [])]);
+    const mainImages = assets.filter((item) => item.type === 'image' && item.group === 'main').map((item) => item.url);
+    const skuImages = assets.filter((item) => item.type === 'image' && item.group === 'sku').map((item) => item.url);
+    const detailImages = assets.filter((item) => item.type === 'image' && item.group === 'detail').map((item) => item.url);
+    const videos = assets.filter((item) => item.type === 'video').map((item) => item.url);
+    const otherImages = assets
+        .filter((item) => item.type === 'image' && !['main', 'sku', 'detail'].includes(item.group))
+        .map((item) => item.url);
+    return {
+        offer_id: offerId,
+        title: cleanText(payload.offerTitle) || cleanText(payload.title) || null,
+        item_url: itemUrl,
+        main_images: mainImages,
+        sku_images: skuImages,
+        detail_images: detailImages,
+        videos,
+        other_images: otherImages,
+        raw_assets: assets,
+        source: [...new Set(assets.map((item) => cleanText(item.source)).filter(Boolean))],
+        main_count: mainImages.length,
+        sku_count: skuImages.length,
+        detail_count: detailImages.length,
+        video_count: videos.length,
+        ...buildProvenance(cleanText(payload.href) || itemUrl),
+    };
+}
+async function readAssetsPayload(page, itemUrl) {
+    const state = await gotoAndReadState(page, itemUrl, 2500, 'assets');
+    assertAuthenticatedState(state, 'assets');
+    await page.autoScroll({ times: 3, delayMs: 400 });
+    await page.wait(1);
+    return await page.evaluate(scriptToReadAssets());
+}
+export async function extractAssetsForInput(page, input) {
+    const itemUrl = buildDetailUrl(String(input ?? ''));
+    const payload = await readAssetsPayload(page, itemUrl);
+    return normalizeAssets(payload);
+}
+cli({
+    site: '1688',
+    name: 'assets',
+    description: '列出 1688 商品页可提取的图片/视频素材',
+    domain: 'www.1688.com',
+    strategy: Strategy.COOKIE,
+    args: [
+        {
+            name: 'input',
+            required: true,
+            positional: true,
+            help: '1688 商品 URL 或 offer ID（如 887904326744）',
+        },
+    ],
+    columns: ['offer_id', 'title', 'main_count', 'sku_count', 'detail_count', 'video_count'],
+    func: async (page, kwargs) => {
+        return [await extractAssetsForInput(page, String(kwargs.input ?? ''))];
+    },
+});
+export const __test__ = {
+    normalizeAssets,
+};

package/dist/clis/1688/assets.test.d.ts

@@ -0,0 +1 @@
+export {};

package/dist/clis/1688/assets.test.js

@@ -0,0 +1,39 @@
+import { describe, expect, it } from 'vitest';
+import { __test__ } from './assets.js';
+import { __test__ as sharedTest } from './shared.js';
+describe('1688 assets normalization', () => {
+    it('normalizes gallery and scanned assets into grouped media lists', () => {
+        const result = __test__.normalizeAssets({
+            href: 'https://detail.1688.com/offer/887904326744.html',
+            title: '测试商品 - 阿里巴巴',
+            offerTitle: '测试商品',
+            offerId: 887904326744,
+            gallery: {
+                mainImage: ['//img.example.com/main-1.jpg'],
+                offerImgList: ['https://img.example.com/main-2.jpg'],
+                wlImageInfos: [{ fullPathImageURI: 'https://img.example.com/main-3.jpg' }],
+            },
+            scannedAssets: [
+                { type: 'image', group: 'sku', url: 'https://img.example.com/sku-1.png', source: 'dom:.sku' },
+                { type: 'image', group: 'detail', url: 'https://img.example.com/detail-1.jpg', source: 'dom:.detail' },
+                { type: 'video', group: 'video', url: 'https://video.example.com/demo.mp4', source: 'script' },
+                { type: 'image', group: 'detail', url: 'blob:https://detail.1688.com/1', source: 'ignore' },
+            ],
+        });
+        expect(result.offer_id).toBe('887904326744');
+        expect(result.main_images).toEqual([
+            'https://img.example.com/main-1.jpg',
+            'https://img.example.com/main-2.jpg',
+            'https://img.example.com/main-3.jpg',
+        ]);
+        expect(result.sku_images).toEqual(['https://img.example.com/sku-1.png']);
+        expect(result.detail_images).toEqual(['https://img.example.com/detail-1.jpg']);
+        expect(result.videos).toEqual(['https://video.example.com/demo.mp4']);
+        expect(result.main_count).toBe(3);
+        expect(result.video_count).toBe(1);
+    });
+    it('normalizes media urls from style syntax and protocol-relative URLs', () => {
+        expect(sharedTest.normalizeMediaUrl('url("//img.example.com/1.jpg")')).toBe('https://img.example.com/1.jpg');
+        expect(sharedTest.normalizeMediaUrl('blob:https://detail.1688.com/1')).toBe('');
+    });
+});

package/dist/clis/1688/download.d.ts

@@ -0,0 +1,9 @@
+import { type MediaItem } from '@jackwener/opencli/download/media-download';
+import { extractAssetsForInput } from './assets.js';
+declare function extFromUrl(url: string, fallback: string): string;
+declare function toDownloadItems(offerId: string, assets: Awaited<ReturnType<typeof extractAssetsForInput>>): MediaItem[];
+export declare const __test__: {
+    extFromUrl: typeof extFromUrl;
+    toDownloadItems: typeof toDownloadItems;
+};
+export {};

package/dist/clis/1688/download.js

@@ -0,0 +1,76 @@
+import * as path from 'node:path';
+import { formatCookieHeader } from '@jackwener/opencli/download';
+import { downloadMedia } from '@jackwener/opencli/download/media-download';
+import { cli, Strategy } from '@jackwener/opencli/registry';
+import { cleanText } from './shared.js';
+import { extractAssetsForInput } from './assets.js';
+function extFromUrl(url, fallback) {
+    try {
+        const ext = path.extname(new URL(url).pathname).toLowerCase();
+        if (ext && ext.length <= 8)
+            return ext;
+    }
+    catch {
+        // ignore
+    }
+    return fallback;
+}
+function toDownloadItems(offerId, assets) {
+    const items = [];
+    const pushImages = (urls, prefix) => {
+        urls.forEach((url, index) => {
+            items.push({
+                type: 'image',
+                url,
+                filename: `${offerId}_${prefix}_${String(index + 1).padStart(2, '0')}${extFromUrl(url, '.jpg')}`,
+            });
+        });
+    };
+    pushImages(assets.main_images, 'main');
+    pushImages(assets.sku_images, 'sku');
+    pushImages(assets.detail_images, 'detail');
+    pushImages(assets.other_images, 'other');
+    assets.videos.forEach((url, index) => {
+        items.push({
+            type: 'video',
+            url,
+            filename: `${offerId}_video_${String(index + 1).padStart(2, '0')}${extFromUrl(url, '.mp4')}`,
+        });
+    });
+    return items;
+}
+cli({
+    site: '1688',
+    name: 'download',
+    description: '批量下载 1688 商品页可提取的图片和视频素材',
+    domain: 'www.1688.com',
+    strategy: Strategy.COOKIE,
+    args: [
+        {
+            name: 'input',
+            required: true,
+            positional: true,
+            help: '1688 商品 URL 或 offer ID（如 887904326744）',
+        },
+        { name: 'output', default: './1688-downloads', help: '输出目录' },
+    ],
+    columns: ['index', 'type', 'status', 'size'],
+    func: async (page, kwargs) => {
+        const assets = await extractAssetsForInput(page, String(kwargs.input ?? ''));
+        const offerId = cleanText(assets.offer_id) || '1688';
+        const items = toDownloadItems(offerId, assets);
+        const browserCookies = await page.getCookies({ domain: '1688.com' });
+        return downloadMedia(items, {
+            output: String(kwargs.output || './1688-downloads'),
+            subdir: offerId,
+            cookies: formatCookieHeader(browserCookies),
+            browserCookies,
+            filenamePrefix: offerId,
+            timeout: 60000,
+        });
+    },
+});
+export const __test__ = {
+    extFromUrl,
+    toDownloadItems,
+};

package/dist/clis/1688/download.test.d.ts

@@ -0,0 +1 @@
+export {};

package/dist/clis/1688/download.test.js

@@ -0,0 +1,31 @@
+import { describe, expect, it } from 'vitest';
+import { __test__ } from './download.js';
+describe('1688 download helpers', () => {
+    it('builds stable filenames for grouped assets', () => {
+        const items = __test__.toDownloadItems('887904326744', {
+            offer_id: '887904326744',
+            title: '测试商品',
+            item_url: 'https://detail.1688.com/offer/887904326744.html',
+            main_images: ['https://img.example.com/a.jpg'],
+            sku_images: ['https://img.example.com/b.png'],
+            detail_images: ['https://img.example.com/c.webp'],
+            videos: ['https://video.example.com/d.mp4'],
+            other_images: [],
+            raw_assets: [],
+            source: [],
+            main_count: 1,
+            sku_count: 1,
+            detail_count: 1,
+            video_count: 1,
+            source_url: 'https://detail.1688.com/offer/887904326744.html',
+            fetched_at: new Date().toISOString(),
+            strategy: 'cookie',
+        });
+        expect(items.map((item) => item.filename)).toEqual([
+            '887904326744_main_01.jpg',
+            '887904326744_sku_01.png',
+            '887904326744_detail_01.webp',
+            '887904326744_video_01.mp4',
+        ]);
+    });
+});

package/dist/clis/1688/shared.d.ts

@@ -43,6 +43,12 @@ export interface SearchCandidate {
     seller_name: string | null;
     seller_url: string | null;
 }
+export interface MediaSource {
+    type: 'image' | 'video';
+    group: 'main' | 'sku' | 'detail' | 'video' | 'unknown';
+    url: string;
+    source?: string;
+}
 export declare function cleanText(value: unknown): string;
 export declare function cleanMultilineText(value: unknown): string;
 export declare function uniqueNonEmpty(values: Array<string | null | undefined>): string[];
@@ -80,6 +86,8 @@ export declare function assertAuthenticatedState(state: PageState, action: strin
 export declare function assertNotCaptcha(state: PageState, action: string): void;
 export declare function toNumber(value: unknown): number | null;
 export declare function limitCandidates<T>(values: T[], limit: number): T[];
+export declare function normalizeMediaUrl(input: unknown): string;
+export declare function uniqueMediaSources(values: MediaSource[]): MediaSource[];
 export declare const __test__: {
     SEARCH_LIMIT_DEFAULT: number;
     SEARCH_LIMIT_MAX: number;
@@ -108,5 +116,7 @@ export declare const __test__: {
     cleanText: typeof cleanText;
     cleanMultilineText: typeof cleanMultilineText;
     uniqueNonEmpty: typeof uniqueNonEmpty;
+    normalizeMediaUrl: typeof normalizeMediaUrl;
+    uniqueMediaSources: typeof uniqueMediaSources;
     limitCandidates: typeof limitCandidates;
 };

package/dist/clis/1688/shared.js

@@ -432,6 +432,47 @@ export function limitCandidates(values, limit) {
     const normalizedLimit = Math.max(1, Math.trunc(limit) || 1);
     return values.slice(0, normalizedLimit);
 }
+export function normalizeMediaUrl(input) {
+    const raw = cleanText(input);
+    if (!raw)
+        return '';
+    let value = raw
+        .replace(/^url\((.*)\)$/i, '$1')
+        .replace(/^['"]|['"]$/g, '')
+        .replace(/\\u002F/g, '/')
+        .replace(/&/g, '&')
+        .trim();
+    if (!value || value.startsWith('data:') || value.startsWith('blob:'))
+        return '';
+    if (value.startsWith('//'))
+        value = `https:${value}`;
+    try {
+        const url = new URL(value);
+        return url.toString();
+    }
+    catch {
+        return '';
+    }
+}
+export function uniqueMediaSources(values) {
+    const seen = new Set();
+    const result = [];
+    for (const value of values) {
+        const url = normalizeMediaUrl(value.url);
+        if (!url)
+            continue;
+        const key = `${value.type}:${url}`;
+        if (seen.has(key))
+            continue;
+        seen.add(key);
+        result.push({
+            ...value,
+            url,
+            source: cleanText(value.source) || undefined,
+        });
+    }
+    return result;
+}
 function normalizeNumericText(value) {
     return value
         .replace(/([¥$€])\s+(?=\d)/g, '$1')
@@ -510,5 +551,7 @@ export const __test__ = {
     cleanText,
     cleanMultilineText,
     uniqueNonEmpty,
+    normalizeMediaUrl,
+    uniqueMediaSources,
     limitCandidates,
 };

package/dist/clis/linux-do/topic-content.d.ts

@@ -0,0 +1,35 @@
+interface LinuxDoTopicPost {
+    post_number?: number;
+    username?: string;
+    raw?: string;
+    cooked?: string;
+    like_count?: number;
+    created_at?: string;
+}
+interface LinuxDoTopicPayload {
+    title?: string;
+    post_stream?: {
+        posts?: LinuxDoTopicPost[];
+    };
+}
+interface TopicContentRow {
+    content: string;
+}
+declare function toLocalTime(utcStr: string): string;
+declare function normalizeTopicPayload(payload: unknown): LinuxDoTopicPayload | null;
+declare function buildTopicMarkdownDocument(params: {
+    title: string;
+    author: string;
+    likes?: number;
+    createdAt: string;
+    url: string;
+    body: string;
+}): string;
+declare function extractTopicContent(payload: unknown, id: number): TopicContentRow;
+export declare const __test__: {
+    buildTopicMarkdownDocument: typeof buildTopicMarkdownDocument;
+    extractTopicContent: typeof extractTopicContent;
+    normalizeTopicPayload: typeof normalizeTopicPayload;
+    toLocalTime: typeof toLocalTime;
+};
+export {};