@jackwener/opencli 1.6.1 → 1.6.2

package/dist/clis/1688/search.js

@@ -0,0 +1,309 @@
+import { CommandExecutionError, EmptyResultError } from '../../errors.js';
+import { cli, Strategy } from '../../registry.js';
+import { FACTORY_BADGE_PATTERNS, SERVICE_BADGE_PATTERNS, assertAuthenticatedState, buildProvenance, buildSearchUrl, canonicalizeItemUrl, canonicalizeSellerUrl, cleanText, extractBadges, extractLocation, extractMemberId, extractOfferId, extractShopId, gotoAndReadState, parseMoqText, parsePriceText, SEARCH_LIMIT_DEFAULT, SEARCH_LIMIT_MAX, parseSearchLimit, uniqueNonEmpty, } from './shared.js';
+const SEARCH_ITEM_URL_PATTERNS = [
+    'detail.1688.com/offer/',
+    'detail.m.1688.com/page/index.html?offerId=',
+];
+const MAX_SEARCH_PAGES = 12;
+function normalizeSearchCandidate(candidate, sourceUrl) {
+    const canonicalItemUrl = canonicalizeItemUrl(cleanText(candidate.item_url));
+    const containerText = cleanText(candidate.container_text);
+    const priceText = firstNonEmpty([
+        normalizeInlineText(candidate.price_text),
+        normalizeInlineText(extractPriceText(candidate.hover_price_text)),
+    ]);
+    const priceRange = parsePriceText(priceText || containerText);
+    const moq = parseMoqText(firstNonEmpty([
+        normalizeInlineText(candidate.moq_text),
+        normalizeInlineText(extractMoqText(containerText)),
+    ]));
+    const canonicalSellerUrl = canonicalizeSellerUrl(cleanText(candidate.seller_url));
+    const evidenceText = uniqueNonEmpty([
+        containerText,
+        ...(candidate.desc_rows ?? []),
+        ...(candidate.tag_items ?? []),
+        ...(candidate.hover_items ?? []),
+    ]).join('\n');
+    const badges = extractBadges(evidenceText, [...FACTORY_BADGE_PATTERNS, ...SERVICE_BADGE_PATTERNS]);
+    const salesText = firstNonEmpty([
+        extractSalesText(candidate.sales_text),
+        extractSalesText(containerText),
+    ]);
+    const returnRateText = extractReturnRateText([...(candidate.tag_items ?? []), ...(candidate.hover_items ?? [])]);
+    const provenance = buildProvenance(sourceUrl);
+    return {
+        rank: 0,
+        offer_id: extractOfferId(canonicalItemUrl ?? '') ?? null,
+        member_id: extractMemberId(canonicalSellerUrl ?? '') ?? null,
+        shop_id: extractShopId(canonicalSellerUrl ?? '') ?? null,
+        title: cleanText(candidate.title) || firstWord(containerText) || null,
+        item_url: canonicalItemUrl,
+        seller_name: cleanText(candidate.seller_name) || null,
+        seller_url: canonicalSellerUrl,
+        price_text: priceRange.price_text || null,
+        price_min: priceRange.price_min,
+        price_max: priceRange.price_max,
+        currency: priceRange.currency,
+        moq_text: moq.moq_text || null,
+        moq_value: moq.moq_value,
+        location: extractLocation(containerText),
+        badges,
+        sales_text: salesText || null,
+        return_rate_text: returnRateText,
+        source_url: provenance.source_url,
+        fetched_at: provenance.fetched_at,
+        strategy: provenance.strategy,
+    };
+}
+function extractMoqText(text) {
+    const normalized = normalizeInlineText(text);
+    return normalized.match(/\d+(?:\.\d+)?\s*(件|个|套|箱|包|双|台|把|只)\s*起批/i)?.[0]
+        ?? normalized.match(/≥\s*\d+(?:\.\d+)?\s*(件|个|套|箱|包|双|台|把|只)?/i)?.[0]
+        ?? normalized.match(/\d+(?:\.\d+)?\s*(?:~|-|至|到)\s*\d+(?:\.\d+)?\s*(件|个|套|箱|包|双|台|把|只)/i)?.[0]
+        ?? '';
+}
+function extractPriceText(text) {
+    const normalized = normalizeInlineText(text);
+    return normalized.match(/[¥$€]\s*\d+(?:\.\d+)?/)?.[0] ?? '';
+}
+function extractSalesText(text) {
+    const normalized = normalizeInlineText(text);
+    if (!normalized)
+        return '';
+    if (/^\d+(?:\.\d+)?\+?\s*(件|套|个|单)$/.test(normalized)) {
+        return normalized;
+    }
+    const match = normalized.match(/(?:已售|销量|售)\s*\d+(?:\.\d+)?\+?\s*(件|套|个|单)?/);
+    return match ? cleanText(match[0]) : '';
+}
+function firstWord(text) {
+    return text.split(/\s+/).find(Boolean) ?? '';
+}
+function firstNonEmpty(values) {
+    return values.map((value) => cleanText(value)).find(Boolean) ?? '';
+}
+function normalizeInlineText(text) {
+    return cleanText(text)
+        .replace(/([¥$€])\s+(?=\d)/g, '$1')
+        .replace(/(\d)\s*\.\s*(\d)/g, '$1.$2')
+        .replace(/\s*([~-])\s*/g, '$1')
+        .trim();
+}
+function extractReturnRateText(values) {
+    return uniqueNonEmpty(values.map((value) => normalizeInlineText(value)))
+        .find((value) => /^回头率\s*\d+(?:\.\d+)?%$/.test(value))
+        ?? null;
+}
+function buildDedupeKey(row) {
+    if (row.offer_id)
+        return `offer:${row.offer_id}`;
+    if (row.item_url)
+        return `url:${row.item_url}`;
+    return null;
+}
+async function readSearchPayload(page, url) {
+    const state = await gotoAndReadState(page, url, 2500, 'search');
+    assertAuthenticatedState(state, 'search');
+    const payload = await page.evaluate(`
+    (() => {
+      const normalizeText = (value) => (value || '').replace(/\\s+/g, ' ').trim();
+      const normalizeUrl = (href) => {
+        if (!href) return '';
+        try {
+          return new URL(href, window.location.href).toString();
+        } catch {
+          return '';
+        }
+      };
+      const isItemHref = (href) => ${JSON.stringify(SEARCH_ITEM_URL_PATTERNS)}
+        .some((pattern) => (href || '').includes(pattern));
+      const uniqueTexts = (values) => [...new Set(values.map((value) => normalizeText(value)).filter(Boolean))];
+      const collectTexts = (root, selector) => uniqueTexts(
+        Array.from(root.querySelectorAll(selector)).map((node) => node.innerText || node.textContent || ''),
+      );
+      const firstText = (root, selectors) => {
+        for (const selector of selectors) {
+          const node = root.querySelector(selector);
+          const value = normalizeText(node ? node.innerText || node.textContent || '' : '');
+          if (value) return value;
+        }
+        return '';
+      };
+      const findMoqText = (values, priceText) => {
+        const moqPattern = /(≥\\s*\\d+(?:\\.\\d+)?\\s*(件|个|套|箱|包|双|台|把|只)?)|(\\d+(?:\\.\\d+)?\\s*(?:~|-|至|到)\\s*\\d+(?:\\.\\d+)?\\s*(件|个|套|箱|包|双|台|把|只))|(\\d+(?:\\.\\d+)?\\s*(件|个|套|箱|包|双|台|把|只)\\s*起批)/i;
+        return values.find((value) => moqPattern.test(value))
+          || normalizeText(priceText).match(moqPattern)?.[0]
+          || '';
+      };
+      const isSellerHref = (href) => {
+        if (!href) return false;
+        try {
+          const url = new URL(href, window.location.href);
+          const host = url.hostname || '';
+          if (!host.endsWith('.1688.com')) return false;
+          if (
+            host === 's.1688.com'
+            || host === 'r.1688.com'
+            || host === 'air.1688.com'
+            || host === 'detail.1688.com'
+            || host === 'detail.m.1688.com'
+            || host === 'dj.1688.com'
+          ) {
+            return false;
+          }
+          return true;
+        } catch {
+          return false;
+        }
+      };
+      const pickContainer = (anchor) => {
+        let node = anchor;
+        while (node && node !== document.body) {
+          const text = normalizeText(node.innerText || node.textContent || '');
+          if (text.length >= 40 && text.length <= 2000) {
+            return node;
+          }
+          node = node.parentElement;
+        }
+        return anchor;
+      };
+      const collectCandidates = () => {
+        const anchors = Array.from(document.querySelectorAll('a')).filter((anchor) => isItemHref(anchor.href || ''));
+        const seen = new Set();
+        const items = [];
+        for (const anchor of anchors) {
+          const href = anchor.href || '';
+          if (!href || seen.has(href)) continue;
+          seen.add(href);
+
+          const container = pickContainer(anchor);
+          const tagItems = collectTexts(container, '.offer-tag-row .offer-desc-item');
+          const hoverItems = collectTexts(container, '.offer-hover-wrapper .offer-desc-item');
+          const sellerAnchor = Array.from(container.querySelectorAll('a'))
+            .find((link) => isSellerHref(link.href || ''));
+          const hoverPriceText = firstText(container, [
+            '.offer-hover-wrapper .hover-price-item',
+            '.offer-hover-wrapper .price-item',
+          ]);
+
+          items.push({
+            item_url: href,
+            title: firstText(container, ['.offer-title-row .title-text', '.offer-title-row'])
+              || normalizeText(anchor.innerText || anchor.textContent || ''),
+            container_text: normalizeText(container.innerText || container.textContent || ''),
+            desc_rows: collectTexts(container, '.offer-desc-row'),
+            price_text: firstText(container, ['.offer-price-row .price-item']),
+            sales_text: firstText(container, ['.offer-price-row .col-desc_after', '.offer-desc-row .col-desc_after']),
+            hover_price_text: hoverPriceText,
+            moq_text: findMoqText(hoverItems, hoverPriceText),
+            tag_items: tagItems,
+            hover_items: hoverItems,
+            seller_name: sellerAnchor ? normalizeText(sellerAnchor.innerText || sellerAnchor.textContent || '') : null,
+            seller_url: sellerAnchor ? sellerAnchor.href : null,
+          });
+        }
+        return items;
+      };
+      const findNextUrl = () => {
+        const selectors = [
+          'a.fui-next:not(.disabled)',
+          'a.next-pagination-item:not(.disabled)',
+          'a[rel="next"]:not(.disabled)',
+          'a[data-role="next"]:not(.disabled)',
+        ];
+        for (const selector of selectors) {
+          const node = document.querySelector(selector);
+          if (!node) continue;
+          const href = normalizeUrl(node.getAttribute('href') || node.href || '');
+          if (href) return href;
+        }
+        const textBased = Array.from(document.querySelectorAll('a'))
+          .find((node) => /下一页|next/i.test(normalizeText(node.textContent || '')));
+        if (!textBased) return '';
+        return normalizeUrl(textBased.getAttribute('href') || textBased.href || '');
+      };
+
+      return {
+        href: window.location.href,
+        title: document.title || '',
+        bodyText: document.body ? document.body.innerText || '' : '',
+        next_url: findNextUrl(),
+        candidates: collectCandidates(),
+      };
+    })()
+  `);
+    if (!payload || typeof payload !== 'object') {
+        throw new CommandExecutionError('1688 search page did not return a readable payload', 'Open the same query in Chrome and verify the page is fully loaded before retrying.');
+    }
+    return payload;
+}
+async function collectSearchRows(page, query, limit) {
+    const rowsByKey = new Map();
+    const seenPages = new Set();
+    let nextUrl = buildSearchUrl(query);
+    let pageCount = 0;
+    while (nextUrl && rowsByKey.size < limit && pageCount < MAX_SEARCH_PAGES) {
+        if (seenPages.has(nextUrl))
+            break;
+        seenPages.add(nextUrl);
+        pageCount += 1;
+        const payload = await readSearchPayload(page, nextUrl);
+        const sourceUrl = cleanText(payload.href) || nextUrl;
+        const candidates = Array.isArray(payload.candidates) ? payload.candidates : [];
+        for (const candidate of candidates) {
+            const row = normalizeSearchCandidate(candidate, sourceUrl);
+            const dedupeKey = buildDedupeKey(row);
+            if (!dedupeKey || rowsByKey.has(dedupeKey))
+                continue;
+            rowsByKey.set(dedupeKey, row);
+            if (rowsByKey.size >= limit)
+                break;
+        }
+        const candidateNextUrl = cleanText(payload.next_url);
+        if (!candidateNextUrl || candidateNextUrl === sourceUrl)
+            break;
+        nextUrl = candidateNextUrl;
+    }
+    if (rowsByKey.size === 0) {
+        throw new EmptyResultError('1688 search', 'No visible results were extracted. Retry with a different query or open the same search page in Chrome first.');
+    }
+    return [...rowsByKey.values()]
+        .slice(0, limit)
+        .map((row, index) => ({ ...row, rank: index + 1 }));
+}
+cli({
+    site: '1688',
+    name: 'search',
+    description: '1688 商品搜索（结果候选、卖家链接、价格/MOQ/销量文本）',
+    domain: 'www.1688.com',
+    strategy: Strategy.COOKIE,
+    navigateBefore: false,
+    args: [
+        {
+            name: 'query',
+            required: true,
+            positional: true,
+            help: '搜索关键词，如 "置物架"',
+        },
+        {
+            name: 'limit',
+            type: 'int',
+            default: SEARCH_LIMIT_DEFAULT,
+            help: `结果数量上限（默认 ${SEARCH_LIMIT_DEFAULT}，最大 ${SEARCH_LIMIT_MAX}）`,
+        },
+    ],
+    columns: ['rank', 'title', 'price_text', 'moq_text', 'seller_name', 'location'],
+    func: async (page, kwargs) => {
+        const query = String(kwargs.query ?? '');
+        const limit = parseSearchLimit(kwargs.limit);
+        return collectSearchRows(page, query, limit);
+    },
+});
+export const __test__ = {
+    normalizeSearchCandidate,
+    extractMoqText,
+    extractSalesText,
+    firstWord,
+    buildDedupeKey,
+};

package/dist/clis/1688/search.test.d.ts

@@ -0,0 +1 @@
+export {};

package/dist/clis/1688/search.test.js

@@ -0,0 +1,75 @@
+import { describe, expect, it } from 'vitest';
+import { __test__ } from './search.js';
+describe('1688 search normalization', () => {
+    it('normalizes search candidates into structured result rows', () => {
+        const result = __test__.normalizeSearchCandidate({
+            item_url: 'https://detail.1688.com/offer/887904326744.html',
+            title: '宿舍置物架桌面加高架',
+            container_text: '宿舍置物架桌面加高架 ¥56.00 2套起批 山东青岛 已售300+套',
+            price_text: '¥ 56 .00',
+            sales_text: '300+套',
+            moq_text: '2套起批',
+            tag_items: ['退货包运费', '回头率52%'],
+            hover_items: ['验厂报告'],
+            seller_name: '青岛沁澜衣品服装有限公司',
+            seller_url: 'https://yinuoweierfushi.1688.com/page/index.html?spm=a123',
+        }, 'https://s.1688.com/selloffer/offer_search.htm?charset=utf8&keywords=置物架');
+        expect(result.rank).toBe(0);
+        expect(result.offer_id).toBe('887904326744');
+        expect(result.shop_id).toBe('yinuoweierfushi');
+        expect(result.item_url).toBe('https://detail.1688.com/offer/887904326744.html');
+        expect(result.seller_url).toBe('https://yinuoweierfushi.1688.com');
+        expect(result.price_text).toBe('¥56.00');
+        expect(result.price_min).toBe(56);
+        expect(result.price_max).toBe(56);
+        expect(result.moq_value).toBe(2);
+        expect(result.location).toBe('山东青岛');
+        expect(result.sales_text).toBe('300+套');
+        expect(result.badges).toEqual(expect.arrayContaining(['退货包运费', '验厂报告']));
+        expect(result.return_rate_text).toBe('回头率52%');
+    });
+    it('does not use hover_price_text as MOQ source', () => {
+        const result = __test__.normalizeSearchCandidate({
+            item_url: 'https://detail.1688.com/offer/887904326744.html',
+            title: 'test',
+            container_text: 'test ¥56.00',
+            price_text: '¥ 56 .00',
+            hover_price_text: '¥56.00 3件起批',
+            moq_text: null,
+        }, 'https://s.1688.com/selloffer/offer_search.htm?charset=utf8&keywords=test');
+        // hover_price_text should not be used for MOQ extraction
+        expect(result.moq_text).toBeNull();
+        expect(result.moq_value).toBeNull();
+    });
+    it('extracts offer id from mobile detail search links', () => {
+        const result = __test__.normalizeSearchCandidate({
+            item_url: 'http://detail.m.1688.com/page/index.html?offerId=910933345396&sortType=&pageId=',
+            title: '',
+            container_text: '桌面书桌办公室工位收纳展示新中式博古架多层茶具厨房摆放置物架 ¥24.3 已售20+件',
+            price_text: '¥ 14 .28',
+            sales_text: '1500+件',
+            moq_text: '≥2个',
+            seller_name: '泰商国际贸易（宁阳）有限公司',
+            seller_url: 'http://tsgjmy.1688.com/',
+        }, 'https://s.1688.com/selloffer/offer_search.htm?charset=utf8&keywords=桌面置物架');
+        expect(result.offer_id).toBe('910933345396');
+        expect(result.shop_id).toBe('tsgjmy');
+        expect(result.item_url).toBe('https://detail.1688.com/offer/910933345396.html');
+        expect(result.title).toContain('桌面书桌办公室工位收纳展示');
+        expect(result.price_text).toBe('¥14.28');
+        expect(result.sales_text).toBe('1500+件');
+        expect(result.moq_text).toBe('≥2个');
+        expect(result.moq_value).toBe(2);
+    });
+    it('prefers offer id and falls back to item url for dedupe key', () => {
+        expect(__test__.buildDedupeKey({
+            offer_id: '123456',
+            item_url: 'https://detail.1688.com/offer/123456.html',
+        })).toBe('offer:123456');
+        expect(__test__.buildDedupeKey({
+            offer_id: null,
+            item_url: 'https://detail.1688.com/offer/123456.html',
+        })).toBe('url:https://detail.1688.com/offer/123456.html');
+        expect(__test__.buildDedupeKey({ offer_id: null, item_url: null })).toBeNull();
+    });
+});

package/dist/clis/1688/shared.d.ts

@@ -0,0 +1,112 @@
+import type { IPage } from '../../types.js';
+export declare const SITE = "1688";
+export declare const HOME_URL = "https://www.1688.com/";
+export declare const SEARCH_URL_PREFIX = "https://s.1688.com/selloffer/offer_search.htm?charset=utf8&keywords=";
+export declare const DETAIL_URL_PREFIX = "https://detail.1688.com/offer/";
+export declare const STORE_MOBILE_URL_PREFIX = "https://winport.m.1688.com/page/index.html?memberId=";
+export declare const STRATEGY = "cookie";
+export declare const SEARCH_LIMIT_DEFAULT = 20;
+export declare const SEARCH_LIMIT_MAX = 100;
+export declare const FACTORY_BADGE_PATTERNS: string[];
+export declare const SERVICE_BADGE_PATTERNS: string[];
+export interface ProvenanceFields {
+    source_url: string;
+    fetched_at: string;
+    strategy: string;
+}
+export interface PageState {
+    href: string;
+    title: string;
+    body_text: string;
+}
+export interface PriceRange {
+    price_text: string;
+    price_min: number | null;
+    price_max: number | null;
+    currency: string | null;
+}
+export interface MoqValue {
+    moq_text: string;
+    moq_value: number | null;
+}
+export interface PriceTier {
+    quantity_text: string;
+    quantity_min: number | null;
+    price_text: string;
+    price: number | null;
+    currency: string | null;
+}
+export interface SearchCandidate {
+    item_url: string;
+    title: string;
+    container_text: string;
+    seller_name: string | null;
+    seller_url: string | null;
+}
+export declare function cleanText(value: unknown): string;
+export declare function cleanMultilineText(value: unknown): string;
+export declare function uniqueNonEmpty(values: Array<string | null | undefined>): string[];
+export declare function parseSearchLimit(input: unknown): number;
+export declare function buildSearchUrl(query: string): string;
+export declare function buildDetailUrl(input: string): string;
+export declare function resolveStoreUrl(input: string): string;
+export declare function canonicalizeStoreUrl(input: string): string;
+export declare function canonicalizeItemUrl(input: string): string | null;
+export declare function canonicalizeSellerUrl(input: string): string | null;
+export declare function extractOfferId(input: string): string | null;
+export declare function extractMemberId(input: string): string | null;
+export declare function extractShopId(input: string): string | null;
+export declare function buildProvenance(sourceUrl: string): ProvenanceFields;
+export declare function parsePriceText(text: string): PriceRange;
+export declare function normalizePriceTiers(rawTiers: Array<{
+    beginAmount?: unknown;
+    price?: unknown;
+}>, unit: string | null): PriceTier[];
+export declare function parseMoqText(text: string): MoqValue;
+export declare function extractLocation(text: string): string | null;
+export declare function extractAddress(text: string): string | null;
+export declare function extractMetric(text: string, label: string): string | null;
+export declare function extractYearsOnPlatform(text: string): string | null;
+export declare function extractMainBusiness(text: string): string | null;
+export declare function extractBadges(text: string, candidates: string[]): string[];
+export declare function guessTopCategories(text: string): string[];
+export declare function isCaptchaState(state: Partial<PageState>): boolean;
+export declare function isLoginState(state: Partial<PageState>): boolean;
+export declare function buildCaptchaHint(action: string): string;
+export declare function readPageState(page: IPage): Promise<PageState>;
+export declare function gotoAndReadState(page: IPage, url: string, settleMs?: number, action?: string): Promise<PageState>;
+export declare function ensure1688Session(page: IPage): Promise<void>;
+export declare function assertAuthenticatedState(state: PageState, action: string): void;
+export declare function assertNotCaptcha(state: PageState, action: string): void;
+export declare function toNumber(value: unknown): number | null;
+export declare function limitCandidates<T>(values: T[], limit: number): T[];
+export declare const __test__: {
+    SEARCH_LIMIT_DEFAULT: number;
+    SEARCH_LIMIT_MAX: number;
+    parseSearchLimit: typeof parseSearchLimit;
+    buildSearchUrl: typeof buildSearchUrl;
+    buildDetailUrl: typeof buildDetailUrl;
+    resolveStoreUrl: typeof resolveStoreUrl;
+    canonicalizeStoreUrl: typeof canonicalizeStoreUrl;
+    canonicalizeItemUrl: typeof canonicalizeItemUrl;
+    canonicalizeSellerUrl: typeof canonicalizeSellerUrl;
+    extractOfferId: typeof extractOfferId;
+    extractMemberId: typeof extractMemberId;
+    extractShopId: typeof extractShopId;
+    parsePriceText: typeof parsePriceText;
+    normalizePriceTiers: typeof normalizePriceTiers;
+    parseMoqText: typeof parseMoqText;
+    extractLocation: typeof extractLocation;
+    extractAddress: typeof extractAddress;
+    extractMetric: typeof extractMetric;
+    extractYearsOnPlatform: typeof extractYearsOnPlatform;
+    extractMainBusiness: typeof extractMainBusiness;
+    extractBadges: typeof extractBadges;
+    guessTopCategories: typeof guessTopCategories;
+    isCaptchaState: typeof isCaptchaState;
+    isLoginState: typeof isLoginState;
+    cleanText: typeof cleanText;
+    cleanMultilineText: typeof cleanMultilineText;
+    uniqueNonEmpty: typeof uniqueNonEmpty;
+    limitCandidates: typeof limitCandidates;
+};