@jackwener/opencli 1.6.0 → 1.6.2

package/src/clis/1688/search.ts

@@ -0,0 +1,402 @@
+import { CommandExecutionError, EmptyResultError } from '../../errors.js';
+import { cli, Strategy } from '../../registry.js';
+import type { IPage } from '../../types.js';
+import {
+  FACTORY_BADGE_PATTERNS,
+  SERVICE_BADGE_PATTERNS,
+  assertAuthenticatedState,
+  buildProvenance,
+  buildSearchUrl,
+  canonicalizeItemUrl,
+  canonicalizeSellerUrl,
+  cleanText,
+  extractBadges,
+  extractLocation,
+  extractMemberId,
+  extractOfferId,
+  extractShopId,
+  gotoAndReadState,
+  parseMoqText,
+  parsePriceText,
+  SEARCH_LIMIT_DEFAULT,
+  SEARCH_LIMIT_MAX,
+  parseSearchLimit,
+  uniqueNonEmpty,
+} from './shared.js';
+
+interface SearchPayload {
+  href?: string;
+  title?: string;
+  bodyText?: string;
+  next_url?: string;
+  candidates?: Array<{
+    item_url?: string;
+    title?: string;
+    container_text?: string;
+    desc_rows?: string[];
+    price_text?: string | null;
+    sales_text?: string | null;
+    hover_price_text?: string | null;
+    moq_text?: string | null;
+    tag_items?: string[];
+    hover_items?: string[];
+    seller_name?: string | null;
+    seller_url?: string | null;
+  }>;
+}
+
+interface SearchRow {
+  rank: number;
+  offer_id: string | null;
+  member_id: string | null;
+  shop_id: string | null;
+  title: string | null;
+  item_url: string | null;
+  seller_name: string | null;
+  seller_url: string | null;
+  price_text: string | null;
+  price_min: number | null;
+  price_max: number | null;
+  currency: string | null;
+  moq_text: string | null;
+  moq_value: number | null;
+  location: string | null;
+  badges: string[];
+  sales_text: string | null;
+  return_rate_text: string | null;
+  source_url: string;
+  fetched_at: string;
+  strategy: string;
+}
+
+const SEARCH_ITEM_URL_PATTERNS = [
+  'detail.1688.com/offer/',
+  'detail.m.1688.com/page/index.html?offerId=',
+];
+const MAX_SEARCH_PAGES = 12;
+
+function normalizeSearchCandidate(
+  candidate: NonNullable<SearchPayload['candidates']>[number],
+  sourceUrl: string,
+): SearchRow {
+  const canonicalItemUrl = canonicalizeItemUrl(cleanText(candidate.item_url));
+  const containerText = cleanText(candidate.container_text);
+  const priceText = firstNonEmpty([
+    normalizeInlineText(candidate.price_text),
+    normalizeInlineText(extractPriceText(candidate.hover_price_text)),
+  ]);
+  const priceRange = parsePriceText(priceText || containerText);
+  const moq = parseMoqText(firstNonEmpty([
+    normalizeInlineText(candidate.moq_text),
+    normalizeInlineText(extractMoqText(containerText)),
+  ]));
+  const canonicalSellerUrl = canonicalizeSellerUrl(cleanText(candidate.seller_url));
+  const evidenceText = uniqueNonEmpty([
+    containerText,
+    ...(candidate.desc_rows ?? []),
+    ...(candidate.tag_items ?? []),
+    ...(candidate.hover_items ?? []),
+  ]).join('\n');
+  const badges = extractBadges(evidenceText, [...FACTORY_BADGE_PATTERNS, ...SERVICE_BADGE_PATTERNS]);
+  const salesText = firstNonEmpty([
+    extractSalesText(candidate.sales_text),
+    extractSalesText(containerText),
+  ]);
+  const returnRateText = extractReturnRateText([...(candidate.tag_items ?? []), ...(candidate.hover_items ?? [])]);
+  const provenance = buildProvenance(sourceUrl);
+
+  return {
+    rank: 0,
+    offer_id: extractOfferId(canonicalItemUrl ?? '') ?? null,
+    member_id: extractMemberId(canonicalSellerUrl ?? '') ?? null,
+    shop_id: extractShopId(canonicalSellerUrl ?? '') ?? null,
+    title: cleanText(candidate.title) || firstWord(containerText) || null,
+    item_url: canonicalItemUrl,
+    seller_name: cleanText(candidate.seller_name) || null,
+    seller_url: canonicalSellerUrl,
+    price_text: priceRange.price_text || null,
+    price_min: priceRange.price_min,
+    price_max: priceRange.price_max,
+    currency: priceRange.currency,
+    moq_text: moq.moq_text || null,
+    moq_value: moq.moq_value,
+    location: extractLocation(containerText),
+    badges,
+    sales_text: salesText || null,
+    return_rate_text: returnRateText,
+    source_url: provenance.source_url,
+    fetched_at: provenance.fetched_at,
+    strategy: provenance.strategy,
+  };
+}
+
+function extractMoqText(text: string | null | undefined): string {
+  const normalized = normalizeInlineText(text);
+  return normalized.match(/\d+(?:\.\d+)?\s*(件|个|套|箱|包|双|台|把|只)\s*起批/i)?.[0]
+    ?? normalized.match(/≥\s*\d+(?:\.\d+)?\s*(件|个|套|箱|包|双|台|把|只)?/i)?.[0]
+    ?? normalized.match(/\d+(?:\.\d+)?\s*(?:~|-|至|到)\s*\d+(?:\.\d+)?\s*(件|个|套|箱|包|双|台|把|只)/i)?.[0]
+    ?? '';
+}
+
+function extractPriceText(text: string | null | undefined): string {
+  const normalized = normalizeInlineText(text);
+  return normalized.match(/[¥$€]\s*\d+(?:\.\d+)?/)?.[0] ?? '';
+}
+
+function extractSalesText(text: string | null | undefined): string {
+  const normalized = normalizeInlineText(text);
+  if (!normalized) return '';
+  if (/^\d+(?:\.\d+)?\+?\s*(件|套|个|单)$/.test(normalized)) {
+    return normalized;
+  }
+  const match = normalized.match(/(?:已售|销量|售)\s*\d+(?:\.\d+)?\+?\s*(件|套|个|单)?/);
+  return match ? cleanText(match[0]) : '';
+}
+
+function firstWord(text: string): string {
+  return text.split(/\s+/).find(Boolean) ?? '';
+}
+
+function firstNonEmpty(values: Array<string | null | undefined>): string {
+  return values.map((value) => cleanText(value)).find(Boolean) ?? '';
+}
+
+function normalizeInlineText(text: string | null | undefined): string {
+  return cleanText(text)
+    .replace(/([¥$€])\s+(?=\d)/g, '$1')
+    .replace(/(\d)\s*\.\s*(\d)/g, '$1.$2')
+    .replace(/\s*([~-])\s*/g, '$1')
+    .trim();
+}
+
+function extractReturnRateText(values: string[]): string | null {
+  return uniqueNonEmpty(values.map((value) => normalizeInlineText(value)))
+    .find((value) => /^回头率\s*\d+(?:\.\d+)?%$/.test(value))
+    ?? null;
+}
+
+function buildDedupeKey(row: Pick<SearchRow, 'offer_id' | 'item_url'>): string | null {
+  if (row.offer_id) return `offer:${row.offer_id}`;
+  if (row.item_url) return `url:${row.item_url}`;
+  return null;
+}
+
+async function readSearchPayload(page: IPage, url: string): Promise<SearchPayload> {
+  const state = await gotoAndReadState(page, url, 2500, 'search');
+  assertAuthenticatedState(state, 'search');
+
+  const payload = await page.evaluate(`
+    (() => {
+      const normalizeText = (value) => (value || '').replace(/\\s+/g, ' ').trim();
+      const normalizeUrl = (href) => {
+        if (!href) return '';
+        try {
+          return new URL(href, window.location.href).toString();
+        } catch {
+          return '';
+        }
+      };
+      const isItemHref = (href) => ${JSON.stringify(SEARCH_ITEM_URL_PATTERNS)}
+        .some((pattern) => (href || '').includes(pattern));
+      const uniqueTexts = (values) => [...new Set(values.map((value) => normalizeText(value)).filter(Boolean))];
+      const collectTexts = (root, selector) => uniqueTexts(
+        Array.from(root.querySelectorAll(selector)).map((node) => node.innerText || node.textContent || ''),
+      );
+      const firstText = (root, selectors) => {
+        for (const selector of selectors) {
+          const node = root.querySelector(selector);
+          const value = normalizeText(node ? node.innerText || node.textContent || '' : '');
+          if (value) return value;
+        }
+        return '';
+      };
+      const findMoqText = (values, priceText) => {
+        const moqPattern = /(≥\\s*\\d+(?:\\.\\d+)?\\s*(件|个|套|箱|包|双|台|把|只)?)|(\\d+(?:\\.\\d+)?\\s*(?:~|-|至|到)\\s*\\d+(?:\\.\\d+)?\\s*(件|个|套|箱|包|双|台|把|只))|(\\d+(?:\\.\\d+)?\\s*(件|个|套|箱|包|双|台|把|只)\\s*起批)/i;
+        return values.find((value) => moqPattern.test(value))
+          || normalizeText(priceText).match(moqPattern)?.[0]
+          || '';
+      };
+      const isSellerHref = (href) => {
+        if (!href) return false;
+        try {
+          const url = new URL(href, window.location.href);
+          const host = url.hostname || '';
+          if (!host.endsWith('.1688.com')) return false;
+          if (
+            host === 's.1688.com'
+            || host === 'r.1688.com'
+            || host === 'air.1688.com'
+            || host === 'detail.1688.com'
+            || host === 'detail.m.1688.com'
+            || host === 'dj.1688.com'
+          ) {
+            return false;
+          }
+          return true;
+        } catch {
+          return false;
+        }
+      };
+      const pickContainer = (anchor) => {
+        let node = anchor;
+        while (node && node !== document.body) {
+          const text = normalizeText(node.innerText || node.textContent || '');
+          if (text.length >= 40 && text.length <= 2000) {
+            return node;
+          }
+          node = node.parentElement;
+        }
+        return anchor;
+      };
+      const collectCandidates = () => {
+        const anchors = Array.from(document.querySelectorAll('a')).filter((anchor) => isItemHref(anchor.href || ''));
+        const seen = new Set();
+        const items = [];
+        for (const anchor of anchors) {
+          const href = anchor.href || '';
+          if (!href || seen.has(href)) continue;
+          seen.add(href);
+
+          const container = pickContainer(anchor);
+          const tagItems = collectTexts(container, '.offer-tag-row .offer-desc-item');
+          const hoverItems = collectTexts(container, '.offer-hover-wrapper .offer-desc-item');
+          const sellerAnchor = Array.from(container.querySelectorAll('a'))
+            .find((link) => isSellerHref(link.href || ''));
+          const hoverPriceText = firstText(container, [
+            '.offer-hover-wrapper .hover-price-item',
+            '.offer-hover-wrapper .price-item',
+          ]);
+
+          items.push({
+            item_url: href,
+            title: firstText(container, ['.offer-title-row .title-text', '.offer-title-row'])
+              || normalizeText(anchor.innerText || anchor.textContent || ''),
+            container_text: normalizeText(container.innerText || container.textContent || ''),
+            desc_rows: collectTexts(container, '.offer-desc-row'),
+            price_text: firstText(container, ['.offer-price-row .price-item']),
+            sales_text: firstText(container, ['.offer-price-row .col-desc_after', '.offer-desc-row .col-desc_after']),
+            hover_price_text: hoverPriceText,
+            moq_text: findMoqText(hoverItems, hoverPriceText),
+            tag_items: tagItems,
+            hover_items: hoverItems,
+            seller_name: sellerAnchor ? normalizeText(sellerAnchor.innerText || sellerAnchor.textContent || '') : null,
+            seller_url: sellerAnchor ? sellerAnchor.href : null,
+          });
+        }
+        return items;
+      };
+      const findNextUrl = () => {
+        const selectors = [
+          'a.fui-next:not(.disabled)',
+          'a.next-pagination-item:not(.disabled)',
+          'a[rel="next"]:not(.disabled)',
+          'a[data-role="next"]:not(.disabled)',
+        ];
+        for (const selector of selectors) {
+          const node = document.querySelector(selector);
+          if (!node) continue;
+          const href = normalizeUrl(node.getAttribute('href') || node.href || '');
+          if (href) return href;
+        }
+        const textBased = Array.from(document.querySelectorAll('a'))
+          .find((node) => /下一页|next/i.test(normalizeText(node.textContent || '')));
+        if (!textBased) return '';
+        return normalizeUrl(textBased.getAttribute('href') || textBased.href || '');
+      };
+
+      return {
+        href: window.location.href,
+        title: document.title || '',
+        bodyText: document.body ? document.body.innerText || '' : '',
+        next_url: findNextUrl(),
+        candidates: collectCandidates(),
+      };
+    })()
+  `) as SearchPayload;
+
+  if (!payload || typeof payload !== 'object') {
+    throw new CommandExecutionError(
+      '1688 search page did not return a readable payload',
+      'Open the same query in Chrome and verify the page is fully loaded before retrying.',
+    );
+  }
+
+  return payload;
+}
+
+async function collectSearchRows(page: IPage, query: string, limit: number): Promise<SearchRow[]> {
+  const rowsByKey = new Map<string, SearchRow>();
+  const seenPages = new Set<string>();
+  let nextUrl = buildSearchUrl(query);
+  let pageCount = 0;
+
+  while (nextUrl && rowsByKey.size < limit && pageCount < MAX_SEARCH_PAGES) {
+    if (seenPages.has(nextUrl)) break;
+    seenPages.add(nextUrl);
+    pageCount += 1;
+
+    const payload = await readSearchPayload(page, nextUrl);
+    const sourceUrl = cleanText(payload.href) || nextUrl;
+    const candidates = Array.isArray(payload.candidates) ? payload.candidates : [];
+
+    for (const candidate of candidates) {
+      const row = normalizeSearchCandidate(candidate, sourceUrl);
+      const dedupeKey = buildDedupeKey(row);
+      if (!dedupeKey || rowsByKey.has(dedupeKey)) continue;
+      rowsByKey.set(dedupeKey, row);
+      if (rowsByKey.size >= limit) break;
+    }
+
+    const candidateNextUrl = cleanText(payload.next_url);
+    if (!candidateNextUrl || candidateNextUrl === sourceUrl) break;
+    nextUrl = candidateNextUrl;
+  }
+
+  if (rowsByKey.size === 0) {
+    throw new EmptyResultError(
+      '1688 search',
+      'No visible results were extracted. Retry with a different query or open the same search page in Chrome first.',
+    );
+  }
+
+  return [...rowsByKey.values()]
+    .slice(0, limit)
+    .map((row, index) => ({ ...row, rank: index + 1 }));
+}
+
+cli({
+  site: '1688',
+  name: 'search',
+  description: '1688 商品搜索（结果候选、卖家链接、价格/MOQ/销量文本）',
+  domain: 'www.1688.com',
+  strategy: Strategy.COOKIE,
+  navigateBefore: false,
+  args: [
+    {
+      name: 'query',
+      required: true,
+      positional: true,
+      help: '搜索关键词，如 "置物架"',
+    },
+    {
+      name: 'limit',
+      type: 'int',
+      default: SEARCH_LIMIT_DEFAULT,
+      help: `结果数量上限（默认 ${SEARCH_LIMIT_DEFAULT}，最大 ${SEARCH_LIMIT_MAX}）`,
+    },
+  ],
+  columns: ['rank', 'title', 'price_text', 'moq_text', 'seller_name', 'location'],
+  func: async (page, kwargs) => {
+    const query = String(kwargs.query ?? '');
+    const limit = parseSearchLimit(kwargs.limit);
+    return collectSearchRows(page, query, limit);
+  },
+});
+
+export const __test__ = {
+  normalizeSearchCandidate,
+  extractMoqText,
+  extractSalesText,
+  firstWord,
+  buildDedupeKey,
+};

package/src/clis/1688/shared.test.ts

@@ -0,0 +1,75 @@
+import { describe, expect, it } from 'vitest';
+import { __test__ } from './shared.js';
+
+describe('1688 shared helpers', () => {
+  it('builds encoded search URLs and validates limit', () => {
+    expect(__test__.buildSearchUrl('置物架')).toBe(
+      'https://s.1688.com/selloffer/offer_search.htm?charset=utf8&keywords=%E7%BD%AE%E7%89%A9%E6%9E%B6',
+    );
+    expect(() => __test__.buildSearchUrl('   ')).toThrowError(/cannot be empty/i);
+
+    expect(__test__.parseSearchLimit(3)).toBe(3);
+    expect(__test__.parseSearchLimit('1000')).toBe(__test__.SEARCH_LIMIT_MAX);
+    expect(() => __test__.parseSearchLimit('0')).toThrowError(/positive integer/i);
+  });
+
+  it('extracts IDs and canonicalizes urls', () => {
+    expect(__test__.extractOfferId('887904326744')).toBe('887904326744');
+    expect(__test__.extractOfferId('https://detail.1688.com/offer/887904326744.html')).toBe('887904326744');
+    expect(__test__.extractMemberId('https://winport.m.1688.com/page/index.html?memberId=b2b-1641351767')).toBe('b2b-1641351767');
+    expect(__test__.extractMemberId('b2b-22154705262941f196')).toBe('b2b-22154705262941f196');
+    expect(__test__.resolveStoreUrl('b2b-22154705262941f196')).toBe(
+      'https://winport.m.1688.com/page/index.html?memberId=b2b-22154705262941f196',
+    );
+    expect(__test__.canonicalizeStoreUrl('https://yinuoweierfushi.1688.com/page/index.html?spm=foo')).toBe(
+      'https://yinuoweierfushi.1688.com',
+    );
+    expect(__test__.canonicalizeItemUrl('http://detail.m.1688.com/page/index.html?offerId=910933345396&spm=x')).toBe(
+      'https://detail.1688.com/offer/910933345396.html',
+    );
+    expect(__test__.canonicalizeSellerUrl('https://yinuoweierfushi.1688.com/page/contactinfo.html?tracelog=1')).toBe(
+      'https://yinuoweierfushi.1688.com',
+    );
+    expect(__test__.extractShopId('https://yinuoweierfushi.1688.com/page/index.html')).toBe('yinuoweierfushi');
+  });
+
+  it('parses price ranges and moq text', () => {
+    expect(__test__.parsePriceText('¥96.00-98.00')).toEqual({
+      price_text: '¥96.00-98.00',
+      price_min: 96,
+      price_max: 98,
+      currency: 'CNY',
+    });
+
+    expect(__test__.parsePriceText('¥ 14 .28')).toEqual({
+      price_text: '¥14.28',
+      price_min: 14.28,
+      price_max: 14.28,
+      currency: 'CNY',
+    });
+
+    expect(__test__.parseMoqText('3套起批')).toEqual({
+      moq_text: '3套起批',
+      moq_value: 3,
+    });
+
+    expect(__test__.parseMoqText('2~999个')).toEqual({
+      moq_text: '2~999个',
+      moq_value: 2,
+    });
+  });
+
+  it('detects captcha and login states', () => {
+    expect(__test__.extractLocation('山东青岛 送至 江苏苏州')).toBe('山东青岛');
+    expect(__test__.isCaptchaState({
+      href: 'https://s.1688.com/_____tmd_____/punish',
+      title: '验证码拦截',
+      body_text: '请拖动下方滑块完成验证',
+    })).toBe(true);
+    expect(__test__.isLoginState({
+      href: 'https://login.taobao.com/member/login.jhtml',
+      title: '账号登录',
+      body_text: '请登录后继续',
+    })).toBe(true);
+  });
+});