@agentrhq/webcmd 0.1.2 → 0.2.0

package/clis/1688/search.js

@@ -1,310 +0,0 @@
-import { CommandExecutionError, EmptyResultError } from '@agentrhq/webcmd/errors';
-import { cli, Strategy } from '@agentrhq/webcmd/registry';
-import { FACTORY_BADGE_PATTERNS, SERVICE_BADGE_PATTERNS, assertAuthenticatedState, buildProvenance, buildSearchUrl, canonicalizeItemUrl, canonicalizeSellerUrl, cleanText, extractBadges, extractLocation, extractMemberId, extractOfferId, extractShopId, gotoAndReadState, parseMoqText, parsePriceText, SEARCH_LIMIT_DEFAULT, SEARCH_LIMIT_MAX, parseSearchLimit, uniqueNonEmpty, } from './shared.js';
-const SEARCH_ITEM_URL_PATTERNS = [
-    'detail.1688.com/offer/',
-    'detail.m.1688.com/page/index.html?offerId=',
-];
-const MAX_SEARCH_PAGES = 12;
-function normalizeSearchCandidate(candidate, sourceUrl) {
-    const canonicalItemUrl = canonicalizeItemUrl(cleanText(candidate.item_url));
-    const containerText = cleanText(candidate.container_text);
-    const priceText = firstNonEmpty([
-        normalizeInlineText(candidate.price_text),
-        normalizeInlineText(extractPriceText(candidate.hover_price_text)),
-    ]);
-    const priceRange = parsePriceText(priceText || containerText);
-    const moq = parseMoqText(firstNonEmpty([
-        normalizeInlineText(candidate.moq_text),
-        normalizeInlineText(extractMoqText(containerText)),
-    ]));
-    const canonicalSellerUrl = canonicalizeSellerUrl(cleanText(candidate.seller_url));
-    const evidenceText = uniqueNonEmpty([
-        containerText,
-        ...(candidate.desc_rows ?? []),
-        ...(candidate.tag_items ?? []),
-        ...(candidate.hover_items ?? []),
-    ]).join('\n');
-    const badges = extractBadges(evidenceText, [...FACTORY_BADGE_PATTERNS, ...SERVICE_BADGE_PATTERNS]);
-    const salesText = firstNonEmpty([
-        extractSalesText(candidate.sales_text),
-        extractSalesText(containerText),
-    ]);
-    const returnRateText = extractReturnRateText([...(candidate.tag_items ?? []), ...(candidate.hover_items ?? [])]);
-    const provenance = buildProvenance(sourceUrl);
-    return {
-        rank: 0,
-        offer_id: extractOfferId(canonicalItemUrl ?? '') ?? null,
-        member_id: extractMemberId(canonicalSellerUrl ?? '') ?? null,
-        shop_id: extractShopId(canonicalSellerUrl ?? '') ?? null,
-        title: cleanText(candidate.title) || firstWord(containerText) || null,
-        item_url: canonicalItemUrl,
-        seller_name: cleanText(candidate.seller_name) || null,
-        seller_url: canonicalSellerUrl,
-        price_text: priceRange.price_text || null,
-        price_min: priceRange.price_min,
-        price_max: priceRange.price_max,
-        currency: priceRange.currency,
-        moq_text: moq.moq_text || null,
-        moq_value: moq.moq_value,
-        location: extractLocation(containerText),
-        badges,
-        sales_text: salesText || null,
-        return_rate_text: returnRateText,
-        source_url: provenance.source_url,
-        fetched_at: provenance.fetched_at,
-        strategy: provenance.strategy,
-    };
-}
-function extractMoqText(text) {
-    const normalized = normalizeInlineText(text);
-    return normalized.match(/\d+(?:\.\d+)?\s*(件|个|套|箱|包|双|台|把|只)\s*起批/i)?.[0]
-        ?? normalized.match(/≥\s*\d+(?:\.\d+)?\s*(件|个|套|箱|包|双|台|把|只)?/i)?.[0]
-        ?? normalized.match(/\d+(?:\.\d+)?\s*(?:~|-|至|到)\s*\d+(?:\.\d+)?\s*(件|个|套|箱|包|双|台|把|只)/i)?.[0]
-        ?? '';
-}
-function extractPriceText(text) {
-    const normalized = normalizeInlineText(text);
-    return normalized.match(/[¥$€]\s*\d+(?:\.\d+)?/)?.[0] ?? '';
-}
-function extractSalesText(text) {
-    const normalized = normalizeInlineText(text);
-    if (!normalized)
-        return '';
-    if (/^\d+(?:\.\d+)?\+?\s*(件|套|个|单)$/.test(normalized)) {
-        return normalized;
-    }
-    const match = normalized.match(/(?:已售|销量|售)\s*\d+(?:\.\d+)?\+?\s*(件|套|个|单)?/);
-    return match ? cleanText(match[0]) : '';
-}
-function firstWord(text) {
-    return text.split(/\s+/).find(Boolean) ?? '';
-}
-function firstNonEmpty(values) {
-    return values.map((value) => cleanText(value)).find(Boolean) ?? '';
-}
-function normalizeInlineText(text) {
-    return cleanText(text)
-        .replace(/([¥$€])\s+(?=\d)/g, '$1')
-        .replace(/(\d)\s*\.\s*(\d)/g, '$1.$2')
-        .replace(/\s*([~-])\s*/g, '$1')
-        .trim();
-}
-function extractReturnRateText(values) {
-    return uniqueNonEmpty(values.map((value) => normalizeInlineText(value)))
-        .find((value) => /^回头率\s*\d+(?:\.\d+)?%$/.test(value))
-        ?? null;
-}
-function buildDedupeKey(row) {
-    if (row.offer_id)
-        return `offer:${row.offer_id}`;
-    if (row.item_url)
-        return `url:${row.item_url}`;
-    return null;
-}
-async function readSearchPayload(page, url) {
-    const state = await gotoAndReadState(page, url, 2500, 'search');
-    assertAuthenticatedState(state, 'search');
-    const payload = await page.evaluate(`
-    (() => {
-      const normalizeText = (value) => (value || '').replace(/\\s+/g, ' ').trim();
-      const normalizeUrl = (href) => {
-        if (!href) return '';
-        try {
-          return new URL(href, window.location.href).toString();
-        } catch {
-          return '';
-        }
-      };
-      const isItemHref = (href) => ${JSON.stringify(SEARCH_ITEM_URL_PATTERNS)}
-        .some((pattern) => (href || '').includes(pattern));
-      const uniqueTexts = (values) => [...new Set(values.map((value) => normalizeText(value)).filter(Boolean))];
-      const collectTexts = (root, selector) => uniqueTexts(
-        Array.from(root.querySelectorAll(selector)).map((node) => node.innerText || node.textContent || ''),
-      );
-      const firstText = (root, selectors) => {
-        for (const selector of selectors) {
-          const node = root.querySelector(selector);
-          const value = normalizeText(node ? node.innerText || node.textContent || '' : '');
-          if (value) return value;
-        }
-        return '';
-      };
-      const findMoqText = (values, priceText) => {
-        const moqPattern = /(≥\\s*\\d+(?:\\.\\d+)?\\s*(件|个|套|箱|包|双|台|把|只)?)|(\\d+(?:\\.\\d+)?\\s*(?:~|-|至|到)\\s*\\d+(?:\\.\\d+)?\\s*(件|个|套|箱|包|双|台|把|只))|(\\d+(?:\\.\\d+)?\\s*(件|个|套|箱|包|双|台|把|只)\\s*起批)/i;
-        return values.find((value) => moqPattern.test(value))
-          || normalizeText(priceText).match(moqPattern)?.[0]
-          || '';
-      };
-      const isSellerHref = (href) => {
-        if (!href) return false;
-        try {
-          const url = new URL(href, window.location.href);
-          const host = url.hostname || '';
-          if (!host.endsWith('.1688.com')) return false;
-          if (
-            host === 's.1688.com'
-            || host === 'r.1688.com'
-            || host === 'air.1688.com'
-            || host === 'detail.1688.com'
-            || host === 'detail.m.1688.com'
-            || host === 'dj.1688.com'
-          ) {
-            return false;
-          }
-          return true;
-        } catch {
-          return false;
-        }
-      };
-      const pickContainer = (anchor) => {
-        let node = anchor;
-        while (node && node !== document.body) {
-          const text = normalizeText(node.innerText || node.textContent || '');
-          if (text.length >= 40 && text.length <= 2000) {
-            return node;
-          }
-          node = node.parentElement;
-        }
-        return anchor;
-      };
-      const collectCandidates = () => {
-        const anchors = Array.from(document.querySelectorAll('a')).filter((anchor) => isItemHref(anchor.href || ''));
-        const seen = new Set();
-        const items = [];
-        for (const anchor of anchors) {
-          const href = anchor.href || '';
-          if (!href || seen.has(href)) continue;
-          seen.add(href);
-
-          const container = pickContainer(anchor);
-          const tagItems = collectTexts(container, '.offer-tag-row .offer-desc-item');
-          const hoverItems = collectTexts(container, '.offer-hover-wrapper .offer-desc-item');
-          const sellerAnchor = Array.from(container.querySelectorAll('a'))
-            .find((link) => isSellerHref(link.href || ''));
-          const hoverPriceText = firstText(container, [
-            '.offer-hover-wrapper .hover-price-item',
-            '.offer-hover-wrapper .price-item',
-          ]);
-
-          items.push({
-            item_url: href,
-            title: firstText(container, ['.offer-title-row .title-text', '.offer-title-row'])
-              || normalizeText(anchor.innerText || anchor.textContent || ''),
-            container_text: normalizeText(container.innerText || container.textContent || ''),
-            desc_rows: collectTexts(container, '.offer-desc-row'),
-            price_text: firstText(container, ['.offer-price-row .price-item']),
-            sales_text: firstText(container, ['.offer-price-row .col-desc_after', '.offer-desc-row .col-desc_after']),
-            hover_price_text: hoverPriceText,
-            moq_text: findMoqText(hoverItems, hoverPriceText),
-            tag_items: tagItems,
-            hover_items: hoverItems,
-            seller_name: sellerAnchor ? normalizeText(sellerAnchor.innerText || sellerAnchor.textContent || '') : null,
-            seller_url: sellerAnchor ? sellerAnchor.href : null,
-          });
-        }
-        return items;
-      };
-      const findNextUrl = () => {
-        const selectors = [
-          'a.fui-next:not(.disabled)',
-          'a.next-pagination-item:not(.disabled)',
-          'a[rel="next"]:not(.disabled)',
-          'a[data-role="next"]:not(.disabled)',
-        ];
-        for (const selector of selectors) {
-          const node = document.querySelector(selector);
-          if (!node) continue;
-          const href = normalizeUrl(node.getAttribute('href') || node.href || '');
-          if (href) return href;
-        }
-        const textBased = Array.from(document.querySelectorAll('a'))
-          .find((node) => /下一页|next/i.test(normalizeText(node.textContent || '')));
-        if (!textBased) return '';
-        return normalizeUrl(textBased.getAttribute('href') || textBased.href || '');
-      };
-
-      return {
-        href: window.location.href,
-        title: document.title || '',
-        bodyText: document.body ? document.body.innerText || '' : '',
-        next_url: findNextUrl(),
-        candidates: collectCandidates(),
-      };
-    })()
-  `);
-    if (!payload || typeof payload !== 'object') {
-        throw new CommandExecutionError('1688 search page did not return a readable payload', 'Open the same query in Chrome and verify the page is fully loaded before retrying.');
-    }
-    return payload;
-}
-async function collectSearchRows(page, query, limit) {
-    const rowsByKey = new Map();
-    const seenPages = new Set();
-    let nextUrl = buildSearchUrl(query);
-    let pageCount = 0;
-    while (nextUrl && rowsByKey.size < limit && pageCount < MAX_SEARCH_PAGES) {
-        if (seenPages.has(nextUrl))
-            break;
-        seenPages.add(nextUrl);
-        pageCount += 1;
-        const payload = await readSearchPayload(page, nextUrl);
-        const sourceUrl = cleanText(payload.href) || nextUrl;
-        const candidates = Array.isArray(payload.candidates) ? payload.candidates : [];
-        for (const candidate of candidates) {
-            const row = normalizeSearchCandidate(candidate, sourceUrl);
-            const dedupeKey = buildDedupeKey(row);
-            if (!dedupeKey || rowsByKey.has(dedupeKey))
-                continue;
-            rowsByKey.set(dedupeKey, row);
-            if (rowsByKey.size >= limit)
-                break;
-        }
-        const candidateNextUrl = cleanText(payload.next_url);
-        if (!candidateNextUrl || candidateNextUrl === sourceUrl)
-            break;
-        nextUrl = candidateNextUrl;
-    }
-    if (rowsByKey.size === 0) {
-        throw new EmptyResultError('1688 search', 'No visible results were extracted. Retry with a different query or open the same search page in Chrome first.');
-    }
-    return [...rowsByKey.values()]
-        .slice(0, limit)
-        .map((row, index) => ({ ...row, rank: index + 1 }));
-}
-cli({
-    site: '1688',
-    name: 'search',
-    access: 'read',
-    description: '1688 商品搜索（结果候选、卖家链接、价格/MOQ/销量文本）',
-    domain: 'www.1688.com',
-    strategy: Strategy.COOKIE,
-    navigateBefore: false,
-    args: [
-        {
-            name: 'query',
-            required: true,
-            positional: true,
-            help: '搜索关键词，如 "置物架"',
-        },
-        {
-            name: 'limit',
-            type: 'int',
-            default: SEARCH_LIMIT_DEFAULT,
-            help: `结果数量上限（默认 ${SEARCH_LIMIT_DEFAULT}，最大 ${SEARCH_LIMIT_MAX}）`,
-        },
-    ],
-    columns: ['rank', 'offer_id', 'title', 'item_url', 'price_text', 'moq_text', 'seller_name', 'member_id', 'location'],
-    func: async (page, kwargs) => {
-        const query = String(kwargs.query ?? '');
-        const limit = parseSearchLimit(kwargs.limit);
-        return collectSearchRows(page, query, limit);
-    },
-});
-export const __test__ = {
-    normalizeSearchCandidate,
-    extractMoqText,
-    extractSalesText,
-    firstWord,
-    buildDedupeKey,
-};

package/clis/1688/search.test.js

@@ -1,75 +0,0 @@
-import { describe, expect, it } from 'vitest';
-import { __test__ } from './search.js';
-describe('1688 search normalization', () => {
-    it('normalizes search candidates into structured result rows', () => {
-        const result = __test__.normalizeSearchCandidate({
-            item_url: 'https://detail.1688.com/offer/887904326744.html',
-            title: '宿舍置物架桌面加高架',
-            container_text: '宿舍置物架桌面加高架 ¥56.00 2套起批 山东青岛 已售300+套',
-            price_text: '¥ 56 .00',
-            sales_text: '300+套',
-            moq_text: '2套起批',
-            tag_items: ['退货包运费', '回头率52%'],
-            hover_items: ['验厂报告'],
-            seller_name: '青岛沁澜衣品服装有限公司',
-            seller_url: 'https://yinuoweierfushi.1688.com/page/index.html?spm=a123',
-        }, 'https://s.1688.com/selloffer/offer_search.htm?charset=utf8&keywords=置物架');
-        expect(result.rank).toBe(0);
-        expect(result.offer_id).toBe('887904326744');
-        expect(result.shop_id).toBe('yinuoweierfushi');
-        expect(result.item_url).toBe('https://detail.1688.com/offer/887904326744.html');
-        expect(result.seller_url).toBe('https://yinuoweierfushi.1688.com');
-        expect(result.price_text).toBe('¥56.00');
-        expect(result.price_min).toBe(56);
-        expect(result.price_max).toBe(56);
-        expect(result.moq_value).toBe(2);
-        expect(result.location).toBe('山东青岛');
-        expect(result.sales_text).toBe('300+套');
-        expect(result.badges).toEqual(expect.arrayContaining(['退货包运费', '验厂报告']));
-        expect(result.return_rate_text).toBe('回头率52%');
-    });
-    it('does not use hover_price_text as MOQ source', () => {
-        const result = __test__.normalizeSearchCandidate({
-            item_url: 'https://detail.1688.com/offer/887904326744.html',
-            title: 'test',
-            container_text: 'test ¥56.00',
-            price_text: '¥ 56 .00',
-            hover_price_text: '¥56.00 3件起批',
-            moq_text: null,
-        }, 'https://s.1688.com/selloffer/offer_search.htm?charset=utf8&keywords=test');
-        // hover_price_text should not be used for MOQ extraction
-        expect(result.moq_text).toBeNull();
-        expect(result.moq_value).toBeNull();
-    });
-    it('extracts offer id from mobile detail search links', () => {
-        const result = __test__.normalizeSearchCandidate({
-            item_url: 'http://detail.m.1688.com/page/index.html?offerId=910933345396&sortType=&pageId=',
-            title: '',
-            container_text: '桌面书桌办公室工位收纳展示新中式博古架多层茶具厨房摆放置物架 ¥24.3 已售20+件',
-            price_text: '¥ 14 .28',
-            sales_text: '1500+件',
-            moq_text: '≥2个',
-            seller_name: '泰商国际贸易（宁阳）有限公司',
-            seller_url: 'http://tsgjmy.1688.com/',
-        }, 'https://s.1688.com/selloffer/offer_search.htm?charset=utf8&keywords=桌面置物架');
-        expect(result.offer_id).toBe('910933345396');
-        expect(result.shop_id).toBe('tsgjmy');
-        expect(result.item_url).toBe('https://detail.1688.com/offer/910933345396.html');
-        expect(result.title).toContain('桌面书桌办公室工位收纳展示');
-        expect(result.price_text).toBe('¥14.28');
-        expect(result.sales_text).toBe('1500+件');
-        expect(result.moq_text).toBe('≥2个');
-        expect(result.moq_value).toBe(2);
-    });
-    it('prefers offer id and falls back to item url for dedupe key', () => {
-        expect(__test__.buildDedupeKey({
-            offer_id: '123456',
-            item_url: 'https://detail.1688.com/offer/123456.html',
-        })).toBe('offer:123456');
-        expect(__test__.buildDedupeKey({
-            offer_id: null,
-            item_url: 'https://detail.1688.com/offer/123456.html',
-        })).toBe('url:https://detail.1688.com/offer/123456.html');
-        expect(__test__.buildDedupeKey({ offer_id: null, item_url: null })).toBeNull();
-    });
-});