@agentrhq/webcmd 0.1.2 → 0.2.0

package/clis/dianping/search.js

@@ -1,157 +0,0 @@
-/**
- * dianping search — search shops/restaurants by keyword on a given city.
- *
- * Targets www.dianping.com (PC site). The PC site renders search results
- * server-side, so the func only needs to parse the SSR DOM after navigate.
- * m.dianping.com (mobile) is intentionally crippled for non-mobile UAs and
- * does not return data without app installation, so it's not used.
- *
- * The DOM extractor is defined as a plain top-level function and injected
- * into `page.evaluate` via `.toString()`, so the same code is exercised by
- * a JSDOM-against-frozen-fixture unit test (see dianping.test.js).
- */
-
-import { cli, Strategy } from '@agentrhq/webcmd/registry';
-import { ArgumentError, CommandExecutionError } from '@agentrhq/webcmd/errors';
-import {
-    SEARCH_COLUMNS,
-    detectAuthOrPageFailure,
-    parsePrice,
-    parseReviewCount,
-    requireSearchLimit,
-    wrapDianpingStep,
-} from './utils.js';
-import { resolveCityIdAsync } from './cityResolver.js';
-
-/**
- * Pure DOM extractor for the dianping search-results page.
- *
- * Uses bare `document` / `location` so it runs identically in:
- *   - the live browser (injected via `${extractSearchRows.toString()}`)
- *   - JSDOM unit tests (which swap `globalThis.document` / `globalThis.location`)
- */
-export function extractSearchRows() {
-    const items = document.querySelectorAll('#shop-all-list li');
-    if (items.length === 0) {
-        const body = document.body;
-        const sampleText = (body && (body.innerText || body.textContent)) || '';
-        return {
-            ok: false,
-            sample: sampleText.slice(0, 800),
-            url: location.href,
-        };
-    }
-    const rows = [];
-    items.forEach((el, i) => {
-        const resultShape = el.querySelector('.tit h4, .review-num, .mean-price, .tag-addr')
-            || el.querySelector('.tit a, a[data-shopid], a[href*="/shop/"]');
-        if (!resultShape) return;
-
-        const link = el.querySelector('.tit a[href*="/shop/"]')
-            || el.querySelector('a[data-shopid]')
-            || el.querySelector('a[href*="/shop/"]');
-        const shopId = (link && link.getAttribute('data-shopid'))
-            || (link && (link.getAttribute('href') || '').match(/\/shop\/([^?#/]+)/)?.[1])
-            || '';
-        const titleEl = el.querySelector('.tit h4');
-        const name = (titleEl && titleEl.textContent && titleEl.textContent.trim())
-            || (link && link.getAttribute('title'))
-            || '';
-        const reviewEl = el.querySelector('.review-num b');
-        const reviewsRaw = (reviewEl && reviewEl.textContent && reviewEl.textContent.trim()) || '';
-        const priceEl = el.querySelector('.mean-price b');
-        const priceRaw = (priceEl && priceEl.textContent && priceEl.textContent.trim()) || '';
-        const tagAddr = Array.from(el.querySelectorAll('.tag-addr .tag'))
-            .map((t) => t.textContent.trim())
-            .filter(Boolean);
-        let starClass = '';
-        const starWrap = el.querySelector('.nebula_star');
-        if (starWrap) {
-            const m = starWrap.outerHTML.match(/star_(\d{2})/g);
-            if (m && m.length) {
-                const last = m[m.length - 1];
-                const lastDigits = last.match(/star_(\d{2})/)[1];
-                starClass = lastDigits;
-            }
-        }
-        rows.push({
-            rank: i + 1,
-            shop_id: shopId,
-            name,
-            starClass,
-            reviewsRaw,
-            priceRaw,
-            cuisine: tagAddr[0] || '',
-            district: tagAddr[1] || '',
-            url: shopId ? 'https://www.dianping.com/shop/' + shopId : '',
-        });
-    });
-    return { ok: true, rows };
-}
-
-cli({
-    site: 'dianping',
-    name: 'search',
-    access: 'read',
-    description: '大众点评店铺搜索（按关键词 + 城市）',
-    domain: 'www.dianping.com',
-    strategy: Strategy.COOKIE,
-    args: [
-        { name: 'keyword', required: true, positional: true, help: '搜索关键词，例如 "火锅"' },
-        { name: 'city', help: '城市名（北京/上海/汕头/beijing/shantou/...）或 cityId 数字。未在静态表中的城市会通过 dianping.com 在线解析。不传则使用 cookie 默认城市' },
-        { name: 'limit', type: 'int', default: 15, help: '返回的店铺数量（最多 15，dianping 单页固定 15 条）' },
-    ],
-    columns: SEARCH_COLUMNS,
-    func: async (page, kwargs) => {
-        const keyword = String(kwargs.keyword || '').trim();
-        if (!keyword) throw new ArgumentError('keyword', 'must be a non-empty string');
-
-        const limit = requireSearchLimit(kwargs.limit);
-
-        const cityId = await wrapDianpingStep(
-            'city resolve',
-            () => resolveCityIdAsync(page, kwargs.city),
-        );
-        const path = cityId
-            ? `/search/keyword/${cityId}/0_${encodeURIComponent(keyword)}`
-            : `/search/keyword/0/0_${encodeURIComponent(keyword)}`;
-        const url = `https://www.dianping.com${path}`;
-
-        await wrapDianpingStep(`search "${keyword}" navigation`, async () => {
-            await page.goto(url);
-            await page.wait(2);
-        });
-
-        const result = await wrapDianpingStep(
-            `search "${keyword}" extraction`,
-            () => page.evaluate(`(${extractSearchRows.toString()})()`),
-        );
-
-        if (!result || !result.ok) {
-            detectAuthOrPageFailure(
-                { text: String(result?.sample || ''), url: String(result?.url || url) },
-                `search "${keyword}"`,
-                { emptyPatterns: [/没有找到|暂无结果|暂无商户|换个关键词|未找到相关/i] },
-            );
-        }
-
-        const rows = (result.rows || []).slice(0, limit);
-        if (rows.length === 0) {
-            throw new CommandExecutionError('dianping search parser found no result-shaped shop cards');
-        }
-        if (rows.some((row) => !row?.shop_id)) {
-            throw new CommandExecutionError('dianping search parser found result cards without shop_id values');
-        }
-        return rows.map((r) => ({
-            rank: r.rank,
-            shop_id: r.shop_id,
-            name: r.name,
-            rating: r.starClass ? Number((Number(r.starClass) / 10).toFixed(1)) : null,
-            reviews: parseReviewCount(r.reviewsRaw),
-            price: parsePrice(r.priceRaw),
-            cuisine: r.cuisine,
-            district: r.district,
-            url: r.url,
-        }));
-    },
-});

package/clis/dianping/shop.js

@@ -1,173 +0,0 @@
-/**
- * dianping shop — read shop detail by shop ID (the alphanumeric handle in
- * `https://www.dianping.com/shop/<shop_id>`).
- *
- * Returns a key/value sheet so the table view stays readable when fields
- * are absent (phone is hidden on PC web — only shows in app — so the row
- * surfaces it as `null` rather than fabricating).
- *
- * The DOM extractor is defined as a plain top-level function and injected
- * into `page.evaluate` via `.toString()`, so the same code is exercised by
- * a JSDOM-against-frozen-fixture unit test (see dianping.test.js). Mocked
- * `page.evaluate` tests can't catch in-browser bugs like the original
- * full-width `【】` vs ASCII `[]` mismatch — fixture tests can.
- */
-
-import { cli, Strategy } from '@agentrhq/webcmd/registry';
-import {
-    SHOP_COLUMNS,
-    detectAuthOrPageFailure,
-    normalizeShopId,
-    parsePrice,
-    parseReviewCount,
-    wrapDianpingStep,
-} from './utils.js';
-
-/**
- * Pure DOM extractor for the dianping shop page.
- *
- * Uses bare `document` / `location` so it runs identically in:
- *   - the live browser (injected via `${extractShopFields.toString()}`)
- *   - JSDOM unit tests (which swap `globalThis.document` / `globalThis.location`)
- *
- * Returns either `{ ok: true, ...fields }` on success or
- * `{ ok: false, sample, url }` so the caller can classify the failure.
- */
-export function extractShopFields() {
-    const head = document.querySelector('.shop-head');
-    if (!head) {
-        const body = document.body;
-        const sampleText = (body && (body.innerText || body.textContent)) || '';
-        return {
-            ok: false,
-            sample: sampleText.slice(0, 800),
-            url: location.href,
-        };
-    }
-    const headText = head.textContent.trim().replace(/\s+/g, ' ');
-    // Shop name: dianping puts it as "【芈重山老火锅(五道口店)】" at
-    // the head of document.title (full-width 【】, not ASCII []).
-    // Try selectors first, then fall back to title parsing.
-    const titleEl = document.querySelector('.shop-name, .shop-head h2, .shop-head h1');
-    let name = (titleEl && titleEl.textContent && titleEl.textContent.trim()) || '';
-    if (!name) {
-        const t = document.title || '';
-        const m = t.match(/【([^】]+)】/);
-        if (m) name = m[1].trim();
-    }
-    const ratingEl = document.querySelector('.star-score');
-    const ratingText = (ratingEl && ratingEl.textContent && ratingEl.textContent.trim()) || '';
-    const features = Array.from(document.querySelectorAll('.shop-feature'))
-        .map((f) => f.textContent.trim())
-        .filter(Boolean);
-    const addressEl = document.querySelector('.desc-info');
-    const address = (addressEl && addressEl.textContent && addressEl.textContent.trim()) || '';
-    const subwayMatch = headText.match(/距(?:地铁)?[^\s]+?步行\d+m/);
-    const subway = subwayMatch ? subwayMatch[0] : '';
-
-    // Reviews: prefer .reviews / .review-num selector — its text is
-    // "21241条" cleanly. Whitespace-collapsed headText fuses the
-    // rating "4.8" with review digits ("4.821241条"), so a
-    // head-wide /\d+条/ regex captures "4.821241" and rounds to 5.
-    const reviewEl = document.querySelector('.reviews, .review-num, .reviewCount, .reviewCountSentence');
-    let reviewsRaw = (reviewEl && reviewEl.textContent && reviewEl.textContent.trim()) || '';
-    if (!reviewsRaw) {
-        const titleReviewEl = document.querySelector('.review-title');
-        const titleText = titleReviewEl && titleReviewEl.textContent;
-        const titleM = titleText && titleText.match(/评价\(([\d.,万]+)\)/);
-        if (titleM) reviewsRaw = titleM[1];
-    }
-
-    // Shop-head text holds price + cuisine + district + rank.
-    const priceMatch = headText.match(/[¥￥]\s*\d+(?:\.\d+)?/);
-
-    // Try to read score breakdown ("口味:4.8 环境:4.8 服务:4.8 食材:4.9").
-    const breakdown = {};
-    const breakKeys = ['口味', '环境', '服务', '食材'];
-    for (const key of breakKeys) {
-        const m = headText.match(new RegExp(key + '[:：]\\s*([0-9.]+)'));
-        if (m) breakdown[key] = Number(m[1]);
-    }
-
-    // Hours: "营业中 11:00-次日02:00" / "今日休息".
-    const hoursMatch = headText.match(/营业中[^\s]*\d{1,2}:\d{2}-(?:次日)?\d{1,2}:\d{2}|今日休息|暂停营业/);
-
-    // Rank line: "海淀区 重庆火锅 口味榜 · 第1名".
-    const rankMatch = headText.match(/[^\s]+?(?:口味|人气|环境|服务)榜\s*[·•]\s*第\d+名/);
-
-    return {
-        ok: true,
-        name,
-        rating: ratingText,
-        reviewsRaw,
-        priceRaw: (priceMatch && priceMatch[0]) || '',
-        breakdown,
-        features,
-        address,
-        subway,
-        hours: (hoursMatch && hoursMatch[0]) || '',
-        rank: (rankMatch && rankMatch[0]) || '',
-        url: location.href,
-    };
-}
-
-cli({
-    site: 'dianping',
-    name: 'shop',
-    access: 'read',
-    aliases: ['detail'],
-    description: '大众点评店铺详情（按 shop_id）',
-    domain: 'www.dianping.com',
-    strategy: Strategy.COOKIE,
-    args: [
-        { name: 'shop_id', required: true, positional: true, help: '店铺 ID（来自 search 的 shop_id 列，或 https://www.dianping.com/shop/<id> URL 段）' },
-    ],
-    columns: SHOP_COLUMNS,
-    func: async (page, kwargs) => {
-        const shopId = normalizeShopId(kwargs.shop_id);
-
-        const url = `https://www.dianping.com/shop/${shopId}`;
-        await wrapDianpingStep(`shop ${shopId} navigation`, async () => {
-            await page.goto(url);
-            await page.wait(3);
-        });
-
-        const data = await wrapDianpingStep(
-            `shop ${shopId} extraction`,
-            () => page.evaluate(`(${extractShopFields.toString()})()`),
-        );
-
-        if (!data || !data.ok) {
-            detectAuthOrPageFailure(
-                { text: String(data?.sample || ''), url: String(data?.url || url) },
-                `shop ${shopId}`,
-                { emptyPatterns: [/商户不存在|店铺不存在|店铺已关闭|页面不存在|404|已下线|没有找到相关商户/i] },
-            );
-        }
-
-        const rating = data.rating ? Number(data.rating) : null;
-        const reviews = parseReviewCount(data.reviewsRaw);
-        const price = parsePrice(data.priceRaw);
-        const breakdown = data.breakdown || {};
-
-        const fields = [
-            ['shop_id', shopId],
-            ['name', data.name || ''],
-            ['rating', Number.isFinite(rating) ? rating : null],
-            ['reviews', reviews],
-            ['price', price],
-            ['rank', data.rank || ''],
-            ['taste', Number.isFinite(breakdown['口味']) ? breakdown['口味'] : null],
-            ['environment', Number.isFinite(breakdown['环境']) ? breakdown['环境'] : null],
-            ['service', Number.isFinite(breakdown['服务']) ? breakdown['服务'] : null],
-            ['ingredients', Number.isFinite(breakdown['食材']) ? breakdown['食材'] : null],
-            ['hours', data.hours || ''],
-            ['address', data.address || ''],
-            ['subway', data.subway || ''],
-            ['features', (data.features || []).join(', ')],
-            ['url', data.url || url],
-        ];
-
-        return fields.map(([field, value]) => ({ field, value }));
-    },
-});

package/clis/dianping/utils.js

@@ -1,157 +0,0 @@
-/**
- * Shared helpers for dianping (大众点评) adapters.
- *
- * Mobile m.dianping.com is intentionally crippled to push users to the
- * native app — body renders as 2 chars when probed from desktop UA.
- * www.dianping.com (PC site) returns the full search HTML server-side
- * and does NOT require JS hydration, so adapters target it directly.
- */
-
-import {
-    ArgumentError,
-    AuthRequiredError,
-    CommandExecutionError,
-    EmptyResultError,
-} from '@agentrhq/webcmd/errors';
-
-/**
- * Common Chinese cities → dianping cityId.
- * Source: dianping.com /citylist URL structure (cityId is the integer in
- * the search path: /search/keyword/{cityId}/0_keyword).
- */
-export const CITY_ID = {
-    beijing: 2, '北京': 2,
-    shanghai: 1, '上海': 1,
-    guangzhou: 4, '广州': 4,
-    shenzhen: 7, '深圳': 7,
-    hangzhou: 3, '杭州': 3,
-    chengdu: 8, '成都': 8,
-    chongqing: 9, '重庆': 9,
-    nanjing: 5, '南京': 5,
-    suzhou: 6, '苏州': 6,
-    xian: 17, '西安': 17,
-    wuhan: 16, '武汉': 16,
-    tianjin: 10, '天津': 10,
-    qingdao: 21, '青岛': 21,
-    changsha: 344, '长沙': 344,
-    dalian: 19, '大连': 19,
-    shenyang: 18, '沈阳': 18,
-    kunming: 25, '昆明': 25,
-    fuzhou: 110, '福州': 110,
-    xiamen: 14, '厦门': 14,
-    hefei: 26, '合肥': 26,
-};
-
-export const SEARCH_COLUMNS = ['rank', 'shop_id', 'name', 'rating', 'reviews', 'price', 'cuisine', 'district', 'url'];
-export const SHOP_COLUMNS = ['field', 'value'];
-
-/**
- * Resolve a city argument (name or id) to a numeric cityId.
- * Returns null when the cookie's default city should be used.
- */
-export function resolveCityId(cityArg) {
-    if (cityArg == null || cityArg === '') return null;
-    const raw = String(cityArg).trim().toLowerCase();
-    if (/^\d+$/.test(raw)) return Number(raw);
-    const id = CITY_ID[raw];
-    if (!id) {
-        const names = Object.keys(CITY_ID).filter((k) => /^[a-z]+$/.test(k)).join(', ');
-        throw new ArgumentError(
-            'city',
-            `unknown city '${cityArg}'. pass a numeric cityId or one of: ${names}`,
-        );
-    }
-    return id;
-}
-
-export function requireSearchLimit(value) {
-    const raw = value == null || value === '' ? 15 : value;
-    const limit = typeof raw === 'number' ? raw : Number(String(raw).trim());
-    if (!Number.isInteger(limit) || limit < 1 || limit > 15) {
-        throw new ArgumentError('limit must be an integer between 1 and 15 (dianping single page)');
-    }
-    return limit;
-}
-
-export function normalizeShopId(rawInput) {
-    const raw = String(rawInput || '').trim();
-    if (!raw) throw new ArgumentError('shop_id must be a non-empty string');
-
-    const idMatch = raw.match(/\/shop\/([^?#/]+)/);
-    const shopId = idMatch ? idMatch[1] : raw;
-    if (!/^[A-Za-z0-9_-]+$/.test(shopId)) {
-        throw new ArgumentError(`'${raw}' does not look like a dianping shop id`);
-    }
-    return shopId;
-}
-
-export function wrapDianpingStep(label, fn) {
-    return Promise.resolve()
-        .then(fn)
-        .catch((err) => {
-            if (err?.code) throw err;
-            const message = err?.message || String(err);
-            throw new CommandExecutionError(`dianping ${label} failed: ${message}`);
-        });
-}
-
-/**
- * Throw the right typed error for a dianping page that didn't render data.
- * The site short-circuits HTML when bot/login checks trip — typically
- * redirects to verify.meituan.com (Yoda icon-tap captcha) or to a login
- * page when the cookie is missing.
- */
-export function detectAuthOrPageFailure({ text = '', url = '' }, contextHint, { emptyPatterns = [] } = {}) {
-    const signal = `${url} ${text}`;
-    if (/verify\.meituan\.com|verifyimg|身份核实|请依次点击|美团安全验证|Yoda/i.test(signal)) {
-        throw new AuthRequiredError(
-            'dianping.com',
-            `dianping ${contextHint} blocked by captcha — open ${url || 'www.dianping.com'} manually in this profile and solve the captcha, then retry`,
-        );
-    }
-    if (/login\.dianping\.com|account\.dianping\.com|请先登录|未登录|请登录/.test(signal)) {
-        throw new AuthRequiredError(
-            'dianping.com',
-            `dianping ${contextHint} requires login — sign in to dianping.com in this profile, then retry`,
-        );
-    }
-    if (emptyPatterns.some((pattern) => pattern.test(signal))) {
-        throw new EmptyResultError(`dianping ${contextHint}`);
-    }
-    const sample = text ? `; sample: ${String(text).slice(0, 160)}` : '';
-    throw new CommandExecutionError(
-        `dianping ${contextHint} did not render expected data${sample}`,
-        'This usually means dianping changed its HTML, returned an unexpected error page, or the browser profile hit an unrecognized anti-bot state.',
-    );
-}
-
-/**
- * Parse "21231" / "1.2万" review-count strings into integers.
- * Returns null when the input has no parseable digits.
- */
-export function parseReviewCount(raw) {
-    if (raw == null) return null;
-    const s = String(raw).trim();
-    if (!s) return null;
-    const wanMatch = s.match(/^([\d.]+)\s*万/);
-    if (wanMatch) {
-        const n = Number(wanMatch[1]);
-        return Number.isFinite(n) ? Math.round(n * 10000) : null;
-    }
-    const plainMatch = s.match(/(\d+(?:\.\d+)?)/);
-    if (!plainMatch) return null;
-    const n = Number(plainMatch[1]);
-    return Number.isFinite(n) ? Math.round(n) : null;
-}
-
-/**
- * Parse "￥109" / "¥109" / "人均￥109" into a numeric yuan value.
- * Returns null when no price is present (some shops omit price entirely).
- */
-export function parsePrice(raw) {
-    if (!raw) return null;
-    const m = String(raw).match(/[¥￥]\s*(\d+(?:\.\d+)?)/);
-    if (!m) return null;
-    const n = Number(m[1]);
-    return Number.isFinite(n) ? n : null;
-}

package/clis/dongchedi/__fixtures__/search.json

@@ -1,60 +0,0 @@
-{
- "data": [
-  {
-   "id": "26_dcd_car_series_v2:5273",
-   "series_id": 5273,
-   "cell_type": 26,
-   "koubei": {
-    "url": "",
-    "tag_list": null
-   },
-   "display": {
-    "title": "宝马X5",
-    "series_name": "宝马X5",
-    "sub_brand_name": "华晨宝马",
-    "agent_price": "51.00-68.00万",
-    "official_price": "59.80-74.80万",
-    "cover_url": "https://p3-dcd.byteimg.com/img/tos-cn-i-dcdx/bf84f7f297c74b4ab69467bef101e7e1~1080x0.png?psm=motor.search.block",
-    "picture_num": 11355,
-    "dealer_text": "询最低价"
-   }
-  },
-  {
-   "id": "200_:7566598885808833062",
-   "series_id": null,
-   "cell_type": 200,
-   "koubei": null,
-   "display": {}
-  },
-  {
-   "id": "200_:7601400898518090302",
-   "series_id": null,
-   "cell_type": 200,
-   "koubei": null,
-   "display": {}
-  },
-  {
-   "id": "200_:7595987626075849241",
-   "series_id": null,
-   "cell_type": 200,
-   "koubei": null,
-   "display": {}
-  },
-  {
-   "id": "200_:7621424306387747352",
-   "series_id": null,
-   "cell_type": 200,
-   "koubei": null,
-   "display": {}
-  },
-  {
-   "id": "200_:7601471677582803518",
-   "series_id": null,
-   "cell_type": 200,
-   "koubei": null,
-   "display": {}
-  }
- ],
- "has_more": 1,
- "return_count": 37
-}