@agentrhq/webcmd 0.1.2 → 0.2.0

package/clis/juejin/utils.js

@@ -1,199 +0,0 @@
-// Shared helpers for the Juejin (`api.juejin.cn`) adapter.
-//
-// Juejin is a Chinese developer community (similar to Dev.to). The public
-// REST API is unauthenticated; all read endpoints are reachable without a
-// browser session. Article URLs round-trip as `https://juejin.cn/post/<id>`.
-import { ArgumentError, CommandExecutionError, EmptyResultError } from '@agentrhq/webcmd/errors';
-
-export const JUEJIN_API_BASE = 'https://api.juejin.cn';
-export const JUEJIN_POST_URL = 'https://juejin.cn/post';
-export const JUEJIN_USER_URL = 'https://juejin.cn/user';
-const UA = 'webcmd-juejin-adapter (+https://github.com/agentrhq/webcmd)';
-
-// Juejin content / article IDs are 19-digit numeric strings.
-const JUEJIN_ID = /^\d{16,20}$/;
-
-// Top-level categories surfaced by `query_category_briefs`. The slugs are
-// stable so the adapter accepts a friendly name in addition to the raw id.
-export const CATEGORY_ALIASES = {
-    backend: '6809637769959178254',
-    frontend: '6809637767543259144',
-    android: '6809635626879549454',
-    ios: '6809635626661445640',
-    ai: '6809637773935378440',
-};
-
-export function requireString(value, label) {
-    const s = String(value ?? '').trim();
-    if (!s) {
-        throw new ArgumentError(`juejin ${label} cannot be empty`);
-    }
-    return s;
-}
-
-export function requireBoundedInt(value, defaultValue, maxValue, label = 'limit') {
-    const raw = value ?? defaultValue;
-    let n;
-    if (typeof raw === 'number') {
-        n = raw;
-    }
-    else if (typeof raw === 'string' && /^[1-9]\d*$/.test(raw)) {
-        n = Number(raw);
-    }
-    else {
-        throw new ArgumentError(`juejin ${label} must be a positive decimal integer`);
-    }
-    if (!Number.isSafeInteger(n) || n <= 0) {
-        throw new ArgumentError(`juejin ${label} must be a positive integer`);
-    }
-    if (n > maxValue) {
-        throw new ArgumentError(`juejin ${label} must be <= ${maxValue}`);
-    }
-    return n;
-}
-
-export function requireCursor(value) {
-    const raw = value ?? '0';
-    if (typeof raw === 'number') {
-        if (Number.isSafeInteger(raw) && raw >= 0) return String(raw);
-        throw new ArgumentError('juejin cursor must be a non-negative decimal integer');
-    }
-    if (typeof raw === 'string' && /^(0|[1-9]\d*)$/.test(raw)) {
-        return raw;
-    }
-    throw new ArgumentError('juejin cursor must be a non-negative decimal integer');
-}
-
-/** Resolve a `--category` arg to the underlying numeric category id. */
-export function resolveCategory(value) {
-    if (value == null) return '';
-    const raw = String(value).trim();
-    if (!raw) return '';
-    if (JUEJIN_ID.test(raw)) return raw;
-    const slug = raw.toLowerCase();
-    if (CATEGORY_ALIASES[slug]) return CATEGORY_ALIASES[slug];
-    throw new ArgumentError(
-        `juejin category "${value}" is not recognised`,
-        `Use a category id (e.g. "${CATEGORY_ALIASES.backend}") or one of: ${Object.keys(CATEGORY_ALIASES).join(', ')}.`,
-    );
-}
-
-/**
- * POST JSON to a Juejin endpoint. The API returns `{ err_no, err_msg, data }`;
- * a non-zero `err_no` is surfaced as a typed `CommandExecutionError`.
- */
-export async function juejinFetch(path, body, label, method = 'POST') {
-    const url = `${JUEJIN_API_BASE}${path}`;
-    let resp;
-    try {
-        const init = {
-            method,
-            headers: { 'user-agent': UA, accept: 'application/json' },
-        };
-        if (method === 'POST') {
-            init.headers['content-type'] = 'application/json';
-            init.body = JSON.stringify(body ?? {});
-        }
-        resp = await fetch(url, init);
-    } catch (err) {
-        throw new CommandExecutionError(
-            `${label} request failed: ${err?.message ?? err}`,
-            'Check that api.juejin.cn is reachable from this network.',
-        );
-    }
-    if (resp.status === 429) {
-        throw new CommandExecutionError(
-            `${label} returned HTTP 429 (rate limited)`,
-            'Juejin throttles bursty traffic; wait a few seconds and retry.',
-        );
-    }
-    if (!resp.ok) {
-        throw new CommandExecutionError(`${label} returned HTTP ${resp.status}`);
-    }
-    let payload;
-    try {
-        payload = await resp.json();
-    } catch (err) {
-        throw new CommandExecutionError(`${label} returned malformed JSON: ${err?.message ?? err}`);
-    }
-    if (!payload || typeof payload !== 'object' || Array.isArray(payload) || !Object.hasOwn(payload, 'err_no')) {
-        throw new CommandExecutionError(`${label} returned a malformed API envelope`);
-    }
-    if (payload.err_no !== 0) {
-        throw new CommandExecutionError(`${label} returned err_no ${payload.err_no}: ${payload.err_msg ?? ''}`);
-    }
-    return payload;
-}
-
-export function readDataArray(payload, label) {
-    if (!payload || typeof payload !== 'object' || Array.isArray(payload) || !Object.hasOwn(payload, 'data')) {
-        throw new CommandExecutionError(`${label} returned a malformed payload`);
-    }
-    if (!Array.isArray(payload.data)) {
-        throw new CommandExecutionError(`${label} returned a non-array data field`);
-    }
-    if (payload.data.length === 0) {
-        throw new EmptyResultError(label, `${label} returned no articles.`);
-    }
-    return payload.data;
-}
-
-function readArticleId(value, label) {
-    const id = String(value ?? '').trim();
-    if (!JUEJIN_ID.test(id)) {
-        throw new CommandExecutionError(`${label} returned a malformed article id`);
-    }
-    return id;
-}
-
-function readOptionalNumber(value, label) {
-    if (value == null) return null;
-    const n = Number(value);
-    if (!Number.isFinite(n)) {
-        throw new CommandExecutionError(`${label} returned a malformed numeric field`);
-    }
-    return n;
-}
-
-/** Map a recommend-feed row (`item_info.article_info` / `author_user_info`) to a flat shape. */
-export function mapFeedItem(row, rank) {
-    const info = row?.item_info ?? {};
-    const article = info.article_info ?? {};
-    const author = info.author_user_info ?? {};
-    const tags = Array.isArray(info.tags)
-        ? info.tags.map(t => t?.tag_name).filter(Boolean).slice(0, 6).join(', ')
-        : '';
-    const articleId = readArticleId(article.article_id, 'juejin recommend');
-    return {
-        rank,
-        article_id: articleId,
-        title: String(article.title ?? '').trim(),
-        brief: String(article.brief_content ?? '').trim(),
-        views: readOptionalNumber(article.view_count, 'juejin recommend'),
-        likes: readOptionalNumber(article.digg_count, 'juejin recommend'),
-        comments: readOptionalNumber(article.comment_count, 'juejin recommend'),
-        author: String(author.user_name ?? '').trim(),
-        tags,
-        url: articleId ? `${JUEJIN_POST_URL}/${articleId}` : '',
-    };
-}
-
-/** Map a hot-list row (different envelope from the feed endpoint) to the same flat shape. */
-export function mapHotItem(row, rank) {
-    const content = row?.content ?? {};
-    const counter = row?.content_counter ?? {};
-    const author = row?.author ?? {};
-    const articleId = readArticleId(content.content_id, 'juejin hot');
-    return {
-        rank,
-        article_id: articleId,
-        title: String(content.title ?? '').trim(),
-        brief: String(content.brief ?? '').trim(),
-        views: readOptionalNumber(counter.view, 'juejin hot'),
-        likes: readOptionalNumber(counter.like, 'juejin hot'),
-        comments: readOptionalNumber(counter.comment_count, 'juejin hot'),
-        hot_rank: readOptionalNumber(counter.hot_rank, 'juejin hot'),
-        author: String(author.name ?? '').trim(),
-        url: articleId ? `${JUEJIN_POST_URL}/${articleId}` : '',
-    };
-}

package/clis/ke/auth.js

@@ -1,46 +0,0 @@
-import { AuthRequiredError, CommandExecutionError } from '@agentrhq/webcmd/errors';
-import { registerSiteAuthCommands } from '../_shared/site-auth.js';
-
-async function hasKeSessionCookie(page) {
-  const cookies = await page.getCookies({ url: 'https://www.ke.com' });
-  return cookies.some(c => c.name === 'lianjia_token' && c.value);
-}
-
-async function verifyKeIdentity(page) {
-  if (!await hasKeSessionCookie(page)) {
-    throw new AuthRequiredError('ke.com', 'Ke lianjia_token cookie missing — anonymous');
-  }
-  await page.goto('https://www.ke.com/');
-  await page.wait(2);
-  const probe = await page.evaluate(`
-    (() => {
-      const loginBtn = document.querySelector('.btn-login, a[class*=actLoginBtn], .login-btn');
-      if (loginBtn && /登录|登陆/.test(loginBtn.innerText || '')) {
-        return { kind: 'auth', detail: 'Ke shows 登录 button — anonymous session' };
-      }
-      const el = document.querySelector('.userNick, .user-name, .myInfo a, [class*=userNick]');
-      const username = (el?.innerText || '').trim();
-      if (!username) {
-        return { kind: 'auth', detail: 'Ke no user-name DOM anchor — anonymous or SSR failed' };
-      }
-      return { ok: true, username };
-    })()
-  `);
-  if (probe?.kind === 'auth') throw new AuthRequiredError('ke.com', probe.detail);
-  if (!probe?.ok) throw new CommandExecutionError(`Unexpected Ke probe: ${JSON.stringify(probe)}`);
-  return { username: probe.username };
-}
-
-registerSiteAuthCommands({
-  site: 'ke',
-  domain: 'ke.com',
-  loginUrl: 'https://clogin.ke.com/login/?service=https%3A%2F%2Fwww.ke.com',
-  columns: ['username'],
-  verify: verifyKeIdentity,
-  poll: async (page) => {
-    if (!await hasKeSessionCookie(page)) {
-      throw new AuthRequiredError('ke.com', 'Waiting for Ke lianjia_token cookie');
-    }
-    return verifyKeIdentity(page);
-  },
-});

package/clis/ke/chengjiao.js

@@ -1,78 +0,0 @@
-import { cli, Strategy } from '@agentrhq/webcmd/registry';
-import { cityUrl, gotoKe } from './utils.js';
-
-cli({
-    site: 'ke',
-    name: 'chengjiao',
-    access: 'read',
-    description: '贝壳找房成交记录',
-    domain: 'ke.com',
-    strategy: Strategy.COOKIE,
-    browser: true,
-    args: [
-        { name: 'city', default: 'bj', help: '城市代码，如 bj(北京), sh(上海), gz(广州), sz(深圳), zs(中山)' },
-        { name: 'district', help: '区域拼音，如 chaoyang, haidian' },
-        { name: 'limit', type: 'int', default: 20, help: '返回数量' },
-    ],
-    columns: ['title', 'community', 'layout', 'area', 'deal_price', 'unit_price', 'deal_date'],
-    func: async (page, kwargs) => {
-        const city = kwargs.city || 'bj';
-        const limit = Number(kwargs.limit) || 20;
-        const base = cityUrl(city);
-
-        let path = '/chengjiao/';
-        if (kwargs.district) {
-            path = `/chengjiao/${kwargs.district}/`;
-        }
-
-        await gotoKe(page, base + path);
-
-        const items = await page.evaluate(`(async () => {
-  // chengjiao page uses .listContent li or similar structure
-  const selectors = [
-    '.listContent li',
-    'ul.listContent li',
-    '.sellListContent li.clear',
-    'li.clear',
-  ];
-  let cards = [];
-  for (const sel of selectors) {
-    cards = document.querySelectorAll(sel);
-    if (cards.length > 0) break;
-  }
-
-  const results = [];
-  for (const card of cards) {
-    const titleEl = card.querySelector('.title a, a.VIEWDATA');
-    if (!titleEl) continue;
-
-    const houseInfoEl = card.querySelector('.houseInfo');
-    const communityEl = card.querySelector('.positionInfo a');
-    const priceEl = card.querySelector('.totalPrice span');
-    const unitPriceEl = card.querySelector('.unitPrice span');
-    const dateEl = card.querySelector('.dealDate');
-    const dealCycleEl = card.querySelector('.dealCycleTxt span');
-
-    const houseText = (houseInfoEl ? houseInfoEl.textContent : '').replace(/\\s+/g, ' ').trim();
-    const houseParts = houseText.split('|').map(s => s.trim());
-
-    const layoutMatch = (houseParts[0] || '').match(/(\\d室\\d厅)/);
-    const layout = layoutMatch ? layoutMatch[1] : (houseParts[0] || '');
-
-    results.push({
-      title: (titleEl.textContent || '').trim(),
-      url: titleEl.href || '',
-      community: (communityEl ? communityEl.textContent : '').trim(),
-      layout: layout,
-      area: (houseParts[1] || '').trim(),
-      deal_price: ((priceEl ? priceEl.textContent : '').trim() || '') + '万',
-      unit_price: (unitPriceEl ? unitPriceEl.textContent : '').trim(),
-      deal_date: (dateEl ? dateEl.textContent : '').replace(/\\s+/g, ' ').trim(),
-    });
-  }
-  return results;
-})()`);
-
-        return (items || []).slice(0, limit);
-    },
-});

package/clis/ke/ershoufang.js

@@ -1,101 +0,0 @@
-import { cli, Strategy } from '@agentrhq/webcmd/registry';
-import { cityUrl, gotoKe } from './utils.js';
-
-cli({
-    site: 'ke',
-    name: 'ershoufang',
-    access: 'read',
-    description: '贝壳找房二手房列表',
-    domain: 'ke.com',
-    strategy: Strategy.COOKIE,
-    browser: true,
-    args: [
-        { name: 'city', default: 'bj', help: '城市代码，如 bj(北京), sh(上海), gz(广州), sz(深圳), zs(中山)' },
-        { name: 'district', help: '区域拼音，如 chaoyang, haidian, tianhe' },
-        { name: 'min-price', type: 'int', help: '最低总价（万元）' },
-        { name: 'max-price', type: 'int', help: '最高总价（万元）' },
-        { name: 'rooms', type: 'int', help: '几居室 (1-5)' },
-        { name: 'limit', type: 'int', default: 20, help: '返回数量' },
-    ],
-    columns: ['title', 'community', 'layout', 'area', 'direction', 'total_price', 'unit_price', 'url'],
-    func: async (page, kwargs) => {
-        const city = kwargs.city || 'bj';
-        const limit = Number(kwargs.limit) || 20;
-        const base = cityUrl(city);
-
-        let path = '/ershoufang/';
-        if (kwargs.district) {
-            path = `/ershoufang/${kwargs.district}/`;
-        }
-
-        const priceParts = [];
-        if (kwargs['min-price'] || kwargs['max-price']) {
-            const min = kwargs['min-price'] || '';
-            const max = kwargs['max-price'] || '';
-            priceParts.push(`p${min}t${max}`);
-        }
-
-        const roomParts = [];
-        if (kwargs.rooms) {
-            roomParts.push(`l${kwargs.rooms}`);
-        }
-
-        const filters = [...priceParts, ...roomParts].join('');
-        const url = base + path + (filters ? filters + '/' : '');
-
-        await gotoKe(page, url);
-
-        const items = await page.evaluate(`(async () => {
-  const cards = document.querySelectorAll('.sellListContent li.clear');
-  const results = [];
-  for (const card of cards) {
-    const titleEl = card.querySelector('.title a');
-    const communityEl = card.querySelector('.positionInfo a');
-    const houseInfoEl = card.querySelector('.houseInfo');
-    const priceEl = card.querySelector('.totalPrice span');
-    const unitPriceEl = card.querySelector('.unitPrice span');
-
-    if (!titleEl) continue;
-
-    // houseInfo text varies:
-    //   "中楼层 (共24层) 4室2厅 | 133.99平米 | 东南"
-    //   "高楼层 (共32层) | 2022年 | 4室2厅 | 110平米"
-    const houseText = (houseInfoEl ? houseInfoEl.textContent : '').replace(/\\s+/g, ' ').trim();
-    const houseParts = houseText.split('|').map(s => s.trim());
-
-    // Extract structured fields from all parts
-    let layout = '', area = '', direction = '', floor = '';
-    for (const part of houseParts) {
-      if (/\\d室\\d厅/.test(part)) {
-        layout = part.match(/(\\d室\\d厅)/)[1];
-      } else if (/平米|㎡/.test(part)) {
-        area = part;
-      } else if (/^[东南西北]+$/.test(part.replace(/\\s/g, ''))) {
-        direction = part;
-      } else if (/楼层/.test(part)) {
-        floor = part;
-      }
-    }
-    // layout might be embedded in the floor part: "中楼层 (共24层) 4室2厅"
-    if (!layout) {
-      const m = houseText.match(/(\\d室\\d厅)/);
-      if (m) layout = m[1];
-    }
-
-    results.push({
-      title: (titleEl.textContent || '').trim(),
-      url: titleEl.href || '',
-      community: (communityEl ? communityEl.textContent : '').trim(),
-      layout: layout,
-      area: area,
-      direction: direction,
-      total_price: ((priceEl ? priceEl.textContent : '').trim() || '') + '万',
-      unit_price: (unitPriceEl ? unitPriceEl.textContent : '').trim(),
-    });
-  }
-  return results;
-})()`);
-
-        return (items || []).slice(0, limit);
-    },
-});

package/clis/ke/utils.js

@@ -1,104 +0,0 @@
-import { AuthRequiredError, CommandExecutionError } from '@agentrhq/webcmd/errors';
-
-const CAPTCHA_TEXT_PATTERNS = [
-    '请拖动下方滑块完成验证',
-    '请按住滑块',
-    '验证码',
-    '安全验证',
-    '访问验证',
-    '滑动验证',
-];
-
-const LOGIN_TEXT_PATTERNS = [
-    '请登录',
-    '登录后',
-    '账号登录',
-    '手机登录',
-    '立即登录',
-    '扫码登录',
-];
-
-function cleanText(value) {
-    return typeof value === 'string'
-        ? value.replace(/\u00a0/g, ' ').replace(/\s+/g, ' ').trim()
-        : '';
-}
-
-export async function readPageState(page) {
-    const result = await page.evaluate(`
-    (() => {
-      try {
-        return {
-          href: window.location.href || '',
-          title: document.title || '',
-          body_text: document.body ? (document.body.innerText || '').substring(0, 2000) : '',
-        };
-      } catch(e) {
-        return { href: '', title: '', body_text: '' };
-      }
-    })()
-  `);
-    if (!result) {
-        return { href: '', title: '', body_text: '' };
-    }
-    return {
-        href: cleanText(result.href),
-        title: cleanText(result.title),
-        body_text: cleanText(result.body_text),
-    };
-}
-
-export function assertNotBlocked(state) {
-    const { href, title, body_text } = state;
-    if (href.includes('hip.ke.com/captcha') || href.includes('/captcha')) {
-        throw new AuthRequiredError('ke.com', '触发了验证码，请先在浏览器中完成验证');
-    }
-    if (CAPTCHA_TEXT_PATTERNS.some(p => title.includes(p) || body_text.includes(p))) {
-        throw new AuthRequiredError('ke.com', '触发了验证码，请先在浏览器中完成滑块验证');
-    }
-    if (LOGIN_TEXT_PATTERNS.some(p => title.includes(p))) {
-        throw new AuthRequiredError('ke.com', '未登录，请先在浏览器中登录贝壳找房');
-    }
-}
-
-export async function gotoKe(page, url) {
-    await page.goto(url, { settleMs: 2500 });
-    await page.wait(2);
-    const state = await readPageState(page);
-    assertNotBlocked(state);
-    return state;
-}
-
-/**
- * Fetch a ke.com JSON API from inside the browser context (credentials included).
- */
-export async function fetchKeJson(page, url) {
-    const result = await page.evaluate(`(async () => {
-    const res = await fetch(${JSON.stringify(url)}, { credentials: 'include' });
-    if (!res.ok) return { __keErr: res.status };
-    try {
-      return await res.json();
-    } catch {
-      return { __keErr: 'parse' };
-    }
-  })()`);
-    const r = result;
-    if (r?.__keErr !== undefined) {
-        const code = r.__keErr;
-        if (code === 401 || code === 403) {
-            throw new AuthRequiredError('ke.com', '未登录或登录已过期，请先在浏览器中登录贝壳找房');
-        }
-        if (code === 'parse') {
-            throw new CommandExecutionError('响应不是有效 JSON', '可能触发了风控，请检查登录状态或稍后重试');
-        }
-        throw new CommandExecutionError(`HTTP ${code}`, '请检查网络连接或登录状态');
-    }
-    return result;
-}
-
-/**
- * Build a ke.com city URL prefix. Default city is 'bj' (Beijing).
- */
-export function cityUrl(city) {
-    return `https://${city}.ke.com`;
-}

package/clis/ke/xiaoqu.js

@@ -1,78 +0,0 @@
-import { cli, Strategy } from '@agentrhq/webcmd/registry';
-import { cityUrl, gotoKe } from './utils.js';
-
-cli({
-    site: 'ke',
-    name: 'xiaoqu',
-    access: 'read',
-    description: '贝壳找房小区列表',
-    domain: 'ke.com',
-    strategy: Strategy.COOKIE,
-    browser: true,
-    args: [
-        { name: 'city', default: 'bj', help: '城市代码，如 bj(北京), sh(上海), gz(广州), sz(深圳), zs(中山)' },
-        { name: 'district', help: '区域拼音，如 chaoyang, haidian' },
-        { name: 'limit', type: 'int', default: 20, help: '返回数量' },
-    ],
-    columns: ['name', 'district', 'avg_price', 'year', 'on_sale'],
-    func: async (page, kwargs) => {
-        const city = kwargs.city || 'bj';
-        const limit = Number(kwargs.limit) || 20;
-        const base = cityUrl(city);
-
-        let path = '/xiaoqu/';
-        if (kwargs.district) {
-            path = `/xiaoqu/${kwargs.district}/`;
-        }
-
-        await gotoKe(page, base + path);
-
-        const items = await page.evaluate(`(async () => {
-  const selectors = [
-    '.xiaoquListItem',
-    'li.xiaoquListItem',
-    '.listContent li',
-    'ul.listContent li',
-  ];
-  let cards = [];
-  for (const sel of selectors) {
-    cards = document.querySelectorAll(sel);
-    if (cards.length > 0) break;
-  }
-
-  const results = [];
-  for (const card of cards) {
-    // Name is in a.img[title] or .title a
-    const imgLink = card.querySelector('a.img[title], a[title]');
-    const titleLink = card.querySelector('.title a');
-    const nameEl = titleLink || imgLink;
-    if (!nameEl) continue;
-
-    const name = (titleLink ? titleLink.textContent : imgLink.getAttribute('title')) || '';
-    const url = nameEl.href || '';
-
-    const priceEl = card.querySelector('.totalPrice span');
-    const districtEl = card.querySelector('.positionInfo a, .district a');
-    const infoEl = card.querySelector('.positionInfo, .houseInfo, .xiaoquListItemInfo');
-    const saleEl = card.querySelector('.xiaoquListItemSellCount a, .houseInfo a');
-
-    const infoText = infoEl ? infoEl.textContent : '';
-    const yearMatch = infoText.match(/(\\d{4})年/);
-
-    const priceText = (priceEl ? priceEl.textContent : '').trim();
-
-    results.push({
-      name: name.trim(),
-      url: url,
-      district: (districtEl ? districtEl.textContent : '').trim(),
-      avg_price: priceText ? priceText + '元/平' : '暂无',
-      year: yearMatch ? yearMatch[1] : '',
-      on_sale: (saleEl ? saleEl.textContent : '').trim(),
-    });
-  }
-  return results;
-})()`);
-
-        return (items || []).slice(0, limit);
-    },
-});