@jackwener/opencli 1.7.2 → 1.7.4

package/clis/grok/image.ts

@@ -0,0 +1,356 @@
+import * as fs from 'node:fs';
+import * as path from 'node:path';
+import * as crypto from 'node:crypto';
+import { cli, Strategy } from '@jackwener/opencli/registry';
+import type { IPage } from '@jackwener/opencli/types';
+
+const GROK_URL = 'https://grok.com/';
+const NO_IMAGE_PREFIX = '[NO IMAGE]';
+const BLOCKED_PREFIX = '[BLOCKED]';
+const SESSION_HINT = 'Likely login/auth/challenge/session issue in the existing grok.com browser session.';
+
+type SendResult = {
+  ok?: boolean;
+  msg?: string;
+  reason?: string;
+  detail?: string;
+};
+
+type BubbleImage = {
+  src: string;
+  w: number;
+  h: number;
+};
+
+type BubbleImageSet = BubbleImage[];
+
+type FetchResult = {
+  ok: boolean;
+  base64?: string;
+  contentType?: string;
+  error?: string;
+};
+
+function normalizeBooleanFlag(value: unknown): boolean {
+  if (typeof value === 'boolean') return value;
+  const normalized = String(value ?? '').trim().toLowerCase();
+  return normalized === 'true' || normalized === '1' || normalized === 'yes' || normalized === 'on';
+}
+
+function dedupeBySrc(images: BubbleImage[]): BubbleImage[] {
+  const seen = new Set<string>();
+  const out: BubbleImage[] = [];
+  for (const img of images) {
+    if (!img.src || seen.has(img.src)) continue;
+    seen.add(img.src);
+    out.push(img);
+  }
+  return out;
+}
+
+function imagesSignature(images: BubbleImage[]): string {
+  return images.map(i => i.src).sort().join('|');
+}
+
+function extFromContentType(ct?: string): string {
+  if (!ct) return 'jpg';
+  if (ct.includes('png')) return 'png';
+  if (ct.includes('webp')) return 'webp';
+  if (ct.includes('gif')) return 'gif';
+  return 'jpg';
+}
+
+function buildFilename(src: string, ct?: string): string {
+  const ext = extFromContentType(ct);
+  const hash = crypto.createHash('sha1').update(src).digest('hex').slice(0, 12);
+  return `grok-${Date.now()}-${hash}.${ext}`;
+}
+
+/** Check whether the tab is already on grok.com (any path). */
+async function isOnGrok(page: IPage): Promise<boolean> {
+  const url = await page.evaluate('window.location.href').catch(() => '');
+  if (typeof url !== 'string' || !url) return false;
+  try {
+    const hostname = new URL(url).hostname;
+    return hostname === 'grok.com' || hostname.endsWith('.grok.com');
+  } catch {
+    return false;
+  }
+}
+
+async function tryStartFreshChat(page: IPage): Promise<void> {
+  await page.evaluate(`(() => {
+    const isVisible = (node) => {
+      if (!(node instanceof HTMLElement)) return false;
+      const rect = node.getBoundingClientRect();
+      const style = window.getComputedStyle(node);
+      return rect.width > 0 && rect.height > 0 && style.visibility !== 'hidden' && style.display !== 'none';
+    };
+    const candidates = Array.from(document.querySelectorAll('a, button')).filter(node => {
+      if (!isVisible(node)) return false;
+      const text = (node.textContent || '').trim().toLowerCase();
+      const aria = (node.getAttribute('aria-label') || '').trim().toLowerCase();
+      const href = node.getAttribute('href') || '';
+      return text.includes('new chat')
+        || text.includes('new conversation')
+        || aria.includes('new chat')
+        || aria.includes('new conversation')
+        || href === '/';
+    });
+    const target = candidates[0];
+    if (target instanceof HTMLElement) target.click();
+  })()`);
+}
+
+async function sendPrompt(page: IPage, prompt: string): Promise<SendResult> {
+  const promptJson = JSON.stringify(prompt);
+  return page.evaluate(`(async () => {
+    try {
+      const waitFor = (ms) => new Promise(resolve => setTimeout(resolve, ms));
+      const composerSelector = '.ProseMirror[contenteditable="true"]';
+      const isVisibleEnabledSubmit = (node) => {
+        if (!(node instanceof HTMLButtonElement)) return false;
+        const rect = node.getBoundingClientRect();
+        const style = window.getComputedStyle(node);
+        return !node.disabled
+          && rect.width > 0
+          && rect.height > 0
+          && style.visibility !== 'hidden'
+          && style.display !== 'none';
+      };
+
+      let pm = null;
+      let box = null;
+      for (let attempt = 0; attempt < 12; attempt += 1) {
+        const composer = document.querySelector(composerSelector);
+        if (composer instanceof HTMLElement) {
+          pm = composer;
+          break;
+        }
+
+        const textarea = document.querySelector('textarea');
+        if (textarea instanceof HTMLTextAreaElement) {
+          box = textarea;
+          break;
+        }
+
+        await waitFor(1000);
+      }
+
+      // Prefer the ProseMirror composer when present (current grok.com UI).
+      if (pm && pm.editor && pm.editor.commands) {
+        try {
+          if (pm.editor.commands.clearContent) pm.editor.commands.clearContent();
+          pm.editor.commands.focus();
+          pm.editor.commands.insertContent(${promptJson});
+          for (let attempt = 0; attempt < 6; attempt += 1) {
+            const sbtn = Array.from(document.querySelectorAll('button[aria-label="Submit"], button[aria-label="\\u63d0\\u4ea4"]'))
+              .find(isVisibleEnabledSubmit);
+            if (sbtn) {
+              sbtn.click();
+              return { ok: true, msg: 'pm-submit' };
+            }
+            await waitFor(500);
+          }
+        } catch (e) { /* fall through to textarea */ }
+      }
+
+      // Fallback: legacy textarea composer.
+      if (!box) return { ok: false, msg: 'no composer (neither ProseMirror nor textarea)' };
+      box.focus(); box.value = '';
+      document.execCommand('selectAll');
+      document.execCommand('insertText', false, ${promptJson});
+      for (let attempt = 0; attempt < 6; attempt += 1) {
+        const btn = Array.from(document.querySelectorAll('button[aria-label="\\u63d0\\u4ea4"], button[aria-label="Submit"]'))
+          .find(isVisibleEnabledSubmit);
+        if (btn) {
+          btn.click();
+          return { ok: true, msg: 'clicked' };
+        }
+
+        const sub = Array.from(document.querySelectorAll('button[type="submit"]'))
+          .find(isVisibleEnabledSubmit);
+        if (sub) {
+          sub.click();
+          return { ok: true, msg: 'clicked-submit' };
+        }
+
+        await waitFor(500);
+      }
+      box.dispatchEvent(new KeyboardEvent('keydown', { key: 'Enter', code: 'Enter', keyCode: 13, bubbles: true }));
+      return { ok: true, msg: 'enter' };
+    } catch (e) { return { ok: false, msg: e && e.toString ? e.toString() : String(e) }; }
+  })()`) as Promise<SendResult>;
+}
+
+/** Read <img> elements from all message bubbles so callers can filter by baseline. */
+async function getBubbleImageSets(page: IPage): Promise<BubbleImageSet[]> {
+  const result = await page.evaluate(`(() => {
+    const bubbles = document.querySelectorAll('div.message-bubble, [data-testid="message-bubble"]');
+    return Array.from(bubbles).map(bubble => Array.from(bubble.querySelectorAll('img'))
+      .map(img => ({
+        src: img.currentSrc || img.src || '',
+        w: img.naturalWidth || img.width || 0,
+        h: img.naturalHeight || img.height || 0,
+      }))
+      .filter(i => i.src && /^https?:/.test(i.src))
+      // Ignore tiny UI/avatar images that may live in the bubble chrome.
+      .filter(i => (i.w === 0 || i.w >= 128) && (i.h === 0 || i.h >= 128)));
+  })()`) as BubbleImageSet[] | undefined;
+
+  const raw = Array.isArray(result) ? result : [];
+  return raw.map(dedupeBySrc);
+}
+
+function pickLatestImageCandidate(
+  bubbleImageSets: BubbleImageSet[],
+  baselineCount: number,
+): BubbleImage[] {
+  const freshSets = bubbleImageSets.slice(Math.max(0, baselineCount));
+  for (let i = freshSets.length - 1; i >= 0; i -= 1) {
+    if (freshSets[i].length) return freshSets[i];
+  }
+  return [];
+}
+
+// Download through the browser's fetch so grok.com cookies and referer are
+// attached automatically — assets.grok.com is gated by Cloudflare and will
+// refuse direct curl/node downloads.
+async function fetchImageAsBase64(page: IPage, url: string): Promise<FetchResult> {
+  const urlJson = JSON.stringify(url);
+  return page.evaluate(`(async () => {
+    try {
+      const res = await fetch(${urlJson}, { credentials: 'include', referrer: 'https://grok.com/' });
+      if (!res.ok) return { ok: false, error: 'HTTP ' + res.status };
+      const blob = await res.blob();
+      const buf = await blob.arrayBuffer();
+      const bytes = new Uint8Array(buf);
+      let binary = '';
+      for (let i = 0; i < bytes.length; i++) binary += String.fromCharCode(bytes[i]);
+      return { ok: true, base64: btoa(binary), contentType: blob.type || 'image/jpeg' };
+    } catch (e) { return { ok: false, error: e && e.message || String(e) }; }
+  })()`) as Promise<FetchResult>;
+}
+
+async function saveImages(
+  page: IPage,
+  images: BubbleImage[],
+  outDir: string,
+): Promise<Array<BubbleImage & { path: string }>> {
+  fs.mkdirSync(outDir, { recursive: true });
+  const results: Array<BubbleImage & { path: string }> = [];
+  for (const img of images) {
+    const fetched = await fetchImageAsBase64(page, img.src);
+    if (!fetched || !fetched.ok) {
+      results.push({ ...img, path: `[DOWNLOAD FAILED] ${fetched?.error || 'unknown'}` });
+      continue;
+    }
+    const filepath = path.join(outDir, buildFilename(img.src, fetched.contentType));
+    fs.writeFileSync(filepath, Buffer.from(fetched.base64 || '', 'base64'));
+    results.push({ ...img, path: filepath });
+  }
+  return results;
+}
+
+function toRow(img: BubbleImage, savedPath = '') {
+  return { url: img.src, width: img.w, height: img.h, path: savedPath };
+}
+
+export const imageCommand = cli({
+  site: 'grok',
+  name: 'image',
+  description: 'Generate images on grok.com and return image URLs',
+  domain: 'grok.com',
+  strategy: Strategy.COOKIE,
+  browser: true,
+  args: [
+    { name: 'prompt', positional: true, type: 'string', required: true, help: 'Image generation prompt' },
+    { name: 'timeout', type: 'int', default: 240, help: 'Max seconds to wait for the image (default: 240)' },
+    { name: 'new', type: 'boolean', default: false, help: 'Start a new chat before sending (default: false)' },
+    { name: 'count', type: 'int', default: 1, help: 'Minimum images to wait for before returning (default: 1)' },
+    { name: 'out', type: 'string', default: '', help: 'Directory to save downloaded images (uses browser session to bypass auth)' },
+  ],
+  columns: ['url', 'width', 'height', 'path'],
+  func: async (page: IPage, kwargs: Record<string, any>) => {
+    const prompt = kwargs.prompt as string;
+    const timeoutMs = ((kwargs.timeout as number) || 240) * 1000;
+    const newChat = normalizeBooleanFlag(kwargs.new);
+    const minCount = Math.max(1, Number(kwargs.count || 1));
+    const outDir = (kwargs.out || '').toString().trim();
+
+    if (newChat) {
+      await page.goto(GROK_URL);
+      await page.wait(2);
+      await tryStartFreshChat(page);
+      await page.wait(2);
+    } else if (!(await isOnGrok(page))) {
+      await page.goto(GROK_URL);
+      await page.wait(3);
+    }
+
+    const baselineBubbleCount = (await getBubbleImageSets(page)).length;
+    const sendResult = await sendPrompt(page, prompt);
+    if (!sendResult || !sendResult.ok) {
+      return [{
+        url: `${BLOCKED_PREFIX} send failed: ${JSON.stringify(sendResult)}. ${SESSION_HINT}`,
+        width: 0,
+        height: 0,
+        path: '',
+      }];
+    }
+
+    const startTime = Date.now();
+    let lastSignature = '';
+    let stableCount = 0;
+    let lastImages: BubbleImage[] = [];
+
+    while (Date.now() - startTime < timeoutMs) {
+      await page.wait(3);
+      const bubbleImageSets = await getBubbleImageSets(page);
+      const images = pickLatestImageCandidate(bubbleImageSets, baselineBubbleCount);
+
+      if (images.length >= minCount) {
+        const signature = imagesSignature(images);
+        if (signature === lastSignature) {
+          stableCount += 1;
+          // Require two consecutive stable reads (~6s) before declaring done.
+          if (stableCount >= 2) {
+            if (outDir) {
+              const saved = await saveImages(page, images, outDir);
+              return saved.map(s => toRow(s, s.path));
+            }
+            return images.map(i => toRow(i));
+          }
+        } else {
+          stableCount = 0;
+          lastSignature = signature;
+          lastImages = images;
+        }
+      }
+    }
+
+    if (lastImages.length) {
+      if (outDir) {
+        const saved = await saveImages(page, lastImages, outDir);
+        return saved.map(s => toRow(s, s.path));
+      }
+      return lastImages.map(i => toRow(i));
+    }
+    return [{
+      url: `${NO_IMAGE_PREFIX} No image appeared within ${Math.round(timeoutMs / 1000)}s.`,
+      width: 0,
+      height: 0,
+      path: '',
+    }];
+  },
+});
+
+export const __test__ = {
+  normalizeBooleanFlag,
+  isOnGrok,
+  dedupeBySrc,
+  imagesSignature,
+  extFromContentType,
+  buildFilename,
+  pickLatestImageCandidate,
+};

package/clis/ke/chengjiao.js

@@ -0,0 +1,77 @@
+import { cli, Strategy } from '@jackwener/opencli/registry';
+import { cityUrl, gotoKe } from './utils.js';
+
+cli({
+    site: 'ke',
+    name: 'chengjiao',
+    description: '贝壳找房成交记录',
+    domain: 'ke.com',
+    strategy: Strategy.COOKIE,
+    browser: true,
+    args: [
+        { name: 'city', default: 'bj', help: '城市代码，如 bj(北京), sh(上海), gz(广州), sz(深圳), zs(中山)' },
+        { name: 'district', help: '区域拼音，如 chaoyang, haidian' },
+        { name: 'limit', type: 'int', default: 20, help: '返回数量' },
+    ],
+    columns: ['title', 'community', 'layout', 'area', 'deal_price', 'unit_price', 'deal_date'],
+    func: async (page, kwargs) => {
+        const city = kwargs.city || 'bj';
+        const limit = Number(kwargs.limit) || 20;
+        const base = cityUrl(city);
+
+        let path = '/chengjiao/';
+        if (kwargs.district) {
+            path = `/chengjiao/${kwargs.district}/`;
+        }
+
+        await gotoKe(page, base + path);
+
+        const items = await page.evaluate(`(async () => {
+  // chengjiao page uses .listContent li or similar structure
+  const selectors = [
+    '.listContent li',
+    'ul.listContent li',
+    '.sellListContent li.clear',
+    'li.clear',
+  ];
+  let cards = [];
+  for (const sel of selectors) {
+    cards = document.querySelectorAll(sel);
+    if (cards.length > 0) break;
+  }
+
+  const results = [];
+  for (const card of cards) {
+    const titleEl = card.querySelector('.title a, a.VIEWDATA');
+    if (!titleEl) continue;
+
+    const houseInfoEl = card.querySelector('.houseInfo');
+    const communityEl = card.querySelector('.positionInfo a');
+    const priceEl = card.querySelector('.totalPrice span');
+    const unitPriceEl = card.querySelector('.unitPrice span');
+    const dateEl = card.querySelector('.dealDate');
+    const dealCycleEl = card.querySelector('.dealCycleTxt span');
+
+    const houseText = (houseInfoEl ? houseInfoEl.textContent : '').replace(/\\s+/g, ' ').trim();
+    const houseParts = houseText.split('|').map(s => s.trim());
+
+    const layoutMatch = (houseParts[0] || '').match(/(\\d室\\d厅)/);
+    const layout = layoutMatch ? layoutMatch[1] : (houseParts[0] || '');
+
+    results.push({
+      title: (titleEl.textContent || '').trim(),
+      url: titleEl.href || '',
+      community: (communityEl ? communityEl.textContent : '').trim(),
+      layout: layout,
+      area: (houseParts[1] || '').trim(),
+      deal_price: ((priceEl ? priceEl.textContent : '').trim() || '') + '万',
+      unit_price: (unitPriceEl ? unitPriceEl.textContent : '').trim(),
+      deal_date: (dateEl ? dateEl.textContent : '').replace(/\\s+/g, ' ').trim(),
+    });
+  }
+  return results;
+})()`);
+
+        return (items || []).slice(0, limit);
+    },
+});

package/clis/ke/ershoufang.js

@@ -0,0 +1,100 @@
+import { cli, Strategy } from '@jackwener/opencli/registry';
+import { cityUrl, gotoKe } from './utils.js';
+
+cli({
+    site: 'ke',
+    name: 'ershoufang',
+    description: '贝壳找房二手房列表',
+    domain: 'ke.com',
+    strategy: Strategy.COOKIE,
+    browser: true,
+    args: [
+        { name: 'city', default: 'bj', help: '城市代码，如 bj(北京), sh(上海), gz(广州), sz(深圳), zs(中山)' },
+        { name: 'district', help: '区域拼音，如 chaoyang, haidian, tianhe' },
+        { name: 'min-price', type: 'int', help: '最低总价（万元）' },
+        { name: 'max-price', type: 'int', help: '最高总价（万元）' },
+        { name: 'rooms', type: 'int', help: '几居室 (1-5)' },
+        { name: 'limit', type: 'int', default: 20, help: '返回数量' },
+    ],
+    columns: ['title', 'community', 'layout', 'area', 'direction', 'total_price', 'unit_price', 'url'],
+    func: async (page, kwargs) => {
+        const city = kwargs.city || 'bj';
+        const limit = Number(kwargs.limit) || 20;
+        const base = cityUrl(city);
+
+        let path = '/ershoufang/';
+        if (kwargs.district) {
+            path = `/ershoufang/${kwargs.district}/`;
+        }
+
+        const priceParts = [];
+        if (kwargs['min-price'] || kwargs['max-price']) {
+            const min = kwargs['min-price'] || '';
+            const max = kwargs['max-price'] || '';
+            priceParts.push(`p${min}t${max}`);
+        }
+
+        const roomParts = [];
+        if (kwargs.rooms) {
+            roomParts.push(`l${kwargs.rooms}`);
+        }
+
+        const filters = [...priceParts, ...roomParts].join('');
+        const url = base + path + (filters ? filters + '/' : '');
+
+        await gotoKe(page, url);
+
+        const items = await page.evaluate(`(async () => {
+  const cards = document.querySelectorAll('.sellListContent li.clear');
+  const results = [];
+  for (const card of cards) {
+    const titleEl = card.querySelector('.title a');
+    const communityEl = card.querySelector('.positionInfo a');
+    const houseInfoEl = card.querySelector('.houseInfo');
+    const priceEl = card.querySelector('.totalPrice span');
+    const unitPriceEl = card.querySelector('.unitPrice span');
+
+    if (!titleEl) continue;
+
+    // houseInfo text varies:
+    //   "中楼层 (共24层) 4室2厅 | 133.99平米 | 东南"
+    //   "高楼层 (共32层) | 2022年 | 4室2厅 | 110平米"
+    const houseText = (houseInfoEl ? houseInfoEl.textContent : '').replace(/\\s+/g, ' ').trim();
+    const houseParts = houseText.split('|').map(s => s.trim());
+
+    // Extract structured fields from all parts
+    let layout = '', area = '', direction = '', floor = '';
+    for (const part of houseParts) {
+      if (/\\d室\\d厅/.test(part)) {
+        layout = part.match(/(\\d室\\d厅)/)[1];
+      } else if (/平米|㎡/.test(part)) {
+        area = part;
+      } else if (/^[东南西北]+$/.test(part.replace(/\\s/g, ''))) {
+        direction = part;
+      } else if (/楼层/.test(part)) {
+        floor = part;
+      }
+    }
+    // layout might be embedded in the floor part: "中楼层 (共24层) 4室2厅"
+    if (!layout) {
+      const m = houseText.match(/(\\d室\\d厅)/);
+      if (m) layout = m[1];
+    }
+
+    results.push({
+      title: (titleEl.textContent || '').trim(),
+      url: titleEl.href || '',
+      community: (communityEl ? communityEl.textContent : '').trim(),
+      layout: layout,
+      area: area,
+      direction: direction,
+      total_price: ((priceEl ? priceEl.textContent : '').trim() || '') + '万',
+      unit_price: (unitPriceEl ? unitPriceEl.textContent : '').trim(),
+    });
+  }
+  return results;
+})()`);
+
+        return (items || []).slice(0, limit);
+    },
+});

package/clis/ke/utils.js

@@ -0,0 +1,104 @@
+import { AuthRequiredError, CommandExecutionError } from '@jackwener/opencli/errors';
+
+const CAPTCHA_TEXT_PATTERNS = [
+    '请拖动下方滑块完成验证',
+    '请按住滑块',
+    '验证码',
+    '安全验证',
+    '访问验证',
+    '滑动验证',
+];
+
+const LOGIN_TEXT_PATTERNS = [
+    '请登录',
+    '登录后',
+    '账号登录',
+    '手机登录',
+    '立即登录',
+    '扫码登录',
+];
+
+function cleanText(value) {
+    return typeof value === 'string'
+        ? value.replace(/\u00a0/g, ' ').replace(/\s+/g, ' ').trim()
+        : '';
+}
+
+export async function readPageState(page) {
+    const result = await page.evaluate(`
+    (() => {
+      try {
+        return {
+          href: window.location.href || '',
+          title: document.title || '',
+          body_text: document.body ? (document.body.innerText || '').substring(0, 2000) : '',
+        };
+      } catch(e) {
+        return { href: '', title: '', body_text: '' };
+      }
+    })()
+  `);
+    if (!result) {
+        return { href: '', title: '', body_text: '' };
+    }
+    return {
+        href: cleanText(result.href),
+        title: cleanText(result.title),
+        body_text: cleanText(result.body_text),
+    };
+}
+
+export function assertNotBlocked(state) {
+    const { href, title, body_text } = state;
+    if (href.includes('hip.ke.com/captcha') || href.includes('/captcha')) {
+        throw new AuthRequiredError('ke.com', '触发了验证码，请先在浏览器中完成验证');
+    }
+    if (CAPTCHA_TEXT_PATTERNS.some(p => title.includes(p) || body_text.includes(p))) {
+        throw new AuthRequiredError('ke.com', '触发了验证码，请先在浏览器中完成滑块验证');
+    }
+    if (LOGIN_TEXT_PATTERNS.some(p => title.includes(p))) {
+        throw new AuthRequiredError('ke.com', '未登录，请先在浏览器中登录贝壳找房');
+    }
+}
+
+export async function gotoKe(page, url) {
+    await page.goto(url, { settleMs: 2500 });
+    await page.wait(2);
+    const state = await readPageState(page);
+    assertNotBlocked(state);
+    return state;
+}
+
+/**
+ * Fetch a ke.com JSON API from inside the browser context (credentials included).
+ */
+export async function fetchKeJson(page, url) {
+    const result = await page.evaluate(`(async () => {
+    const res = await fetch(${JSON.stringify(url)}, { credentials: 'include' });
+    if (!res.ok) return { __keErr: res.status };
+    try {
+      return await res.json();
+    } catch {
+      return { __keErr: 'parse' };
+    }
+  })()`);
+    const r = result;
+    if (r?.__keErr !== undefined) {
+        const code = r.__keErr;
+        if (code === 401 || code === 403) {
+            throw new AuthRequiredError('ke.com', '未登录或登录已过期，请先在浏览器中登录贝壳找房');
+        }
+        if (code === 'parse') {
+            throw new CommandExecutionError('响应不是有效 JSON', '可能触发了风控，请检查登录状态或稍后重试');
+        }
+        throw new CommandExecutionError(`HTTP ${code}`, '请检查网络连接或登录状态');
+    }
+    return result;
+}
+
+/**
+ * Build a ke.com city URL prefix. Default city is 'bj' (Beijing).
+ */
+export function cityUrl(city) {
+    return `https://${city}.ke.com`;
+}