@jackwener/opencli 1.6.2 → 1.6.4

package/src/clis/xueqiu/comments.ts

@@ -1,461 +0,0 @@
-import type { IPage } from '../../types.js';
-import { cli, Strategy } from '../../registry.js';
-import { ArgumentError, AuthRequiredError, CommandExecutionError, EmptyResultError } from '../../errors.js';
-import { log } from '../../logger.js';
-import { isRecord } from '../../utils.js';
-
-/**
- * Minimal browser-response shape used by the classifier.
- */
-export interface XueqiuCommentsResponse {
-  status: number;
-  contentType: string;
-  json: unknown;
-  textSnippet: string;
-}
-
-/**
- * Minimal normalized row shape used during pagination and deduplication.
- */
-export interface XueqiuCommentRow {
-  id: string;
-  author: string;
-  text?: string;
-  likes?: number;
-  replies?: number;
-  retweets?: number;
-  created_at?: string | null;
-  url?: string | null;
-}
-
-/**
- * Public CLI row shape. This intentionally omits the internal stable ID used
- * only for deduplication, so machine-readable output matches the command
- * contract and table columns.
- */
-export type XueqiuCommentOutputRow = Omit<XueqiuCommentRow, 'id'>;
-
-/**
- * Pagination options for collecting enough rows to satisfy `--limit`.
- */
-export interface CollectCommentRowsOptions {
-  symbol: string;
-  limit: number;
-  pageSize: number;
-  maxRequests: number;
-  fetchPage: (pageNumber: number, pageSize: number) => Promise<XueqiuCommentsResponse>;
-  warn?: (message: string) => void;
-}
-
-type XueqiuCommentsKind = 'auth' | 'anti-bot' | 'argument' | 'empty' | 'incompatible' | 'unknown';
-
-const XUEQIU_SYMBOL_PATTERN = /^(?:[A-Z]{2}\d{5,6}|\d{4,6}|[A-Z]{1,5}(?:[.-][A-Z]{1,2})?)$/;
-
-const FAILURE_REASON_BY_KIND: Record<XueqiuCommentsKind, string> = {
-  auth: 'auth failure',
-  'anti-bot': 'anti-bot challenge',
-  argument: 'invalid symbol',
-  empty: 'no more discussion data',
-  incompatible: 'unexpected response shape',
-  unknown: 'unknown request failure',
-};
-
-function getCommentList(json: Record<string, unknown>): unknown[] | null {
-  if (Array.isArray(json.list)) return json.list;
-  if (isRecord(json.data) && Array.isArray(json.data.list)) return json.data.list;
-  return null;
-}
-
-function isAntiBotHtml(response: XueqiuCommentsResponse, envelopeText: string): boolean {
-  const htmlText = `${envelopeText} ${response.textSnippet}`.toLowerCase();
-  return response.contentType.includes('text/html')
-    && (
-      /captcha|challenge|aliyun_waf|risk/i.test(htmlText)
-      || /_WAF_|_waf_|renderData|aliyun_waf/i.test(response.textSnippet)
-    );
-}
-
-function toFiniteCount(value: unknown): number {
-  const count = Number(value ?? 0);
-  return Number.isFinite(count) ? count : 0;
-}
-
-function normalizeIdentifier(value: unknown): string {
-  if (typeof value === 'string') return value.trim();
-  if (typeof value === 'number' && Number.isFinite(value)) return String(value);
-  return '';
-}
-
-function buildPaginationStopMessage(
-  requestNumber: number,
-  collected: number,
-  target: number,
-  reason: string,
-): string {
-  return `xueqiu comments pagination stopped after request ${requestNumber}, `
-    + `collected ${collected}/${target} items, `
-    + `reason: ${reason}`;
-}
-
-function throwFirstPageFailure(kind: XueqiuCommentsKind, symbol: string): never {
-  if (kind === 'auth' || kind === 'anti-bot') {
-    throw new AuthRequiredError('xueqiu.com', 'Stock discussions require login or challenge clearance');
-  }
-  if (kind === 'argument') {
-    throw new ArgumentError(`xueqiu comments received an invalid symbol: ${symbol}`);
-  }
-  if (kind === 'empty') {
-    throw new EmptyResultError(
-      `xueqiu/comments ${symbol}`,
-      `No discussion data found for ${symbol}`,
-    );
-  }
-  throw new CommandExecutionError(
-    `Unexpected response while loading xueqiu comments for ${symbol}`,
-    'Run the command again with --verbose to inspect the raw site response.',
-  );
-}
-
-/**
- * Extract the raw item list from one classified JSON payload.
- *
- * @param json Raw parsed JSON payload from browser fetch.
- * @returns Discussion items when the response shape is usable.
- */
-export function getCommentItems(json: unknown): Record<string, any>[] {
-  if (!isRecord(json)) return [];
-
-  const list = getCommentList(json) ?? [];
-
-  return list.filter((item): item is Record<string, any> => !!item && typeof item === 'object');
-}
-
-/**
- * Classify one raw browser response before command-level error handling.
- *
- * @param response Structured browser response payload.
- * @returns Tagged result describing the response class.
- */
-export function classifyXueqiuCommentsResponse(response: XueqiuCommentsResponse): { kind: XueqiuCommentsKind } {
-  const jsonRecord = isRecord(response.json) ? response.json : null;
-  const commentList = jsonRecord ? getCommentList(jsonRecord) : null;
-  const envelopeText = [
-    jsonRecord?.error,
-    jsonRecord?.errors,
-    jsonRecord?.code,
-    jsonRecord?.message,
-    jsonRecord?.msg,
-  ].filter(Boolean).join(' ').toLowerCase();
-  const responseText = `${envelopeText} ${response.textSnippet}`.toLowerCase();
-
-  if (isAntiBotHtml(response, envelopeText)) {
-    return { kind: 'anti-bot' };
-  }
-  if (response.status === 401 || response.status === 403) {
-    return { kind: 'auth' };
-  }
-  if (/login required|unauthorized|unauthorised|forbidden|not logged in|need login/.test(responseText)) {
-    return { kind: 'auth' };
-  }
-  if (/invalid symbol|invalid code|bad symbol/.test(envelopeText)) {
-    return { kind: 'argument' };
-  }
-  if (/no data|no result|not found|no matching/.test(envelopeText)) {
-    return { kind: 'empty' };
-  }
-  if (commentList && commentList.length === 0) {
-    return { kind: 'empty' };
-  }
-  if (response.contentType.includes('application/json') && jsonRecord && commentList === null) {
-    return { kind: 'incompatible' };
-  }
-  return { kind: 'unknown' };
-}
-
-/**
- * Merge one new page of rows while preserving the first occurrence of each ID.
- *
- * @param current Rows already collected.
- * @param incoming Rows from the next page.
- * @returns Deduplicated merged rows.
- */
-export function mergeUniqueCommentRows(
-  current: XueqiuCommentRow[],
-  incoming: XueqiuCommentRow[],
-): XueqiuCommentRow[] {
-  const merged = [...current];
-  const seen = new Set(current.map(item => item.id));
-
-  for (const row of incoming) {
-    if (seen.has(row.id)) continue;
-    seen.add(row.id);
-    merged.push(row);
-  }
-  return merged;
-}
-
-/**
- * Normalize one raw xueqiu discussion item into the CLI row shape.
- *
- * Returned rows represent stock-scoped discussion posts, not replies under
- * one parent post.
- *
- * @param item Raw API item.
- * @returns Cleaned CLI row.
- */
-export function normalizeCommentItem(item: Record<string, any>): XueqiuCommentRow {
-  const text = String(item.description ?? '')
-    .replace(/<[^>]+>/g, ' ')
-    .replace(/&nbsp;/g, ' ')
-    .replace(/&amp;/g, '&')
-    .replace(/&lt;/g, '<')
-    .replace(/&gt;/g, '>')
-    .replace(/\s+/g, ' ')
-    .trim();
-
-  const id = normalizeIdentifier(item.id);
-  const userId = normalizeIdentifier(item.user?.id);
-
-  const createdAtDate = item.created_at ? new Date(item.created_at) : null;
-  const createdAt = createdAtDate && !Number.isNaN(createdAtDate.getTime())
-    ? createdAtDate.toISOString()
-    : null;
-
-  return {
-    id,
-    author: String(item.user?.screen_name ?? ''),
-    text,
-    likes: toFiniteCount(item.fav_count),
-    replies: toFiniteCount(item.reply_count),
-    retweets: toFiniteCount(item.retweet_count),
-    created_at: createdAt,
-    url: userId && id ? `https://xueqiu.com/${userId}/${id}` : null,
-  };
-}
-
-/**
- * Remove internal-only fields before returning rows to the CLI renderer.
- *
- * @param row Internal row shape used during pagination.
- * @returns Public output row that matches the documented command contract.
- */
-export function toCommentOutputRow(row: XueqiuCommentRow): XueqiuCommentOutputRow {
-  const { id: _id, ...outputRow } = row;
-  return outputRow;
-}
-
-/**
- * Convert response classification into a compact warning phrase.
- *
- * @param kind Classifier result kind.
- * @returns Human-readable reason fragment for stderr warnings.
- */
-export function describeFailureKind(kind: XueqiuCommentsKind): string {
-  return FAILURE_REASON_BY_KIND[kind];
-}
-
-/**
- * Fetch one discussion page from inside the browser context so cookies and
- * any site-side request state stay attached to the request.
- *
- * @param page Active browser page.
- * @param symbol Normalized stock symbol.
- * @param pageNumber Internal page counter, starting from 1.
- * @param pageSize Item count per internal request.
- * @returns Structured response for command-side classification.
- */
-export async function fetchCommentsPage(
-  page: IPage,
-  symbol: string,
-  pageNumber: number,
-  pageSize: number,
-): Promise<XueqiuCommentsResponse> {
-  const url = new URL('https://xueqiu.com/query/v1/symbol/search/status');
-  url.searchParams.set('symbol', symbol);
-  url.searchParams.set('count', String(pageSize));
-  url.searchParams.set('page', String(pageNumber));
-  url.searchParams.set('sort', 'time');
-
-  return page.evaluate(`
-    (async () => {
-      try {
-        const response = await fetch(${JSON.stringify(url.toString())}, {
-          credentials: 'include',
-          headers: {
-            'accept': 'application/json, text/plain, */*',
-            'x-requested-with': 'XMLHttpRequest',
-          },
-          referrer: ${JSON.stringify(`https://xueqiu.com/S/${symbol}`)},
-          referrerPolicy: 'strict-origin-when-cross-origin',
-        });
-        const contentType = response.headers.get('content-type') || '';
-        const text = await response.text();
-        let json = null;
-        if (contentType.includes('application/json')) {
-          try {
-            json = JSON.parse(text);
-          } catch {
-            json = null;
-          }
-        }
-        return {
-          status: response.status,
-          contentType,
-          json,
-          textSnippet: text.slice(0, 2000),
-        };
-      } catch (error) {
-        return {
-          status: 0,
-          contentType: 'text/plain',
-          json: null,
-          textSnippet: error instanceof Error ? error.message : String(error),
-        };
-      }
-    })()
-  `) as Promise<XueqiuCommentsResponse>;
-}
-
-/**
- * Collect enough stock discussion rows to satisfy the requested limit.
- *
- * This helper owns the internal pagination policy so the public command
- * contract can stay small and expose only `--limit`.
- *
- * @param options Pagination inputs and a page-fetch callback.
- * @returns Deduplicated normalized rows, possibly partial with a warning.
- */
-export async function collectCommentRows(options: CollectCommentRowsOptions): Promise<XueqiuCommentRow[]> {
-  const warn = options.warn ?? log.warn;
-  let rows: XueqiuCommentRow[] = [];
-  const seenIds = new Set<string>();
-
-  for (let requestNumber = 1; requestNumber <= options.maxRequests; requestNumber += 1) {
-    const response = await options.fetchPage(requestNumber, options.pageSize);
-    const classified = classifyXueqiuCommentsResponse(response);
-
-    if (requestNumber === 1 && classified.kind !== 'unknown') {
-      throwFirstPageFailure(classified.kind, options.symbol);
-    } else if (classified.kind === 'empty') {
-      break;
-    } else if (classified.kind !== 'unknown') {
-      warn(buildPaginationStopMessage(
-        requestNumber,
-        rows.length,
-        options.limit,
-        describeFailureKind(classified.kind),
-      ));
-      break;
-    }
-
-    const rawItems = getCommentItems(response.json);
-    const pageRows = rawItems
-      .map(item => normalizeCommentItem(item))
-      .filter(row => row.id);
-    if (pageRows.length === 0) {
-      if (requestNumber === 1) {
-        throw new CommandExecutionError(
-          `Unexpected response while loading xueqiu comments for ${options.symbol}`,
-          'Run the command again with --verbose to inspect the raw site response.',
-        );
-      }
-      if (classified.kind === 'unknown') {
-        warn(buildPaginationStopMessage(
-          requestNumber,
-          rows.length,
-          options.limit,
-          describeFailureKind(classified.kind),
-        ));
-      }
-      break;
-    }
-
-    let advanced = false;
-    for (const row of pageRows) {
-      if (seenIds.has(row.id)) continue;
-      seenIds.add(row.id);
-      rows.push(row);
-      advanced = true;
-    }
-
-    if (rows.length >= options.limit) {
-      return rows.slice(0, options.limit);
-    }
-    if (rawItems.length < options.pageSize) {
-      break;
-    }
-    if (!advanced) {
-      warn(buildPaginationStopMessage(
-        requestNumber,
-        rows.length,
-        options.limit,
-        'pagination did not advance',
-      ));
-      break;
-    }
-    if (requestNumber === options.maxRequests) {
-      warn(buildPaginationStopMessage(requestNumber, rows.length, options.limit, 'reached safety cap'));
-    }
-  }
-
-  return rows.slice(0, options.limit);
-}
-
-cli({
-  site: 'xueqiu',
-  name: 'comments',
-  description: '获取单只股票的讨论动态',
-  domain: 'xueqiu.com',
-  strategy: Strategy.COOKIE,
-  browser: true,
-  navigateBefore: false,
-  args: [
-    {
-      name: 'symbol',
-      positional: true,
-      required: true,
-      help: 'Stock symbol, e.g. SH600519, AAPL, or 00700',
-    },
-    { name: 'limit', type: 'int', default: 20, help: 'Number of discussion posts to return' },
-  ],
-  columns: ['author', 'text', 'likes', 'replies', 'retweets', 'created_at', 'url'],
-  func: async (page, args) => {
-    const symbol = normalizeSymbolInput(args.symbol);
-    const limit = Number(args.limit);
-    if (!Number.isInteger(limit) || limit <= 0) {
-      throw new ArgumentError('xueqiu comments requires --limit to be a positive integer');
-    }
-    if (limit > 100) {
-      throw new ArgumentError('xueqiu comments supports --limit up to 100');
-    }
-    const pageSize = Math.min(limit, 20);
-    await page.goto('https://xueqiu.com');
-    const rows = await collectCommentRows({
-      symbol,
-      limit,
-      pageSize,
-      maxRequests: 5,
-      fetchPage: (pageNumber, currentPageSize) => fetchCommentsPage(page, symbol, pageNumber, currentPageSize),
-      warn: log.warn,
-    });
-    return rows.map(row => toCommentOutputRow(row));
-  },
-});
-
-/**
- * Convert raw CLI input into a normalized stock symbol.
- *
- * @param raw User-provided CLI argument.
- * @returns Upper-cased symbol string.
- */
-export function normalizeSymbolInput(raw: unknown): string {
-  const symbol = String(raw ?? '').trim().toUpperCase();
-  if (!symbol) throw new ArgumentError('xueqiu comments requires a symbol');
-  if (/^HTTPS?:\/\//.test(symbol)) {
-    throw new ArgumentError('xueqiu comments only accepts a symbol, not a URL');
-  }
-  if (!XUEQIU_SYMBOL_PATTERN.test(symbol)) {
-    throw new ArgumentError(`xueqiu comments received an invalid symbol: ${symbol}`);
-  }
-  return symbol;
-}

package/src/clis/xueqiu/danjuan-utils.test.ts

@@ -1,49 +0,0 @@
-import { describe, expect, it, vi } from 'vitest';
-import { fetchDanjuanAll } from './danjuan-utils.js';
-
-describe('fetchDanjuanAll', () => {
-  it('throws when no Danjuan accounts are visible', async () => {
-    const mockPage = {
-      evaluate: vi.fn().mockResolvedValue({ _emptyAccounts: true }),
-    } as any;
-
-    await expect(fetchDanjuanAll(mockPage)).rejects.toThrow('No fund accounts found');
-  });
-
-  it('throws when any account detail request fails', async () => {
-    const mockPage = {
-      evaluate: vi.fn().mockResolvedValue({
-        detailErrors: [
-          { accountName: '默认账户', accountId: 'acc-1', error: 403 },
-        ],
-      }),
-    } as any;
-
-    await expect(fetchDanjuanAll(mockPage)).rejects.toThrow(
-      'Failed to fetch Danjuan account details: 默认账户 (acc-1): 403',
-    );
-  });
-
-  it('returns the combined snapshot when all account details succeed', async () => {
-    const snapshot = {
-      asOf: '2026-03-25',
-      totalAssetAmount: 100,
-      totalAssetDailyGain: 1,
-      totalAssetHoldGain: 2,
-      totalAssetTotalGain: 3,
-      totalFundMarketValue: 80,
-      accounts: [{ accountId: 'acc-1', accountName: '默认账户' }],
-      holdings: [{ accountId: 'acc-1', fdCode: '000001', fdName: '示例基金' }],
-      detailErrors: [],
-    };
-    const mockPage = {
-      evaluate: vi.fn().mockResolvedValue(snapshot),
-    } as any;
-
-    await expect(fetchDanjuanAll(mockPage)).resolves.toMatchObject({
-      asOf: '2026-03-25',
-      accounts: [{ accountId: 'acc-1', accountName: '默认账户' }],
-      holdings: [{ accountId: 'acc-1', fdCode: '000001', fdName: '示例基金' }],
-    });
-  });
-});

package/src/clis/xueqiu/danjuan-utils.ts

@@ -1,176 +0,0 @@
-/**
- * Shared helpers for Danjuan (蛋卷基金) adapters.
- *
- * Core design: a single page.evaluate call fetches the gain overview AND
- * all per-account holdings in parallel (Promise.all), minimising Node↔Browser
- * round-trips to exactly one.
- */
-
-import type { IPage } from '../../types.js';
-
-export const DANJUAN_DOMAIN = 'danjuanfunds.com';
-export const DANJUAN_ASSET_PAGE = `https://${DANJUAN_DOMAIN}/my-money`;
-
-const GAIN_URL = `https://${DANJUAN_DOMAIN}/djapi/fundx/profit/assets/gain?gains=%5B%22private%22%5D`;
-const SUMMARY_URL = `https://${DANJUAN_DOMAIN}/djapi/fundx/profit/assets/summary?invest_account_id=`;
-
-// ---------------------------------------------------------------------------
-// Types — keep everything explicit so TS consumers get autocomplete.
-// ---------------------------------------------------------------------------
-
-export interface DanjuanAccount {
-  accountId: string;
-  accountName: string;
-  accountType: string;
-  accountCode: string;
-  marketValue: number | null;
-  dailyGain: number | null;
-  mainFlag: boolean;
-}
-
-export interface DanjuanHolding {
-  accountId: string;
-  accountName: string;
-  accountType: string;
-  fdCode: string;
-  fdName: string;
-  category: string;
-  marketValue: number | null;
-  volume: number | null;
-  usableRemainShare: number | null;
-  dailyGain: number | null;
-  holdGain: number | null;
-  holdGainRate: number | null;
-  totalGain: number | null;
-  nav: number | null;
-  marketPercent: number | null;
-}
-
-export interface DanjuanSnapshot {
-  asOf: string | null;
-  totalAssetAmount: number | null;
-  totalAssetDailyGain: number | null;
-  totalAssetHoldGain: number | null;
-  totalAssetTotalGain: number | null;
-  totalFundMarketValue: number | null;
-  accounts: DanjuanAccount[];
-  holdings: DanjuanHolding[];
-}
-
-// ---------------------------------------------------------------------------
-// Single-evaluate fetcher
-// ---------------------------------------------------------------------------
-
-/**
- * Fetch the complete Danjuan fund picture in ONE browser round-trip.
- *
- * Inside the browser context we:
- *   1. Fetch the gain/assets overview (contains account list)
- *   2. Promise.all → fetch every account's holdings in parallel
- *   3. Return the combined result to Node
- */
-export async function fetchDanjuanAll(page: IPage): Promise<DanjuanSnapshot> {
-  const raw: any = await page.evaluate(`
-    (async () => {
-      const f = async (u) => {
-        const r = await fetch(u, { credentials: 'include' });
-        if (!r.ok) return { _err: r.status };
-        try { return await r.json(); } catch { return { _err: 'parse' }; }
-      };
-      const n = (v) => { const x = Number(v); return Number.isFinite(x) ? x : null; };
-
-      const gain = await f(${JSON.stringify(GAIN_URL)});
-      if (gain._err) return { _httpError: gain._err };
-
-      const root = gain.data || {};
-      const fundSec = (root.items || []).find(i => i && i.summary_type === 'FUND');
-      const rawAccs = fundSec && Array.isArray(fundSec.invest_account_list)
-        ? fundSec.invest_account_list : [];
-
-      const accounts = rawAccs.map(a => ({
-        accountId:   String(a.invest_account_id || ''),
-        accountName: a.invest_account_name || '',
-        accountType: a.invest_account_type || '',
-        accountCode: a.invest_account_code || '',
-        marketValue: n(a.market_value),
-        dailyGain:   n(a.daily_gain),
-        mainFlag:    !!a.main_flag,
-      }));
-
-      if (!accounts.length) {
-        return { _emptyAccounts: true };
-      }
-
-      const details = await Promise.all(
-        accounts.map(a => f(${JSON.stringify(SUMMARY_URL)} + encodeURIComponent(a.accountId)))
-      );
-
-      const holdings = [];
-      const detailErrors = [];
-      for (let i = 0; i < accounts.length; i++) {
-        const d = details[i];
-        if (d._err) {
-          detailErrors.push({
-            accountId: accounts[i].accountId,
-            accountName: accounts[i].accountName,
-            error: d._err,
-          });
-          continue;
-        }
-        const data = d.data || {};
-        const funds = Array.isArray(data.items) ? data.items : [];
-        const acc = accounts[i];
-        for (const fd of funds) {
-          holdings.push({
-            accountId:        acc.accountId,
-            accountName:      data.invest_account_name || acc.accountName,
-            accountType:      data.invest_account_type || acc.accountType,
-            fdCode:           fd.fd_code || '',
-            fdName:           fd.fd_name || '',
-            category:         fd.category_text || fd.category || '',
-            marketValue:      n(fd.market_value),
-            volume:           n(fd.volume),
-            usableRemainShare:n(fd.usable_remain_share),
-            dailyGain:        n(fd.daily_gain),
-            holdGain:         n(fd.hold_gain),
-            holdGainRate:     n(fd.hold_gain_rate),
-            totalGain:        n(fd.total_gain),
-            nav:              n(fd.nav),
-            marketPercent:    n(fd.market_percent),
-          });
-        }
-      }
-
-      return {
-        asOf:                root.daily_gain_date || null,
-        totalAssetAmount:    n(root.amount),
-        totalAssetDailyGain: n(root.daily_gain),
-        totalAssetHoldGain:  n(root.hold_gain),
-        totalAssetTotalGain: n(root.total_gain),
-        totalFundMarketValue:n(fundSec && fundSec.amount),
-        accounts,
-        holdings,
-        detailErrors,
-      };
-    })()
-  `);
-
-  if (raw?._httpError) {
-    throw new Error(`HTTP ${raw._httpError} — Hint: not logged in to ${DANJUAN_DOMAIN}?`);
-  }
-  if (raw?._emptyAccounts) {
-    throw new Error(`No fund accounts found — Hint: not logged in to ${DANJUAN_DOMAIN}?`);
-  }
-  if (Array.isArray(raw?.detailErrors) && raw.detailErrors.length > 0) {
-    const failedAccounts = raw.detailErrors
-      .map((item: { accountName?: string; accountId?: string; error?: string | number }) => {
-        const label = item.accountName && item.accountId
-          ? `${item.accountName} (${item.accountId})`
-          : item.accountName || item.accountId || 'unknown account';
-        return `${label}: ${item.error}`;
-      })
-      .join(', ');
-    throw new Error(`Failed to fetch Danjuan account details: ${failedAccounts}`);
-  }
-  return raw as DanjuanSnapshot;
-}