@jackwener/opencli 1.6.1 → 1.6.2

package/src/clis/1688/store.ts

@@ -0,0 +1,300 @@
+import { CommandExecutionError, EmptyResultError } from '../../errors.js';
+import { cli, Strategy } from '../../registry.js';
+import type { IPage } from '../../types.js';
+import {
+  FACTORY_BADGE_PATTERNS,
+  SERVICE_BADGE_PATTERNS,
+  assertAuthenticatedState,
+  buildDetailUrl,
+  buildProvenance,
+  canonicalizeSellerUrl,
+  canonicalizeStoreUrl,
+  cleanMultilineText,
+  cleanText,
+  extractAddress,
+  extractBadges,
+  extractMemberId,
+  extractMetric,
+  extractOfferId,
+  extractShopId,
+  extractYearsOnPlatform,
+  gotoAndReadState,
+  guessTopCategories,
+  resolveStoreUrl,
+  uniqueNonEmpty,
+} from './shared.js';
+
+interface StoreBrowserPayload {
+  href?: string;
+  title?: string;
+  bodyText?: string;
+  offerLinks?: string[];
+  contactLinks?: string[];
+}
+
+interface StoreItemSeed {
+  href?: string;
+  bodyText?: string;
+  seller?: {
+    companyName?: string;
+    memberId?: string;
+    winportUrl?: string;
+    sellerWinportUrlMap?: Record<string, string>;
+  };
+  services?: Array<{ serviceName?: string }>;
+}
+
+function normalizeStorePayload(input: {
+  resolvedUrl: string;
+  storePayload: StoreBrowserPayload | null;
+  contactPayload: StoreBrowserPayload | null;
+  seed: StoreItemSeed | null;
+  explicitMemberId: string | null;
+}): Record<string, unknown> {
+  const storePayload = input.storePayload;
+  const contactPayload = input.contactPayload;
+  const seed = input.seed;
+
+  const contactText = cleanMultilineText(contactPayload?.bodyText);
+  const storeText = cleanMultilineText(storePayload?.bodyText);
+  const seedText = cleanMultilineText(seed?.bodyText);
+  const combinedText = [contactText, storeText, seedText].filter(Boolean).join('\n');
+
+  const sellerUrlRaw = cleanText(
+    seed?.seller?.winportUrl
+      ?? seed?.seller?.sellerWinportUrlMap?.defaultUrl
+      ?? storePayload?.href
+      ?? input.resolvedUrl,
+  );
+  const storeUrl = safeCanonicalStoreUrl(sellerUrlRaw || input.resolvedUrl) ?? input.resolvedUrl;
+  const sellerUrl = canonicalizeSellerUrl(sellerUrlRaw) ?? storeUrl;
+  const companyUrl = pickCompanyUrl(contactPayload?.href, storeUrl);
+  const memberId = cleanText(seed?.seller?.memberId)
+    || input.explicitMemberId
+    || extractMemberId(input.resolvedUrl)
+    || extractMemberId(storePayload?.href ?? '')
+    || null;
+  const shopId = extractShopId(sellerUrl) ?? extractShopId(storeUrl);
+  const companyName = cleanText(seed?.seller?.companyName)
+    || firstNamedLine(contactText)
+    || firstNamedLine(storeText)
+    || null;
+  const serviceBadges = uniqueNonEmpty([
+    ...extractBadges(combinedText, SERVICE_BADGE_PATTERNS),
+    ...((seed?.services ?? []).map((service) => cleanText(service.serviceName))),
+  ]);
+  const factoryBadges = extractBadges(combinedText, FACTORY_BADGE_PATTERNS);
+
+  return {
+    member_id: memberId,
+    shop_id: shopId,
+    store_name: companyName,
+    store_url: storeUrl,
+    company_name: companyName,
+    company_url: companyUrl,
+    business_model_text: firstMetric(combinedText, ['经营模式', '生产加工', '主营产品']),
+    years_on_platform_text: extractYearsOnPlatform(combinedText),
+    location: extractAddress(contactText) ?? extractAddress(storeText),
+    staff_size_text: firstMetric(combinedText, ['员工人数', '员工总数']),
+    factory_badges: factoryBadges,
+    service_badges: serviceBadges,
+    response_rate_text: firstMetric(combinedText, ['响应率', '回复率', '响应速度']),
+    return_rate_text: extractReturnRate(combinedText),
+    top_categories: guessTopCategories(combinedText),
+    phone_text: extractMetric(contactText, '电话'),
+    mobile_text: extractMetric(contactText, '手机'),
+    ...buildProvenance(cleanText(contactPayload?.href) || cleanText(storePayload?.href) || input.resolvedUrl),
+  };
+}
+
+function safeCanonicalStoreUrl(url: string): string | null {
+  try {
+    return canonicalizeStoreUrl(url);
+  } catch {
+    return null;
+  }
+}
+
+function pickCompanyUrl(contactHref: string | undefined, storeUrl: string): string | null {
+  const fromPage = cleanText(contactHref);
+  if (fromPage) {
+    const normalized = buildContactUrl(fromPage);
+    if (normalized) return normalized;
+  }
+  return buildContactUrl(storeUrl);
+}
+
+function buildContactUrl(storeUrl: string): string | null {
+  try {
+    const parsed = new URL(storeUrl);
+    if (!parsed.hostname.endsWith('.1688.com')) return null;
+    return `${parsed.protocol}//${parsed.hostname}/page/contactinfo.html`;
+  } catch {
+    return null;
+  }
+}
+
+function firstNamedLine(text: string): string | null {
+  return text
+    .split('\n')
+    .map((line) => cleanText(line))
+    .find((line) => line.includes('有限公司') || line.includes('商行') || line.includes('工厂'))
+    ?? null;
+}
+
+function firstMetric(text: string, labels: string[]): string | null {
+  for (const label of labels) {
+    const value = extractMetric(text, label);
+    if (value) return value;
+  }
+  return null;
+}
+
+function extractReturnRate(text: string): string | null {
+  const inline = text.match(/回头率\s*([0-9.]+%)/);
+  if (inline) return cleanText(inline[0]);
+  const multiline = text.match(/回头率\s*\n\s*([0-9.]+%)/);
+  if (!multiline) return null;
+  return `回头率${cleanText(multiline[1])}`;
+}
+
+function firstOfferId(links: string[]): string | null {
+  for (const link of links) {
+    const offerId = extractOfferId(link);
+    if (offerId) return offerId;
+  }
+  return null;
+}
+
+function firstContactUrl(links: string[]): string | null {
+  for (const link of links) {
+    const url = buildContactUrl(link);
+    if (url) return url;
+  }
+  return null;
+}
+
+async function readStorePayload(page: IPage, url: string, action: string): Promise<StoreBrowserPayload> {
+  const state = await gotoAndReadState(page, url, 2500, action);
+  assertAuthenticatedState(state, action);
+
+  return await page.evaluate(`
+    (() => ({
+      href: window.location.href,
+      title: document.title || '',
+      bodyText: document.body ? document.body.innerText || '' : '',
+      offerLinks: Array.from(document.querySelectorAll('a[href*="detail.1688.com/offer/"], a[href*="offerId="]'))
+        .map((anchor) => anchor.href)
+        .filter(Boolean),
+      contactLinks: Array.from(document.querySelectorAll('a[href*="contactinfo"]'))
+        .map((anchor) => anchor.href)
+        .filter(Boolean),
+    }))()
+  `) as StoreBrowserPayload;
+}
+
+async function readItemSeed(page: IPage, offerId: string): Promise<StoreItemSeed> {
+  const itemUrl = buildDetailUrl(offerId);
+  const state = await gotoAndReadState(page, itemUrl, 2500, 'store seed item');
+  assertAuthenticatedState(state, 'store seed item');
+
+  const seed = await page.evaluate(`
+    (() => {
+      const model = window.context?.result?.global?.globalData?.model ?? null;
+      const toJson = (value) => JSON.parse(JSON.stringify(value ?? null));
+      return {
+        href: window.location.href,
+        bodyText: document.body ? document.body.innerText || '' : '',
+        seller: toJson(model?.sellerModel),
+        services: toJson(model?.shippingServices?.fields?.buyerProtectionModel ?? []),
+      };
+    })()
+  `) as StoreItemSeed;
+
+  const hasSellerContext = !!cleanText(seed?.seller?.memberId) || !!cleanText(seed?.seller?.winportUrl);
+  if (!hasSellerContext) {
+    throw new CommandExecutionError(
+      '1688 store seed item did not expose seller context',
+      '当前 tab 非商品详情上下文，请切到 detail.1688.com 商品页并重试',
+    );
+  }
+
+  return seed;
+}
+
+function hasAnyEvidence(
+  storePayload: StoreBrowserPayload | null,
+  contactPayload: StoreBrowserPayload | null,
+  seed: StoreItemSeed | null,
+): boolean {
+  return !!cleanText(storePayload?.bodyText)
+    || !!cleanText(contactPayload?.bodyText)
+    || !!cleanText(seed?.bodyText);
+}
+
+cli({
+  site: '1688',
+  name: 'store',
+  description: '1688 店铺/供应商公开信息（联系方式、主营、入驻年限、公开服务信号）',
+  domain: 'www.1688.com',
+  strategy: Strategy.COOKIE,
+  navigateBefore: false,
+  args: [
+    {
+      name: 'input',
+      required: true,
+      positional: true,
+      help: '1688 店铺 URL 或 member ID（如 b2b-22154705262941f196）',
+    },
+  ],
+  columns: ['store_name', 'years_on_platform_text', 'location', 'return_rate_text'],
+  func: async (page, kwargs) => {
+    const rawInput = String(kwargs.input ?? '');
+    const resolvedUrl = resolveStoreUrl(rawInput);
+    const explicitMemberId = extractMemberId(rawInput);
+
+    const storePayload = await readStorePayload(page, resolvedUrl, 'store');
+    const contactUrl = firstContactUrl(storePayload.contactLinks ?? []) || buildContactUrl(storePayload.href || resolvedUrl);
+    const contactPayload = contactUrl ? await readStorePayload(page, contactUrl, 'store contact') : null;
+    const offerId = extractOfferId(rawInput)
+      || firstOfferId(storePayload.offerLinks ?? [])
+      || firstOfferId(contactPayload?.offerLinks ?? []);
+
+    let seed: StoreItemSeed | null = null;
+    if (offerId) {
+      try {
+        seed = await readItemSeed(page, offerId);
+      } catch (error) {
+        if (!(error instanceof CommandExecutionError)) throw error;
+      }
+    }
+
+    if (!hasAnyEvidence(storePayload, contactPayload, seed)) {
+      throw new EmptyResultError(
+        '1688 store',
+        'Store page is reachable but no visible fields were extracted. Open the store page in Chrome and retry.',
+      );
+    }
+
+    return [
+      normalizeStorePayload({
+        resolvedUrl,
+        storePayload,
+        contactPayload,
+        seed,
+        explicitMemberId,
+      }),
+    ];
+  },
+});
+
+export const __test__ = {
+  normalizeStorePayload,
+  safeCanonicalStoreUrl,
+  buildContactUrl,
+  firstNamedLine,
+  firstMetric,
+  extractReturnRate,
+  firstOfferId,
+  firstContactUrl,
+};

package/src/clis/amazon/bestsellers.test.ts

@@ -1,9 +1,9 @@
 import { describe, expect, it } from 'vitest';
-import { __test__ } from './bestsellers.js';
+import { __test__ } from './rankings.js';
 
 describe('amazon bestsellers normalization', () => {
   it('normalizes bestseller cards and infers review counts from card text', () => {
-    const result = __test__.normalizeBestsellerCandidate({
+    const result = __test__.normalizeRankingCandidate({
       asin: 'B0DR31GC3D',
       title: '',
       href: 'https://www.amazon.com/NUTIKAS-Shelves-Desktop-Orgnizer-Shlef/dp/B0DR31GC3D/ref=zg_bs',
@@ -11,7 +11,16 @@ describe('amazon bestsellers normalization', () => {
       rating_text: '4.3 out of 5 stars',
       review_count_text: '',
       card_text: 'Desk Shelves Desktop Organizer Shlef\n4.3 out of 5 stars\n435\n$25.92',
-    }, 2, 'Amazon Best Sellers: Best Desktop & Off-Surface Shelves', 'https://www.amazon.com/example');
+    }, {
+      listType: 'bestsellers',
+      rankFallback: 2,
+      listTitle: 'Amazon Best Sellers: Best Desktop & Off-Surface Shelves',
+      sourceUrl: 'https://www.amazon.com/example',
+      categoryTitle: null,
+      categoryUrl: 'https://www.amazon.com/example',
+      categoryPath: [],
+      visibleCategoryLinks: [],
+    });
 
     expect(result.rank).toBe(2);
     expect(result.asin).toBe('B0DR31GC3D');

package/src/clis/amazon/bestsellers.ts

@@ -1,180 +1,8 @@
-import { CommandExecutionError } from '../../errors.js';
-import { cli, Strategy } from '../../registry.js';
-import type { IPage } from '../../types.js';
-import {
-  buildProvenance,
-  cleanText,
-  extractAsin,
-  extractReviewCountFromCardText,
-  firstMeaningfulLine,
-  normalizeProductUrl,
-  parsePriceText,
-  parseRatingValue,
-  parseReviewCount,
-  resolveBestsellersUrl,
-  uniqueNonEmpty,
-  assertUsableState,
-  gotoAndReadState,
-} from './shared.js';
+import { cli } from '../../registry.js';
+import { createRankingCliOptions } from './rankings.js';
 
-interface BestsellersPagePayload {
-  href?: string;
-  title?: string;
-  list_title?: string;
-  cards?: Array<{
-    rank_text?: string | null;
-    asin?: string | null;
-    title?: string | null;
-    href?: string | null;
-    price_text?: string | null;
-    rating_text?: string | null;
-    review_count_text?: string | null;
-    card_text?: string | null;
-  }>;
-  page_links?: string[];
-}
-
-function normalizeBestsellerCandidate(
-  candidate: NonNullable<BestsellersPagePayload['cards']>[number],
-  rank: number,
-  listTitle: string | null,
-  sourceUrl: string,
-): Record<string, unknown> {
-  const productUrl = normalizeProductUrl(candidate.href);
-  const asin = extractAsin(candidate.asin ?? '') ?? extractAsin(productUrl ?? '') ?? null;
-  const title = cleanText(candidate.title) || firstMeaningfulLine(candidate.card_text);
-  const price = parsePriceText(cleanText(candidate.price_text) || candidate.card_text);
-  const ratingText = cleanText(candidate.rating_text) || null;
-  const reviewCountText = cleanText(candidate.review_count_text)
-    || extractReviewCountFromCardText(candidate.card_text)
-    || null;
-  const provenance = buildProvenance(sourceUrl);
-
-  return {
-    rank,
-    asin,
-    title: title || null,
-    product_url: productUrl,
-    list_title: listTitle,
-    ...provenance,
-    price_text: price.price_text,
-    price_value: price.price_value,
-    currency: price.currency,
-    rating_text: ratingText,
-    rating_value: parseRatingValue(ratingText),
-    review_count_text: reviewCountText,
-    review_count: parseReviewCount(reviewCountText),
-  };
-}
-
-async function readBestsellersPage(page: IPage, url: string): Promise<BestsellersPagePayload> {
-  const state = await gotoAndReadState(page, url, 2500, 'bestsellers');
-  assertUsableState(state, 'bestsellers');
-
-  return await page.evaluate(`
-    (() => ({
-      href: window.location.href,
-      title: document.title || '',
-      list_title:
-        document.querySelector('#zg_banner_text')?.textContent
-        || document.querySelector('h1')?.textContent
-        || '',
-      cards: Array.from(document.querySelectorAll('.p13n-sc-uncoverable-faceout'))
-        .map((card) => ({
-          rank_text:
-            card.querySelector('.zg-bdg-text')?.textContent
-            || card.querySelector('[class*="rank"]')?.textContent
-            || '',
-          asin: card.id || '',
-          title:
-            card.querySelector('[class*="line-clamp"]')?.textContent
-            || card.querySelector('img')?.getAttribute('alt')
-            || '',
-          href: card.querySelector('a[href*="/dp/"]')?.href || '',
-          price_text: card.querySelector('.a-price .a-offscreen')?.textContent || '',
-          rating_text: card.querySelector('[aria-label*="out of 5 stars"]')?.getAttribute('aria-label') || '',
-          review_count_text:
-            card.querySelector('a[href*="#customerReviews"]')?.textContent
-            || card.querySelector('.a-size-small')?.textContent
-            || '',
-          card_text: card.innerText || '',
-        })),
-      page_links: Array.from(document.querySelectorAll('li.a-normal a, li.a-selected a'))
-        .map((anchor) => anchor.href || '')
-        .filter((href) => /\\/zgbs\\//.test(href) && /(?:[?&]pg=|ref=zg_bs_pg_)/.test(href)),
-    }))()
-  `) as BestsellersPagePayload;
-}
-
-cli({
-  site: 'amazon',
-  name: 'bestsellers',
+cli(createRankingCliOptions({
+  commandName: 'bestsellers',
+  listType: 'bestsellers',
   description: 'Amazon Best Sellers pages for category candidate discovery',
-  domain: 'amazon.com',
-  strategy: Strategy.COOKIE,
-  navigateBefore: false,
-  args: [
-    {
-      name: 'input',
-      positional: true,
-      help: 'Best sellers URL or /zgbs path. Omit to use the root Best Sellers page.',
-    },
-    {
-      name: 'limit',
-      type: 'int',
-      default: 100,
-      help: 'Maximum number of ranked items to return (default 100)',
-    },
-  ],
-  columns: ['rank', 'asin', 'title', 'price_text', 'rating_value', 'review_count'],
-  func: async (page, kwargs) => {
-    const limit = Math.max(1, Number(kwargs.limit) || 100);
-    const initialUrl = resolveBestsellersUrl(typeof kwargs.input === 'string' ? kwargs.input : undefined);
-
-    const queue = [initialUrl];
-    const visited = new Set<string>();
-    const seenAsins = new Set<string>();
-    const results: Record<string, unknown>[] = [];
-    let listTitle: string | null = null;
-
-    while (queue.length > 0 && results.length < limit) {
-      const nextUrl = queue.shift()!;
-      if (visited.has(nextUrl)) continue;
-      visited.add(nextUrl);
-
-      const payload = await readBestsellersPage(page, nextUrl);
-      const sourceUrl = cleanText(payload.href) || nextUrl;
-      listTitle = cleanText(payload.list_title) || cleanText(payload.title) || listTitle;
-      const cards = payload.cards ?? [];
-
-      for (const card of cards) {
-        const normalized = normalizeBestsellerCandidate(card, results.length + 1, listTitle, sourceUrl);
-        const asin = cleanText(String(normalized.asin ?? ''));
-        if (!asin || seenAsins.has(asin)) continue;
-        seenAsins.add(asin);
-        results.push(normalized);
-        if (results.length >= limit) break;
-      }
-
-      const pageLinks = uniqueNonEmpty(payload.page_links ?? []);
-      for (const href of pageLinks) {
-        if (!visited.has(href) && !queue.includes(href)) {
-          queue.push(href);
-        }
-      }
-    }
-
-    if (results.length === 0) {
-      throw new CommandExecutionError(
-        'amazon bestsellers did not expose any ranked items',
-        'Open the same best sellers page in Chrome, verify it is a real Amazon ranking page, and retry.',
-      );
-    }
-
-    return results.slice(0, limit);
-  },
-});
-
-export const __test__ = {
-  normalizeBestsellerCandidate,
-};
+}));

package/src/clis/amazon/movers-shakers.ts

@@ -0,0 +1,8 @@
+import { cli } from '../../registry.js';
+import { createRankingCliOptions } from './rankings.js';
+
+cli(createRankingCliOptions({
+  commandName: 'movers-shakers',
+  listType: 'movers_shakers',
+  description: 'Amazon Movers & Shakers pages for short-term growth signals',
+}));

package/src/clis/amazon/new-releases.ts

@@ -0,0 +1,8 @@
+import { cli } from '../../registry.js';
+import { createRankingCliOptions } from './rankings.js';
+
+cli(createRankingCliOptions({
+  commandName: 'new-releases',
+  listType: 'new_releases',
+  description: 'Amazon New Releases pages for early momentum discovery',
+}));

package/src/clis/amazon/rankings.test.ts

@@ -0,0 +1,47 @@
+import { describe, expect, it } from 'vitest';
+import { __test__ } from './rankings.js';
+
+describe('amazon rankings helpers', () => {
+  it('normalizes ranking candidates with unified schema', () => {
+    const result = __test__.normalizeRankingCandidate(
+      {
+        rank_text: '#3',
+        asin: 'B0DR31GC3D',
+        title: 'Desk Shelves Desktop Organizer',
+        href: 'https://www.amazon.com/dp/B0DR31GC3D/ref=zg_bs',
+        price_text: '$25.92',
+        rating_text: '4.3 out of 5 stars',
+        review_count_text: '435',
+      },
+      {
+        listType: 'new_releases',
+        rankFallback: 3,
+        listTitle: 'Amazon New Releases',
+        sourceUrl: 'https://www.amazon.com/gp/new-releases',
+        categoryTitle: 'Home & Kitchen',
+        categoryUrl: 'https://www.amazon.com/gp/new-releases/home-garden',
+        categoryPath: ['Home & Kitchen'],
+        visibleCategoryLinks: [{ title: 'Storage', url: 'https://www.amazon.com/gp/new-releases/storage', node_id: null }],
+      },
+    );
+
+    expect(result.list_type).toBe('new_releases');
+    expect(result.rank).toBe(3);
+    expect(result.asin).toBe('B0DR31GC3D');
+    expect(result.product_url).toBe('https://www.amazon.com/dp/B0DR31GC3D');
+    expect(result.category_title).toBe('Home & Kitchen');
+    expect(result.visible_category_links).toEqual([
+      { title: 'Storage', url: 'https://www.amazon.com/gp/new-releases/storage', node_id: null },
+    ]);
+  });
+
+  it('deduplicates category links and parses rank fallback', () => {
+    const links = __test__.normalizeVisibleCategoryLinks([
+      { title: 'Kitchen', url: '/gp/new-releases/home-garden' },
+      { title: 'Kitchen', url: 'https://www.amazon.com/gp/new-releases/home-garden' },
+      { title: 'Storage', url: '/gp/new-releases/storage', node_id: '1064954' },
+    ]);
+    expect(links.length).toBe(2);
+    expect(__test__.parseRank('N/A', 8)).toBe(8);
+  });
+});