@zenalexa/unicli 0.219.0 → 0.220.1

package/src/adapters/dianping/shop.ts

@@ -0,0 +1,261 @@
+/**
+ * @owner   src/adapters/dianping/shop.ts
+ * @does    Register agent-facing Dianping shop detail extraction implemented with site-specific safety checks.
+ * @needs   Dianping shop page DOM and a browser profile that can pass Dianping anti-bot checks.
+ * @feeds   surface coverage ledger and restaurant search-to-detail workflows.
+ * @breaks  Dianping shop-head DOM drift, login redirects, captcha gates, or deleted shops can hide detail fields.
+ */
+
+import { cli, Strategy } from "../../registry.js";
+import type { IPage } from "../../types.js";
+import { str } from "../_shared/browser-tools.js";
+
+const DIANPING_ORIGIN = "https://www.dianping.com";
+const SHOP_ID_RE = /^[A-Za-z0-9_-]+$/;
+const SHOP_FIELDS = [
+  "shop_id",
+  "name",
+  "rating",
+  "reviews",
+  "price",
+  "rank",
+  "taste",
+  "environment",
+  "service",
+  "ingredients",
+  "hours",
+  "address",
+  "subway",
+  "features",
+  "url",
+] as const;
+
+type DianpingShopField = (typeof SHOP_FIELDS)[number];
+
+interface DianpingShopBreakdown {
+  口味?: unknown;
+  环境?: unknown;
+  服务?: unknown;
+  食材?: unknown;
+}
+
+interface DianpingShopData {
+  name?: unknown;
+  rating?: unknown;
+  reviewsRaw?: unknown;
+  priceRaw?: unknown;
+  rank?: unknown;
+  breakdown?: DianpingShopBreakdown;
+  hours?: unknown;
+  address?: unknown;
+  subway?: unknown;
+  features?: unknown;
+  url?: unknown;
+}
+
+interface DianpingShopResult extends DianpingShopData {
+  ok?: unknown;
+  sample?: unknown;
+}
+
+export function normalizeDianpingShopId(value: unknown): string {
+  const raw = str(value).trim();
+  if (!raw) throw new Error("Dianping shop_id must be a non-empty string.");
+  const match = raw.match(/\/shop\/([^/?#]+)/);
+  const shopId = match?.[1] ?? raw;
+  if (!SHOP_ID_RE.test(shopId)) {
+    throw new Error(
+      `Dianping shop_id must be an alphanumeric shop ID or /shop/<id> URL.`,
+    );
+  }
+  return shopId;
+}
+
+export function parseDianpingReviewCount(value: unknown): number | null {
+  const text = str(value).trim();
+  if (!text) return null;
+  const wan = text.match(/^([\d.]+)\s*万/);
+  if (wan) {
+    const count = Number(wan[1]);
+    return Number.isFinite(count) ? Math.round(count * 10000) : null;
+  }
+  const plain = text.match(/(\d+(?:\.\d+)?)/);
+  if (!plain) return null;
+  const count = Number(plain[1]);
+  return Number.isFinite(count) ? Math.round(count) : null;
+}
+
+export function parseDianpingPrice(value: unknown): number | null {
+  const match = str(value).match(/[¥￥]\s*(\d+(?:\.\d+)?)/);
+  if (!match) return null;
+  const price = Number(match[1]);
+  return Number.isFinite(price) ? price : null;
+}
+
+function numberOrNull(value: unknown): number | null {
+  if (typeof value === "number" && Number.isFinite(value)) return value;
+  const text = str(value).trim();
+  if (!text) return null;
+  const valueNumber = Number(text);
+  return Number.isFinite(valueNumber) ? valueNumber : null;
+}
+
+function featuresText(value: unknown): string {
+  if (Array.isArray(value))
+    return value
+      .map((item) => str(item).trim())
+      .filter(Boolean)
+      .join(", ");
+  return str(value).trim();
+}
+
+export function mapDianpingShopFields(
+  shopId: string,
+  data: DianpingShopData,
+  fallbackUrl: string,
+): Array<{ field: DianpingShopField; value: unknown }> {
+  const rating = numberOrNull(data.rating);
+  const breakdown = data.breakdown ?? {};
+  const values: Record<DianpingShopField, unknown> = {
+    shop_id: shopId,
+    name: str(data.name).trim(),
+    rating,
+    reviews: parseDianpingReviewCount(data.reviewsRaw),
+    price: parseDianpingPrice(data.priceRaw),
+    rank: str(data.rank).trim(),
+    taste: numberOrNull(breakdown["口味"]),
+    environment: numberOrNull(breakdown["环境"]),
+    service: numberOrNull(breakdown["服务"]),
+    ingredients: numberOrNull(breakdown["食材"]),
+    hours: str(data.hours).trim(),
+    address: str(data.address).trim(),
+    subway: str(data.subway).trim(),
+    features: featuresText(data.features),
+    url: str(data.url).trim() || fallbackUrl,
+  };
+  return SHOP_FIELDS.map((field) => ({ field, value: values[field] }));
+}
+
+export function classifyDianpingShopFailure(
+  sample: unknown,
+  url: string,
+  shopId: string,
+): never {
+  const signal = `${url} ${str(sample)}`;
+  if (
+    /verify\.meituan\.com|verifyimg|身份核实|请依次点击|美团安全验证|Yoda/i.test(
+      signal,
+    )
+  ) {
+    throw new Error(
+      `Dianping shop ${shopId} is blocked by captcha; solve the Dianping captcha in this browser profile and retry.`,
+    );
+  }
+  if (
+    /login\.dianping\.com|account\.dianping\.com|请先登录|未登录|请登录/.test(
+      signal,
+    )
+  ) {
+    throw new Error(
+      `Dianping shop ${shopId} requires login; sign in to dianping.com in this browser profile and retry.`,
+    );
+  }
+  if (
+    /商户不存在|店铺不存在|店铺已关闭|页面不存在|404|已下线|没有找到相关商户/i.test(
+      signal,
+    )
+  ) {
+    throw new Error(`Dianping shop ${shopId} was not found or is unavailable.`);
+  }
+  const preview = str(sample).slice(0, 160);
+  throw new Error(
+    `Dianping shop ${shopId} did not render expected shop data${preview ? `; sample: ${preview}` : ""}.`,
+  );
+}
+
+function buildDianpingShopEvaluate(): string {
+  return `(() => {
+    const clean = (value) => value == null ? '' : String(value).replace(/\\s+/g, ' ').trim();
+    const head = document.querySelector('.shop-head');
+    if (!head) {
+      const sample = clean(document.body?.innerText || document.body?.textContent || '').slice(0, 800);
+      return { ok: false, sample, url: location.href };
+    }
+    const headText = clean(head.textContent);
+    const titleEl = document.querySelector('.shop-name, .shop-head h2, .shop-head h1');
+    let name = clean(titleEl?.textContent);
+    if (!name) {
+      const match = String(document.title || '').match(/【([^】]+)】/);
+      if (match) name = clean(match[1]);
+    }
+    const rating = clean(document.querySelector('.star-score')?.textContent);
+    const reviewEl = document.querySelector('.reviews, .review-num, .reviewCount, .reviewCountSentence');
+    let reviewsRaw = clean(reviewEl?.textContent);
+    if (!reviewsRaw) {
+      const titleText = clean(document.querySelector('.review-title')?.textContent);
+      const titleMatch = titleText.match(/评价\\(([\\d.,万]+)\\)/);
+      if (titleMatch) reviewsRaw = titleMatch[1];
+    }
+    const priceMatch = headText.match(/[¥￥]\\s*\\d+(?:\\.\\d+)?/);
+    const breakdown = {};
+    for (const key of ['口味', '环境', '服务', '食材']) {
+      const match = headText.match(new RegExp(key + '[:：]\\\\s*([0-9.]+)'));
+      if (match) breakdown[key] = Number(match[1]);
+    }
+    const features = Array.from(document.querySelectorAll('.shop-feature')).map((node) => clean(node.textContent)).filter(Boolean);
+    const subwayMatch = headText.match(/距(?:地铁)?[^\\s]+?步行\\d+m/);
+    const hoursMatch = headText.match(/营业中[^\\s]*\\d{1,2}:\\d{2}-(?:次日)?\\d{1,2}:\\d{2}|今日休息|暂停营业/);
+    const rankMatch = headText.match(/[^\\s]+?(?:口味|人气|环境|服务)榜\\s*[·•]\\s*第\\d+名/);
+    return {
+      ok: true,
+      name,
+      rating,
+      reviewsRaw,
+      priceRaw: priceMatch?.[0] || '',
+      rank: rankMatch?.[0] || '',
+      breakdown,
+      hours: hoursMatch?.[0] || '',
+      address: clean(document.querySelector('.desc-info')?.textContent),
+      subway: subwayMatch?.[0] || '',
+      features,
+      url: location.href,
+    };
+  })()`;
+}
+
+cli({
+  site: "dianping",
+  name: "shop",
+  description: "Read Dianping shop details by shop ID or shop URL",
+  domain: "www.dianping.com",
+  strategy: Strategy.COOKIE,
+  browser: true,
+  args: [
+    {
+      name: "shop_id",
+      type: "str",
+      required: true,
+      positional: true,
+      description: "Dianping shop ID or https://www.dianping.com/shop/<id> URL",
+    },
+  ],
+  columns: ["field", "value"],
+  func: async (page, kwargs) => {
+    const shopId = normalizeDianpingShopId(kwargs.shop_id);
+    const url = `${DIANPING_ORIGIN}/shop/${shopId}`;
+    const p = page as IPage;
+    await p.goto(url, { waitUntil: "load", settleMs: 3000 });
+    await p.wait(3);
+    const result = (await p.evaluate(
+      buildDianpingShopEvaluate(),
+    )) as DianpingShopResult;
+    if (!result?.ok) {
+      classifyDianpingShopFailure(
+        result?.sample,
+        str(result?.url) || url,
+        shopId,
+      );
+    }
+    return mapDianpingShopFields(shopId, result, url);
+  },
+});

package/src/adapters/dlsite/web.test.ts

@@ -0,0 +1,132 @@
+import { describe, expect, it } from "vitest";
+import { resolveCommand } from "../../registry.js";
+import {
+  decodeDlsiteHtml,
+  dlsiteCreatorUrl,
+  dlsiteGenreUrl,
+  dlsiteMakerUrl,
+  dlsiteSearchUrl,
+  parseDlsiteDetailHtml,
+  parseDlsiteSearchHtml,
+} from "./web.js";
+
+describe("dlsite public commands", () => {
+  it("registers search, type shortcuts, and detail", () => {
+    expect(
+      Object.keys(resolveCommand("dlsite", "search")!.adapter.commands),
+    ).toEqual(
+      expect.arrayContaining([
+        "search",
+        "manga",
+        "cg",
+        "game",
+        "maker",
+        "creator",
+        "genre",
+        "work",
+      ]),
+    );
+  });
+
+  it("builds search URLs with sort and work type filters", () => {
+    const args = resolveCommand("dlsite", "search")!.command.adapterArgs;
+    expect(args).toEqual(
+      expect.arrayContaining([
+        expect.objectContaining({
+          name: "sort",
+          choices: ["release", "hot", "rating", "reviews", "price", "title"],
+        }),
+        expect.objectContaining({
+          name: "type",
+          choices: ["all", "manga", "cg", "game", "novel", "voice", "video"],
+        }),
+      ]),
+    );
+
+    expect(
+      dlsiteSearchUrl({
+        query: "花火",
+        sort: "hot",
+        type: "cg",
+        page: 2,
+      }),
+    ).toBe(
+      "https://www.dlsite.com/maniax/fsr/=/work_type/ICG/keyword/%E8%8A%B1%E7%81%AB/order/dl_d/page/2",
+    );
+    expect(
+      dlsiteMakerUrl({ maker_id: "RG01012594", sort: "hot", page: 1 }),
+    ).toBe(
+      "https://www.dlsite.com/maniax/circle/profile/=/page/1/maker_id/RG01012594.html/order/dl_d",
+    );
+    expect(dlsiteMakerUrl({ maker_id: "VG02994" })).toBe(
+      "https://www.dlsite.com/maniax/circle/profile/=/page/1/maker_id/VG02994.html/order/release",
+    );
+    expect(dlsiteCreatorUrl({ creator: "わかなはなび" })).toBe(
+      "https://www.dlsite.com/maniax/fsr/=/keyword_creater/%22%E3%82%8F%E3%81%8B%E3%81%AA%E3%81%AF%E3%81%AA%E3%81%B3%22/ana_flg/all/order/release/page/1",
+    );
+    expect(dlsiteGenreUrl({ genre: "001", sort: "rating" })).toBe(
+      "https://www.dlsite.com/maniax/fsr/=/genre/001/order/rate/page/1",
+    );
+  });
+
+  it("parses search result cards", () => {
+    const rows = parseDlsiteSearchHtml(
+      `<div data-list_item_product_id="RJ005751" class="search_result_img_box_inner">
+        <thumb-with-ng-filter-block link="https://www.dlsite.com/maniax/work/=/product_id/RJ005751.html"
+          :thumb-candidates="['//img.dlsite.jp/resize/images2/work/doujin/RJ006000/RJ005751_img_main_240x240.webp']"></thumb-with-ng-filter-block>
+        <dd class="work_category_free_sample"><div class="work_category type_MNG"><a>マンガ</a></div></dd>
+        <dd class="work_name"><div><a href="https://www.dlsite.com/maniax/work/=/product_id/RJ005751.html" title="Komm.susser Tod.">Komm.susser Tod.</a></div></dd>
+        <dd class="maker_name"><a href="https://www.dlsite.com/maniax/circle/profile/=/maker_id/RG01444.html">スタジオきゃうん</a></dd>
+        <span class="work_price_base">770</span>
+        <dd class="work_genre"><span class="icon_ADL" title="R18">R18</span></dd>
+        <dd class="work_dl">販売数:&nbsp;<span>745</span></dd>
+        <dd class="work_rating"><div class="star_rating star_40 mini">(327)</div></dd>
+      </div>`,
+      5,
+    );
+    expect(rows).toEqual([
+      {
+        rank: 1,
+        product_id: "RJ005751",
+        title: "Komm.susser Tod.",
+        maker: "スタジオきゃうん",
+        maker_id: "RG01444",
+        work_type: "マンガ",
+        age: "R18",
+        price_jpy: "770",
+        sales: "745",
+        rating: "star_40",
+        reviews: "327",
+        thumb:
+          "https://img.dlsite.jp/resize/images2/work/doujin/RJ006000/RJ005751_img_main_240x240.webp",
+        url: "https://www.dlsite.com/maniax/work/=/product_id/RJ005751.html",
+      },
+    ]);
+  });
+
+  it("parses detail page metadata", () => {
+    expect(
+      parseDlsiteDetailHtml(
+        `<meta property="og:image" content="https://img.dlsite.jp/main.jpg">
+         <h1 itemprop="name" id="work_name">Komm.susser Tod.</h1>
+         <span itemprop="brand" class="maker_name"><a>スタジオきゃうん</a></span>
+         <table id="work_outline"><tr><th>販売日</th><td>2001年10月02日</td></tr>
+         <tr><th>作品形式</th><td><span title="マンガ">マンガ</span></td></tr>
+         <tr><th>ページ数</th><td>36ページ</td></tr></table>
+         <div hidden class="ga4_event_item_RJ005751" data-product_id="RJ005751" data-maker_id="RG01444" data-work_type="MNG" data-price="770"></div>`,
+        "RJ005751",
+      ),
+    ).toMatchObject({
+      product_id: "RJ005751",
+      title: "Komm.susser Tod.",
+      maker: "スタジオきゃうん",
+      maker_id: "RG01444",
+      work_type: "MNG",
+      release_date: "2001年10月02日",
+      pages: "36ページ",
+      price_jpy: "770",
+      image: "https://img.dlsite.jp/main.jpg",
+    });
+    expect(decodeDlsiteHtml("A &amp; B<br>C")).toBe("A & B C");
+  });
+});