@zenalexa/unicli 0.220.0 → 0.220.1

package/src/adapters/mangadex/web.ts

@@ -0,0 +1,210 @@
+/**
+ * @owner   src/adapters/mangadex/web.ts
+ * @does    Register MangaDex public manga and author search commands.
+ * @needs   MangaDex public API and relationship include semantics.
+ * @feeds   Manga/doujin-adjacent title and creator discovery.
+ * @breaks  MangaDex API schema or public rate limits can block lookup.
+ */
+
+import { USER_AGENT } from "../../constants.js";
+import { cli, Strategy } from "../../registry.js";
+
+const API = "https://api.mangadex.org";
+
+function str(value: unknown): string {
+  return value === undefined || value === null ? "" : String(value);
+}
+
+function required(value: unknown, label: string): string {
+  const text = str(value).trim();
+  if (!text) throw new Error(`mangadex ${label} cannot be empty.`);
+  return text;
+}
+
+function requireLimit(value: unknown): number {
+  if (value === undefined || value === null || value === "") return 10;
+  const n = Number(value);
+  if (!Number.isInteger(n) || n < 1 || n > 50) {
+    throw new Error("mangadex limit must be an integer in [1, 50].");
+  }
+  return n;
+}
+
+function optionalYear(value: unknown): number | undefined {
+  if (value === undefined || value === null || value === "") return undefined;
+  const n = Number(value);
+  if (!Number.isInteger(n) || n < 1900 || n > 2100) {
+    throw new Error("mangadex year must be an integer in [1900, 2100].");
+  }
+  return n;
+}
+
+const SORT_PARAMS: Record<string, [string, string] | undefined> = {
+  relevance: undefined,
+  latest: ["order[latestUploadedChapter]", "desc"],
+  followed: ["order[followedCount]", "desc"],
+  year: ["order[year]", "desc"],
+};
+
+function applySort(url: URL, value: unknown): void {
+  const key = String(value ?? "relevance").trim();
+  const spec = SORT_PARAMS[key];
+  if (!(key in SORT_PARAMS)) {
+    throw new Error(
+      `mangadex sort must be one of: ${Object.keys(SORT_PARAMS).join(", ")}.`,
+    );
+  }
+  if (spec) url.searchParams.set(spec[0], spec[1]);
+}
+
+const CONTENT_RATINGS = new Set([
+  "safe",
+  "suggestive",
+  "erotica",
+  "pornographic",
+  "all",
+]);
+
+function applyContentRating(url: URL, value: unknown): void {
+  if (value === undefined || value === null || value === "") return;
+  const rating = String(value).trim();
+  if (!CONTENT_RATINGS.has(rating)) {
+    throw new Error(
+      `mangadex content_rating must be one of: ${Array.from(CONTENT_RATINGS).join(", ")}.`,
+    );
+  }
+  if (rating !== "all") url.searchParams.append("contentRating[]", rating);
+}
+
+async function getJson(url: URL): Promise<unknown[]> {
+  const response = await fetch(url, {
+    headers: { Accept: "application/json", "User-Agent": USER_AGENT },
+  });
+  if (!response.ok)
+    throw new Error(`mangadex request failed with HTTP ${response.status}.`);
+  const data = (await response.json()) as { data?: unknown[]; result?: string };
+  if (data.result && data.result !== "ok")
+    throw new Error(`mangadex API returned ${data.result}.`);
+  return data.data ?? [];
+}
+
+function localized(values: unknown): string {
+  const obj =
+    values && typeof values === "object"
+      ? (values as Record<string, unknown>)
+      : {};
+  return str(
+    obj.en || obj.ja || obj["ja-ro"] || obj.zh || Object.values(obj)[0],
+  );
+}
+
+export function mapMangaDexManga(rows: unknown[]): Record<string, unknown>[] {
+  return rows.map((row, index) => {
+    const item = row as Record<string, unknown>;
+    const attrs =
+      (item.attributes as Record<string, unknown> | undefined) ?? {};
+    return {
+      rank: index + 1,
+      id: item.id,
+      title: localized(attrs.title),
+      status: str(attrs.status),
+      year: attrs.year ?? null,
+      content_rating: str(attrs.contentRating),
+      latest_uploaded_chapter: str(attrs.latestUploadedChapter),
+      description: localized(attrs.description).slice(0, 700),
+      url: `https://mangadex.org/title/${item.id}`,
+    };
+  });
+}
+
+export function mapMangaDexAuthors(rows: unknown[]): Record<string, unknown>[] {
+  return rows.map((row, index) => {
+    const item = row as Record<string, unknown>;
+    const attrs =
+      (item.attributes as Record<string, unknown> | undefined) ?? {};
+    return {
+      rank: index + 1,
+      id: item.id,
+      name: str(attrs.name),
+      twitter: str(attrs.twitter),
+      pixiv: str(attrs.pixiv),
+      website: str(attrs.website),
+      url: `https://mangadex.org/author/${item.id}`,
+    };
+  });
+}
+
+async function searchManga(kwargs: Record<string, unknown>) {
+  const query = required(kwargs.query, "query");
+  const url = new URL(`${API}/manga`);
+  url.searchParams.set("title", query);
+  url.searchParams.set("limit", String(requireLimit(kwargs.limit)));
+  const year = optionalYear(kwargs.year);
+  if (year) url.searchParams.set("year", String(year));
+  applySort(url, kwargs.sort);
+  applyContentRating(url, kwargs["content-rating"]);
+  url.searchParams.append("includes[]", "author");
+  url.searchParams.append("includes[]", "artist");
+  const rows = mapMangaDexManga(await getJson(url));
+  if (rows.length === 0)
+    throw new Error(`No MangaDex manga found for "${query}".`);
+  return rows;
+}
+
+async function searchAuthors(kwargs: Record<string, unknown>) {
+  const query = required(kwargs.query, "query");
+  const url = new URL(`${API}/author`);
+  url.searchParams.set("name", query);
+  url.searchParams.set("limit", String(requireLimit(kwargs.limit)));
+  const rows = mapMangaDexAuthors(await getJson(url));
+  if (rows.length === 0)
+    throw new Error(`No MangaDex authors found for "${query}".`);
+  return rows;
+}
+
+const SEARCH_ARGS = [
+  { name: "query", type: "str" as const, required: true, positional: true },
+  { name: "limit", type: "int" as const, default: 10 },
+];
+
+const MANGA_ARGS = [
+  { name: "query", type: "str" as const, required: true, positional: true },
+  { name: "limit", type: "int" as const, default: 10 },
+  { name: "year", type: "int" as const },
+  {
+    name: "sort",
+    type: "str" as const,
+    default: "relevance",
+    choices: ["relevance", "latest", "followed", "year"],
+  },
+  {
+    name: "content-rating",
+    type: "str" as const,
+    choices: ["safe", "suggestive", "erotica", "pornographic", "all"],
+  },
+];
+
+cli({
+  site: "mangadex",
+  name: "manga",
+  description:
+    "Search MangaDex manga by Japanese title, romaji, alias, or keyword",
+  domain: "mangadex.org",
+  strategy: Strategy.PUBLIC,
+  browser: false,
+  args: MANGA_ARGS,
+  columns: ["rank", "id", "title", "status", "year", "content_rating", "url"],
+  func: async (_page, kwargs) => searchManga(kwargs),
+});
+
+cli({
+  site: "mangadex",
+  name: "authors",
+  description: "Search MangaDex authors and artists by public name or romaji",
+  domain: "mangadex.org",
+  strategy: Strategy.PUBLIC,
+  browser: false,
+  args: SEARCH_ARGS,
+  columns: ["rank", "id", "name", "twitter", "pixiv", "website", "url"],
+  func: async (_page, kwargs) => searchAuthors(kwargs),
+});

package/src/adapters/moegirl/web.test.ts

@@ -0,0 +1,87 @@
+import { describe, expect, it } from "vitest";
+import { resolveCommand } from "../../registry.js";
+import {
+  mapMoegirlOpenSearch,
+  parseMoegirlLinksHtml,
+  parseMoegirlPageHtml,
+} from "./web.js";
+
+describe("moegirl public commands", () => {
+  it("registers search, page, and links", () => {
+    expect(
+      Object.keys(resolveCommand("moegirl", "search")!.adapter.commands),
+    ).toEqual(expect.arrayContaining(["search", "page", "links"]));
+  });
+
+  it("maps OpenSearch results", () => {
+    expect(
+      mapMoegirlOpenSearch(
+        [
+          "花火",
+          ["花火", "花火(星穹铁道)"],
+          ["", ""],
+          [
+            "https://zh.moegirl.org.cn/%E8%8A%B1%E7%81%AB",
+            "https://zh.moegirl.org.cn/%E8%8A%B1%E7%81%AB(%E6%98%9F%E7%A9%B9%E9%93%81%E9%81%93)",
+          ],
+        ],
+        5,
+      ),
+    ).toEqual([
+      {
+        rank: 1,
+        title: "花火",
+        description: "",
+        url: "https://zh.moegirl.org.cn/%E8%8A%B1%E7%81%AB",
+      },
+      {
+        rank: 2,
+        title: "花火(星穹铁道)",
+        description: "",
+        url: "https://zh.moegirl.org.cn/%E8%8A%B1%E7%81%AB(%E6%98%9F%E7%A9%B9%E9%93%81%E9%81%93)",
+      },
+    ]);
+  });
+
+  it("parses page text and metadata", () => {
+    const row = parseMoegirlPageHtml(
+      `<title>花火(星穹铁道) - 萌娘百科 万物皆可萌的百科全书</title>
+       <meta name="description" content="花火是游戏《崩坏：星穹铁道》中的登场角色。">
+       <link rel="canonical" href="https://zh.moegirl.org.cn/%E8%8A%B1%E7%81%AB(%E6%98%9F%E7%A9%B9%E9%93%81%E9%81%93)">
+       <script>RLCONF={"wgCategories":["崩坏：星穹铁道角色","游戏角色"]};</script>
+       <template id="MOE_SKIN_TEMPLATE_BODYCONTENT"><div class="mw-parser-output">
+       <p><b>花火</b>是游戏《崩坏：星穹铁道》中的角色。</p>
+       <p>她属于假面愚者。</p></div></template>`,
+      "花火(星穹铁道)",
+      1,
+    );
+    expect(row).toMatchObject({
+      title: "花火(星穹铁道)",
+      description: "花火是游戏《崩坏：星穹铁道》中的登场角色。",
+      categories: ["崩坏：星穹铁道角色", "游戏角色"],
+      paragraphs: 1,
+      url: "https://zh.moegirl.org.cn/%E8%8A%B1%E7%81%AB(%E6%98%9F%E7%A9%B9%E9%93%81%E9%81%93)",
+    });
+    expect(String(row.extract)).toContain("崩坏：星穹铁道");
+  });
+
+  it("extracts disambiguation links with contains filter", () => {
+    expect(
+      parseMoegirlLinksHtml(
+        `<template id="MOE_SKIN_TEMPLATE_BODYCONTENT"><div>
+          <a href="/%E8%8A%B1%E7%81%AB" title="花火">花火</a>
+          <a href="/%E8%8A%B1%E7%81%AB(%E6%98%9F%E7%A9%B9%E9%93%81%E9%81%93)" title="花火(星穹铁道)">花火(星穹铁道)</a>
+          <a href="/index.php?title=X&amp;action=edit&amp;redlink=1" title="X（页面不存在）">X</a>
+        </div></template>`,
+        5,
+        "星穹铁道",
+      ),
+    ).toEqual([
+      {
+        rank: 1,
+        title: "花火(星穹铁道)",
+        url: "https://zh.moegirl.org.cn/%E8%8A%B1%E7%81%AB(%E6%98%9F%E7%A9%B9%E9%93%81%E9%81%93)",
+      },
+    ]);
+  });
+});

package/src/adapters/moegirl/web.ts

@@ -0,0 +1,343 @@
+/**
+ * @owner   src/adapters/moegirl/web.ts
+ * @does    Register Moegirl public search, page reading, and article link extraction commands.
+ * @needs   Moegirl public OpenSearch endpoint and rendered MediaWiki HTML.
+ * @feeds   ACG entity discovery, character disambiguation, and wiki-backed content research.
+ * @breaks  Moegirl skin/template markup drift can reduce page and link extraction quality.
+ */
+
+import { USER_AGENT } from "../../constants.js";
+import { cli, Strategy } from "../../registry.js";
+
+const ORIGIN = "https://zh.moegirl.org.cn";
+
+interface SearchRow {
+  rank: number;
+  title: string;
+  description: string;
+  url: string;
+}
+
+interface LinkRow {
+  rank: number;
+  title: string;
+  url: string;
+}
+
+function str(value: unknown): string {
+  return value === undefined || value === null ? "" : String(value);
+}
+
+function decodeHtml(value: unknown): string {
+  return decodeHtmlEntities(value).replace(/\s+/g, " ").trim();
+}
+
+function decodeHtmlText(value: unknown): string {
+  return decodeHtmlEntities(value)
+    .replace(/[ \t\f\v]+/g, " ")
+    .replace(/\n[ \t]+/g, "\n")
+    .replace(/\n{3,}/g, "\n\n")
+    .trim();
+}
+
+function decodeHtmlEntities(value: unknown): string {
+  return str(value)
+    .replace(/&#(\d+);/g, (_m, code: string) =>
+      String.fromCodePoint(Number(code)),
+    )
+    .replace(/&#x([0-9a-f]+);/gi, (_m, code: string) =>
+      String.fromCodePoint(Number.parseInt(code, 16)),
+    )
+    .replace(/ /g, " ")
+    .replace(/&/g, "&")
+    .replace(/&lt;/g, "<")
+    .replace(/&gt;/g, ">")
+    .replace(/&quot;/g, '"')
+    .replace(/&#039;/g, "'")
+    .replace(/&#39;/g, "'");
+}
+
+function requireText(value: unknown, label: string): string {
+  const text = str(value).trim();
+  if (!text) throw new Error(`moegirl ${label} cannot be empty.`);
+  return text;
+}
+
+function requireLimit(value: unknown, fallback = 10): number {
+  if (value === undefined || value === null || value === "") return fallback;
+  const n = Number(value);
+  if (!Number.isInteger(n) || n < 1 || n > 50) {
+    throw new Error("moegirl limit must be an integer in [1, 50].");
+  }
+  return n;
+}
+
+function requireParagraphCap(value: unknown): number {
+  if (value === undefined || value === null || value === "") return 0;
+  const n = Number(value);
+  if (!Number.isInteger(n) || n < 0 || n > 100) {
+    throw new Error("moegirl paragraphs must be an integer in [0, 100].");
+  }
+  return n;
+}
+
+function articleUrl(title: string): string {
+  return `${ORIGIN}/${encodeURIComponent(title.replace(/ /g, "_"))}`;
+}
+
+async function fetchJson(url: string): Promise<unknown> {
+  const response = await fetch(url, {
+    headers: {
+      Accept: "application/json",
+      "User-Agent": USER_AGENT,
+    },
+  });
+  if (!response.ok) {
+    throw new Error(`moegirl request failed with HTTP ${response.status}.`);
+  }
+  return response.json();
+}
+
+async function fetchText(url: string): Promise<string> {
+  const response = await fetch(url, {
+    headers: {
+      Accept: "text/html,application/xhtml+xml",
+      "User-Agent": USER_AGENT,
+    },
+  });
+  if (!response.ok) {
+    throw new Error(`moegirl request failed with HTTP ${response.status}.`);
+  }
+  return response.text();
+}
+
+export function mapMoegirlOpenSearch(
+  data: unknown,
+  limit: number,
+): SearchRow[] {
+  if (
+    !Array.isArray(data) ||
+    !Array.isArray(data[1]) ||
+    !Array.isArray(data[3])
+  ) {
+    throw new Error("moegirl OpenSearch response shape changed.");
+  }
+  const titles = data[1] as unknown[];
+  const descriptions = Array.isArray(data[2]) ? (data[2] as unknown[]) : [];
+  const urls = data[3] as unknown[];
+  return titles.slice(0, limit).map((title, index) => ({
+    rank: index + 1,
+    title: str(title),
+    description: str(descriptions[index]),
+    url: str(urls[index]),
+  }));
+}
+
+function firstMatch(value: string, re: RegExp): string {
+  const match = value.match(re);
+  return match ? decodeHtml(match[1]) : "";
+}
+
+function bodyHtml(html: string): string {
+  return (
+    firstRawMatch(
+      html,
+      /<template id="MOE_SKIN_TEMPLATE_BODYCONTENT">([\s\S]*?)<\/template>/,
+    ) ||
+    firstRawMatch(
+      html,
+      /<div[^>]+class="[^"]*\bmw-parser-output\b[^"]*"[^>]*>([\s\S]*?)<\/div>\s*<\/div>/,
+    ) ||
+    html
+  );
+}
+
+function firstRawMatch(value: string, re: RegExp): string {
+  const match = value.match(re);
+  return match ? match[1] : "";
+}
+
+function cleanArticleText(html: string): string {
+  return decodeHtmlText(
+    bodyHtml(html)
+      .replace(/<style[\s\S]*?<\/style>/gi, " ")
+      .replace(/<script[\s\S]*?<\/script>/gi, " ")
+      .replace(/<sup[\s\S]*?<\/sup>/gi, " ")
+      .replace(/<\/(?:p|li|h[1-6]|tr)>/gi, "\n\n")
+      .replace(/<(?:br|div)\b[^>]*>/gi, "\n")
+      .replace(/<[^>]+>/g, " ")
+      .replace(/\n\s+/g, "\n")
+      .replace(/\n{3,}/g, "\n\n"),
+  );
+}
+
+export function parseMoegirlPageHtml(
+  html: string,
+  requestedTitle: string,
+  paragraphCap: number,
+): Record<string, unknown> {
+  const pageTitle =
+    firstMatch(html, /<title>([^<]+?)(?: - 萌娘百科[^<]*)?<\/title>/) ||
+    firstMatch(
+      html,
+      /<meta property="og:title" content="(?:&lt;[^&]+&gt;)?([^"<]+)(?:&lt;\/[^&]+&gt;)?"/,
+    ) ||
+    requestedTitle;
+  const title = requestedTitle.includes("(") ? requestedTitle : pageTitle;
+  const description =
+    firstMatch(html, /<meta name="description" content="([^"]*)"/) ||
+    firstMatch(html, /<meta property="og:description" content="([^"]*)"/);
+  const url =
+    firstMatch(html, /<link rel="canonical" href="([^"]+)"/) ||
+    articleUrl(title);
+  const categories = [...html.matchAll(/"wgCategories":\[(.*?)\]/g)][0]?.[1]
+    ?.split(",")
+    .map((item) => decodeHtml(item.replace(/^"|"$/g, "")))
+    .filter(Boolean);
+  const text = cleanArticleText(html);
+  if (!text) {
+    throw new Error(
+      `Moegirl article "${requestedTitle}" has no readable text.`,
+    );
+  }
+  const paragraphs = text
+    .split(/\n+/)
+    .map((part) => part.trim())
+    .filter(Boolean);
+  const selected =
+    paragraphCap > 0 ? paragraphs.slice(0, paragraphCap) : paragraphs;
+  return {
+    title,
+    description,
+    categories: categories ?? [],
+    paragraphs: selected.length,
+    extract: selected.join("\n\n"),
+    url,
+  };
+}
+
+export function parseMoegirlLinksHtml(
+  html: string,
+  limit: number,
+  contains: string,
+): LinkRow[] {
+  const rows: LinkRow[] = [];
+  const seen = new Set<string>();
+  const needle = contains.trim();
+  for (const match of bodyHtml(html).matchAll(
+    /<a\b(?=[^>]*\bhref="([^"]+)")(?=[^>]*\btitle="([^"]+)")[^>]*>/g,
+  )) {
+    const href = decodeHtml(match[1]);
+    const title = decodeHtml(match[2]);
+    if (!href.startsWith("/") || href.includes("redlink=1")) continue;
+    if (title.includes("页面不存在")) continue;
+    if (needle && !title.includes(needle)) continue;
+    const key = `${title}\n${href}`;
+    if (seen.has(key)) continue;
+    seen.add(key);
+    rows.push({
+      rank: rows.length + 1,
+      title,
+      url: `${ORIGIN}${href}`,
+    });
+    if (rows.length >= limit) break;
+  }
+  return rows;
+}
+
+cli({
+  site: "moegirl",
+  name: "search",
+  description:
+    "Search Moegirl ACG wiki articles for characters, works, songs, games, and studios",
+  domain: "zh.moegirl.org.cn",
+  strategy: Strategy.PUBLIC,
+  browser: false,
+  args: [
+    { name: "query", type: "str", required: true, positional: true },
+    { name: "limit", type: "int", default: 10 },
+  ],
+  columns: ["rank", "title", "description", "url"],
+  func: async (_page, kwargs) => {
+    const query = requireText(kwargs.query, "query");
+    const limit = requireLimit(kwargs.limit);
+    const url = new URL(`${ORIGIN}/api.php`);
+    url.searchParams.set("action", "opensearch");
+    url.searchParams.set("search", query);
+    url.searchParams.set("limit", String(limit));
+    url.searchParams.set("namespace", "0");
+    url.searchParams.set("format", "json");
+    const rows = mapMoegirlOpenSearch(await fetchJson(url.toString()), limit);
+    if (rows.length === 0)
+      throw new Error(`No Moegirl pages found for "${query}".`);
+    return rows;
+  },
+});
+
+cli({
+  site: "moegirl",
+  name: "page",
+  description: "Read a Moegirl article as plain text with metadata",
+  domain: "zh.moegirl.org.cn",
+  strategy: Strategy.PUBLIC,
+  browser: false,
+  args: [
+    { name: "title", type: "str", required: true, positional: true },
+    {
+      name: "paragraphs",
+      type: "int",
+      default: 0,
+      description: "Paragraph cap, 0 means full",
+    },
+  ],
+  columns: [
+    "title",
+    "description",
+    "categories",
+    "paragraphs",
+    "extract",
+    "url",
+  ],
+  func: async (_page, kwargs) => {
+    const title = requireText(kwargs.title, "title");
+    const paragraphCap = requireParagraphCap(kwargs.paragraphs);
+    return [
+      parseMoegirlPageHtml(
+        await fetchText(articleUrl(title)),
+        title,
+        paragraphCap,
+      ),
+    ];
+  },
+});
+
+cli({
+  site: "moegirl",
+  name: "links",
+  description:
+    "Extract internal Moegirl article links from a page, useful for disambiguation pages",
+  domain: "zh.moegirl.org.cn",
+  strategy: Strategy.PUBLIC,
+  browser: false,
+  args: [
+    { name: "title", type: "str", required: true, positional: true },
+    { name: "limit", type: "int", default: 20 },
+    {
+      name: "contains",
+      type: "str",
+      description: "Only return links whose title contains this text",
+    },
+  ],
+  columns: ["rank", "title", "url"],
+  func: async (_page, kwargs) => {
+    const title = requireText(kwargs.title, "title");
+    const rows = parseMoegirlLinksHtml(
+      await fetchText(articleUrl(title)),
+      requireLimit(kwargs.limit),
+      str(kwargs.contains),
+    );
+    if (rows.length === 0)
+      throw new Error(`No Moegirl links found on "${title}".`);
+    return rows;
+  },
+});

package/src/adapters/pdf/read.yaml

@@ -0,0 +1,49 @@
+site: pdf
+name: read
+description: Extract text from a local PDF file
+type: desktop
+strategy: public
+binary: pdftotext
+detect: which pdftotext
+
+args:
+  file:
+    type: str
+    required: true
+    positional: true
+    description: Local PDF file path
+    x-unicli-kind: path
+  first_page:
+    type: int
+    default: 1
+    description: First page to read
+  last_page:
+    type: int
+    default: 20
+    description: Last page to read
+
+pipeline:
+  - exec:
+      command: pdftotext
+      args:
+        - "-layout"
+        - "-enc"
+        - "UTF-8"
+        - "-f"
+        - "${{ args.first_page }}"
+        - "-l"
+        - "${{ args.last_page }}"
+        - "${{ args.file }}"
+        - "-"
+      parse: text
+      timeout: 60000
+
+columns: []
+
+# schema-v2 metadata — injected by `unicli migrate schema-v2`
+capabilities: ["subprocess.exec"]
+minimum_capability: subprocess.exec
+trust: user
+confidentiality: private
+quarantine: false
+schema_version: v2