@zenalexa/unicli 0.219.0 → 0.220.0

package/src/adapters/aibase/news.test.ts

@@ -0,0 +1,42 @@
+import { describe, expect, it } from "vitest";
+import {
+  buildExtractAibaseNewsJs,
+  mapAibaseNewsPayload,
+  requireNewsLimit,
+} from "./news.js";
+
+describe("aibase agent-facing news command", () => {
+  it("validates limits and exposes selector extraction code", () => {
+    expect(requireNewsLimit(undefined)).toBe(20);
+    expect(requireNewsLimit("50")).toBe(50);
+    expect(() => requireNewsLimit("0")).toThrow("limit must be");
+    expect(buildExtractAibaseNewsJs()).toContain("/zh/daily/");
+  });
+
+  it("maps extracted rows and rejects selector drift", () => {
+    expect(
+      mapAibaseNewsPayload(
+        {
+          ok: true,
+          rows: [
+            { title: " AI  news ", url: " https://www.aibase.com/zh/daily/1 " },
+            { title: "", url: "https://example.test/empty" },
+          ],
+        },
+        10,
+      ),
+    ).toEqual([
+      {
+        rank: 1,
+        title: "AI news",
+        url: "https://www.aibase.com/zh/daily/1",
+      },
+    ]);
+    expect(() =>
+      mapAibaseNewsPayload({ ok: false, reason: "selector-missing" }, 10),
+    ).toThrow("selector drift");
+    expect(() => mapAibaseNewsPayload({ ok: true, rows: [] }, 10)).toThrow(
+      "no article rows",
+    );
+  });
+});

package/src/adapters/aibase/news.ts

@@ -0,0 +1,118 @@
+/**
+ * @owner   src/adapters/aibase/news.ts
+ * @does    Register agent-facing AIbase daily-news browser extraction command.
+ * @needs   Public AIbase daily page, browser runtime, selector-drift error reporting.
+ * @feeds   surface coverage ledger, AI news discovery, Chinese AI daily brief surface.
+ * @breaks  AIbase DOM drift or silent empty extraction hides news coverage failures.
+ */
+
+import { cli, Strategy } from "../../registry.js";
+import type { IPage } from "../../types.js";
+
+const AIBASE_DAILY_URL = "https://www.aibase.com/zh/daily";
+
+function normalizeText(value: unknown): string {
+  return String(value ?? "")
+    .replace(/\s+/g, " ")
+    .trim();
+}
+
+export function requireNewsLimit(value: unknown, fallback = 20): number {
+  if (value === undefined || value === null || value === "") return fallback;
+  const n = Number(value);
+  if (!Number.isInteger(n) || n < 1 || n > 50) {
+    throw new Error("limit must be an integer in [1, 50].");
+  }
+  return n;
+}
+
+export function buildExtractAibaseNewsJs(): string {
+  return `
+    (() => {
+      const anchors = Array.from(document.querySelectorAll('.bg-white .grid a[href], a[href*="/zh/daily/"]'))
+        .filter((anchor) => {
+          const href = anchor.getAttribute('href') || '';
+          const text = (anchor.innerText || anchor.textContent || '').trim();
+          return text && href && !href.endsWith('/zh/daily') && !href.endsWith('/zh/daily/');
+        });
+      if (anchors.length === 0) {
+        return {
+          ok: false,
+          reason: 'selector-missing',
+          title: document.title || '',
+          bodyText: (document.body?.innerText || document.body?.textContent || '').slice(0, 500),
+        };
+      }
+      const seen = new Set();
+      const rows = [];
+      for (const anchor of anchors) {
+        const url = new URL(anchor.getAttribute('href'), location.href).href;
+        if (seen.has(url)) continue;
+        seen.add(url);
+        rows.push({
+          title: anchor.innerText || anchor.textContent || '',
+          url,
+        });
+      }
+      return { ok: true, rows };
+    })()
+  `;
+}
+
+interface BrowserExtractPayload {
+  ok?: unknown;
+  reason?: unknown;
+  title?: unknown;
+  rows?: Array<{ title?: unknown; url?: unknown }>;
+}
+
+export function mapAibaseNewsPayload(
+  payload: BrowserExtractPayload,
+  limit: number,
+): Array<Record<string, unknown>> {
+  if (!payload || typeof payload !== "object") {
+    throw new Error("AIbase daily page returned an unreadable payload.");
+  }
+  if (payload.ok !== true) {
+    const reason = normalizeText(payload.reason) || "selector-drift";
+    const title = normalizeText(payload.title);
+    throw new Error(
+      `AIbase daily selector drift: ${reason}${title ? ` (${title})` : ""}`,
+    );
+  }
+  const rows = (Array.isArray(payload.rows) ? payload.rows : [])
+    .map((row, index) => ({
+      rank: index + 1,
+      title: normalizeText(row.title),
+      url: normalizeText(row.url),
+    }))
+    .filter((row) => row.title && row.url);
+  if (rows.length === 0) {
+    throw new Error(
+      "AIbase daily page loaded, but no article rows were extracted.",
+    );
+  }
+  return rows
+    .slice(0, limit)
+    .map((row, index) => ({ ...row, rank: index + 1 }));
+}
+
+cli({
+  site: "aibase",
+  name: "news",
+  description: "AIbase daily AI industry news",
+  domain: "www.aibase.com",
+  strategy: Strategy.PUBLIC,
+  browser: true,
+  args: [{ name: "limit", type: "int", default: 20, description: "Max rows" }],
+  columns: ["rank", "title", "url"],
+  func: async (page, kwargs) => {
+    const p = page as IPage;
+    const limit = requireNewsLimit(kwargs.limit);
+    await p.goto(AIBASE_DAILY_URL, { waitUntil: "load", settleMs: 3000 });
+    const payload = (await p.evaluate(
+      buildExtractAibaseNewsJs(),
+    )) as BrowserExtractPayload;
+    return mapAibaseNewsPayload(payload, limit);
+  },
+});

package/src/adapters/arxiv/papers.test.ts

@@ -0,0 +1,59 @@
+import { describe, expect, it } from "vitest";
+import {
+  decodeArxivEntities,
+  parseArxivEntries,
+  requireArxivAuthor,
+  requireArxivCategory,
+  requireArxivLimit,
+} from "./papers.js";
+
+describe("arxiv agent-facing author and recent commands", () => {
+  it("validates author, category, and limit inputs", () => {
+    expect(requireArxivAuthor(" Yoshua Bengio ")).toBe("Yoshua Bengio");
+    expect(() => requireArxivAuthor("")).toThrow("cannot be empty");
+    expect(requireArxivCategory("cs.CL")).toBe("cs.CL");
+    expect(requireArxivCategory("q-bio.NC")).toBe("q-bio.NC");
+    expect(() => requireArxivCategory("../cs.CL")).toThrow("Invalid arXiv");
+    expect(requireArxivLimit(undefined, 20)).toBe(20);
+    expect(requireArxivLimit("50", 20)).toBe(50);
+    expect(() => requireArxivLimit("51", 20)).toThrow("arxiv limit");
+  });
+
+  it("decodes entities and parses Atom entries", () => {
+    expect(decodeArxivEntities("A &amp; B &lt; C")).toBe("A & B < C");
+    expect(
+      parseArxivEntries(`
+        <feed>
+          <entry>
+            <id>http://arxiv.org/abs/1706.03762v7</id>
+            <title> Attention   Is All You Need </title>
+            <summary> We present   transformers. </summary>
+            <published>2017-06-12T17:57:34Z</published>
+            <updated>2023-08-02T00:00:00Z</updated>
+            <author><name>Alice &amp; Bob</name></author>
+            <author><name>Carol</name></author>
+            <arxiv:primary_category term="cs.CL" />
+            <category term="cs.CL" />
+            <category term="cs.LG" />
+            <arxiv:comment>15 pages</arxiv:comment>
+            <link title="pdf" href="https://arxiv.org/pdf/1706.03762" rel="related" />
+          </entry>
+        </feed>
+      `),
+    ).toEqual([
+      {
+        id: "1706.03762",
+        title: "Attention Is All You Need",
+        authors: "Alice & Bob, Carol",
+        abstract: "We present transformers.",
+        published: "2017-06-12",
+        updated: "2023-08-02",
+        primary_category: "cs.CL",
+        categories: "cs.CL, cs.LG",
+        comment: "15 pages",
+        pdf: "https://arxiv.org/pdf/1706.03762",
+        url: "https://arxiv.org/abs/1706.03762",
+      },
+    ]);
+  });
+});

package/src/adapters/arxiv/papers.ts

@@ -0,0 +1,226 @@
+/**
+ * @owner   src/adapters/arxiv/papers.ts
+ * @does    Register agent-facing arXiv author and recent category commands.
+ * @needs   export.arxiv.org Atom API, category validation, conservative XML parsing.
+ * @feeds   surface coverage ledger, scholarly search workflow, arXiv category monitoring.
+ * @breaks  arXiv Atom shape drift, weak category parsing, or silent empty feeds hide paper discovery failures.
+ */
+
+import { cli, Strategy } from "../../registry.js";
+
+const ARXIV_BASE = "https://export.arxiv.org/api/query";
+const CATEGORY_RE = /^[a-z]+(?:-[a-z]+)*(?:\.[A-Za-z0-9]+(?:-[A-Za-z0-9]+)*)?$/;
+
+interface ArxivEntry {
+  id: string;
+  title: string;
+  authors: string;
+  abstract: string;
+  published: string;
+  updated: string;
+  primary_category: string;
+  categories: string;
+  comment: string;
+  pdf: string;
+  url: string;
+}
+
+export function requireArxivLimit(
+  value: unknown,
+  fallback: number,
+  max = 50,
+): number {
+  if (value === undefined || value === null || value === "") return fallback;
+  const n = Number(value);
+  if (!Number.isInteger(n) || n < 1 || n > max) {
+    throw new Error(`arxiv limit must be an integer in [1, ${max}].`);
+  }
+  return n;
+}
+
+export function requireArxivAuthor(value: unknown): string {
+  const author = String(value ?? "").trim();
+  if (!author) throw new Error("arxiv author cannot be empty.");
+  return author;
+}
+
+export function requireArxivCategory(value: unknown): string {
+  const category = String(value ?? "").trim();
+  if (!CATEGORY_RE.test(category)) {
+    throw new Error(`Invalid arXiv category "${String(value)}".`);
+  }
+  return category;
+}
+
+export function decodeArxivEntities(value: string): string {
+  return value
+    .replace(/&amp;/g, "&")
+    .replace(/&lt;/g, "<")
+    .replace(/&gt;/g, ">")
+    .replace(/&quot;/g, '"')
+    .replace(/&apos;/g, "'")
+    .replace(/&#39;/g, "'");
+}
+
+function extractFirst(xml: string, tag: string): string {
+  const match = xml.match(new RegExp(`<${tag}[^>]*>([\\s\\S]*?)<\\/${tag}>`));
+  return match ? match[1].trim() : "";
+}
+
+function extractAll(xml: string, tag: string): string[] {
+  const out: string[] = [];
+  const re = new RegExp(`<${tag}[^>]*>([\\s\\S]*?)<\\/${tag}>`, "g");
+  let match: RegExpExecArray | null;
+  while ((match = re.exec(xml)) !== null) out.push(match[1].trim());
+  return out;
+}
+
+function extractAttr(xml: string, tag: string, attr: string): string {
+  const match = xml.match(new RegExp(`<${tag}\\b[^>]*?\\b${attr}="([^"]*)"`));
+  return match ? match[1] : "";
+}
+
+function extractAllAttr(xml: string, tag: string, attr: string): string[] {
+  const out: string[] = [];
+  const re = new RegExp(`<${tag}\\b[^>]*?\\b${attr}="([^"]*)"`, "g");
+  let match: RegExpExecArray | null;
+  while ((match = re.exec(xml)) !== null) out.push(match[1]);
+  return out;
+}
+
+function findLinkHref(xml: string, rel: string): string {
+  const re = /<link\b([^>]*)\/?>/g;
+  let match: RegExpExecArray | null;
+  while ((match = re.exec(xml)) !== null) {
+    const attrs = match[1];
+    if (!new RegExp(`\\brel="${rel}"`).test(attrs)) continue;
+    const href = attrs.match(/\bhref="([^"]*)"/);
+    if (href) return href[1];
+  }
+  return "";
+}
+
+export function parseArxivEntries(xml: string): ArxivEntry[] {
+  const out: ArxivEntry[] = [];
+  const re = /<entry>([\s\S]*?)<\/entry>/g;
+  let match: RegExpExecArray | null;
+  while ((match = re.exec(xml)) !== null) {
+    const entry = match[1];
+    const rawId = extractFirst(entry, "id");
+    const id = rawId
+      .replace(/^https?:\/\/arxiv\.org\/abs\//, "")
+      .replace(/v\d+$/, "");
+    const title = decodeArxivEntities(
+      extractFirst(entry, "title").replace(/\s+/g, " "),
+    ).trim();
+    out.push({
+      id,
+      title,
+      authors: decodeArxivEntities(extractAll(entry, "name").join(", ")),
+      abstract: decodeArxivEntities(
+        extractFirst(entry, "summary").replace(/\s+/g, " "),
+      ).trim(),
+      published: extractFirst(entry, "published").slice(0, 10),
+      updated: extractFirst(entry, "updated").slice(0, 10),
+      primary_category: extractAttr(entry, "arxiv:primary_category", "term"),
+      categories: extractAllAttr(entry, "category", "term").join(", "),
+      comment: decodeArxivEntities(
+        extractFirst(entry, "arxiv:comment").replace(/\s+/g, " "),
+      ).trim(),
+      pdf: findLinkHref(entry, "related") || `https://arxiv.org/pdf/${id}`,
+      url: `https://arxiv.org/abs/${id}`,
+    });
+  }
+  return out;
+}
+
+async function fetchArxiv(params: URLSearchParams): Promise<string> {
+  const response = await fetch(`${ARXIV_BASE}?${params.toString()}`, {
+    headers: {
+      "User-Agent": "unicli-arxiv/1.0 (https://github.com/olo-dot-io/Uni-CLI)",
+      Accept: "application/atom+xml, application/xml, text/xml",
+    },
+  });
+  if (!response.ok)
+    throw new Error(`arXiv API returned HTTP ${response.status}.`);
+  return response.text();
+}
+
+function compactRows(entries: ArxivEntry[]): Array<Record<string, unknown>> {
+  return entries.map((entry) => ({
+    id: entry.id,
+    title: entry.title,
+    authors: entry.authors,
+    published: entry.published,
+    primary_category: entry.primary_category,
+    url: entry.url,
+  }));
+}
+
+cli({
+  site: "arxiv",
+  name: "author",
+  description: "List arXiv papers by a given author",
+  domain: "export.arxiv.org",
+  strategy: Strategy.PUBLIC,
+  args: [
+    {
+      name: "author",
+      type: "str",
+      required: true,
+      positional: true,
+      description: "Author name",
+    },
+    { name: "limit", type: "int", default: 20, description: "Max papers" },
+  ],
+  columns: ["id", "title", "authors", "published", "primary_category", "url"],
+  func: async (_page, kwargs) => {
+    const author = requireArxivAuthor(kwargs.author);
+    const limit = requireArxivLimit(kwargs.limit, 20);
+    const params = new URLSearchParams({
+      search_query: `au:"${author}"`,
+      max_results: String(limit),
+      sortBy: "submittedDate",
+      sortOrder: "descending",
+    });
+    const rows = compactRows(parseArxivEntries(await fetchArxiv(params)));
+    if (rows.length === 0) {
+      throw new Error(`No arXiv papers found for author "${author}".`);
+    }
+    return rows;
+  },
+});
+
+cli({
+  site: "arxiv",
+  name: "recent",
+  description: "List recent arXiv submissions in a category",
+  domain: "export.arxiv.org",
+  strategy: Strategy.PUBLIC,
+  args: [
+    {
+      name: "category",
+      type: "str",
+      required: true,
+      positional: true,
+      description: "arXiv category",
+    },
+    { name: "limit", type: "int", default: 10, description: "Max papers" },
+  ],
+  columns: ["id", "title", "authors", "published", "primary_category", "url"],
+  func: async (_page, kwargs) => {
+    const category = requireArxivCategory(kwargs.category);
+    const limit = requireArxivLimit(kwargs.limit, 10);
+    const params = new URLSearchParams({
+      search_query: `cat:${category}`,
+      max_results: String(limit),
+      sortBy: "submittedDate",
+      sortOrder: "descending",
+    });
+    const rows = compactRows(parseArxivEntries(await fetchArxiv(params)));
+    if (rows.length === 0) {
+      throw new Error(`No recent arXiv papers found in ${category}.`);
+    }
+    return rows;
+  },
+});

package/src/adapters/bbc/topic.test.ts

@@ -0,0 +1,52 @@
+import { describe, expect, it } from "vitest";
+import {
+  bbcPubDateToIso,
+  decodeBbcEntities,
+  extractRssTag,
+  parseBbcRssItems,
+  requireBbcLimit,
+  requireBbcTopic,
+} from "./topic.js";
+
+describe("bbc agent-facing topic command", () => {
+  it("validates topics and limits", () => {
+    expect(requireBbcTopic("science-and-environment")).toBe(
+      "science_and_environment",
+    );
+    expect(requireBbcTopic(" Entertainment and Arts ")).toBe(
+      "entertainment_and_arts",
+    );
+    expect(() => requireBbcTopic("sport")).toThrow("not supported");
+    expect(requireBbcLimit(undefined)).toBe(20);
+    expect(requireBbcLimit("50")).toBe(50);
+    expect(() => requireBbcLimit("51")).toThrow("bbc limit");
+  });
+
+  it("parses RSS items with CDATA and entities", () => {
+    expect(decodeBbcEntities("A & B 'C'")).toBe("A & B 'C'");
+    const block = "<title><![CDATA[World &amp; News]]></title>";
+    expect(extractRssTag(block, "title")).toBe("World &amp; News");
+    expect(
+      parseBbcRssItems(`
+        <rss><channel>
+          <item>
+            <title><![CDATA[World &amp; News]]></title>
+            <description>Summary &lt;one&gt;</description>
+            <link>https://www.bbc.com/news/world-1</link>
+            <pubDate>Tue, 12 May 2026 10:00:00 GMT</pubDate>
+            <guid>guid-1</guid>
+          </item>
+        </channel></rss>
+      `),
+    ).toEqual([
+      {
+        title: "World & News",
+        description: "Summary <one>",
+        link: "https://www.bbc.com/news/world-1",
+        pubDate: "Tue, 12 May 2026 10:00:00 GMT",
+        guid: "guid-1",
+      },
+    ]);
+    expect(bbcPubDateToIso("Tue, 12 May 2026 10:00:00 GMT")).toBe("2026-05-12");
+  });
+});

package/src/adapters/bbc/topic.ts

@@ -0,0 +1,149 @@
+/**
+ * @owner   src/adapters/bbc/topic.ts
+ * @does    Register agent-facing BBC topic RSS command.
+ * @needs   feeds.bbci.co.uk public RSS feeds, topic whitelist, conservative RSS parsing.
+ * @feeds   surface coverage ledger, BBC section headline discovery.
+ * @breaks  BBC RSS envelope drift, loose topic parsing, or silent empty feeds hide news coverage failures.
+ */
+
+import { cli, Strategy } from "../../registry.js";
+
+const BBC_FEED_BASE = "https://feeds.bbci.co.uk/news";
+const TOPICS = [
+  "world",
+  "business",
+  "politics",
+  "health",
+  "education",
+  "science_and_environment",
+  "technology",
+  "entertainment_and_arts",
+] as const;
+
+type BbcTopic = (typeof TOPICS)[number];
+
+interface RssItem {
+  title: string;
+  description: string;
+  link: string;
+  pubDate: string;
+  guid: string;
+}
+
+export function requireBbcLimit(value: unknown, fallback = 20): number {
+  if (value === undefined || value === null || value === "") return fallback;
+  const n = Number(value);
+  if (!Number.isInteger(n) || n < 1 || n > 50) {
+    throw new Error("bbc limit must be an integer in [1, 50].");
+  }
+  return n;
+}
+
+export function requireBbcTopic(value: unknown): BbcTopic {
+  const topic = String(value ?? "")
+    .trim()
+    .toLowerCase()
+    .replace(/[\s-]+/g, "_");
+  if (!TOPICS.includes(topic as BbcTopic)) {
+    throw new Error(`bbc topic "${String(value)}" is not supported.`);
+  }
+  return topic as BbcTopic;
+}
+
+export function decodeBbcEntities(value: unknown): string {
+  return String(value ?? "")
+    .replace(/&#x([0-9a-fA-F]+);/g, (_m, hex: string) =>
+      String.fromCodePoint(Number.parseInt(hex, 16)),
+    )
+    .replace(/&#(\d+);/g, (_m, dec: string) =>
+      String.fromCodePoint(Number.parseInt(dec, 10)),
+    )
+    .replace(/&amp;/g, "&")
+    .replace(/&lt;/g, "<")
+    .replace(/&gt;/g, ">")
+    .replace(/&quot;/g, '"')
+    .replace(/&apos;/g, "'")
+    .replace(/&#39;/g, "'")
+    .replace(/&nbsp;/g, " ");
+}
+
+export function extractRssTag(block: string, tag: string): string {
+  const cdata = block.match(
+    new RegExp(
+      `<${tag}[^>]*>\\s*<!\\[CDATA\\[([\\s\\S]*?)\\]\\]>\\s*<\\/${tag}>`,
+    ),
+  );
+  if (cdata) return cdata[1];
+  const plain = block.match(new RegExp(`<${tag}[^>]*>([\\s\\S]*?)<\\/${tag}>`));
+  return plain ? plain[1] : "";
+}
+
+export function parseBbcRssItems(xml: string): RssItem[] {
+  const items: RssItem[] = [];
+  const re = /<item[^>]*>([\s\S]*?)<\/item>/g;
+  let match: RegExpExecArray | null;
+  while ((match = re.exec(xml)) !== null) {
+    const block = match[1];
+    items.push({
+      title: decodeBbcEntities(extractRssTag(block, "title")).trim(),
+      description: decodeBbcEntities(
+        extractRssTag(block, "description"),
+      ).trim(),
+      link: decodeBbcEntities(extractRssTag(block, "link")).trim(),
+      pubDate: decodeBbcEntities(extractRssTag(block, "pubDate")).trim(),
+      guid: decodeBbcEntities(extractRssTag(block, "guid")).trim(),
+    });
+  }
+  return items;
+}
+
+export function bbcPubDateToIso(value: unknown): string {
+  const date = new Date(String(value ?? ""));
+  return Number.isNaN(date.getTime()) ? "" : date.toISOString().slice(0, 10);
+}
+
+async function fetchBbcRss(topic: BbcTopic): Promise<string> {
+  const response = await fetch(`${BBC_FEED_BASE}/${topic}/rss.xml`, {
+    headers: {
+      "User-Agent": "unicli-bbc/1.0 (https://github.com/olo-dot-io/Uni-CLI)",
+      Accept: "application/rss+xml, application/xml",
+    },
+  });
+  if (!response.ok)
+    throw new Error(`bbc topic ${topic} returned HTTP ${response.status}.`);
+  return response.text();
+}
+
+cli({
+  site: "bbc",
+  name: "topic",
+  description: "BBC News headlines for a specific section",
+  domain: "feeds.bbci.co.uk",
+  strategy: Strategy.PUBLIC,
+  args: [
+    {
+      name: "topic",
+      type: "str",
+      required: true,
+      positional: true,
+      choices: [...TOPICS],
+      description: "BBC section name",
+    },
+    { name: "limit", type: "int", default: 20, description: "Max headlines" },
+  ],
+  columns: ["rank", "title", "description", "pubDate", "url"],
+  func: async (_page, kwargs) => {
+    const topic = requireBbcTopic(kwargs.topic);
+    const limit = requireBbcLimit(kwargs.limit);
+    const items = parseBbcRssItems(await fetchBbcRss(topic));
+    if (items.length === 0)
+      throw new Error(`BBC ${topic} feed returned no items.`);
+    return items.slice(0, limit).map((item, index) => ({
+      rank: index + 1,
+      title: item.title,
+      description: item.description,
+      pubDate: bbcPubDateToIso(item.pubDate),
+      url: item.link,
+    }));
+  },
+});