@zenalexa/unicli 0.225.2 → 0.225.3

package/src/adapters/arxiv/papers.ts

@@ -1,15 +1,28 @@
 /**
  * @owner   src/adapters/arxiv/papers.ts
- * @does    Register agent-facing arXiv author and recent category commands.
- * @needs   export.arxiv.org Atom API, category validation, conservative XML parsing.
- * @feeds   surface coverage ledger, scholarly search workflow, arXiv category monitoring.
- * @breaks  arXiv Atom shape drift, weak category parsing, or silent empty feeds hide paper discovery failures.
+ * @does    Register agent-facing arXiv author, recent category, and PDF text-read commands.
+ * @needs   export.arxiv.org Atom API, arxiv.org PDF URLs, category/id validation, conservative XML parsing, pdftotext.
+ * @feeds   surface coverage ledger, scholarly search/read workflow, arXiv category monitoring.
+ * @breaks  arXiv Atom/PDF shape drift, weak category/id parsing, denied PDF downloads, missing pdftotext, or silent empty feeds hide paper discovery/read failures.
+ * @invariants  arXiv ids are normalized before URL construction; read returns PDF-derived text only and labels `text_source=pdf`.
+ * @side-effects HTTPS egress to export.arxiv.org and arxiv.org; read writes PDFs under the requested output directory and executes pdftotext.
+ * @perf        O(limit) for Atom discovery; O(PDF bytes + extracted pages) for read.
+ * @concurrency safe - per-command local state only
+ * @test        src/adapters/arxiv/papers.test.ts, tests/unit/commands/scholar.test.ts
+ * @stability   experimental
+ * @since       0.225.2
  */
 
+import { execFile } from "node:child_process";
+import { join, resolve } from "node:path";
+import { promisify } from "node:util";
+
 import { cli, Strategy } from "../../registry.js";
+import { httpDownload, sanitizeFilename } from "../../engine/download.js";
 
 const ARXIV_BASE = "https://export.arxiv.org/api/query";
 const CATEGORY_RE = /^[a-z]+(?:-[a-z]+)*(?:\.[A-Za-z0-9]+(?:-[A-Za-z0-9]+)*)?$/;
+const execFileAsync = promisify(execFile);
 
 interface ArxivEntry {
   id: string;
@@ -44,6 +57,20 @@ export function requireArxivAuthor(value: unknown): string {
   return author;
 }
 
+export function normalizeArxivId(value: unknown): string {
+  const id = String(value ?? "")
+    .trim()
+    .replace(/^arxiv:/i, "")
+    .replace(/^https?:\/\/(?:www\.)?arxiv\.org\/(?:abs|pdf)\//i, "")
+    .replace(/\.pdf$/i, "");
+  if (
+    !/^(?:\d{4}\.\d{4,5}|[a-z-]+(?:\.[A-Z]{2})?\/\d{7})(?:v\d+)?$/i.test(id)
+  ) {
+    throw new Error(`Invalid arXiv id "${String(value ?? "")}".`);
+  }
+  return id;
+}
+
 export function requireArxivCategory(value: unknown): string {
   const category = String(value ?? "").trim();
   if (!CATEGORY_RE.test(category)) {
@@ -157,6 +184,160 @@ function compactRows(entries: ArxivEntry[]): Array<Record<string, unknown>> {
   }));
 }
 
+function arxivPdfUrl(id: string): string {
+  return `https://arxiv.org/pdf/${id}`;
+}
+
+function arxivAbsUrl(id: string): string {
+  return `https://arxiv.org/abs/${id.replace(/v\d+$/i, "")}`;
+}
+
+export function arxivArtifactFilename(input: {
+  id: string;
+  title?: unknown;
+}): string {
+  const title = String(input.title ?? "")
+    .replace(/\s+/g, " ")
+    .trim()
+    .replace(/[^A-Za-z0-9._-]+/g, "-")
+    .replace(/^-+|-+$/g, "")
+    .slice(0, 96);
+  return sanitizeFilename(`${input.id}${title ? `-${title}` : ""}.pdf`);
+}
+
+export function requireArxivPageRange(
+  firstPage: unknown,
+  lastPage: unknown,
+): { firstPage: number; lastPage: number } {
+  const first = Number(firstPage ?? 1);
+  const last = Number(lastPage ?? 20);
+  if (!Number.isInteger(first) || first < 1) {
+    throw new Error("arxiv first-page must be an integer >= 1.");
+  }
+  if (!Number.isInteger(last) || last < first) {
+    throw new Error("arxiv last-page must be an integer >= first-page.");
+  }
+  return { firstPage: first, lastPage: last };
+}
+
+export function requireArxivMaxChars(
+  value: unknown,
+  fallback = 40_000,
+): number {
+  if (value === undefined || value === null || value === "") return fallback;
+  const n = Number(value);
+  if (!Number.isInteger(n) || n < 1_000 || n > 1_000_000) {
+    throw new Error(
+      `arxiv max-chars must be an integer in [1000, 1000000]. Got: ${String(value)}`,
+    );
+  }
+  return n;
+}
+
+function truncateText(
+  text: string,
+  maxChars: number,
+): {
+  text: string;
+  truncated: boolean;
+  originalChars: number;
+} {
+  if (text.length <= maxChars) {
+    return { text, truncated: false, originalChars: text.length };
+  }
+  return {
+    text: `${text.slice(0, maxChars).trimEnd()}\n\n[truncated at ${maxChars} characters]`,
+    truncated: true,
+    originalChars: text.length,
+  };
+}
+
+async function fetchArxivEntryById(id: string): Promise<ArxivEntry> {
+  const params = new URLSearchParams({ id_list: id });
+  const rows = parseArxivEntries(await fetchArxiv(params));
+  const row = rows[0];
+  if (!row) throw new Error(`No arXiv paper found for ${id}.`);
+  return row;
+}
+
+export async function readArxivPaper(
+  kwargs: Record<string, unknown>,
+): Promise<Record<string, unknown>> {
+  const id = normalizeArxivId(kwargs.id ?? kwargs.arxiv_id ?? kwargs.ref);
+  const entry = await fetchArxivEntryById(id);
+  const canonicalId = entry.id || id.replace(/v\d+$/i, "");
+  const pdfUrl = arxivPdfUrl(id);
+  const outputDir = resolve(String(kwargs.output ?? "./arxiv-downloads"));
+  const path = join(
+    outputDir,
+    arxivArtifactFilename({ id, title: entry.title }),
+  );
+  const download = await httpDownload(pdfUrl, path, {
+    Accept: "application/pdf,*/*",
+    Referer: arxivAbsUrl(canonicalId),
+    "User-Agent": "unicli-arxiv/1.0 (https://github.com/olo-dot-io/Uni-CLI)",
+  });
+  if (download.status === "failed" || !download.path) {
+    throw new Error(
+      `arXiv PDF download failed for ${id}: ${download.error ?? "no path"}.`,
+    );
+  }
+
+  const { firstPage, lastPage } = requireArxivPageRange(
+    kwargs["first-page"] ?? kwargs.firstPage,
+    kwargs["last-page"] ?? kwargs.lastPage,
+  );
+  const maxChars = requireArxivMaxChars(
+    kwargs["max-chars"] ?? kwargs.maxChars,
+    40_000,
+  );
+  const { stdout } = await execFileAsync(
+    "pdftotext",
+    [
+      "-layout",
+      "-enc",
+      "UTF-8",
+      "-f",
+      String(firstPage),
+      "-l",
+      String(lastPage),
+      download.path,
+      "-",
+    ],
+    { timeout: 60_000, maxBuffer: 10 * 1024 * 1024 },
+  );
+  const text = stdout.trim();
+  if (!text) {
+    throw new Error(
+      `pdftotext returned no text for arXiv ${id} pages ${firstPage}-${lastPage}.`,
+    );
+  }
+  const truncated = truncateText(text, maxChars);
+  return {
+    id: canonicalId,
+    title: entry.title,
+    authors: entry.authors
+      .split(/\s*,\s*/)
+      .map((author) => author.trim())
+      .filter(Boolean),
+    year: Number(entry.published.slice(0, 4)) || undefined,
+    date: entry.published,
+    venue: "arXiv",
+    type: "preprint",
+    abstract: entry.abstract,
+    arxiv_id: canonicalId,
+    source_adapter: "arxiv",
+    source_url: arxivAbsUrl(canonicalId),
+    pdf_url: pdfUrl,
+    path: download.path,
+    text: truncated.text,
+    text_chars: truncated.originalChars,
+    text_truncated: truncated.truncated,
+    text_source: "pdf",
+    retrieved_at: new Date().toISOString(),
+  };
+}
+
 cli({
   site: "arxiv",
   name: "author",
@@ -192,6 +373,72 @@ cli({
   },
 });
 
+cli({
+  site: "arxiv",
+  name: "read",
+  description: "Download an arXiv PDF by ID and extract text with pdftotext",
+  domain: "arxiv.org",
+  strategy: Strategy.PUBLIC,
+  args: [
+    {
+      name: "id",
+      type: "str",
+      required: true,
+      positional: true,
+      description: "arXiv paper ID (e.g. 1706.03762)",
+      "x-unicli-kind": "id",
+      "x-unicli-accepts": ["url"],
+    },
+    {
+      name: "output",
+      type: "str",
+      default: "./arxiv-downloads",
+      description: "Output directory",
+      "x-unicli-kind": "path",
+    },
+    {
+      name: "first-page",
+      type: "int",
+      default: 1,
+      description: "First PDF page to extract",
+    },
+    {
+      name: "last-page",
+      type: "int",
+      default: 20,
+      description: "Last PDF page to extract",
+    },
+    {
+      name: "max-chars",
+      type: "int",
+      default: 40000,
+      description: "Maximum extracted text characters",
+    },
+  ],
+  columns: [
+    "id",
+    "title",
+    "source_adapter",
+    "source_url",
+    "pdf_url",
+    "path",
+    "text_source",
+    "text",
+    "text_chars",
+    "text_truncated",
+  ],
+  capabilities: [
+    "http.fetch",
+    "http.download",
+    "subprocess.exec",
+    "scholar.fulltext",
+    "scholar.pdf",
+  ],
+  executables: ["pdftotext"],
+  minimum_capability: "subprocess.exec",
+  func: async (_page, kwargs) => [await readArxivPaper(kwargs)],
+});
+
 cli({
   site: "arxiv",
   name: "recent",

package/src/adapters/baidu-scholar/search.ts

@@ -1,7 +1,26 @@
+/**
+ * @owner       src::adapters::baidu-scholar::search
+ * @does        Registers Baidu Scholar public browser search as a discovery-only scholarly source.
+ * @needs       xueshu.baidu.com current `/ndscholar/browse/search` result DOM, src/registry.ts, src/types.ts, browser tools
+ * @feeds       src/commands/scholar.ts capability discovery, `unicli baidu-scholar search`, `unicli scholar coverage/doctor`
+ * @breaks      Baidu Scholar route or result-card DOM drift can return empty rows or navigation errors.
+ * @invariants  Search is discovery-only; source/provider links are hints, not PDF/full-text proof.
+ * @side-effects Navigates a Uni-CLI managed browser page to Baidu Scholar public search.
+ * @perf        O(limit) DOM extraction after one page navigation.
+ * @concurrency safe — command state is page-local
+ * @test        live smoke via `unicli baidu-scholar search <query>`; URL contract in tests/unit/adapters/scholar-sources.test.ts
+ * @stability   experimental
+ * @since       2026-06-27
+ */
+
 import { cli, Strategy } from "../../registry.js";
 import type { IPage } from "../../types.js";
 import { intArg, js, str } from "../_shared/browser-tools.js";
 
+export function buildBaiduScholarSearchUrl(query: string): string {
+  return `https://xueshu.baidu.com/ndscholar/browse/search?wd=${encodeURIComponent(query)}`;
+}
+
 cli({
   site: "baidu-scholar",
   name: "search",
@@ -13,7 +32,15 @@ cli({
     { name: "query", type: "str", required: true, positional: true },
     { name: "limit", type: "int", default: 10 },
   ],
-  columns: ["title", "authors", "source", "url"],
+  columns: [
+    "id",
+    "title",
+    "authors",
+    "source",
+    "year",
+    "cited_by_count",
+    "source_url",
+  ],
   capabilities: [
     "mcp-browser.navigate",
     "mcp-browser.evaluate",
@@ -22,19 +49,55 @@ cli({
   func: async (page, kwargs) => {
     const p = page as IPage;
     const limit = intArg(kwargs.limit, 10, 50);
-    await p.goto(
-      `https://xueshu.baidu.com/s?wd=${encodeURIComponent(str(kwargs.query))}`,
-      { settleMs: 2500 },
-    );
+    await p.goto(buildBaiduScholarSearchUrl(str(kwargs.query)), {
+      settleMs: 3000,
+    });
     const rows = await p.evaluate(`(() => {
-      const cards = [...document.querySelectorAll('.result, .sc_content, .result-item')];
+      const normalize = (value) => (value || '').replace(/\\s+/g, ' ').trim();
+      const cleanAuthor = (value) => normalize(value).replace(/[，,]+$/g, '');
+      const paperId = (url) => {
+        try {
+          return new URL(url, location.href).searchParams.get('paperid') || '';
+        } catch {
+          return '';
+        }
+      };
+      const cards = [...document.querySelectorAll('.paper-wrap.result, .result, .sc_content, .result-item')];
       return cards.map((card) => {
-        const link = card.querySelector('h3 a, .t a, a[href]');
+        const link = card.querySelector('.paper-title a[href], h3 a[href], .t a[href], a[href]');
+        const url = link ? new URL(link.getAttribute('href') || '', location.href).href : '';
+        const info = card.querySelector('.paper-info');
+        const infoText = normalize(info?.textContent);
+        const authors = [...(info?.querySelectorAll('a[href*="author"]') || [])]
+          .map((node) => cleanAuthor(node.textContent))
+          .filter(Boolean);
+        const source = normalize(
+          [...(info?.querySelectorAll('a[href]') || [])]
+            .find((node) => {
+              const href = node.getAttribute('href') || '';
+              return !href.includes('author%3A') && !href.includes('refpaperuri');
+            })?.textContent
+        ).replace(/^《|》$/g, '');
+        const sourceLinks = [...card.querySelectorAll('.paper-source a[href]')]
+          .map((node) => ({
+            label: normalize(node.textContent),
+            url: new URL(node.getAttribute('href') || '', location.href).href
+          }))
+          .filter((item) => item.label && item.url && !item.url.startsWith('javascript:'));
+        const citedText = normalize(card.querySelector('.paper-info a[href*="refpaperuri"]')?.textContent);
         return {
-          title: (link?.textContent || '').replace(/\\s+/g, ' ').trim(),
-          authors: (card.querySelector('.author_text, .sc_info, .c_font')?.textContent || '').replace(/\\s+/g, ' ').trim(),
-          source: (card.querySelector('.journal_title, .sc_info')?.textContent || '').replace(/\\s+/g, ' ').trim(),
-          url: link ? new URL(link.getAttribute('href') || '', location.href).href : ''
+          id: paperId(url) || url || normalize(link?.textContent),
+          title: normalize(link?.textContent),
+          authors: authors.join(', '),
+          source,
+          venue: source,
+          type: normalize(card.querySelector('.paper-type')?.textContent),
+          year: (infoText.match(/(19|20)\\d{2}/) || [])[0] || '',
+          abstract: normalize(card.querySelector('.paper-abstract')?.textContent).replace(/\\s*查看全部>>$/, ''),
+          cited_by_count: citedText.match(/\\d+/)?.[0] || '',
+          source_url: url,
+          url,
+          source_links: sourceLinks
         };
       }).filter((row) => row.title).slice(0, ${js(limit)});
     })()`);

package/src/adapters/biorxiv/preprints.ts

@@ -0,0 +1,112 @@
+/**
+ * @owner       src::adapters::biorxiv::preprints
+ * @does        Registers bioRxiv recent/search, DOI metadata, PDF download, and read commands backed by the official xRxiv API helpers.
+ * @needs       src/adapters/rxiv/preprints.ts, api.biorxiv.org, bioRxiv PDF/JATS asset URLs.
+ * @feeds       surface coverage ledger, scholarly preprint discovery/search, scholar DOI read/download routing.
+ * @breaks      bioRxiv API drift, date-window search exhaustion, Cloudflare denial on source assets, or missing pdftotext stops read/download rather than fabricating text.
+ */
+
+import { cli, Strategy } from "../../registry.js";
+import {
+  downloadRxivPdf,
+  fetchPaperRow,
+  fetchRecentRows,
+  fetchSearchRows,
+  readRxivPaper,
+  RXIV_DOWNLOAD_ARGS,
+  RXIV_DOWNLOAD_CAPABILITIES,
+  RXIV_DOWNLOAD_COLUMNS,
+  RXIV_PAPER_ARGS,
+  RXIV_PAPER_CAPABILITIES,
+  RXIV_PAPER_COLUMNS,
+  RXIV_READ_ARGS,
+  RXIV_READ_CAPABILITIES,
+  RXIV_READ_COLUMNS,
+  RXIV_RECENT_ARGS,
+  RXIV_RECENT_CAPABILITIES,
+  RXIV_RECENT_COLUMNS,
+  RXIV_SEARCH_ARGS,
+  RXIV_SEARCH_CAPABILITIES,
+  RXIV_SEARCH_COLUMNS,
+  type RxivConfig,
+} from "../rxiv/preprints.js";
+
+const CONFIG: RxivConfig = {
+  site: "biorxiv",
+  label: "bioRxiv",
+  apiServer: "biorxiv",
+  webOrigin: "https://www.biorxiv.org",
+};
+const DOMAIN = "api.biorxiv.org";
+
+cli({
+  site: "biorxiv",
+  name: "recent",
+  description: "List recent bioRxiv preprints from the official API",
+  domain: DOMAIN,
+  strategy: Strategy.PUBLIC,
+  args: RXIV_RECENT_ARGS,
+  columns: RXIV_RECENT_COLUMNS,
+  capabilities: RXIV_RECENT_CAPABILITIES,
+  func: async (_page, kwargs) => fetchRecentRows(CONFIG, kwargs),
+});
+
+cli({
+  site: "biorxiv",
+  name: "search",
+  description:
+    "Search bioRxiv official API metadata within a bounded date window",
+  domain: DOMAIN,
+  strategy: Strategy.PUBLIC,
+  args: RXIV_SEARCH_ARGS,
+  columns: RXIV_SEARCH_COLUMNS,
+  capabilities: RXIV_SEARCH_CAPABILITIES,
+  func: async (_page, kwargs) => fetchSearchRows(CONFIG, kwargs),
+});
+
+cli({
+  site: "biorxiv",
+  name: "paper",
+  description: "Fetch bioRxiv preprint metadata by DOI",
+  domain: DOMAIN,
+  strategy: Strategy.PUBLIC,
+  args: RXIV_PAPER_ARGS,
+  columns: RXIV_PAPER_COLUMNS,
+  capabilities: RXIV_PAPER_CAPABILITIES,
+  func: async (_page, kwargs) => [
+    await fetchPaperRow(CONFIG, kwargs.doi ?? kwargs.id ?? kwargs.ref),
+  ],
+});
+
+cli({
+  site: "biorxiv",
+  name: "download",
+  description: "Download a bioRxiv preprint PDF by DOI",
+  domain: DOMAIN,
+  strategy: Strategy.PUBLIC,
+  args: RXIV_DOWNLOAD_ARGS,
+  columns: RXIV_DOWNLOAD_COLUMNS,
+  capabilities: RXIV_DOWNLOAD_CAPABILITIES,
+  minimum_capability: "http.download",
+  func: async (_page, kwargs) => [
+    await downloadRxivPdf(
+      CONFIG,
+      await fetchPaperRow(CONFIG, kwargs.doi ?? kwargs.id ?? kwargs.ref),
+      kwargs.output,
+    ),
+  ],
+});
+
+cli({
+  site: "biorxiv",
+  name: "read",
+  description:
+    "Read bioRxiv preprint text by DOI, preferring JATS XML before PDF extraction",
+  domain: DOMAIN,
+  strategy: Strategy.PUBLIC,
+  args: RXIV_READ_ARGS,
+  columns: RXIV_READ_COLUMNS,
+  capabilities: RXIV_READ_CAPABILITIES,
+  minimum_capability: "subprocess.exec",
+  func: async (_page, kwargs) => [await readRxivPaper(CONFIG, kwargs)],
+});