@zenalexa/unicli 0.225.2 → 0.225.3

package/src/adapters/huggingface-papers/search.yaml

@@ -17,24 +17,58 @@ args:
 
 pipeline:
   - fetch:
-      url: https://huggingface.co/api/papers
+      url: https://huggingface.co/api/papers/search
       params:
-        search: "${{ args.query }}"
+        q: "${{ args.query }}"
         limit: "${{ args.limit }}"
 
   - map:
-      title: "${{ item.title || '' }}"
-      authors: "${{ item.authors ? item.authors.slice(0,3).map(a => a.name || a).join(', ') : '' }}"
-      published: "${{ item.publishedAt || '' }}"
-      upvotes: "${{ item.upvotes || 0 }}"
-      url: "${{ 'https://huggingface.co/papers/' + (item.id || '') }}"
+      id: "${{ (item.paper || item).id || '' }}"
+      title: "${{ (item.paper || item).title || item.title || '' }}"
+      authors: "${{ (item.paper || item).authors ? (item.paper || item).authors.slice(0,3).map(a => a.name || a).join(', ') : '' }}"
+      published: "${{ (item.paper || item).publishedAt || item.publishedAt || '' }}"
+      upvotes: "${{ (item.paper || item).upvotes || item.upvotes || 0 }}"
+      url: "${{ 'https://huggingface.co/papers/' + ((item.paper || item).id || '') }}"
+      source_url: "${{ (item.paper || item).id ? 'https://huggingface.co/papers/' + (item.paper || item).id : '' }}"
+      pdf_url: "${{ (item.paper || item).id ? 'https://arxiv.org/pdf/' + (item.paper || item).id : '' }}"
+      code_url: "${{ (item.paper || item).githubRepo || '' }}"
+      github_stars: "${{ (item.paper || item).githubStars || '' }}"
+      project_url: "${{ (item.paper || item).projectPage || '' }}"
+      dataset_url: "${{ (item.paper || item).linkedDatasets && (item.paper || item).linkedDatasets[0] ? 'https://huggingface.co/datasets/' + (item.paper || item).linkedDatasets[0].id : '' }}"
+      model_urls: "${{ (item.paper || item).linkedModels ? (item.paper || item).linkedModels.map(x => 'https://huggingface.co/' + x.id).join(', ') : '' }}"
+      dataset_urls: "${{ (item.paper || item).linkedDatasets ? (item.paper || item).linkedDatasets.map(x => 'https://huggingface.co/datasets/' + x.id).join(', ') : '' }}"
+      space_urls: "${{ (item.paper || item).linkedSpaces ? (item.paper || item).linkedSpaces.map(x => 'https://huggingface.co/spaces/' + x.id).join(', ') : '' }}"
+      num_models: "${{ (item.paper || item).numTotalModels !== null && (item.paper || item).numTotalModels !== undefined ? (item.paper || item).numTotalModels : '' }}"
+      num_datasets: "${{ (item.paper || item).numTotalDatasets !== null && (item.paper || item).numTotalDatasets !== undefined ? (item.paper || item).numTotalDatasets : '' }}"
+      num_spaces: "${{ (item.paper || item).numTotalSpaces !== null && (item.paper || item).numTotalSpaces !== undefined ? (item.paper || item).numTotalSpaces : '' }}"
 
   - limit: ${{ args.limit }}
 
-columns: [title, authors, upvotes, published, url]
+columns:
+  [
+    id,
+    title,
+    authors,
+    upvotes,
+    published,
+    url,
+    source_url,
+    pdf_url,
+    code_url,
+    github_stars,
+    project_url,
+    dataset_url,
+    model_urls,
+    dataset_urls,
+    space_urls,
+    num_models,
+    num_datasets,
+    num_spaces,
+  ]
 
 # schema-v2 metadata — injected by `unicli migrate schema-v2`
-capabilities: ["http.fetch", "scholar.search", "scholar.code"]
+capabilities:
+  ["http.fetch", "scholar.search", "scholar.code", "scholar.datasets"]
 minimum_capability: http.fetch
 trust: public
 confidentiality: public

package/src/adapters/medrxiv/preprints.ts

@@ -0,0 +1,112 @@
+/**
+ * @owner       src::adapters::medrxiv::preprints
+ * @does        Registers medRxiv recent/search, DOI metadata, PDF download, and read commands backed by the official xRxiv API helpers.
+ * @needs       src/adapters/rxiv/preprints.ts, api.biorxiv.org medRxiv endpoints, medRxiv PDF/JATS asset URLs.
+ * @feeds       surface coverage ledger, clinical preprint discovery/search, scholar DOI read/download routing.
+ * @breaks      medRxiv API drift, date-window search exhaustion, source-asset denial, or missing pdftotext stops read/download rather than fabricating text.
+ */
+
+import { cli, Strategy } from "../../registry.js";
+import {
+  downloadRxivPdf,
+  fetchPaperRow,
+  fetchRecentRows,
+  fetchSearchRows,
+  readRxivPaper,
+  RXIV_DOWNLOAD_ARGS,
+  RXIV_DOWNLOAD_CAPABILITIES,
+  RXIV_DOWNLOAD_COLUMNS,
+  RXIV_PAPER_ARGS,
+  RXIV_PAPER_CAPABILITIES,
+  RXIV_PAPER_COLUMNS,
+  RXIV_READ_ARGS,
+  RXIV_READ_CAPABILITIES,
+  RXIV_READ_COLUMNS,
+  RXIV_RECENT_ARGS,
+  RXIV_RECENT_CAPABILITIES,
+  RXIV_RECENT_COLUMNS,
+  RXIV_SEARCH_ARGS,
+  RXIV_SEARCH_CAPABILITIES,
+  RXIV_SEARCH_COLUMNS,
+  type RxivConfig,
+} from "../rxiv/preprints.js";
+
+const CONFIG: RxivConfig = {
+  site: "medrxiv",
+  label: "medRxiv",
+  apiServer: "medrxiv",
+  webOrigin: "https://www.medrxiv.org",
+};
+const DOMAIN = "api.biorxiv.org";
+
+cli({
+  site: "medrxiv",
+  name: "recent",
+  description: "List recent medRxiv preprints from the official API",
+  domain: DOMAIN,
+  strategy: Strategy.PUBLIC,
+  args: RXIV_RECENT_ARGS,
+  columns: RXIV_RECENT_COLUMNS,
+  capabilities: RXIV_RECENT_CAPABILITIES,
+  func: async (_page, kwargs) => fetchRecentRows(CONFIG, kwargs),
+});
+
+cli({
+  site: "medrxiv",
+  name: "search",
+  description:
+    "Search medRxiv official API metadata within a bounded date window",
+  domain: DOMAIN,
+  strategy: Strategy.PUBLIC,
+  args: RXIV_SEARCH_ARGS,
+  columns: RXIV_SEARCH_COLUMNS,
+  capabilities: RXIV_SEARCH_CAPABILITIES,
+  func: async (_page, kwargs) => fetchSearchRows(CONFIG, kwargs),
+});
+
+cli({
+  site: "medrxiv",
+  name: "paper",
+  description: "Fetch medRxiv preprint metadata by DOI",
+  domain: DOMAIN,
+  strategy: Strategy.PUBLIC,
+  args: RXIV_PAPER_ARGS,
+  columns: RXIV_PAPER_COLUMNS,
+  capabilities: RXIV_PAPER_CAPABILITIES,
+  func: async (_page, kwargs) => [
+    await fetchPaperRow(CONFIG, kwargs.doi ?? kwargs.id ?? kwargs.ref),
+  ],
+});
+
+cli({
+  site: "medrxiv",
+  name: "download",
+  description: "Download a medRxiv preprint PDF by DOI",
+  domain: DOMAIN,
+  strategy: Strategy.PUBLIC,
+  args: RXIV_DOWNLOAD_ARGS,
+  columns: RXIV_DOWNLOAD_COLUMNS,
+  capabilities: RXIV_DOWNLOAD_CAPABILITIES,
+  minimum_capability: "http.download",
+  func: async (_page, kwargs) => [
+    await downloadRxivPdf(
+      CONFIG,
+      await fetchPaperRow(CONFIG, kwargs.doi ?? kwargs.id ?? kwargs.ref),
+      kwargs.output,
+    ),
+  ],
+});
+
+cli({
+  site: "medrxiv",
+  name: "read",
+  description:
+    "Read medRxiv preprint text by DOI, preferring JATS XML before PDF extraction",
+  domain: DOMAIN,
+  strategy: Strategy.PUBLIC,
+  args: RXIV_READ_ARGS,
+  columns: RXIV_READ_COLUMNS,
+  capabilities: RXIV_READ_CAPABILITIES,
+  minimum_capability: "subprocess.exec",
+  func: async (_page, kwargs) => [await readRxivPaper(CONFIG, kwargs)],
+});

package/src/adapters/neurips/proceedings.ts

@@ -1,12 +1,12 @@
 /**
  * @owner       src::adapters::neurips::proceedings
- * @does        Registers NeurIPS proceedings search over the official yearly paper list.
- * @needs       proceedings.neurips.cc static HTML, src/registry.ts
- * @feeds       src/commands/scholar.ts via scholar.search, scholar.pdf, and scholar.venue
- * @breaks      NeurIPS markup drift surfaces as empty parse output; no unrelated source fallback is used.
- * @invariants  Year is explicit; paper URLs are absolutized against proceedings.neurips.cc.
- * @side-effects HTTPS egress to proceedings.neurips.cc only
- * @perf        O(N) over one proceedings HTML page
+ * @does        Registers NeurIPS proceedings search, paper detail retrieval, and PDF text reading over official paper pages.
+ * @needs       proceedings.neurips.cc static HTML/PDFs, src/adapters/scholar-artifacts/pdf-read.ts, src/registry.ts
+ * @feeds       src/commands/scholar.ts via scholar.search, scholar.get, scholar.pdf, scholar.fulltext, and scholar.venue
+ * @breaks      NeurIPS markup/PDF drift, denied downloads, or missing pdftotext surface as explicit adapter errors; no unrelated source fallback is used.
+ * @invariants  Year is explicit; paper detail prefers citation_* metadata and official /file/ PDF URLs.
+ * @side-effects HTTPS egress to proceedings.neurips.cc; read writes one PDF artifact and executes pdftotext.
+ * @perf        O(N) over one proceedings HTML page; read is O(PDF bytes + selected pages)
  * @concurrency safe
  * @test        tests/unit/adapters/scholar-sources.test.ts
  * @stability   experimental
@@ -15,8 +15,18 @@
 
 import { cli, Strategy } from "../../registry.js";
 import type { ScholarlyWorkRecord } from "../../types/scholarly.js";
+import { readScholarPdf } from "../scholar-artifacts/pdf-read.js";
 
 const ORIGIN = "https://proceedings.neurips.cc";
+const NEURIPS_USER_AGENT =
+  "unicli-neurips/1.0 (https://github.com/olo-dot-io/Uni-CLI)";
+
+type NeuripsActionableError = Error & {
+  code?: string;
+  suggestion?: string;
+  retryable?: boolean;
+  alternatives?: string[];
+};
 
 function decode(value: string): string {
   return value
@@ -35,6 +45,21 @@ function absolute(path: string): string {
     : `${ORIGIN}${path.startsWith("/") ? "" : "/"}${path}`;
 }
 
+function metaContents(html: string, name: string): string[] {
+  const values: string[] = [];
+  const re = new RegExp(
+    `<meta\\s+name=["']${name}["']\\s+content=["']([^"']*)["'][^>]*>`,
+    "gi",
+  );
+  let match: RegExpExecArray | null;
+  while ((match = re.exec(html)) !== null) values.push(decode(match[1]));
+  return values;
+}
+
+function firstMetaContent(html: string, name: string): string {
+  return metaContents(html, name)[0] ?? "";
+}
+
 function requireYear(value: unknown): string {
   const year = String(value ?? "").trim();
   if (!/^\d{4}$/.test(year))
@@ -42,6 +67,90 @@ function requireYear(value: unknown): string {
   return year;
 }
 
+function requireNeuripsPaperId(value: unknown): string {
+  const raw = String(value ?? "").trim();
+  const id =
+    raw
+      .match(/\/(?:hash|file)\/([^/?#]+\.html)/)?.[1]
+      ?.replace(/\.html$/, "") ?? raw.replace(/\.html$/, "");
+  if (!/^[A-Za-z0-9_.-]+$/.test(id)) {
+    throw new Error(`NeurIPS paper id "${raw}" is not valid.`);
+  }
+  return id;
+}
+
+function abstractUrl(id: string, year: string): string {
+  return `${ORIGIN}/paper_files/paper/${year}/hash/${id}.html`;
+}
+
+function pdfUrlFromAbstractUrl(sourceUrl: string): string {
+  return sourceUrl
+    .replace("/hash/", "/file/")
+    .replace("-Abstract-", "-Paper-")
+    .replace(/\.html$/, ".pdf");
+}
+
+function neuripsUpstreamError(
+  label: string,
+  detail: string,
+): NeuripsActionableError {
+  const error = new Error(
+    `${label} failed: ${detail}.`,
+  ) as NeuripsActionableError;
+  error.code = "upstream_error";
+  error.suggestion =
+    "NeurIPS proceedings did not return the expected public paper page on this network path; retry later or verify the official proceedings.neurips.cc page manually.";
+  error.retryable =
+    /fetch failed|timeout|ECONNRESET|ETIMEDOUT|HTTP (429|5\d\d)/i.test(detail);
+  error.alternatives = [];
+  return error;
+}
+
+export function parseNeuripsPaperPage(
+  html: string,
+  sourceUrl: string,
+): ScholarlyWorkRecord {
+  const title =
+    firstMetaContent(html, "citation_title") ||
+    decode(
+      html
+        .match(/<h1 class="paper-title">([\s\S]*?)<\/h1>/i)?.[1]
+        ?.replace(/<[^>]+>/g, " ") ?? "",
+    );
+  if (!title) throw new Error("NeurIPS paper page did not expose a title.");
+  const year =
+    firstMetaContent(html, "citation_publication_date").match(/\d{4}/)?.[0] ??
+    sourceUrl.match(/\/paper\/(\d{4})\//)?.[1];
+  const id =
+    sourceUrl
+      .split("/")
+      .pop()
+      ?.replace(/\.html$/, "") ?? title;
+  const pdfUrl =
+    firstMetaContent(html, "citation_pdf_url") ||
+    html.match(/href=["']([^"']+-Paper-[^"']+\.pdf)["']/i)?.[1] ||
+    "";
+  return {
+    id,
+    title,
+    authors: metaContents(html, "citation_author"),
+    year: year ? Number(year) : undefined,
+    venue: "NeurIPS",
+    type: firstMetaContent(html, "citation_journal_title") || undefined,
+    doi: firstMetaContent(html, "citation_doi") || undefined,
+    abstract:
+      decode(
+        html
+          .match(/<p class="paper-abstract">([\s\S]*?)<\/p>\s*<\/p>/i)?.[1]
+          ?.replace(/<[^>]+>/g, " ") ?? "",
+      ) || undefined,
+    pdf_url: pdfUrl ? absolute(pdfUrl) : undefined,
+    source_adapter: "neurips",
+    source_url: sourceUrl,
+    retrieved_at: new Date().toISOString(),
+  };
+}
+
 export function parseNeuripsRows(
   html: string,
   year = "2024",
@@ -65,9 +174,7 @@ export function parseNeuripsRows(
         .filter(Boolean),
       year: Number(year),
       venue: "NeurIPS",
-      pdf_url: sourceUrl
-        .replace("-Abstract-", "-Paper-")
-        .replace(/\.html$/, ".pdf"),
+      pdf_url: pdfUrlFromAbstractUrl(sourceUrl),
       source_adapter: "neurips",
       source_url: sourceUrl,
       retrieved_at: new Date().toISOString(),
@@ -76,6 +183,54 @@ export function parseNeuripsRows(
   return out;
 }
 
+async function fetchNeuripsHtml(url: string, label: string): Promise<string> {
+  let response: Response;
+  try {
+    response = await fetch(url, {
+      headers: {
+        Accept: "text/html",
+        "User-Agent": NEURIPS_USER_AGENT,
+      },
+    });
+  } catch (error) {
+    throw neuripsUpstreamError(
+      label,
+      error instanceof Error ? error.message : String(error),
+    );
+  }
+  if (response.status === 404) throw new Error(`${label} returned no page.`);
+  if (!response.ok)
+    throw neuripsUpstreamError(label, `HTTP ${response.status}`);
+  return response.text();
+}
+
+async function readNeuripsPaperPdf(
+  row: ScholarlyWorkRecord,
+  kwargs: Record<string, unknown>,
+): Promise<Record<string, unknown>> {
+  if (!row.pdf_url) throw new Error(`NeurIPS paper ${row.id} has no PDF URL.`);
+  return readScholarPdf(
+    {
+      id: row.id,
+      title: row.title,
+      source_adapter: "neurips",
+      source_url: row.source_url,
+      pdf_url: row.pdf_url,
+      output: kwargs.output,
+      filename: kwargs.filename,
+      "first-page": kwargs["first-page"] ?? kwargs.firstPage,
+      "last-page": kwargs["last-page"] ?? kwargs.lastPage,
+      "max-chars": kwargs["max-chars"] ?? kwargs.maxChars,
+    },
+    {
+      site: "neurips",
+      command: "read",
+      defaultOutput: "./neurips-downloads",
+      userAgent: NEURIPS_USER_AGENT,
+    },
+  );
+}
+
 cli({
   site: "neurips",
   name: "search",
@@ -100,19 +255,14 @@ cli({
       .toLowerCase();
     if (!query) throw new Error("neurips search query cannot be empty.");
     const year = requireYear(kwargs.year);
-    const response = await fetch(`${ORIGIN}/paper_files/paper/${year}`, {
-      headers: {
-        Accept: "text/html",
-        "User-Agent":
-          "unicli-neurips/1.0 (https://github.com/olo-dot-io/Uni-CLI)",
-      },
-    });
-    if (response.status === 404)
-      throw new Error(`NeurIPS ${year} returned no proceedings page.`);
-    if (!response.ok)
-      throw new Error(`NeurIPS ${year} returned HTTP ${response.status}.`);
     const limit = Math.min(Math.max(Number(kwargs.limit ?? 20), 1), 200);
-    const rows = parseNeuripsRows(await response.text(), year)
+    const rows = parseNeuripsRows(
+      await fetchNeuripsHtml(
+        `${ORIGIN}/paper_files/paper/${year}`,
+        `NeurIPS ${year}`,
+      ),
+      year,
+    )
       .filter((row) =>
         `${row.title} ${row.authors?.join(" ") ?? ""}`
           .toLowerCase()
@@ -124,3 +274,97 @@ cli({
     return rows;
   },
 });
+
+cli({
+  site: "neurips",
+  name: "paper",
+  description: "Fetch NeurIPS proceedings paper metadata by page id",
+  domain: "proceedings.neurips.cc",
+  strategy: Strategy.PUBLIC,
+  args: [
+    { name: "id", type: "str", required: true, positional: true },
+    { name: "year", type: "str", default: "2024" },
+  ],
+  columns: [
+    "id",
+    "title",
+    "authors",
+    "year",
+    "venue",
+    "doi",
+    "pdf_url",
+    "source_url",
+  ],
+  capabilities: ["http.fetch", "scholar.get", "scholar.pdf"],
+  func: async (_page, kwargs) => {
+    const year = requireYear(kwargs.year);
+    const id = requireNeuripsPaperId(kwargs.id ?? kwargs.ref);
+    const url = abstractUrl(id, year);
+    return [
+      parseNeuripsPaperPage(
+        await fetchNeuripsHtml(url, `NeurIPS paper ${id}`),
+        url,
+      ),
+    ];
+  },
+});
+
+cli({
+  site: "neurips",
+  name: "read",
+  description:
+    "Download a NeurIPS proceedings paper PDF by page id and extract text",
+  domain: "proceedings.neurips.cc",
+  strategy: Strategy.PUBLIC,
+  args: [
+    { name: "id", type: "str", required: true, positional: true },
+    { name: "year", type: "str", default: "2024" },
+    {
+      name: "output",
+      type: "str",
+      default: "./neurips-downloads",
+      description: "Output directory for the downloaded PDF",
+      "x-unicli-kind": "path",
+    },
+    { name: "filename", type: "str", description: "Output PDF filename" },
+    { name: "first-page", type: "int", default: 1, description: "First page" },
+    { name: "last-page", type: "int", default: 20, description: "Last page" },
+    {
+      name: "max-chars",
+      type: "int",
+      default: 40000,
+      description: "Maximum extracted text characters",
+    },
+  ],
+  columns: [
+    "id",
+    "title",
+    "source_adapter",
+    "source_url",
+    "pdf_url",
+    "path",
+    "text_source",
+    "text",
+    "text_chars",
+    "text_truncated",
+  ],
+  capabilities: [
+    "http.fetch",
+    "http.download",
+    "subprocess.exec",
+    "scholar.fulltext",
+    "scholar.pdf",
+  ],
+  executables: ["pdftotext"],
+  minimum_capability: "subprocess.exec",
+  func: async (_page, kwargs) => {
+    const year = requireYear(kwargs.year);
+    const id = requireNeuripsPaperId(kwargs.id ?? kwargs.ref);
+    const url = abstractUrl(id, year);
+    const row = parseNeuripsPaperPage(
+      await fetchNeuripsHtml(url, `NeurIPS paper ${id}`),
+      url,
+    );
+    return [await readNeuripsPaperPdf(row, kwargs)];
+  },
+});

package/src/adapters/openalex/works.test.ts

@@ -40,8 +40,12 @@ describe("openalex agent-facing commands", () => {
           publication_year: 2026,
           cited_by_count: 5,
           authorships: [{ author: { display_name: "Ada" } }],
-          primary_location: { source: { display_name: "Journal" } },
-          open_access: { is_oa: true },
+          primary_location: {
+            landing_page_url: "https://publisher.test/paper",
+            pdf_url: "https://publisher.test/paper.pdf",
+            source: { display_name: "Journal" },
+          },
+          open_access: { is_oa: true, oa_url: "https://publisher.test/paper" },
           type: "article",
         },
       ],
@@ -62,7 +66,8 @@ describe("openalex agent-facing commands", () => {
       is_open_access: true,
       type: "article",
       doi: "10.1/example",
-      pdf_url: "",
+      pdf_url: "https://publisher.test/paper.pdf",
+      landing_url: "https://publisher.test/paper",
       openalex_id: "W1234",
       source_adapter: "openalex",
       source_url: "https://openalex.org/W1234",
@@ -86,7 +91,11 @@ describe("openalex agent-facing commands", () => {
           { author: { display_name: "Ada" } },
           { author: { display_name: "Grace" } },
         ],
-        primary_location: { source: { display_name: "Journal" } },
+        primary_location: {
+          landing_page_url: "https://publisher.test/paper",
+          source: { display_name: "Journal" },
+        },
+        best_oa_location: { pdf_url: "https://repository.test/paper.pdf" },
         cited_by_count: 5,
         open_access: { is_oa: true, oa_url: "https://example.test/pdf" },
         referenced_works: ["W1", "W2"],
@@ -98,6 +107,8 @@ describe("openalex agent-facing commands", () => {
       authors: "Ada, Grace",
       referencedCount: 2,
       doi: "10.1/example",
+      pdf_url: "https://repository.test/paper.pdf",
+      landing_url: "https://publisher.test/paper",
       abstract: "hello world",
     });
     expect(() => mapOpenAlexWorkRow({})).toThrow(