@zenalexa/unicli 0.225.2 → 0.225.3

package/src/adapters/unpaywall/works.ts

@@ -1,12 +1,12 @@
 /**
  * @owner       src::adapters::unpaywall::works
- * @does        Registers Unpaywall DOI open-access lookup for PDF availability.
- * @needs       api.unpaywall.org v2, UNPAYWALL_EMAIL or --email, src/registry.ts
- * @feeds       src/commands/scholar.ts via scholar.pdf and scholar.get
- * @breaks      Missing email is an explicit invalid-input error; Unpaywall drift surfaces as adapter error, never as a fabricated PDF.
- * @invariants  Only DOI-shaped references are accepted; best_oa_location is preferred for PDF and landing URLs.
- * @side-effects HTTPS egress to api.unpaywall.org only
- * @perf        O(1) per DOI
+ * @does        Registers Unpaywall DOI open-access lookup and source PDF read commands.
+ * @needs       api.unpaywall.org v2, UNPAYWALL_EMAIL or --email, src/adapters/scholar-artifacts/pdf-read.ts, pdftotext
+ * @feeds       src/commands/scholar.ts via scholar.pdf, scholar.get, and scholar.fulltext
+ * @breaks      Missing email is an explicit invalid-input error; Unpaywall drift, missing best OA PDF URLs, or pdftotext failures surface as adapter errors.
+ * @invariants  Only DOI-shaped references are accepted; best_oa_location is preferred for PDF and landing URLs; read requires a source-provided PDF URL.
+ * @side-effects HTTPS egress to api.unpaywall.org and source PDF hosts; read writes one PDF and executes pdftotext.
+ * @perf        O(1) DOI lookup plus O(PDF bytes + extracted page range) for read
  * @concurrency safe
  * @test        tests/unit/adapters/scholar-sources.test.ts
  * @stability   experimental
@@ -15,9 +15,17 @@
 
 import { cli, Strategy } from "../../registry.js";
 import type { ScholarlyWorkRecord } from "../../types/scholarly.js";
+import { readScholarPdf } from "../scholar-artifacts/pdf-read.js";
 
 const API = "https://api.unpaywall.org/v2";
 
+type UnpaywallActionableError = Error & {
+  code?: string;
+  suggestion?: string;
+  retryable?: boolean;
+  alternatives?: string[];
+};
+
 interface OaLocation {
   url_for_pdf?: unknown;
   url_for_landing_page?: unknown;
@@ -44,11 +52,37 @@ function bareDoi(value: unknown): string {
     .replace(/^https?:\/\/(?:dx\.)?doi\.org\//i, "");
 }
 
+function unpaywallInputError(
+  message: string,
+  suggestion: string,
+): UnpaywallActionableError {
+  const error = new Error(message) as UnpaywallActionableError;
+  error.code = "invalid_input";
+  error.suggestion = suggestion;
+  error.retryable = false;
+  error.alternatives = [];
+  return error;
+}
+
+function unpaywallUpstreamError(
+  message: string,
+  retryable: boolean,
+): UnpaywallActionableError {
+  const error = new Error(message) as UnpaywallActionableError;
+  error.code = "upstream_error";
+  error.suggestion =
+    "Unpaywall did not return a usable open-access response on this request; retry later, provide a valid requester email, or fall back to OpenAlex/Semantic Scholar.";
+  error.retryable = retryable;
+  error.alternatives = [];
+  return error;
+}
+
 export function requireUnpaywallDoi(value: unknown): string {
   const doi = bareDoi(value);
   if (!/^10\.\S+\/\S+/.test(doi)) {
-    throw new Error(
+    throw unpaywallInputError(
       `unpaywall DOI "${String(value ?? "")}" is not recognised.`,
+      "Pass a DOI such as 10.1038/nature12373 or https://doi.org/10.1038/nature12373.",
     );
   }
   return doi;
@@ -57,7 +91,10 @@ export function requireUnpaywallDoi(value: unknown): string {
 function requireEmail(value: unknown): string {
   const email = str(value) || process.env.UNPAYWALL_EMAIL?.trim() || "";
   if (!/^[^\s@]+@[^\s@]+\.[^\s@]+$/.test(email)) {
-    throw new Error("unpaywall lookup requires --email or UNPAYWALL_EMAIL.");
+    throw unpaywallInputError(
+      "unpaywall lookup requires --email or UNPAYWALL_EMAIL.",
+      "Pass --email <requester-email> to `unicli unpaywall oa`, or --unpaywall-email <requester-email> through `unicli scholar pdf/read/download`.",
+    );
   }
   return email;
 }
@@ -86,6 +123,32 @@ export function mapUnpaywallWork(
   };
 }
 
+async function readUnpaywallWorkPdf(
+  row: ScholarlyWorkRecord,
+  kwargs: Record<string, unknown>,
+): Promise<Record<string, unknown>> {
+  const pdfUrl = str(row.pdf_url);
+  if (!pdfUrl) {
+    throw new Error(`Unpaywall work ${row.id} has no source PDF URL.`);
+  }
+  return readScholarPdf(
+    {
+      ...kwargs,
+      id: row.id,
+      title: row.title,
+      source_adapter: row.source_adapter,
+      source_url: row.source_url,
+      pdf_url: pdfUrl,
+    },
+    {
+      site: "unpaywall",
+      command: "read",
+      defaultOutput: "./unpaywall-downloads",
+      userAgent: "unicli-unpaywall/1.0 (https://github.com/olo-dot-io/Uni-CLI)",
+    },
+  );
+}
+
 async function fetchUnpaywall(
   doi: string,
   email: string,
@@ -103,10 +166,17 @@ async function fetchUnpaywall(
   if (response.status === 404)
     throw new Error(`Unpaywall returned no result for ${doi}.`);
   if (response.status === 422)
-    throw new Error("Unpaywall rejected the email parameter.");
-  if (response.status === 429) throw new Error("Unpaywall returned HTTP 429.");
+    throw unpaywallInputError(
+      "Unpaywall rejected the email parameter.",
+      "Provide a valid requester email address; Unpaywall requires a real contact email for API use.",
+    );
+  if (response.status === 429)
+    throw unpaywallUpstreamError("Unpaywall returned HTTP 429.", true);
   if (!response.ok)
-    throw new Error(`Unpaywall returned HTTP ${response.status}.`);
+    throw unpaywallUpstreamError(
+      `Unpaywall returned HTTP ${response.status}.`,
+      response.status >= 500,
+    );
   return response.json() as Promise<UnpaywallWork>;
 }
 
@@ -136,3 +206,62 @@ cli({
     return [mapUnpaywallWork(await fetchUnpaywall(doi, email), "unpaywall")];
   },
 });
+
+cli({
+  site: "unpaywall",
+  name: "read",
+  description: "Download an Unpaywall open-access PDF by DOI and extract text",
+  domain: "api.unpaywall.org",
+  strategy: Strategy.PUBLIC,
+  args: [
+    { name: "doi", type: "str", required: true, positional: true },
+    { name: "email", type: "str", description: "Unpaywall requester email" },
+    {
+      name: "output",
+      type: "str",
+      default: "./unpaywall-downloads",
+      description: "Output directory for the downloaded PDF",
+      "x-unicli-kind": "path",
+    },
+    { name: "filename", type: "str", description: "Output PDF filename" },
+    { name: "first-page", type: "int", default: 1, description: "First page" },
+    { name: "last-page", type: "int", default: 20, description: "Last page" },
+    {
+      name: "max-chars",
+      type: "int",
+      default: 40000,
+      description: "Maximum extracted text characters",
+    },
+  ],
+  columns: [
+    "id",
+    "title",
+    "source_adapter",
+    "source_url",
+    "pdf_url",
+    "path",
+    "text_source",
+    "text",
+    "text_chars",
+    "text_truncated",
+  ],
+  capabilities: [
+    "http.fetch",
+    "http.download",
+    "subprocess.exec",
+    "scholar.fulltext",
+    "scholar.pdf",
+  ],
+  executables: ["pdftotext"],
+  minimum_capability: "subprocess.exec",
+  func: async (_page, kwargs) => {
+    const doi = requireUnpaywallDoi(kwargs.doi ?? kwargs.id ?? kwargs.ref);
+    const email = requireEmail(kwargs.email);
+    return [
+      await readUnpaywallWorkPdf(
+        mapUnpaywallWork(await fetchUnpaywall(doi, email), "unpaywall"),
+        kwargs,
+      ),
+    ];
+  },
+});

package/src/adapters/wanfang/search.ts

@@ -1,3 +1,18 @@
+/**
+ * @owner       src::adapters::wanfang::search
+ * @does        Registers the Wanfang browser search adapter as a discovery-only scholarly source with normalized paper rows.
+ * @needs       src/registry.ts, src/types.ts, src/adapters/_shared/browser-tools.ts, Wanfang public search pages
+ * @feeds       src/commands/scholar.ts capability discovery, `unicli wanfang search`, `unicli scholar coverage/doctor`
+ * @breaks      Upstream DOM changes can return empty search rows; missing id/source_url prevents scholar-layer normalization.
+ * @invariants  Search is discovery-only and never claims metadata-get, PDF, full-text, citation, review, code, or dataset evidence.
+ * @side-effects Navigates a Uni-CLI managed browser page to Wanfang public search.
+ * @perf        O(limit) DOM extraction after one page navigation.
+ * @concurrency safe — command state is page-local
+ * @test        tests/unit/commands/scholar.test.ts
+ * @stability   experimental
+ * @since       2026-06-27
+ */
+
 import { cli, Strategy } from "../../registry.js";
 import type { IPage } from "../../types.js";
 import { intArg, js, str } from "../_shared/browser-tools.js";
@@ -13,7 +28,12 @@ cli({
     { name: "query", type: "str", required: true, positional: true },
     { name: "limit", type: "int", default: 10 },
   ],
-  columns: ["title", "authors", "source", "url"],
+  columns: ["id", "title", "authors", "source", "year", "source_url"],
+  capabilities: [
+    "mcp-browser.navigate",
+    "mcp-browser.evaluate",
+    "scholar.search",
+  ],
   func: async (page, kwargs) => {
     const p = page as IPage;
     const limit = intArg(kwargs.limit, 10, 50);
@@ -22,14 +42,44 @@ cli({
       { settleMs: 2500 },
     );
     const rows = await p.evaluate(`(() => {
-      const cards = [...document.querySelectorAll('.normal-list .item, .result-list .item, .paper-item, .record-item')];
+      const normalize = (value) => (value || '').replace(/\\s+/g, ' ').trim();
+      const detailUrl = (id) => {
+        const parts = String(id || '').split('_');
+        if (parts.length < 2) return '';
+        const type = parts[0];
+        const key = parts.slice(1).join('_');
+        return key ? 'https://d.wanfangdata.com.cn/' + type + '/' + key : '';
+      };
+      const cards = [...document.querySelectorAll('.normal-list')].filter((card) =>
+        card.querySelector('.title-area .title, .title-id-hidden')
+      );
       return cards.map((card) => {
-        const link = card.querySelector('a[href], span.title a');
+        const id = normalize(card.querySelector('.title-id-hidden')?.textContent);
+        const title = normalize(card.querySelector('.title-area .title')?.textContent);
+        const authorArea = card.querySelector('.author-area');
+        const authorTexts = [...(authorArea?.querySelectorAll('.authors') || [])]
+          .map((node) => normalize(node.textContent))
+          .filter((text) => text && !/(19|20)\\d{2}年/.test(text) && text !== '等');
+        const source = normalize(authorArea?.querySelector('.periodical-title')?.textContent).replace(/^《|》$/g, '');
+        const type = normalize(authorArea?.querySelector('.essay-type')?.textContent);
+        const authorText = normalize(authorArea?.textContent);
+        const year = (authorText.match(/(19|20)\\d{2}/) || [])[0] || '';
+        const abstract = normalize(card.querySelector('.abstract-area')?.textContent).replace(/^摘要：?/, '');
+        const metrics = normalize(card.querySelector('.button-area')?.textContent);
+        const cited = (metrics.match(/被引[:：]?\\s*(\\d+)/) || [])[1] || '';
+        const url = detailUrl(id);
         return {
-          title: (card.querySelector('span.title, .title, a[href]')?.textContent || '').replace(/\\s+/g, ' ').trim(),
-          authors: (card.querySelector('span.authors, .authors, .author')?.textContent || '').replace(/\\s+/g, ' ').trim(),
-          source: (card.querySelector('.source, .journal, .info')?.textContent || '').replace(/\\s+/g, ' ').trim(),
-          url: link ? new URL(link.getAttribute('href') || '', location.href).href : ''
+          id: url || title,
+          title,
+          authors: authorTexts.join(', '),
+          source,
+          venue: source,
+          type,
+          year,
+          abstract,
+          cited_by_count: cited,
+          source_url: url,
+          url
         };
       }).filter((row) => row.title).slice(0, ${js(limit)});
     })()`);

package/src/adapters/cnki/search.yaml

@@ -1,49 +0,0 @@
-site: cnki
-name: search
-description: Search CNKI academic papers
-domain: scholar.cnki.net
-type: web-api
-strategy: public
-lint_listing_detail: skip
-
-args:
-  query:
-    type: str
-    required: true
-    positional: true
-    description: Search query
-  limit:
-    type: int
-    default: 20
-    description: Maximum results
-
-pipeline:
-  - fetch:
-      url: https://scholar.cnki.net/api/search
-      params:
-        q: "${{ args.query }}"
-        page: "1"
-        size: "${{ args.limit }}"
-        sort: relevant
-
-  - select: data.items
-
-  - map:
-      rank: ${{ index + 1 }}
-      title: "${{ (item.title || '') | strip_html }}"
-      authors: "${{ (item.authors || []).map(a => a.name).join(', ') }}"
-      source: "${{ item.publicationName || '' }}"
-      year: "${{ item.year || '' }}"
-      citations: ${{ item.citedCount || 0 }}
-
-  - limit: ${{ args.limit }}
-
-columns: [rank, title, authors, source, year, citations]
-
-# schema-v2 metadata — injected by `unicli migrate schema-v2`
-capabilities: ["http.fetch"]
-minimum_capability: http.fetch
-trust: public
-confidentiality: public
-quarantine: false
-schema_version: v2