@zenalexa/unicli 0.225.2 → 0.225.3

package/src/adapters/scholar-artifacts/pdf-read.ts

@@ -0,0 +1,277 @@
+/**
+ * @owner       src::adapters::scholar-artifacts::pdf-read
+ * @does        Provides side-effect-free scholarly PDF download and pdftotext extraction helpers for source adapters.
+ * @needs       src/engine/executor.ts download/exec steps, src/engine/download.ts, pdftotext
+ * @feeds       src/adapters/scholar-artifacts/pdf.ts and source-specific scholarly read commands
+ * @breaks      Invalid PDF URLs, denied download paths, missing pdftotext, or empty extracted text throw before claim text reaches agents.
+ * @invariants  Helpers register no commands; callers pass the owning site/command so operation policy and resource attribution stay source-scoped.
+ * @side-effects HTTPS/HTTP egress to the supplied PDF URL; writes one PDF under the requested output directory; executes pdftotext.
+ * @perf        O(PDF bytes + extracted page range); page range defaults to first 20 pages.
+ * @concurrency safe — each invocation writes one deterministic artifact path.
+ * @test        tests/unit/adapters/scholar-artifacts.test.ts
+ * @stability   experimental
+ * @since       2026-06-27
+ */
+
+import { join, resolve } from "node:path";
+
+import { runPipeline } from "../../engine/executor.js";
+import { sanitizeFilename } from "../../engine/download.js";
+import { Strategy } from "../../registry.js";
+
+export interface ScholarArtifactDownloadRow {
+  id: string;
+  title: string;
+  source_adapter: string;
+  source_url?: string;
+  pdf_url: string;
+  path?: string;
+  _download?: unknown;
+}
+
+export interface ScholarPdfReadOptions {
+  site: string;
+  command: string;
+  defaultOutput: string;
+  userAgent: string;
+}
+
+function stringField(value: unknown): string {
+  return typeof value === "string" ? value.trim() : "";
+}
+
+export function requireScholarPdfUrl(value: unknown): string {
+  const url = stringField(value);
+  if (!url) throw new Error("scholar PDF URL is required.");
+  let parsed: URL;
+  try {
+    parsed = new URL(url);
+  } catch {
+    throw new Error(`scholar PDF URL "${url}" is not a valid URL.`);
+  }
+  if (parsed.protocol !== "https:" && parsed.protocol !== "http:") {
+    throw new Error(`scholar PDF URL "${url}" must use http or https.`);
+  }
+  return parsed.toString();
+}
+
+export function requireScholarPageRange(
+  firstPage: unknown,
+  lastPage: unknown,
+): { firstPage: number; lastPage: number } {
+  const first = Number(firstPage ?? 1);
+  const last = Number(lastPage ?? 20);
+  if (!Number.isInteger(first) || first < 1) {
+    throw new Error("first-page must be a positive integer.");
+  }
+  if (!Number.isInteger(last) || last < first) {
+    throw new Error(
+      "last-page must be an integer greater than or equal to first-page.",
+    );
+  }
+  return { firstPage: first, lastPage: last };
+}
+
+export function requireScholarMaxChars(
+  value: unknown,
+  fallback = 40_000,
+): number {
+  const raw =
+    value === undefined || value === null || value === "" ? fallback : value;
+  const n = typeof raw === "number" ? raw : Number(raw);
+  if (!Number.isInteger(n) || n < 1_000 || n > 1_000_000) {
+    throw new Error(
+      `scholar max-chars must be an integer in [1000, 1000000]. Got: ${String(value)}`,
+    );
+  }
+  return n;
+}
+
+export function truncateScholarText(
+  text: string,
+  maxChars: number,
+): { text: string; truncated: boolean; originalChars: number } {
+  const originalChars = text.length;
+  if (originalChars <= maxChars) {
+    return { text, truncated: false, originalChars };
+  }
+  return {
+    text: `${text.slice(0, maxChars).trimEnd()}\n\n[truncated at ${maxChars} characters]`,
+    truncated: true,
+    originalChars,
+  };
+}
+
+export function scholarArtifactFilename(input: {
+  source_adapter?: unknown;
+  id?: unknown;
+  title?: unknown;
+  filename?: unknown;
+}): string {
+  const explicit = stringField(input.filename);
+  if (explicit) {
+    const safe = sanitizeFilename(explicit);
+    return safe.toLowerCase().endsWith(".pdf") ? safe : `${safe}.pdf`;
+  }
+
+  const source = sanitizeFilename(
+    stringField(input.source_adapter) || "scholar",
+  );
+  const id = sanitizeFilename(stringField(input.id) || "paper");
+  const title = sanitizeFilename(stringField(input.title))
+    .replace(/\s+/g, "_")
+    .slice(0, 96);
+  const stem = title ? `${source}-${id}-${title}` : `${source}-${id}`;
+  return `${stem.slice(0, 180)}.pdf`;
+}
+
+function downloadPath(row: unknown): string {
+  if (!row || typeof row !== "object") return "";
+  const download = (row as { _download?: unknown })._download;
+  if (!download || typeof download !== "object") return "";
+  return stringField((download as { path?: unknown }).path);
+}
+
+function downloadFailure(row: unknown): string {
+  if (!row || typeof row !== "object") return "download step returned no row";
+  const download = (row as { _download?: unknown })._download;
+  if (!download || typeof download !== "object") {
+    return "download step returned no _download metadata";
+  }
+  const status = stringField((download as { status?: unknown }).status);
+  const error = stringField((download as { error?: unknown }).error);
+  return [status ? `status=${status}` : "", error ? `error=${error}` : ""]
+    .filter(Boolean)
+    .join(", ");
+}
+
+export async function downloadScholarPdf(
+  kwargs: Record<string, unknown>,
+  options: ScholarPdfReadOptions,
+): Promise<ScholarArtifactDownloadRow> {
+  const pdfUrl = requireScholarPdfUrl(kwargs.pdf_url);
+  const output = resolve(stringField(kwargs.output) || options.defaultOutput);
+  const filename = scholarArtifactFilename(kwargs);
+  const id = stringField(kwargs.id) || pdfUrl;
+  const title = stringField(kwargs.title) || id;
+  const sourceAdapter = stringField(kwargs.source_adapter) || options.site;
+  const sourceUrl = stringField(kwargs.source_url);
+
+  const rows = await runPipeline(
+    [
+      {
+        download: {
+          url: "${{ args.pdf_url }}",
+          dir: "${{ args.output }}",
+          filename: "${{ args.filename }}",
+          type: "document",
+          headers: {
+            Accept: "application/pdf,*/*",
+            "User-Agent": options.userAgent,
+          },
+        },
+      },
+    ],
+    {
+      args: {
+        pdf_url: pdfUrl,
+        output,
+        filename,
+      },
+      source: "internal",
+    },
+    undefined,
+    {
+      site: options.site,
+      command: options.command,
+      strategy: Strategy.PUBLIC,
+      domain: new URL(pdfUrl).hostname,
+      surface: "cli",
+    },
+  );
+  const row = rows[0];
+  const path = downloadPath(row);
+  if (!path) {
+    throw new Error(
+      `scholar PDF download failed for ${pdfUrl}: ${downloadFailure(row)}.`,
+    );
+  }
+
+  return {
+    id,
+    title,
+    source_adapter: sourceAdapter,
+    ...(sourceUrl ? { source_url: sourceUrl } : {}),
+    pdf_url: pdfUrl,
+    path,
+    _download:
+      row && typeof row === "object"
+        ? (row as { _download?: unknown })._download
+        : undefined,
+  };
+}
+
+export async function readScholarPdf(
+  kwargs: Record<string, unknown>,
+  options: ScholarPdfReadOptions,
+): Promise<Record<string, unknown>> {
+  const downloaded = await downloadScholarPdf(kwargs, options);
+  const { firstPage, lastPage } = requireScholarPageRange(
+    kwargs["first-page"] ?? kwargs.firstPage ?? kwargs.first_page,
+    kwargs["last-page"] ?? kwargs.lastPage ?? kwargs.last_page,
+  );
+  const maxChars = requireScholarMaxChars(
+    kwargs["max-chars"] ?? kwargs.maxChars ?? kwargs.max_chars,
+  );
+  const [text] = await runPipeline(
+    [
+      {
+        exec: {
+          command: "pdftotext",
+          args: [
+            "-layout",
+            "-enc",
+            "UTF-8",
+            "-f",
+            "${{ args.first_page }}",
+            "-l",
+            "${{ args.last_page }}",
+            "${{ args.file }}",
+            "-",
+          ],
+          parse: "text",
+          timeout: 60000,
+        },
+      },
+    ],
+    {
+      args: {
+        file: downloaded.path,
+        first_page: firstPage,
+        last_page: lastPage,
+      },
+      source: "internal",
+    },
+    undefined,
+    {
+      site: options.site,
+      command: options.command,
+      strategy: Strategy.PUBLIC,
+      surface: "cli",
+    },
+  );
+  const extracted = stringField(text);
+  if (!extracted) {
+    throw new Error(
+      `pdftotext extracted no text from ${downloaded.path ?? join(".", scholarArtifactFilename(kwargs))}.`,
+    );
+  }
+  const truncated = truncateScholarText(extracted, maxChars);
+  return {
+    ...downloaded,
+    text: truncated.text,
+    text_chars: truncated.originalChars,
+    text_truncated: truncated.truncated,
+    text_source: "pdf",
+  };
+}

package/src/adapters/scholar-artifacts/pdf.ts

@@ -0,0 +1,133 @@
+/**
+ * @owner       src::adapters::scholar-artifacts::pdf
+ * @does        Registers source-agnostic scholarly PDF artifact download and text extraction commands.
+ * @needs       src/engine/executor.ts download/exec steps, pdftotext, scholarly adapters that expose pdf_url
+ * @feeds       src/commands/scholar.ts generic scholar download/read workflows
+ * @breaks      PDF URL drift, denied download paths, missing pdftotext, or empty extracted text stop the artifact loop.
+ * @invariants  PDF bytes are downloaded through pipeline resource guards; text extraction uses the same pdftotext contract as pdf/read.
+ * @side-effects HTTPS egress to the supplied PDF URL; writes PDF files under the requested output directory; executes pdftotext for read-pdf.
+ * @perf        O(PDF bytes + extracted pages); page range defaults to the first 20 pages.
+ * @concurrency safe — each command invocation writes one deterministic PDF path
+ * @test        tests/unit/adapters/scholar-artifacts.test.ts
+ * @stability   experimental
+ * @since       2026-06-26
+ */
+
+import { cli, Strategy } from "../../registry.js";
+import { downloadScholarPdf, readScholarPdf } from "./pdf-read.js";
+
+export {
+  requireScholarMaxChars,
+  requireScholarPageRange,
+  requireScholarPdfUrl,
+  scholarArtifactFilename,
+  truncateScholarText,
+} from "./pdf-read.js";
+
+const SCHOLAR_ARTIFACT_PDF_OPTIONS = {
+  site: "scholar-artifacts",
+  command: "read-pdf",
+  defaultOutput: "./scholar-downloads",
+  userAgent:
+    "unicli-scholar-artifacts/1.0 (https://github.com/olo-dot-io/Uni-CLI)",
+};
+
+cli({
+  site: "scholar-artifacts",
+  name: "download-pdf",
+  description: "Download a scholarly PDF URL with artifact metadata",
+  domain: "scholarly-pdf",
+  strategy: Strategy.PUBLIC,
+  args: [
+    {
+      name: "pdf_url",
+      type: "str",
+      required: true,
+      positional: true,
+      description: "Open scholarly PDF URL",
+      format: "uri",
+    },
+    { name: "title", type: "str", description: "Paper title" },
+    { name: "id", type: "str", description: "Source-local paper id" },
+    { name: "source_adapter", type: "str", description: "Source adapter name" },
+    { name: "source_url", type: "str", description: "Landing page URL" },
+    {
+      name: "output",
+      type: "str",
+      default: "./scholar-downloads",
+      description: "Output directory",
+    },
+    { name: "filename", type: "str", description: "Output PDF filename" },
+  ],
+  columns: [
+    "id",
+    "title",
+    "source_adapter",
+    "pdf_url",
+    "source_url",
+    "path",
+    "_download",
+  ],
+  capabilities: ["http.download"],
+  minimum_capability: "http.download",
+  func: async (_page, kwargs) => [
+    await downloadScholarPdf(kwargs, {
+      ...SCHOLAR_ARTIFACT_PDF_OPTIONS,
+      command: "download-pdf",
+    }),
+  ],
+});
+
+cli({
+  site: "scholar-artifacts",
+  name: "read-pdf",
+  description: "Download a scholarly PDF URL and extract text with pdftotext",
+  domain: "scholarly-pdf",
+  strategy: Strategy.PUBLIC,
+  args: [
+    {
+      name: "pdf_url",
+      type: "str",
+      required: true,
+      positional: true,
+      description: "Open scholarly PDF URL",
+      format: "uri",
+    },
+    { name: "title", type: "str", description: "Paper title" },
+    { name: "id", type: "str", description: "Source-local paper id" },
+    { name: "source_adapter", type: "str", description: "Source adapter name" },
+    { name: "source_url", type: "str", description: "Landing page URL" },
+    {
+      name: "output",
+      type: "str",
+      default: "./scholar-downloads",
+      description: "Output directory",
+    },
+    { name: "filename", type: "str", description: "Output PDF filename" },
+    { name: "first-page", type: "int", default: 1, description: "First page" },
+    { name: "last-page", type: "int", default: 20, description: "Last page" },
+    {
+      name: "max-chars",
+      type: "int",
+      default: 40000,
+      description: "Maximum extracted text characters",
+    },
+  ],
+  columns: [
+    "id",
+    "title",
+    "source_adapter",
+    "pdf_url",
+    "source_url",
+    "path",
+    "text",
+    "text_chars",
+    "text_truncated",
+  ],
+  capabilities: ["http.download", "subprocess.exec"],
+  executables: ["pdftotext"],
+  minimum_capability: "subprocess.exec",
+  func: async (_page, kwargs) => [
+    await readScholarPdf(kwargs, SCHOLAR_ARTIFACT_PDF_OPTIONS),
+  ],
+});

package/src/adapters/semantic-scholar/papers.ts

@@ -1,12 +1,12 @@
 /**
  * @owner       src::adapters::semantic-scholar::papers
- * @does        Registers Semantic Scholar Graph API paper search, detail, citations, references, and PDF discovery commands.
- * @needs       api.semanticscholar.org Graph v1, optional SEMANTIC_SCHOLAR_API_KEY, src/registry.ts
+ * @does        Registers Semantic Scholar Graph API paper search, detail, citations, references, and source PDF read commands.
+ * @needs       api.semanticscholar.org Graph v1, optional SEMANTIC_SCHOLAR_API_KEY, src/adapters/scholar-artifacts/pdf-read.ts, pdftotext
  * @feeds       src/commands/scholar.ts via scholar.* capability tags
- * @breaks      Graph API rate limits or response-shape drift surface as explicit adapter errors; no cached fallback is used.
- * @invariants  Paper references are normalized to Semantic Scholar's accepted DOI:/ARXIV:/paperId formats; output maps to ScholarlyWorkRecord.
- * @side-effects HTTPS egress to api.semanticscholar.org only
- * @perf        O(limit) JSON mapping per command
+ * @breaks      Graph API rate limits, response-shape drift, missing OA PDF URLs, or pdftotext failures surface as explicit adapter errors; no cached fallback is used.
+ * @invariants  Paper references are normalized to Semantic Scholar's accepted DOI:/ARXIV:/paperId formats; read requires openAccessPdf.url before text is claimed.
+ * @side-effects HTTPS egress to api.semanticscholar.org and source PDF hosts; read writes one PDF and executes pdftotext.
+ * @perf        O(limit) JSON mapping per command; O(PDF bytes + extracted page range) for read
  * @concurrency safe
  * @test        tests/unit/adapters/scholar-sources.test.ts
  * @stability   experimental
@@ -15,6 +15,7 @@
 
 import { cli, Strategy } from "../../registry.js";
 import type { ScholarlyWorkRecord } from "../../types/scholarly.js";
+import { readScholarPdf } from "../scholar-artifacts/pdf-read.js";
 
 const API = "https://api.semanticscholar.org/graph/v1";
 const FIELDS = [
@@ -149,6 +150,33 @@ export function mapSemanticScholarPaper(
   };
 }
 
+async function readSemanticScholarPaperPdf(
+  row: ScholarlyWorkRecord,
+  kwargs: Record<string, unknown>,
+): Promise<Record<string, unknown>> {
+  const pdfUrl = str(row.pdf_url);
+  if (!pdfUrl) {
+    throw new Error(`Semantic Scholar paper ${row.id} has no source PDF URL.`);
+  }
+  return readScholarPdf(
+    {
+      ...kwargs,
+      id: row.id,
+      title: row.title,
+      source_adapter: row.source_adapter,
+      source_url: row.source_url,
+      pdf_url: pdfUrl,
+    },
+    {
+      site: "semantic-scholar",
+      command: "read",
+      defaultOutput: "./semantic-scholar-downloads",
+      userAgent:
+        "unicli-semantic-scholar/1.0 (https://github.com/olo-dot-io/Uni-CLI)",
+    },
+  );
+}
+
 function rows(
   papers: unknown,
   source = "semantic-scholar",
@@ -224,6 +252,70 @@ cli({
   },
 });
 
+cli({
+  site: "semantic-scholar",
+  name: "read",
+  description:
+    "Download a Semantic Scholar open-access paper PDF and extract text",
+  domain: "api.semanticscholar.org",
+  strategy: Strategy.PUBLIC,
+  args: [
+    { name: "id", type: "str", required: true, positional: true },
+    {
+      name: "output",
+      type: "str",
+      default: "./semantic-scholar-downloads",
+      description: "Output directory for the downloaded PDF",
+      "x-unicli-kind": "path",
+    },
+    { name: "filename", type: "str", description: "Output PDF filename" },
+    { name: "first-page", type: "int", default: 1, description: "First page" },
+    { name: "last-page", type: "int", default: 20, description: "Last page" },
+    {
+      name: "max-chars",
+      type: "int",
+      default: 40000,
+      description: "Maximum extracted text characters",
+    },
+  ],
+  columns: [
+    "id",
+    "title",
+    "source_adapter",
+    "source_url",
+    "pdf_url",
+    "path",
+    "text_source",
+    "text",
+    "text_chars",
+    "text_truncated",
+  ],
+  capabilities: [
+    "http.fetch",
+    "http.download",
+    "subprocess.exec",
+    "scholar.fulltext",
+    "scholar.pdf",
+  ],
+  executables: ["pdftotext"],
+  minimum_capability: "subprocess.exec",
+  func: async (_page, kwargs) => {
+    const ref = requireSemanticScholarPaperRef(
+      kwargs.id ?? kwargs.ref ?? kwargs.doi ?? kwargs.arxiv_id,
+    );
+    const paper = (await fetchS2(
+      `/paper/${encodeURIComponent(ref)}?fields=${encodeURIComponent(FIELDS)}`,
+      `semantic-scholar paper ${ref}`,
+    )) as S2Paper;
+    return [
+      await readSemanticScholarPaperPdf(
+        mapSemanticScholarPaper(paper, "semantic-scholar"),
+        kwargs,
+      ),
+    ];
+  },
+});
+
 for (const [name, path, cap] of [
   ["citations", "citations", "scholar.citations"],
   ["references", "references", "scholar.references"],