@zenalexa/unicli 0.220.1 → 0.221.1

package/src/adapters/pubmed/articles.ts

@@ -304,6 +304,7 @@ cli({
     { name: "limit", type: "int", default: 20, description: "Max results" },
   ],
   columns: SUMMARY_COLUMNS,
+  capabilities: ["http.fetch", "scholar.search"],
   func: async (_page, kwargs) => {
     const query = requirePubMedText(kwargs.query, "query");
     const limit = requirePubMedLimit(kwargs.limit);
@@ -345,6 +346,7 @@ cli({
     },
   ],
   columns: ["field", "value"],
+  capabilities: ["http.fetch", "scholar.get"],
   func: async (_page, kwargs) => {
     const pmid = requirePmid(kwargs.pmid);
     const xml = String(
@@ -371,6 +373,7 @@ cli({
     { name: "limit", type: "int", default: 20, description: "Max results" },
   ],
   columns: SUMMARY_COLUMNS,
+  capabilities: ["http.fetch", "scholar.author", "scholar.search"],
   func: async (_page, kwargs) => {
     const name = requirePubMedText(kwargs.name, "author");
     const limit = requirePubMedLimit(kwargs.limit);
@@ -414,6 +417,7 @@ cli({
     { name: "limit", type: "int", default: 20, description: "Max results" },
   ],
   columns: SUMMARY_COLUMNS,
+  capabilities: ["http.fetch", "scholar.citations", "scholar.references"],
   func: async (_page, kwargs) => {
     const pmid = requirePmid(kwargs.pmid);
     const direction = requireChoice(
@@ -459,6 +463,7 @@ cli({
     { name: "limit", type: "int", default: 20, description: "Max results" },
   ],
   columns: RELATED_COLUMNS,
+  capabilities: ["http.fetch", "scholar.search"],
   func: async (_page, kwargs) => {
     const pmid = requirePmid(kwargs.pmid);
     const limit = requirePubMedLimit(kwargs.limit);

package/src/adapters/reddit/comments-tree.test.ts

@@ -0,0 +1,79 @@
+import { describe, expect, it } from "vitest";
+
+import { extractRedditCommentRows } from "./comments.js";
+import { normalizeCommentRows } from "../../social/comments.js";
+
+describe("reddit comments tree extraction", () => {
+  it("preserves nested reply hierarchy from Reddit JSON listings", () => {
+    const rawRows = extractRedditCommentRows(
+      [
+        { data: { children: [] } },
+        {
+          data: {
+            children: [
+              {
+                kind: "t1",
+                data: {
+                  id: "root",
+                  name: "t1_root",
+                  parent_id: "t3_post",
+                  author: "alice",
+                  body: "root body",
+                  score: 10,
+                  created_utc: 1,
+                  replies: {
+                    data: {
+                      children: [
+                        {
+                          kind: "t1",
+                          data: {
+                            id: "child",
+                            name: "t1_child",
+                            parent_id: "t1_root",
+                            author: "bob",
+                            body: "child body",
+                            score: 3,
+                            created_utc: 2,
+                            replies: "",
+                          },
+                        },
+                      ],
+                    },
+                  },
+                },
+              },
+            ],
+          },
+        },
+      ],
+      "r/test/comments/post/title",
+    );
+
+    const rows = normalizeCommentRows(rawRows, {
+      platform: "reddit",
+      contentId: "r/test/comments/post/title",
+    });
+
+    expect(rows).toEqual([
+      expect.objectContaining({
+        comment_id: "t1_root",
+        parent_id: "",
+        depth: 0,
+        path: "0001",
+        author: "alice",
+        text: "root body",
+        likes: 10,
+        replies: 1,
+      }),
+      expect.objectContaining({
+        comment_id: "t1_child",
+        parent_id: "t1_root",
+        depth: 1,
+        path: "0001.0001",
+        author: "bob",
+        text: "child body",
+        likes: 3,
+      }),
+    ]);
+  });
+});

package/src/adapters/reddit/comments.ts

@@ -0,0 +1,159 @@
+/**
+ * @owner   Reddit comments adapter.
+ * @does    Fetches a Reddit post comment listing and emits normalized nested comment rows.
+ * @needs   Public reddit JSON listing endpoint and a normal web User-Agent.
+ * @feeds   Social comment tree analysis and Reddit read workflows.
+ * @breaks  Reddit JSON schema or public endpoint access changes can block unauthenticated reads.
+ */
+
+import { USER_AGENT } from "../../constants.js";
+import { cli, Strategy } from "../../registry.js";
+import { normalizeCommentRows } from "../../social/comments.js";
+
+interface RedditListing {
+  data?: {
+    children?: RedditThing[];
+  };
+}
+
+interface RedditThing {
+  kind?: string;
+  data?: RedditCommentData;
+}
+
+interface RedditCommentData {
+  id?: string;
+  name?: string;
+  parent_id?: string;
+  author?: string;
+  body?: string;
+  score?: number;
+  created_utc?: number;
+  replies?: "" | RedditListing;
+}
+
+function normalizePostPath(input: string): string {
+  const raw = input.trim();
+  if (!raw) throw new Error("Reddit post URL or ID is required");
+  if (/^https?:\/\//i.test(raw)) {
+    const url = new URL(raw);
+    return url.pathname.replace(/^\/+/, "").replace(/\.json$/, "");
+  }
+  return raw.replace(/^\/+/, "").replace(/\.json$/, "");
+}
+
+function collectCommentRows(
+  listing: RedditListing,
+  contentId: string,
+  parentId = "",
+): Array<Record<string, unknown>> {
+  const rows: Array<Record<string, unknown>> = [];
+  for (const thing of listing.data?.children ?? []) {
+    if (thing.kind !== "t1") continue;
+    const data = thing.data;
+    if (!data?.id) continue;
+    const commentId = data.name ?? `t1_${data.id}`;
+    rows.push({
+      id: commentId,
+      parent_id:
+        data.parent_id && data.parent_id.startsWith("t1_")
+          ? data.parent_id
+          : parentId,
+      author: data.author ?? "",
+      body: data.body ?? "",
+      text: data.body ?? "",
+      score: data.score ?? 0,
+      replies:
+        data.replies && typeof data.replies === "object"
+          ? (data.replies.data?.children?.filter((child) => child.kind === "t1")
+              .length ?? 0)
+          : 0,
+      created: data.created_utc
+        ? new Date(data.created_utc * 1000).toISOString()
+        : "",
+      content_id: contentId,
+    });
+    if (data.replies && typeof data.replies === "object") {
+      rows.push(...collectCommentRows(data.replies, contentId, commentId));
+    }
+  }
+  return rows;
+}
+
+export function extractRedditCommentRows(
+  response: unknown,
+  contentId: string,
+): Array<Record<string, unknown>> {
+  if (!Array.isArray(response) || response.length < 2) {
+    throw new Error("Unexpected Reddit comments response");
+  }
+  return collectCommentRows(response[1] as RedditListing, contentId);
+}
+
+cli({
+  site: "reddit",
+  name: "comments",
+  description: "Reddit post comments with nested reply hierarchy",
+  domain: "www.reddit.com",
+  strategy: Strategy.PUBLIC,
+  args: [
+    {
+      name: "url",
+      required: true,
+      positional: true,
+      description: "Reddit post URL, path, or comments ID",
+      "x-unicli-accepts": ["url", "id"],
+    },
+    {
+      name: "sort",
+      default: "top",
+      choices: ["best", "top", "new", "controversial", "old", "qa"],
+      description: "Comment sort order",
+    },
+    {
+      name: "limit",
+      type: "int",
+      default: 20,
+      description: "Number of top-level comments",
+    },
+  ],
+  columns: [
+    "platform",
+    "content_id",
+    "comment_id",
+    "parent_id",
+    "depth",
+    "path",
+    "author",
+    "text",
+    "likes",
+    "replies",
+    "created",
+    "body",
+    "score",
+  ],
+  socialCapabilities: ["read", "comments", "comment_replies"],
+  func: async (_page, kwargs) => {
+    const postPath = normalizePostPath(String(kwargs.url ?? ""));
+    const sort = String(kwargs.sort ?? "top");
+    const limit = Number(kwargs.limit ?? 20);
+    const response = await fetch(
+      `https://www.reddit.com/${postPath}.json?sort=${encodeURIComponent(sort)}&limit=${encodeURIComponent(String(limit))}&raw_json=1`,
+      {
+        headers: {
+          "User-Agent": USER_AGENT,
+          Accept: "application/json",
+        },
+      },
+    );
+    if (!response.ok) {
+      throw new Error(`Reddit comments request failed: ${response.status}`);
+    }
+    const json = await response.json();
+    const rows = extractRedditCommentRows(json, postPath);
+    return normalizeCommentRows(rows, {
+      platform: "reddit",
+      contentId: postPath,
+    });
+  },
+});

package/src/adapters/semantic-scholar/papers.ts

@@ -0,0 +1,268 @@
+/**
+ * @owner       src::adapters::semantic-scholar::papers
+ * @does        Registers Semantic Scholar Graph API paper search, detail, citations, references, and PDF discovery commands.
+ * @needs       api.semanticscholar.org Graph v1, optional SEMANTIC_SCHOLAR_API_KEY, src/registry.ts
+ * @feeds       src/commands/scholar.ts via scholar.* capability tags
+ * @breaks      Graph API rate limits or response-shape drift surface as explicit adapter errors; no cached fallback is used.
+ * @invariants  Paper references are normalized to Semantic Scholar's accepted DOI:/ARXIV:/paperId formats; output maps to ScholarlyWorkRecord.
+ * @side-effects HTTPS egress to api.semanticscholar.org only
+ * @perf        O(limit) JSON mapping per command
+ * @concurrency safe
+ * @test        tests/unit/adapters/scholar-sources.test.ts
+ * @stability   experimental
+ * @since       2026-05-19
+ */
+
+import { cli, Strategy } from "../../registry.js";
+import type { ScholarlyWorkRecord } from "../../types/scholarly.js";
+
+const API = "https://api.semanticscholar.org/graph/v1";
+const FIELDS = [
+  "paperId",
+  "title",
+  "abstract",
+  "year",
+  "authors",
+  "citationCount",
+  "referenceCount",
+  "venue",
+  "publicationVenue",
+  "url",
+  "openAccessPdf",
+  "externalIds",
+].join(",");
+
+interface S2Paper {
+  paperId?: unknown;
+  title?: unknown;
+  abstract?: unknown;
+  year?: unknown;
+  authors?: Array<{ name?: unknown }>;
+  citationCount?: unknown;
+  referenceCount?: unknown;
+  venue?: unknown;
+  url?: unknown;
+  openAccessPdf?: { url?: unknown };
+  externalIds?: Record<string, unknown>;
+}
+
+function str(value: unknown): string {
+  return typeof value === "string" ? value.trim() : "";
+}
+
+function num(value: unknown): number | undefined {
+  return typeof value === "number" && Number.isFinite(value)
+    ? value
+    : undefined;
+}
+
+function bareDoi(value: unknown): string {
+  return str(value)
+    .replace(/^doi:/i, "")
+    .replace(/^https?:\/\/(?:dx\.)?doi\.org\//i, "");
+}
+
+function bareArxiv(value: unknown): string {
+  return str(value)
+    .replace(/^arxiv:/i, "")
+    .replace(/^https?:\/\/arxiv\.org\/(?:abs|pdf)\//i, "")
+    .replace(/\.pdf$/i, "")
+    .replace(/v\d+$/i, "");
+}
+
+export function requireSemanticScholarPaperRef(value: unknown): string {
+  const raw = String(value ?? "").trim();
+  if (!raw) throw new Error("semantic-scholar paper reference is required.");
+  const doi = bareDoi(raw);
+  if (/^10\.\S+\/\S+/.test(doi)) return `DOI:${doi}`;
+  if (
+    /^(?:arxiv:|https?:\/\/arxiv\.org\/(?:abs|pdf)\/|\d{4}\.\d{4,5})/i.test(raw)
+  ) {
+    return `ARXIV:${bareArxiv(raw)}`;
+  }
+  if (/^[a-f0-9]{40}$/i.test(raw)) return raw;
+  throw new Error(
+    `semantic-scholar paper reference "${raw}" is not recognised.`,
+  );
+}
+
+function headers(): Record<string, string> {
+  const out: Record<string, string> = {
+    Accept: "application/json",
+    "User-Agent":
+      "unicli-semantic-scholar/1.0 (https://github.com/olo-dot-io/Uni-CLI)",
+  };
+  const key = process.env.SEMANTIC_SCHOLAR_API_KEY?.trim();
+  if (key) out["x-api-key"] = key;
+  return out;
+}
+
+async function fetchS2(path: string, label: string): Promise<unknown> {
+  const response = await fetch(`${API}${path}`, { headers: headers() });
+  if (response.status === 404) throw new Error(`${label} returned no result.`);
+  if (response.status === 429) {
+    throw new Error(
+      `${label} returned HTTP 429; set SEMANTIC_SCHOLAR_API_KEY or retry later.`,
+    );
+  }
+  if (!response.ok)
+    throw new Error(`${label} returned HTTP ${response.status}.`);
+  const json = (await response.json()) as {
+    error?: unknown;
+    message?: unknown;
+  };
+  if (json.error || json.message) {
+    throw new Error(
+      `${label} returned API error: ${String(json.error ?? json.message)}.`,
+    );
+  }
+  return json;
+}
+
+export function mapSemanticScholarPaper(
+  paper: S2Paper,
+  source: string,
+): ScholarlyWorkRecord {
+  const id = str(paper.paperId);
+  if (!id)
+    throw new Error("Semantic Scholar returned a paper without paperId.");
+  const doi = bareDoi(paper.externalIds?.DOI);
+  const arxiv = bareArxiv(paper.externalIds?.ArXiv);
+  return {
+    id,
+    title: str(paper.title),
+    abstract: str(paper.abstract) || undefined,
+    authors: Array.isArray(paper.authors)
+      ? paper.authors.map((author) => str(author.name)).filter(Boolean)
+      : undefined,
+    year: num(paper.year),
+    venue: str(paper.venue) || undefined,
+    doi: doi || undefined,
+    arxiv_id: arxiv || undefined,
+    semantic_scholar_id: id,
+    cited_by_count: num(paper.citationCount),
+    references_count: num(paper.referenceCount),
+    pdf_url: str(paper.openAccessPdf?.url) || undefined,
+    source_adapter: source,
+    source_url: str(paper.url) || `https://www.semanticscholar.org/paper/${id}`,
+    retrieved_at: new Date().toISOString(),
+  };
+}
+
+function rows(
+  papers: unknown,
+  source = "semantic-scholar",
+): ScholarlyWorkRecord[] {
+  const list = Array.isArray(papers) ? papers : [];
+  return list.map((paper) => mapSemanticScholarPaper(paper as S2Paper, source));
+}
+
+cli({
+  site: "semantic-scholar",
+  name: "search",
+  description: "Search Semantic Scholar papers",
+  domain: "api.semanticscholar.org",
+  strategy: Strategy.PUBLIC,
+  args: [
+    { name: "query", type: "str", required: true, positional: true },
+    { name: "limit", type: "int", default: 20 },
+  ],
+  columns: [
+    "id",
+    "title",
+    "authors",
+    "year",
+    "venue",
+    "doi",
+    "pdf_url",
+    "source_url",
+  ],
+  capabilities: ["http.fetch", "scholar.search"],
+  func: async (_page, kwargs) => {
+    const query = String(kwargs.query ?? "").trim();
+    if (!query)
+      throw new Error("semantic-scholar search query cannot be empty.");
+    const limit = Math.min(Math.max(Number(kwargs.limit ?? 20), 1), 100);
+    const body = (await fetchS2(
+      `/paper/search?query=${encodeURIComponent(query)}&limit=${limit}&fields=${encodeURIComponent(FIELDS)}`,
+      "semantic-scholar search",
+    )) as { data?: S2Paper[] };
+    const out = rows(body.data);
+    if (out.length === 0)
+      throw new Error(`No Semantic Scholar papers matched "${query}".`);
+    return out;
+  },
+});
+
+cli({
+  site: "semantic-scholar",
+  name: "paper",
+  description: "Fetch one Semantic Scholar paper by paperId, DOI, or arXiv id",
+  domain: "api.semanticscholar.org",
+  strategy: Strategy.PUBLIC,
+  args: [{ name: "id", type: "str", required: true, positional: true }],
+  columns: [
+    "id",
+    "title",
+    "authors",
+    "year",
+    "venue",
+    "doi",
+    "pdf_url",
+    "source_url",
+  ],
+  capabilities: ["http.fetch", "scholar.get", "scholar.pdf"],
+  func: async (_page, kwargs) => {
+    const ref = requireSemanticScholarPaperRef(
+      kwargs.id ?? kwargs.ref ?? kwargs.doi ?? kwargs.arxiv_id,
+    );
+    const paper = (await fetchS2(
+      `/paper/${encodeURIComponent(ref)}?fields=${encodeURIComponent(FIELDS)}`,
+      `semantic-scholar paper ${ref}`,
+    )) as S2Paper;
+    return [mapSemanticScholarPaper(paper, "semantic-scholar")];
+  },
+});
+
+for (const [name, path, cap] of [
+  ["citations", "citations", "scholar.citations"],
+  ["references", "references", "scholar.references"],
+] as const) {
+  cli({
+    site: "semantic-scholar",
+    name,
+    description: `List Semantic Scholar paper ${name}`,
+    domain: "api.semanticscholar.org",
+    strategy: Strategy.PUBLIC,
+    args: [
+      { name: "id", type: "str", required: true, positional: true },
+      { name: "limit", type: "int", default: 20 },
+    ],
+    columns: [
+      "id",
+      "title",
+      "authors",
+      "year",
+      "venue",
+      "doi",
+      "pdf_url",
+      "source_url",
+    ],
+    capabilities: ["http.fetch", cap],
+    func: async (_page, kwargs) => {
+      const ref = requireSemanticScholarPaperRef(kwargs.id ?? kwargs.ref);
+      const limit = Math.min(Math.max(Number(kwargs.limit ?? 20), 1), 100);
+      const body = (await fetchS2(
+        `/paper/${encodeURIComponent(ref)}/${path}?limit=${limit}&fields=${encodeURIComponent(FIELDS)}`,
+        `semantic-scholar ${name} ${ref}`,
+      )) as { data?: Array<{ citingPaper?: S2Paper; citedPaper?: S2Paper }> };
+      const papers = (body.data ?? []).map((item) =>
+        name === "citations" ? item.citingPaper : item.citedPaper,
+      );
+      const out = rows(papers);
+      if (out.length === 0)
+        throw new Error(`No Semantic Scholar ${name} found for ${ref}.`);
+      return out;
+    },
+  });
+}

package/src/adapters/threads/post.test.ts

@@ -0,0 +1,64 @@
+import { describe, expect, it } from "vitest";
+
+import { parseThreadsPostHtml, parseThreadsProfileHtml } from "./post.js";
+
+const postHtml = String.raw`
+  <html>
+    <head>
+      <meta property="og:title" content="Mark Zuckerberg (@zuck) on Threads" />
+      <meta property="og:description" content="Today we&#039;re starting to roll out Incognito Chat &amp; private message requests." />
+      <meta property="og:url" content="https://www.threads.com/@zuck/post/DYSAIo_FL77" />
+      <meta property="og:image" content="https://scontent.cdninstagram.com/v/t51.2885-15/threads-image.jpg" />
+      <meta property="og:image:width" content="1920" />
+      <meta property="og:image:height" content="1440" />
+      <meta property="al:ios:url" content="barcelona://media?shortcode=DYSAIo_FL77" />
+      <link rel="alternate" href="https://threads.net/ap/users/17841401746480004/post/17955322232962801/" type="application/activity+json" />
+    </head>
+  </html>
+`;
+
+const profileHtml = String.raw`
+  <html>
+    <head>
+      <meta property="og:title" content="Mark Zuckerberg (@zuck) • Threads, Say more" />
+      <meta property="og:description" content="5.5M Followers • 145 Threads • Mostly superintelligence and MMA takes. See the latest conversations with @zuck." />
+      <meta property="og:url" content="https://www.threads.com/@zuck" />
+      <meta property="og:image" content="https://scontent.cdninstagram.com/v/t51.2885-19/profile.jpg" />
+    </head>
+  </html>
+`;
+
+describe("Threads public page metadata", () => {
+  it("extracts a public post from Threads metadata", () => {
+    expect(
+      parseThreadsPostHtml(
+        postHtml,
+        "https://www.threads.net/@zuck/post/DYSAIo_FL77",
+      ),
+    ).toEqual({
+      activity_json_url:
+        "https://threads.net/ap/users/17841401746480004/post/17955322232962801/",
+      author: "Mark Zuckerberg",
+      handle: "@zuck",
+      image_height: 1440,
+      image_url:
+        "https://scontent.cdninstagram.com/v/t51.2885-15/threads-image.jpg",
+      image_width: 1920,
+      shortcode: "DYSAIo_FL77",
+      text: "Today we're starting to roll out Incognito Chat & private message requests.",
+      url: "https://www.threads.com/@zuck/post/DYSAIo_FL77",
+    });
+  });
+
+  it("extracts a public profile from Threads metadata", () => {
+    expect(parseThreadsProfileHtml(profileHtml, "zuck")).toEqual({
+      avatar_url: "https://scontent.cdninstagram.com/v/t51.2885-19/profile.jpg",
+      bio: "Mostly superintelligence and MMA takes.",
+      followers: "5.5M",
+      handle: "@zuck",
+      name: "Mark Zuckerberg",
+      threads: "145",
+      url: "https://www.threads.com/@zuck",
+    });
+  });
+});