@zenalexa/unicli 0.219.0 → 0.220.1

package/src/adapters/lobsters/read-domain.ts

@@ -0,0 +1,400 @@
+/**
+ * @owner   src/adapters/lobsters/read-domain.ts
+ * @does    Register agent-facing Lobsters read and domain commands.
+ * @needs   Public lobste.rs JSON endpoints, strict ids/domains, bounded comment tree rendering.
+ * @feeds   surface coverage ledger, Lobsters story readers, source-domain research workflows.
+ * @breaks  Lobsters JSON shape drift, weak tree bounds, or lossy HTML conversion degrades reads.
+ */
+
+import { cli, Strategy } from "../../registry.js";
+
+const LOBSTERS_BASE = "https://lobste.rs";
+const DOMAIN_PATTERN =
+  /^[a-z0-9]([a-z0-9-]{0,61}[a-z0-9])?(\.[a-z0-9]([a-z0-9-]{0,61}[a-z0-9])?)+$/i;
+
+interface LobstersComment {
+  short_id?: unknown;
+  parent_comment?: unknown;
+  comment?: unknown;
+  comment_plain?: unknown;
+  commenting_user?: unknown;
+  score?: unknown;
+  is_deleted?: unknown;
+  is_moderated?: unknown;
+}
+
+interface LobstersStory {
+  short_id?: unknown;
+  title?: unknown;
+  description?: unknown;
+  description_plain?: unknown;
+  submitter_user?: unknown;
+  score?: unknown;
+  comment_count?: unknown;
+  comments?: unknown;
+  created_at?: unknown;
+  tags?: unknown;
+  url?: unknown;
+  comments_url?: unknown;
+}
+
+function stringField(value: unknown): string {
+  return typeof value === "string" ? value : "";
+}
+
+function numberOrNull(value: unknown): number | null {
+  return typeof value === "number" && Number.isFinite(value) ? value : null;
+}
+
+function numberOrBlank(value: unknown): number | "" {
+  return typeof value === "number" && Number.isFinite(value) ? value : "";
+}
+
+export function requireLobstersShortId(value: unknown): string {
+  const id = String(value ?? "").trim();
+  if (!/^[a-z0-9]+$/.test(id)) {
+    throw new Error(`Invalid Lobsters short_id: ${String(value)}.`);
+  }
+  return id;
+}
+
+export function requireLobstersDomain(value: unknown): string {
+  const domain = String(value ?? "")
+    .trim()
+    .toLowerCase();
+  if (!domain) throw new Error("lobsters domain is required.");
+  if (!DOMAIN_PATTERN.test(domain)) {
+    throw new Error(
+      `lobsters domain "${String(value)}" is not a valid hostname.`,
+    );
+  }
+  return domain;
+}
+
+export function requireLobstersLimit(
+  value: unknown,
+  fallback: number,
+  max: number | null,
+  label: string,
+): number {
+  const raw =
+    value === undefined || value === null || value === "" ? fallback : value;
+  const limit = Number(raw);
+  if (!Number.isInteger(limit) || limit <= 0) {
+    throw new Error(`lobsters ${label} must be a positive integer.`);
+  }
+  if (max !== null && limit > max) {
+    throw new Error(`lobsters ${label} must be <= ${max}.`);
+  }
+  return limit;
+}
+
+export function requireLobstersMinInt(
+  value: unknown,
+  fallback: number,
+  min: number,
+  label: string,
+): number {
+  const raw =
+    value === undefined || value === null || value === "" ? fallback : value;
+  const n = Number(raw);
+  if (!Number.isInteger(n) || n < min) {
+    throw new Error(`lobsters ${label} must be an integer >= ${min}.`);
+  }
+  return n;
+}
+
+export function lobstersHtmlToText(value: unknown): string {
+  return String(value ?? "")
+    .replace(/<p>/gi, "\n\n")
+    .replace(/<\/p>/gi, "")
+    .replace(/<br\s*\/?>/gi, "\n")
+    .replace(/<i>(.*?)<\/i>/gi, "$1")
+    .replace(/<em>(.*?)<\/em>/gi, "$1")
+    .replace(/<strong>(.*?)<\/strong>/gi, "$1")
+    .replace(/<a[^>]*href="([^"]*)"[^>]*>(.*?)<\/a>/gi, "$2 ($1)")
+    .replace(/<pre><code>([\s\S]*?)<\/code><\/pre>/gi, "\n$1\n")
+    .replace(/<code>(.*?)<\/code>/gi, "`$1`")
+    .replace(/<[^>]+>/g, "")
+    .replace(/&#x27;/g, "'")
+    .replace(/&apos;/g, "'")
+    .replace(/&quot;/g, '"')
+    .replace(/&lt;/g, "<")
+    .replace(/&gt;/g, ">")
+    .replace(/&amp;/g, "&")
+    .replace(/&nbsp;/g, " ")
+    .replace(/&#x2F;/g, "/")
+    .trim();
+}
+
+function truncate(text: string, maxLength: number, suffix: string): string {
+  return text.length > maxLength
+    ? `${text.slice(0, maxLength)}${suffix}`
+    : text;
+}
+
+function indentLines(text: string, depth: number): string {
+  if (depth <= 0) return text;
+  const prefix = `${"  ".repeat(depth)}> `;
+  return text
+    .split("\n")
+    .map((line) => `${prefix}${line}`)
+    .join("\n");
+}
+
+function commentShortId(comment: LobstersComment): string {
+  return stringField(comment.short_id);
+}
+
+function commentParentId(comment: LobstersComment): string {
+  return stringField(comment.parent_comment);
+}
+
+function commentBody(comment: LobstersComment): string {
+  return (
+    stringField(comment.comment_plain) || lobstersHtmlToText(comment.comment)
+  );
+}
+
+function storyBody(story: LobstersStory): string {
+  return (
+    stringField(story.description_plain) ||
+    lobstersHtmlToText(story.description)
+  );
+}
+
+function visibleComments(story: LobstersStory): LobstersComment[] {
+  return Array.isArray(story.comments)
+    ? story.comments.filter((comment): comment is LobstersComment => {
+        return typeof comment === "object" && comment !== null;
+      })
+    : [];
+}
+
+export function buildLobstersReadRows(
+  story: LobstersStory | null | undefined,
+  options: {
+    limit: number;
+    maxDepth: number;
+    maxReplies: number;
+    maxLength: number;
+  },
+): Array<Record<string, unknown>> {
+  if (!story || !story.short_id) throw new Error("Lobsters story not found.");
+  const rows: Array<Record<string, unknown>> = [];
+  const body = truncate(
+    storyBody(story).trim(),
+    options.maxLength,
+    "\n... [truncated]",
+  );
+  const storyParts = [stringField(story.title)];
+  if (body) storyParts.push(`\n${body}`);
+  if (story.url) storyParts.push(`\n${stringField(story.url)}`);
+  rows.push({
+    type: "POST",
+    author: stringField(story.submitter_user) || "[deleted]",
+    score: numberOrNull(story.score) ?? 0,
+    text: storyParts.join("").trim(),
+  });
+
+  const children = new Map<string, LobstersComment[]>();
+  for (const comment of visibleComments(story)) {
+    const parent = commentParentId(comment);
+    const bucket = children.get(parent) ?? [];
+    bucket.push(comment);
+    children.set(parent, bucket);
+  }
+
+  function emit(comment: LobstersComment, depth: number): void {
+    if (comment.is_deleted || comment.is_moderated) return;
+    const id = commentShortId(comment);
+    rows.push({
+      type: depth === 0 ? "L0" : `L${depth}`,
+      author: stringField(comment.commenting_user) || "[deleted]",
+      score: numberOrBlank(comment.score),
+      text: indentLines(
+        truncate(commentBody(comment).trim(), options.maxLength, "..."),
+        depth,
+      ),
+    });
+    const kids = id ? (children.get(id) ?? []) : [];
+    if (depth + 1 >= options.maxDepth) {
+      if (kids.length > 0) {
+        rows.push({
+          type: `L${depth + 1}`,
+          author: "",
+          score: "",
+          text: `${"  ".repeat(depth + 1)}[+${kids.length} more replies]`,
+        });
+      }
+      return;
+    }
+    const visible = kids.slice(0, options.maxReplies);
+    for (const kid of visible) emit(kid, depth + 1);
+    const hidden = kids.length - visible.length;
+    if (hidden > 0) {
+      rows.push({
+        type: `L${depth + 1}`,
+        author: "",
+        score: "",
+        text: `${"  ".repeat(depth + 1)}[+${hidden} more replies]`,
+      });
+    }
+  }
+
+  const topLevel = children.get("") ?? [];
+  const visibleTop = topLevel.slice(0, options.limit);
+  for (const comment of visibleTop) emit(comment, 0);
+  const hiddenTop = topLevel.length - visibleTop.length;
+  if (hiddenTop > 0) {
+    rows.push({
+      type: "",
+      author: "",
+      score: "",
+      text: `[+${hiddenTop} more top-level comments]`,
+    });
+  }
+  return rows;
+}
+
+export function mapLobstersDomainRows(
+  stories: LobstersStory[],
+): Array<Record<string, unknown>> {
+  return stories.map((item, index) => ({
+    rank: index + 1,
+    id: stringField(item.short_id),
+    title: stringField(item.title),
+    score: numberOrNull(item.score),
+    author: stringField(item.submitter_user),
+    comments: numberOrNull(item.comment_count),
+    created_at: stringField(item.created_at).slice(0, 10),
+    tags: Array.isArray(item.tags) ? item.tags.map(String).join(", ") : "",
+    submission_url: stringField(item.url),
+    comments_url: stringField(item.comments_url),
+  }));
+}
+
+async function fetchLobstersJson<T>(url: string, label: string): Promise<T> {
+  const response = await fetch(url, {
+    headers: {
+      Accept: "application/json",
+      "User-Agent":
+        "unicli-lobsters/1.0 (https://github.com/olo-dot-io/Uni-CLI)",
+    },
+  });
+  if (response.status === 404) throw new Error(`${label} not found.`);
+  if (!response.ok)
+    throw new Error(`Lobsters returned HTTP ${response.status} for ${label}.`);
+  return (await response.json()) as T;
+}
+
+cli({
+  site: "lobsters",
+  name: "read",
+  description: "Read a Lobste.rs story and its comment tree",
+  domain: "lobste.rs",
+  strategy: Strategy.PUBLIC,
+  browser: false,
+  args: [
+    {
+      name: "id",
+      type: "str",
+      required: true,
+      positional: true,
+      description: "Lobste.rs short_id",
+    },
+    {
+      name: "limit",
+      type: "int",
+      default: 25,
+      description: "Max top-level comments",
+    },
+    { name: "depth", type: "int", default: 2, description: "Max reply depth" },
+    {
+      name: "replies",
+      type: "int",
+      default: 5,
+      description: "Max replies per comment",
+    },
+    {
+      name: "max-length",
+      type: "int",
+      default: 2000,
+      description: "Max characters per comment body",
+    },
+  ],
+  columns: ["type", "author", "score", "text"],
+  func: async (_page, kwargs) => {
+    const id = requireLobstersShortId(kwargs.id);
+    const story = await fetchLobstersJson<LobstersStory>(
+      `${LOBSTERS_BASE}/s/${id}.json`,
+      `lobsters/${id}`,
+    );
+    return buildLobstersReadRows(story, {
+      limit: requireLobstersLimit(kwargs.limit, 25, null, "read limit"),
+      maxDepth: requireLobstersLimit(kwargs.depth, 2, null, "read depth"),
+      maxReplies: requireLobstersLimit(kwargs.replies, 5, null, "read replies"),
+      maxLength: requireLobstersMinInt(
+        kwargs["max-length"] ?? kwargs.maxLength,
+        2000,
+        100,
+        "read max-length",
+      ),
+    });
+  },
+});
+
+cli({
+  site: "lobsters",
+  name: "domain",
+  description: "Lobste.rs stories submitted from a specific domain",
+  domain: "lobste.rs",
+  strategy: Strategy.PUBLIC,
+  browser: false,
+  args: [
+    {
+      name: "domain",
+      type: "str",
+      required: true,
+      positional: true,
+      description: "Source domain",
+    },
+    {
+      name: "limit",
+      type: "int",
+      default: 20,
+      description: "Number of stories",
+    },
+  ],
+  columns: [
+    "rank",
+    "id",
+    "title",
+    "score",
+    "author",
+    "comments",
+    "created_at",
+    "tags",
+    "submission_url",
+    "comments_url",
+  ],
+  func: async (_page, kwargs) => {
+    const domain = requireLobstersDomain(kwargs.domain);
+    const limit = requireLobstersLimit(kwargs.limit, 20, 25, "domain limit");
+    const stories = await fetchLobstersJson<unknown>(
+      `${LOBSTERS_BASE}/domains/${encodeURIComponent(domain)}.json`,
+      `lobsters domain ${domain}`,
+    );
+    if (!Array.isArray(stories) || stories.length === 0) {
+      throw new Error(`No Lobste.rs stories found for domain "${domain}".`);
+    }
+    return mapLobstersDomainRows(
+      stories
+        .filter(
+          (story): story is LobstersStory =>
+            typeof story === "object" && story !== null,
+        )
+        .slice(0, limit),
+    );
+  },
+});

package/src/adapters/mangadex/web.test.ts

@@ -0,0 +1,46 @@
+import { describe, expect, it } from "vitest";
+import { resolveCommand } from "../../registry.js";
+import { mapMangaDexAuthors, mapMangaDexManga } from "./web.js";
+
+describe("mangadex public commands", () => {
+  it("registers manga and author search", () => {
+    expect(
+      Object.keys(resolveCommand("mangadex", "manga")!.adapter.commands),
+    ).toEqual(expect.arrayContaining(["manga", "authors"]));
+  });
+
+  it("exposes year, sort, and content rating controls for recent manga lookup", () => {
+    expect(resolveCommand("mangadex", "manga")!.command.adapterArgs).toEqual(
+      expect.arrayContaining([
+        expect.objectContaining({ name: "year", type: "int" }),
+        expect.objectContaining({
+          name: "sort",
+          choices: expect.arrayContaining(["latest", "followed", "relevance"]),
+        }),
+        expect.objectContaining({
+          name: "content-rating",
+          choices: expect.arrayContaining(["safe", "suggestive", "erotica"]),
+        }),
+      ]),
+    );
+  });
+
+  it("maps manga rows", () => {
+    expect(
+      mapMangaDexManga([
+        {
+          id: "m1",
+          attributes: { title: { en: "Hanabi" }, status: "ongoing" },
+        },
+      ]),
+    ).toMatchObject([
+      { rank: 1, id: "m1", title: "Hanabi", status: "ongoing" },
+    ]);
+  });
+
+  it("maps author rows", () => {
+    expect(
+      mapMangaDexAuthors([{ id: "a1", attributes: { name: "Yuzusoft" } }]),
+    ).toMatchObject([{ rank: 1, id: "a1", name: "Yuzusoft" }]);
+  });
+});

package/src/adapters/mangadex/web.ts

@@ -0,0 +1,210 @@
+/**
+ * @owner   src/adapters/mangadex/web.ts
+ * @does    Register MangaDex public manga and author search commands.
+ * @needs   MangaDex public API and relationship include semantics.
+ * @feeds   Manga/doujin-adjacent title and creator discovery.
+ * @breaks  MangaDex API schema or public rate limits can block lookup.
+ */
+
+import { USER_AGENT } from "../../constants.js";
+import { cli, Strategy } from "../../registry.js";
+
+const API = "https://api.mangadex.org";
+
+function str(value: unknown): string {
+  return value === undefined || value === null ? "" : String(value);
+}
+
+function required(value: unknown, label: string): string {
+  const text = str(value).trim();
+  if (!text) throw new Error(`mangadex ${label} cannot be empty.`);
+  return text;
+}
+
+function requireLimit(value: unknown): number {
+  if (value === undefined || value === null || value === "") return 10;
+  const n = Number(value);
+  if (!Number.isInteger(n) || n < 1 || n > 50) {
+    throw new Error("mangadex limit must be an integer in [1, 50].");
+  }
+  return n;
+}
+
+function optionalYear(value: unknown): number | undefined {
+  if (value === undefined || value === null || value === "") return undefined;
+  const n = Number(value);
+  if (!Number.isInteger(n) || n < 1900 || n > 2100) {
+    throw new Error("mangadex year must be an integer in [1900, 2100].");
+  }
+  return n;
+}
+
+const SORT_PARAMS: Record<string, [string, string] | undefined> = {
+  relevance: undefined,
+  latest: ["order[latestUploadedChapter]", "desc"],
+  followed: ["order[followedCount]", "desc"],
+  year: ["order[year]", "desc"],
+};
+
+function applySort(url: URL, value: unknown): void {
+  const key = String(value ?? "relevance").trim();
+  const spec = SORT_PARAMS[key];
+  if (!(key in SORT_PARAMS)) {
+    throw new Error(
+      `mangadex sort must be one of: ${Object.keys(SORT_PARAMS).join(", ")}.`,
+    );
+  }
+  if (spec) url.searchParams.set(spec[0], spec[1]);
+}
+
+const CONTENT_RATINGS = new Set([
+  "safe",
+  "suggestive",
+  "erotica",
+  "pornographic",
+  "all",
+]);
+
+function applyContentRating(url: URL, value: unknown): void {
+  if (value === undefined || value === null || value === "") return;
+  const rating = String(value).trim();
+  if (!CONTENT_RATINGS.has(rating)) {
+    throw new Error(
+      `mangadex content_rating must be one of: ${Array.from(CONTENT_RATINGS).join(", ")}.`,
+    );
+  }
+  if (rating !== "all") url.searchParams.append("contentRating[]", rating);
+}
+
+async function getJson(url: URL): Promise<unknown[]> {
+  const response = await fetch(url, {
+    headers: { Accept: "application/json", "User-Agent": USER_AGENT },
+  });
+  if (!response.ok)
+    throw new Error(`mangadex request failed with HTTP ${response.status}.`);
+  const data = (await response.json()) as { data?: unknown[]; result?: string };
+  if (data.result && data.result !== "ok")
+    throw new Error(`mangadex API returned ${data.result}.`);
+  return data.data ?? [];
+}
+
+function localized(values: unknown): string {
+  const obj =
+    values && typeof values === "object"
+      ? (values as Record<string, unknown>)
+      : {};
+  return str(
+    obj.en || obj.ja || obj["ja-ro"] || obj.zh || Object.values(obj)[0],
+  );
+}
+
+export function mapMangaDexManga(rows: unknown[]): Record<string, unknown>[] {
+  return rows.map((row, index) => {
+    const item = row as Record<string, unknown>;
+    const attrs =
+      (item.attributes as Record<string, unknown> | undefined) ?? {};
+    return {
+      rank: index + 1,
+      id: item.id,
+      title: localized(attrs.title),
+      status: str(attrs.status),
+      year: attrs.year ?? null,
+      content_rating: str(attrs.contentRating),
+      latest_uploaded_chapter: str(attrs.latestUploadedChapter),
+      description: localized(attrs.description).slice(0, 700),
+      url: `https://mangadex.org/title/${item.id}`,
+    };
+  });
+}
+
+export function mapMangaDexAuthors(rows: unknown[]): Record<string, unknown>[] {
+  return rows.map((row, index) => {
+    const item = row as Record<string, unknown>;
+    const attrs =
+      (item.attributes as Record<string, unknown> | undefined) ?? {};
+    return {
+      rank: index + 1,
+      id: item.id,
+      name: str(attrs.name),
+      twitter: str(attrs.twitter),
+      pixiv: str(attrs.pixiv),
+      website: str(attrs.website),
+      url: `https://mangadex.org/author/${item.id}`,
+    };
+  });
+}
+
+async function searchManga(kwargs: Record<string, unknown>) {
+  const query = required(kwargs.query, "query");
+  const url = new URL(`${API}/manga`);
+  url.searchParams.set("title", query);
+  url.searchParams.set("limit", String(requireLimit(kwargs.limit)));
+  const year = optionalYear(kwargs.year);
+  if (year) url.searchParams.set("year", String(year));
+  applySort(url, kwargs.sort);
+  applyContentRating(url, kwargs["content-rating"]);
+  url.searchParams.append("includes[]", "author");
+  url.searchParams.append("includes[]", "artist");
+  const rows = mapMangaDexManga(await getJson(url));
+  if (rows.length === 0)
+    throw new Error(`No MangaDex manga found for "${query}".`);
+  return rows;
+}
+
+async function searchAuthors(kwargs: Record<string, unknown>) {
+  const query = required(kwargs.query, "query");
+  const url = new URL(`${API}/author`);
+  url.searchParams.set("name", query);
+  url.searchParams.set("limit", String(requireLimit(kwargs.limit)));
+  const rows = mapMangaDexAuthors(await getJson(url));
+  if (rows.length === 0)
+    throw new Error(`No MangaDex authors found for "${query}".`);
+  return rows;
+}
+
+const SEARCH_ARGS = [
+  { name: "query", type: "str" as const, required: true, positional: true },
+  { name: "limit", type: "int" as const, default: 10 },
+];
+
+const MANGA_ARGS = [
+  { name: "query", type: "str" as const, required: true, positional: true },
+  { name: "limit", type: "int" as const, default: 10 },
+  { name: "year", type: "int" as const },
+  {
+    name: "sort",
+    type: "str" as const,
+    default: "relevance",
+    choices: ["relevance", "latest", "followed", "year"],
+  },
+  {
+    name: "content-rating",
+    type: "str" as const,
+    choices: ["safe", "suggestive", "erotica", "pornographic", "all"],
+  },
+];
+
+cli({
+  site: "mangadex",
+  name: "manga",
+  description:
+    "Search MangaDex manga by Japanese title, romaji, alias, or keyword",
+  domain: "mangadex.org",
+  strategy: Strategy.PUBLIC,
+  browser: false,
+  args: MANGA_ARGS,
+  columns: ["rank", "id", "title", "status", "year", "content_rating", "url"],
+  func: async (_page, kwargs) => searchManga(kwargs),
+});
+
+cli({
+  site: "mangadex",
+  name: "authors",
+  description: "Search MangaDex authors and artists by public name or romaji",
+  domain: "mangadex.org",
+  strategy: Strategy.PUBLIC,
+  browser: false,
+  args: SEARCH_ARGS,
+  columns: ["rank", "id", "name", "twitter", "pixiv", "website", "url"],
+  func: async (_page, kwargs) => searchAuthors(kwargs),
+});