ants-move 0.0.1 → 0.0.2

package/README.md

@@ -1,6 +1,6 @@
 # ants-move
 
-`ants-move` is an open-source TypeScript CLI for moving data between systems. Its commands are small workers: collectors bring data in, and future automation commands can move that data into forms and other systems. The first release collects data from 36Kr, Toutiao, and Hacker News.
+`ants-move` is an open-source TypeScript CLI for moving data between systems. Its commands are small workers: collectors bring data in, and future automation commands can move that data into forms and other systems. The current collectors read 36Kr, Toutiao, Hacker News, and GitHub Trending data.
 
 ## Installation and requirements
 
@@ -86,11 +86,31 @@ ants hn search <query> [--limit 1..100] [--sort relevance|date] [--format json|t
 
 Hacker News uses at most eight concurrent detail requests. ID-list and search responses are limited to 5 MB, while each Firebase story detail is limited to 256 KB.
 
+## GitHub Trending commands
+
+Fetch every repository shown on GitHub's all-language Trending page. The period defaults to `daily`:
+
+```bash
+ants github trending
+ants github trending --since daily
+ants github trending --since weekly
+ants github trending --since monthly
+```
+
+JSON is the default output. Use `--format table` or `-t` for a terminal table:
+
+```bash
+ants github trending --since weekly --format table
+ants github trending --since monthly -t
+```
+
+The collector reads the official server-rendered HTML with one GitHub request and does not use a third-party Trending API or Playwright. The request has a 30-second timeout and a 5 MB HTML limit. Parsing accepts at most 100 repositories and retains at most ten displayed contributors per repository. The command has no language filter, result limit, automatic retry, or cache; a successful zero-row parse is rejected as a page-structure error instead of returning a misleading empty ranking.
+
 ## High-concurrency usage warning
 
-Resource use is bounded within one process, but identical commands running in separate processes are not globally deduplicated or rate limited. At `Q` concurrent invocations, upstream work can approach `20Q` requests for a 36Kr list, `201Q` requests for a Hacker News list, or `105Q` browser navigations plus page subresources for a Toutiao author collection.
+Resource use is bounded within one process, but identical commands running in separate processes are not globally deduplicated or rate limited. At `Q` concurrent invocations, upstream work can approach `20Q` requests for a 36Kr list, `201Q` requests for a Hacker News list, `105Q` browser navigations plus page subresources for a Toutiao author collection, or `Q` requests for GitHub Trending.
 
-Retained collector data is also bounded per invocation: 5 MB for a 36Kr list, approximately 50 MB across 200 Hacker News candidate details before result filtering, and 10 MB for Toutiao author article results. Toutiao may additionally hold up to five active 5 MB feed buffers inside one browser session; browser process overhead and required page subresources are separate from these payload limits.
+Retained collector data is also bounded per invocation: 5 MB for a 36Kr list, approximately 50 MB across 200 Hacker News candidate details before result filtering, 10 MB for Toutiao author article results, and one 5 MB GitHub HTML response before Cheerio parsing. Toutiao may additionally hold up to five active 5 MB feed buffers inside one browser session; browser process overhead, required page subresources, and HTML parser object overhead are separate from these payload limits.
 
 Do not place the CLI directly on a high-QPS request path. Online services must use an external bounded queue and a shared rate limiter to apply backpressure across processes and instances. This project has no distributed lock, shared cache, retry queue, or multi-instance single-flight mechanism.
 

package/dist/index.d.ts

@@ -124,6 +124,36 @@ interface ToutiaoRuntime {
     withSession: <T>(operation: (session: ToutiaoSession) => Promise<T>) => Promise<T>;
 }
 
+type GitHubTrendingPeriod = 'daily' | 'weekly' | 'monthly';
+interface GitHubTrendingContributor {
+    avatarUrl: string;
+    url: string;
+    username: string;
+}
+interface GitHubTrendingRepository {
+    builtBy?: GitHubTrendingContributor[];
+    description?: string;
+    forks: number;
+    fullName: string;
+    language?: string;
+    name: string;
+    owner: string;
+    rank: number;
+    stars: number;
+    starsInPeriod: number;
+    url: string;
+}
+interface GitHubTrendingRuntimeResult {
+    items: GitHubTrendingRepository[];
+    period: GitHubTrendingPeriod;
+    sourceUrl: string;
+}
+interface GitHubRuntime {
+    fetchTrending: (options: {
+        period: GitHubTrendingPeriod;
+    }) => Promise<GitHubTrendingRuntimeResult>;
+}
+
 interface CliIo {
     stderr: (value: string) => void;
     stdout: (value: string) => void;
@@ -132,6 +162,7 @@ interface CliRunner {
     run: (args: string[]) => Promise<number>;
 }
 interface CreateCliOptions extends Partial<CliIo> {
+    githubRuntime?: GitHubRuntime;
     hackerNewsRuntime?: HackerNewsRuntime;
     kr36Runtime?: Kr36Runtime;
     toutiaoRuntime?: ToutiaoRuntime;

package/dist/index.js

@@ -2119,6 +2119,440 @@ async function loadPlaywright() {
   return await import("playwright");
 }
 
+// src/github/command.ts
+import { z as z4 } from "zod";
+
+// src/github/output.ts
+function renderGitHubTrendingAsJson(result) {
+  return JSON.stringify(
+    {
+      ok: true,
+      data: result
+    },
+    null,
+    2
+  );
+}
+function renderGitHubTrendingAsTable(result) {
+  return renderSafeTable([
+    [
+      "rank",
+      "repository",
+      "language",
+      "stars",
+      "forks",
+      "period stars",
+      "description",
+      "url"
+    ],
+    ...result.items.map((item) => [
+      String(item.rank),
+      item.fullName,
+      item.language ?? "",
+      String(item.stars),
+      String(item.forks),
+      String(item.starsInPeriod),
+      item.description ?? "",
+      item.url
+    ])
+  ]);
+}
+function renderGitHubCommandErrorAsJson(code, message, details) {
+  return JSON.stringify(
+    {
+      ok: false,
+      error: {
+        code,
+        message,
+        ...details === void 0 ? {} : { details }
+      }
+    },
+    null,
+    2
+  );
+}
+
+// src/github/types.ts
+var GitHubCommandError = class extends Error {
+  constructor(code, message, exitCode, details) {
+    super(message);
+    this.code = code;
+    this.exitCode = exitCode;
+    this.details = details;
+    this.name = "GitHubCommandError";
+  }
+};
+
+// src/github/service.ts
+var SUPPORTED_PERIODS = ["daily", "weekly", "monthly"];
+var SUPPORTED_PERIOD_SET = new Set(SUPPORTED_PERIODS);
+var GitHubService = class {
+  constructor(dependencies) {
+    this.dependencies = dependencies;
+  }
+  async trending(options) {
+    const period = parsePeriod(options.period);
+    const result = await this.dependencies.runtime.fetchTrending({ period });
+    return {
+      ...result,
+      meta: {
+        totalItems: result.items.length
+      }
+    };
+  }
+};
+function parsePeriod(period) {
+  const value = period ?? "daily";
+  if (SUPPORTED_PERIOD_SET.has(value)) {
+    return value;
+  }
+  throw new GitHubCommandError(
+    "GITHUB_INVALID_PERIOD",
+    "GitHub Trending period must be daily, weekly, or monthly.",
+    2,
+    {
+      period: value,
+      supportedPeriods: [...SUPPORTED_PERIODS]
+    }
+  );
+}
+
+// src/github/command.ts
+var trendingOptionsSchema = z4.object({
+  format: z4.enum(["json", "table"]).default("json"),
+  since: z4.string().default("daily"),
+  table: z4.boolean().optional()
+});
+function registerGitHubCommands(program, dependencies) {
+  const github = program.command("github").description("Fetch GitHub Trending repositories.");
+  const service = new GitHubService({ runtime: dependencies.runtime });
+  github.command("trending").description("Fetch the all-language GitHub Trending page.").option(
+    "--since <period>",
+    "trending period: daily, weekly, or monthly",
+    "daily"
+  ).option("--format <format>", "output format", "json").option("-t, --table", "render as a table").action(async (options) => {
+    const parsed = trendingOptionsSchema.parse(applyTableShortcut4(options));
+    const result = await service.trending({ period: parsed.since });
+    dependencies.stdout(
+      parsed.format === "table" ? `${renderGitHubTrendingAsTable(result)}
+` : `${renderGitHubTrendingAsJson(result)}
+`
+    );
+  });
+}
+function handleGitHubCommandError(error, dependencies) {
+  if (error instanceof GitHubCommandError) {
+    dependencies.stderr(
+      `${renderGitHubCommandErrorAsJson(error.code, error.message, error.details)}
+`
+    );
+    return error.exitCode;
+  }
+  return void 0;
+}
+function applyTableShortcut4(options) {
+  if (options.table) {
+    return {
+      ...options,
+      format: "table"
+    };
+  }
+  return {
+    ...options,
+    format: options.format
+  };
+}
+
+// src/github/parser.ts
+import { load } from "cheerio";
+var GITHUB_ORIGIN = "https://github.com";
+var PERIOD_LABEL = {
+  daily: "today",
+  monthly: "this month",
+  weekly: "this week"
+};
+function parseGitHubTrendingHtml(html, options) {
+  const $ = load(html);
+  const items = [];
+  $("article.Box-row").each((_, element) => {
+    const row = $(element);
+    const repository = parseRepositoryLink(
+      row.find("h2 a[href]").first().attr("href")
+    );
+    if (!repository) {
+      return;
+    }
+    if (items.length >= options.maxItems) {
+      throw new GitHubCommandError(
+        "GITHUB_PARSE_FAILED",
+        "GitHub Trending returned more repositories than the configured limit.",
+        2,
+        {
+          maxItems: options.maxItems,
+          sourceUrl: options.sourceUrl
+        }
+      );
+    }
+    const description = normalizeText(row.find("p.col-9").first().text());
+    const language = normalizeText(
+      row.find('[itemprop="programmingLanguage"]').first().text()
+    );
+    const builtBy = parseContributors(
+      $,
+      row.find('a[data-hovercard-type="user"][href]'),
+      options.maxBuiltBy
+    );
+    items.push({
+      forks: parseCount(findRepositoryLinkText(
+        $,
+        row.find("a[href]"),
+        `${repository.path}/forks`
+      )),
+      fullName: `${repository.owner}/${repository.name}`,
+      name: repository.name,
+      owner: repository.owner,
+      rank: items.length + 1,
+      stars: parseCount(findRepositoryLinkText(
+        $,
+        row.find("a[href]"),
+        `${repository.path}/stargazers`
+      )),
+      starsInPeriod: parsePeriodStars(
+        row.find("span.float-sm-right").first().text(),
+        options.period
+      ),
+      url: repository.url,
+      ...builtBy.length === 0 ? {} : { builtBy },
+      ...description.length === 0 ? {} : { description },
+      ...language.length === 0 ? {} : { language }
+    });
+  });
+  if (items.length === 0) {
+    throw new GitHubCommandError(
+      "GITHUB_PARSE_FAILED",
+      "Failed to parse GitHub Trending repositories.",
+      2,
+      {
+        sourceUrl: options.sourceUrl
+      }
+    );
+  }
+  return {
+    items,
+    period: options.period,
+    sourceUrl: options.sourceUrl
+  };
+}
+function parseRepositoryLink(href) {
+  const url = parseGitHubUrl(href);
+  if (!url) return void 0;
+  const segments = url.pathname.split("/").filter(Boolean);
+  if (segments.length !== 2) return void 0;
+  const [owner, name] = segments;
+  const path = `/${owner}/${name}`;
+  return {
+    name,
+    owner,
+    path,
+    url: `${GITHUB_ORIGIN}${path}`
+  };
+}
+function parseContributors($, links, maxBuiltBy) {
+  const contributors = [];
+  links.each((_, element) => {
+    if (contributors.length >= maxBuiltBy) return false;
+    const link = $(element);
+    const user = parseGitHubUser(link.attr("href"));
+    const avatarUrl = parseHttpUrl(link.find("img").first().attr("src"));
+    if (!user || !avatarUrl) return;
+    contributors.push({
+      avatarUrl,
+      url: `${GITHUB_ORIGIN}/${user}`,
+      username: user
+    });
+  });
+  return contributors;
+}
+function parseGitHubUser(href) {
+  const url = parseGitHubUrl(href);
+  if (!url) return void 0;
+  const segments = url.pathname.split("/").filter(Boolean);
+  return segments.length === 1 ? segments[0] : void 0;
+}
+function parseGitHubUrl(value) {
+  if (!value) return void 0;
+  try {
+    const url = new URL(value, GITHUB_ORIGIN);
+    return url.origin === GITHUB_ORIGIN ? url : void 0;
+  } catch {
+    return void 0;
+  }
+}
+function parseHttpUrl(value) {
+  if (!value) return void 0;
+  try {
+    const url = new URL(value, GITHUB_ORIGIN);
+    return url.protocol === "http:" || url.protocol === "https:" ? url.toString() : void 0;
+  } catch {
+    return void 0;
+  }
+}
+function findRepositoryLinkText($, links, expectedPath) {
+  let text = "";
+  links.each((_, element) => {
+    const link = $(element);
+    const url = parseGitHubUrl(link.attr("href"));
+    if (url?.pathname === expectedPath) {
+      text = link.text();
+      return false;
+    }
+  });
+  return text;
+}
+function parsePeriodStars(value, period) {
+  const suffix = new RegExp(`\\s+stars?\\s+${PERIOD_LABEL[period]}\\s*$`, "i");
+  if (!suffix.test(value)) return 0;
+  return parseCount(value.replace(suffix, ""));
+}
+function parseCount(value) {
+  const digits = value.replace(/[,\s]/g, "");
+  if (!/^\d+$/.test(digits)) return 0;
+  const count = Number(digits);
+  return Number.isSafeInteger(count) ? count : 0;
+}
+function normalizeText(value) {
+  return value.replace(/\s+/g, " ").trim();
+}
+
+// src/github/runtime.ts
+var DEFAULT_MAX_RESPONSE_BYTES2 = 5 * 1024 * 1024;
+var DEFAULT_REQUEST_TIMEOUT_MS2 = 3e4;
+var MAX_BUILT_BY = 10;
+var MAX_ITEMS = 100;
+function createDefaultGitHubRuntime(options = {}) {
+  const dependencies = {
+    fetch: options.fetch ?? fetch,
+    maxResponseBytes: options.maxResponseBytes ?? DEFAULT_MAX_RESPONSE_BYTES2,
+    requestTimeoutMs: options.requestTimeoutMs ?? DEFAULT_REQUEST_TIMEOUT_MS2
+  };
+  return {
+    fetchTrending: async ({ period }) => fetchTrending(dependencies, period)
+  };
+}
+async function fetchTrending(dependencies, period) {
+  const sourceUrl = createSourceUrl(period);
+  const controller = new AbortController();
+  const timeout = setTimeout(
+    () => controller.abort(),
+    dependencies.requestTimeoutMs
+  );
+  try {
+    let response;
+    try {
+      response = await dependencies.fetch(sourceUrl, {
+        headers: {
+          accept: "text/html,application/xhtml+xml",
+          "user-agent": "ants-move GitHub collector"
+        },
+        redirect: "error",
+        signal: controller.signal
+      });
+    } catch {
+      throw fetchFailed(sourceUrl);
+    }
+    if (!response.ok) {
+      await cancelResponseBody2(response);
+      throw new GitHubCommandError(
+        "GITHUB_FETCH_FAILED",
+        `Failed to fetch GitHub Trending data. HTTP ${response.status}.`,
+        2,
+        {
+          status: response.status,
+          url: sourceUrl
+        }
+      );
+    }
+    let body;
+    try {
+      body = await readBoundedBody2(
+        response,
+        dependencies.maxResponseBytes,
+        sourceUrl
+      );
+    } catch (error) {
+      if (error instanceof GitHubCommandError) throw error;
+      throw fetchFailed(sourceUrl);
+    }
+    return parseGitHubTrendingHtml(new TextDecoder().decode(body), {
+      maxBuiltBy: MAX_BUILT_BY,
+      maxItems: MAX_ITEMS,
+      period,
+      sourceUrl
+    });
+  } finally {
+    clearTimeout(timeout);
+  }
+}
+function createSourceUrl(period) {
+  const url = new URL("/trending", "https://github.com");
+  url.searchParams.set("since", period);
+  return url.toString();
+}
+async function readBoundedBody2(response, maxBytes, url) {
+  const declaredLength = Number(response.headers.get("content-length"));
+  if (Number.isFinite(declaredLength) && declaredLength > maxBytes) {
+    await cancelResponseBody2(response);
+    throw responseTooLarge2(url, maxBytes);
+  }
+  const reader = response.body?.getReader();
+  if (!reader) {
+    return new Uint8Array();
+  }
+  const chunks = [];
+  let totalBytes = 0;
+  while (true) {
+    const { done, value } = await reader.read();
+    if (done) break;
+    totalBytes += value.byteLength;
+    if (totalBytes > maxBytes) {
+      await reader.cancel().catch(() => void 0);
+      throw responseTooLarge2(url, maxBytes);
+    }
+    chunks.push(value);
+  }
+  const body = new Uint8Array(totalBytes);
+  let offset = 0;
+  for (const chunk of chunks) {
+    body.set(chunk, offset);
+    offset += chunk.byteLength;
+  }
+  return body;
+}
+async function cancelResponseBody2(response) {
+  await response.body?.cancel().catch(() => void 0);
+}
+function fetchFailed(url) {
+  return new GitHubCommandError(
+    "GITHUB_FETCH_FAILED",
+    "Failed to fetch GitHub Trending data.",
+    2,
+    {
+      url
+    }
+  );
+}
+function responseTooLarge2(url, maxBytes) {
+  return new GitHubCommandError(
+    "GITHUB_RESPONSE_TOO_LARGE",
+    "GitHub Trending response exceeded the configured size limit.",
+    2,
+    {
+      maxBytes,
+      url
+    }
+  );
+}
+
 // src/cli.ts
 function createCli(options = {}) {
   const io = {
@@ -2141,6 +2575,11 @@ function createCli(options = {}) {
     stderr: io.stderr,
     stdout: io.stdout
   });
+  registerGitHubCommands(program, {
+    runtime: options.githubRuntime ?? createDefaultGitHubRuntime(),
+    stderr: io.stderr,
+    stdout: io.stdout
+  });
   registerToutiaoCommands(program, {
     runtime: options.toutiaoRuntime ?? createDefaultToutiaoRuntime(),
     stderr: io.stderr,
@@ -2155,6 +2594,10 @@ function createCli(options = {}) {
         if (error instanceof CommanderError && error.exitCode === 0) {
           return 0;
         }
+        const githubExitCode = handleGitHubCommandError(error, io);
+        if (githubExitCode !== void 0) {
+          return githubExitCode;
+        }
         const toutiaoExitCode = handleToutiaoCommandError(error, io);
         if (toutiaoExitCode !== void 0) {
           return toutiaoExitCode;