@folio-agent/handler 0.7.0 → 0.8.1

package/dist/ingest/generate.js

@@ -1,4 +1,5 @@
 import { readFile, stat } from "node:fs/promises";
+import { collectGithubRepos, fetchGithubReadmes } from "./github.js";
 import { createUrlMatcher } from "./glob.js";
 import { htmlToText } from "./html-to-text.js";
 import { readTextFile, scanDist, scanKnowledgeDir } from "./scan-dist.js";
@@ -19,9 +20,13 @@ export async function generateKnowledge(config) {
     const matchesUrl = createUrlMatcher(config);
     const pages = [];
     const warnings = [];
+    const githubRepos = new Set();
     const distFiles = await scanDist(config.distDir);
     for (const file of distFiles.filter((f) => matchesUrl(f.urlPath))) {
         const html = await readTextFile(file.absolutePath);
+        if (config.github)
+            for (const repo of collectGithubRepos(html))
+                githubRepos.add(repo);
         const { title, text } = htmlToText(html);
         pages.push({ url: file.urlPath, source: "dist", title, text });
     }
@@ -44,6 +49,26 @@ export async function generateKnowledge(config) {
             warnings.push(`articlesDir not found: ${config.zenn.articlesDir} — skipping zenn ingest`);
         }
     }
+    if (config.github) {
+        const { pages: githubPages, failures } = await fetchGithubReadmes(githubRepos);
+        pages.push(...githubPages);
+        if (failures.length > 0) {
+            // 初回のビルドではまだスナップショットが無いことがある。無ければ空として扱い、各リポを警告に回す
+            const snapshotPath = config.github.snapshotPath;
+            const snapshot = snapshotPath && (await pathExists(snapshotPath))
+                ? JSON.parse(await readFile(snapshotPath, "utf-8"))
+                : [];
+            for (const failure of failures) {
+                const fallback = snapshot.find((p) => p.url === failure.repoUrl);
+                if (fallback) {
+                    pages.push(fallback);
+                }
+                else {
+                    warnings.push(`${failure.error} — skipping ${failure.repoUrl}`);
+                }
+            }
+        }
+    }
     pages.sort((a, b) => a.url.localeCompare(b.url));
     const estimatedTokens = estimateTokens(pages.map((p) => p.text).join("\n"));
     const threshold = config.tokenWarningThreshold ?? DEFAULT_TOKEN_WARNING_THRESHOLD;

package/dist/ingest/github.d.ts

@@ -0,0 +1,18 @@
+import type { IngestConfig, KnowledgePage } from "./types.js";
+/** Returns the canonical `https://github.com/<owner>/<repo>` for a repo link, or undefined. */
+export declare function toGithubRepoUrl(href: string): string | undefined;
+/** Collects GitHub repo URLs linked from `<a href>` in the given HTML, without duplicates. */
+export declare function collectGithubRepos(html: string): string[];
+/** Fetches the default README of a repo and turns it into a knowledge page. Throws on failure. */
+export declare function fetchGithubReadme(repoUrl: string): Promise<KnowledgePage>;
+export interface GithubFetchResult {
+    pages: KnowledgePage[];
+    failures: {
+        repoUrl: string;
+        error: string;
+    }[];
+}
+/** Fetches READMEs for all repos sequentially, separating successes from failures. */
+export declare function fetchGithubReadmes(repoUrls: Iterable<string>): Promise<GithubFetchResult>;
+/** Collects repos linked from the included (non-excluded) pages of `config.distDir`. */
+export declare function collectGithubReposFromDist(config: IngestConfig): Promise<string[]>;

package/dist/ingest/github.js

@@ -0,0 +1,106 @@
+import { parse } from "node-html-parser";
+import { createUrlMatcher } from "./glob.js";
+import { readTextFile, scanDist } from "./scan-dist.js";
+// リポではない github.com のトップレベルのパス。網羅はしない（取りこぼしは README 取得の 404 で warnings に出る）
+const RESERVED_OWNERS = new Set([
+    "about",
+    "apps",
+    "collections",
+    "customer-stories",
+    "enterprise",
+    "events",
+    "explore",
+    "features",
+    "login",
+    "marketplace",
+    "new",
+    "notifications",
+    "orgs",
+    "pricing",
+    "pulls",
+    "issues",
+    "search",
+    "security",
+    "settings",
+    "signup",
+    "site",
+    "sponsors",
+    "team",
+    "topics",
+    "trending",
+]);
+const NAME_PATTERN = /^[A-Za-z0-9._-]+$/;
+/** Returns the canonical `https://github.com/<owner>/<repo>` for a repo link, or undefined. */
+export function toGithubRepoUrl(href) {
+    let url;
+    try {
+        url = new URL(href);
+    }
+    catch {
+        return undefined;
+    }
+    if (url.protocol !== "https:" && url.protocol !== "http:")
+        return undefined;
+    const host = url.hostname.toLowerCase();
+    if (host !== "github.com" && host !== "www.github.com")
+        return undefined;
+    const [owner, rawRepo] = url.pathname.split("/").filter(Boolean);
+    if (!owner || !rawRepo)
+        return undefined;
+    const repo = rawRepo.replace(/\.git$/, "");
+    if (!NAME_PATTERN.test(owner) || !NAME_PATTERN.test(repo) || repo === "." || repo === "..")
+        return undefined;
+    if (RESERVED_OWNERS.has(owner.toLowerCase()))
+        return undefined;
+    return `https://github.com/${owner}/${repo}`;
+}
+/** Collects GitHub repo URLs linked from `<a href>` in the given HTML, without duplicates. */
+export function collectGithubRepos(html) {
+    const repos = new Set();
+    for (const anchor of parse(html).querySelectorAll("a[href]")) {
+        const repoUrl = toGithubRepoUrl(anchor.getAttribute("href") ?? "");
+        if (repoUrl)
+            repos.add(repoUrl);
+    }
+    return [...repos];
+}
+/** Fetches the default README of a repo and turns it into a knowledge page. Throws on failure. */
+export async function fetchGithubReadme(repoUrl) {
+    const slug = repoUrl.replace(/^https:\/\/github\.com\//, "");
+    const headers = {
+        Accept: "application/vnd.github.raw",
+        "User-Agent": "folio-agent-ingest",
+    };
+    const token = process.env.GITHUB_TOKEN;
+    if (token)
+        headers.Authorization = `Bearer ${token}`;
+    const response = await fetch(`https://api.github.com/repos/${slug}/readme`, { headers });
+    if (!response.ok) {
+        throw new Error(`failed to fetch README for ${slug}: HTTP ${response.status}`);
+    }
+    const text = (await response.text()).trim();
+    return { url: repoUrl, source: "github", title: slug, text };
+}
+/** Fetches READMEs for all repos sequentially, separating successes from failures. */
+export async function fetchGithubReadmes(repoUrls) {
+    const result = { pages: [], failures: [] };
+    for (const repoUrl of repoUrls) {
+        try {
+            result.pages.push(await fetchGithubReadme(repoUrl));
+        }
+        catch (error) {
+            result.failures.push({ repoUrl, error: error instanceof Error ? error.message : String(error) });
+        }
+    }
+    return result;
+}
+/** Collects repos linked from the included (non-excluded) pages of `config.distDir`. */
+export async function collectGithubReposFromDist(config) {
+    const matchesUrl = createUrlMatcher(config);
+    const repos = new Set();
+    for (const file of (await scanDist(config.distDir)).filter((f) => matchesUrl(f.urlPath))) {
+        for (const repo of collectGithubRepos(await readTextFile(file.absolutePath)))
+            repos.add(repo);
+    }
+    return [...repos];
+}

package/dist/ingest/types.d.ts

@@ -17,6 +17,11 @@ export interface IngestConfig {
      * separate/private repo that isn't checked out).
      */
     zennSnapshotPath?: string;
+    /**
+     * READMEs of GitHub repos linked from the included pages. Omit to keep ingest offline
+     * (no request is made to GitHub).
+     */
+    github?: GithubIngestConfig;
     /** Token threshold above which the CLI warns that CAG may no longer fit. */
     tokenWarningThreshold?: number;
     /** UI language captured by `folio-agent-init`. Ignored by ingest itself. */
@@ -38,7 +43,14 @@ export interface ZennIngestConfig {
     /** Base used to build public article URLs, e.g. "https://zenn.dev/username/articles". No trailing slash. */
     baseUrl: string;
 }
-export type KnowledgeSource = "dist" | "knowledge" | "zenn";
+export interface GithubIngestConfig {
+    /**
+     * Path to a snapshot JSON produced by `folio-agent-sync-github`. Used in place of repos
+     * whose README could not be fetched.
+     */
+    snapshotPath?: string;
+}
+export type KnowledgeSource = "dist" | "knowledge" | "zenn" | "github";
 export interface KnowledgePage {
     url: string;
     source: KnowledgeSource;

package/dist/sync/github-cli.d.ts

@@ -0,0 +1,8 @@
+#!/usr/bin/env node
+import type { KnowledgePage } from "../ingest/types.js";
+/**
+ * Reads `configPath`, collects GitHub repos linked from the included pages of `distDir`,
+ * fetches their READMEs, and writes the successful ones to `outputPath` as `KnowledgePage[]`.
+ * Failed repos are reported on stderr and left out. Throws if `config.github` is not set.
+ */
+export declare function syncGithubSnapshot(configPath: string, outputPath: string): Promise<KnowledgePage[]>;

package/dist/sync/github-cli.js

@@ -0,0 +1,42 @@
+#!/usr/bin/env node
+import { realpathSync } from "node:fs";
+import { readFile, writeFile } from "node:fs/promises";
+import { fileURLToPath } from "node:url";
+import { collectGithubReposFromDist, fetchGithubReadmes } from "../ingest/github.js";
+/**
+ * Reads `configPath`, collects GitHub repos linked from the included pages of `distDir`,
+ * fetches their READMEs, and writes the successful ones to `outputPath` as `KnowledgePage[]`.
+ * Failed repos are reported on stderr and left out. Throws if `config.github` is not set.
+ */
+export async function syncGithubSnapshot(configPath, outputPath) {
+    const config = JSON.parse(await readFile(configPath, "utf-8"));
+    if (!config.github) {
+        throw new Error("config.github is not set");
+    }
+    const repos = await collectGithubReposFromDist(config);
+    const { pages, failures } = await fetchGithubReadmes(repos);
+    for (const failure of failures) {
+        console.error(`skipped ${failure.repoUrl}: ${failure.error}`);
+    }
+    await writeFile(outputPath, JSON.stringify(pages, null, 2));
+    return pages;
+}
+async function main() {
+    const configPath = process.argv[2];
+    const outputPath = process.argv[3];
+    if (!configPath || !outputPath) {
+        console.error("usage: folio-agent-sync-github <config.json> <output.json>");
+        console.error("Run after building the site: repos are collected from links in distDir.");
+        process.exitCode = 1;
+        return;
+    }
+    const pages = await syncGithubSnapshot(configPath, outputPath);
+    console.log(`wrote ${pages.length} github page(s) -> ${outputPath}`);
+}
+// node_modules/.bin のシンボリックリンク経由でも動くよう realpath で比べる（sync/cli.ts と同じ理由）
+if (process.argv[1] && fileURLToPath(import.meta.url) === realpathSync(process.argv[1])) {
+    main().catch((error) => {
+        console.error(error);
+        process.exitCode = 1;
+    });
+}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@folio-agent/handler",
-  "version": "0.7.0",
+  "version": "0.8.1",
   "description": "Chat handler (LangGraph.js on Cloudflare Workers) + knowledge ingestion CLI",
   "license": "MIT",
   "type": "module",
@@ -13,7 +13,8 @@
   "bin": {
     "folio-agent-ingest": "dist/ingest/cli.js",
     "folio-agent-init": "dist/init/cli.js",
-    "folio-agent-sync-zenn": "dist/sync/cli.js"
+    "folio-agent-sync-zenn": "dist/sync/cli.js",
+    "folio-agent-sync-github": "dist/sync/github-cli.js"
   },
   "repository": {
     "type": "git",
@@ -39,14 +40,14 @@
     "prepublishOnly": "tsc -b --force"
   },
   "dependencies": {
-    "@google/genai": "^2.23.0",
-    "@langchain/langgraph": "^1.4.16",
-    "node-html-parser": "^9.0.4",
+    "@google/genai": "^2.31.0",
+    "@langchain/langgraph": "^1.5.2",
+    "node-html-parser": "^9.0.5",
     "picomatch": "^4.0.7"
   },
   "devDependencies": {
-    "@cloudflare/workers-types": "^5.20260920.1",
+    "@cloudflare/workers-types": "^5.20261025.1",
     "@types/picomatch": "^4.0.2",
-    "wrangler": "^4.135.0"
+    "wrangler": "^4.152.0"
   }
 }