@checkstack/gitops-backend 0.1.0

package/src/scrapers/github-scraper.test.ts

@@ -0,0 +1,355 @@
+import { describe, it, expect } from "bun:test";
+import { githubScraper } from "./github-scraper";
+import type { ScraperOptions, FetchFn } from "./types";
+import type { Logger } from "@checkstack/backend-api";
+
+const mockLogger: Logger = {
+  info: () => {},
+  error: () => {},
+  warn: () => {},
+  debug: () => {},
+};
+
+/**
+ * Creates a mock fetch that returns pre-configured responses based on URL patterns.
+ */
+function createMockFetch(
+  handlers: Array<{
+    pattern: string | RegExp;
+    response: unknown;
+    status?: number;
+    headers?: Record<string, string>;
+  }>,
+): FetchFn {
+  return async (input: RequestInfo | URL) => {
+    const url = typeof input === "string" ? input : input.toString();
+
+    for (const handler of handlers) {
+      const matches =
+        typeof handler.pattern === "string"
+          ? url.includes(handler.pattern)
+          : handler.pattern.test(url);
+
+      if (matches) {
+        return new Response(JSON.stringify(handler.response), {
+          status: handler.status ?? 200,
+          headers: {
+            "Content-Type": "application/json",
+            ...handler.headers,
+          },
+        });
+      }
+    }
+
+    return new Response("Not Found", { status: 404 });
+  };
+}
+
+const BASE_OPTIONS: Omit<ScraperOptions, "fetch"> = {
+  target: "my-org",
+  pathPattern: ".checkstack/**/*.yaml",
+  authToken: "ghp_test_token",
+  logger: mockLogger,
+};
+
+describe("githubScraper", () => {
+  it("discovers files from a single repo target", async () => {
+    const mockFetch = createMockFetch([
+      {
+        pattern: /repos\/my-org\/my-repo\/git\/trees/,
+        response: {
+          sha: "abc",
+          tree: [
+            { path: ".checkstack/systems.yaml", type: "blob" },
+            { path: "README.md", type: "blob" },
+            { path: "src", type: "tree" },
+          ],
+          truncated: false,
+        },
+      },
+      {
+        pattern: /repos\/my-org\/my-repo\/contents\//,
+        response: {
+          content: btoa("apiVersion: checkstack.io/v1alpha1"),
+          encoding: "base64",
+        },
+      },
+      {
+        pattern: /repos\/my-org\/my-repo$/,
+        response: { full_name: "my-org/my-repo", default_branch: "main" },
+      },
+    ]);
+
+    const files = await githubScraper.discoverFiles({
+      ...BASE_OPTIONS,
+      target: "my-org/my-repo",
+      fetch: mockFetch,
+    });
+
+    expect(files).toHaveLength(1);
+    expect(files[0].repository).toBe("my-org/my-repo");
+    expect(files[0].filePath).toBe(".checkstack/systems.yaml");
+    expect(files[0].content).toBe("apiVersion: checkstack.io/v1alpha1");
+    expect(files[0].branch).toBe("main");
+  });
+
+  it("enumerates repos from an org target", async () => {
+    const mockFetch = createMockFetch([
+      {
+        pattern: "orgs/my-org/repos",
+        response: [
+          { full_name: "my-org/repo-a", default_branch: "main" },
+          { full_name: "my-org/repo-b", default_branch: "develop" },
+        ],
+      },
+      {
+        pattern: "my-org/repo-a/git/trees",
+        response: {
+          sha: "abc",
+          tree: [{ path: ".checkstack/sys.yaml", type: "blob" }],
+          truncated: false,
+        },
+      },
+      {
+        pattern: "my-org/repo-b/git/trees",
+        response: { sha: "def", tree: [], truncated: false },
+      },
+      {
+        pattern: "repo-a/contents",
+        response: { content: btoa("yaml-content"), encoding: "base64" },
+      },
+    ]);
+
+    const files = await githubScraper.discoverFiles({
+      ...BASE_OPTIONS,
+      fetch: mockFetch,
+    });
+
+    expect(files).toHaveLength(1);
+    expect(files[0].repository).toBe("my-org/repo-a");
+    expect(files[0].branch).toBe("main");
+  });
+
+  it("falls back to user endpoint when org returns 404", async () => {
+    let userEndpointCalled = false;
+
+    const mockFetch = createMockFetch([
+      {
+        pattern: "orgs/my-user/repos",
+        response: { message: "Not Found" },
+        status: 404,
+      },
+      {
+        pattern: "users/my-user/repos",
+        response: [
+          { full_name: "my-user/personal-repo", default_branch: "main" },
+        ],
+      },
+      {
+        pattern: "personal-repo/git/trees",
+        response: { sha: "abc", tree: [], truncated: false },
+      },
+    ]);
+
+    // Wrap to detect user endpoint call
+    const wrappedFetch: FetchFn = async (input, init) => {
+      const url = typeof input === "string" ? input : input.toString();
+      if (url.includes("users/my-user")) userEndpointCalled = true;
+      return mockFetch(input, init);
+    };
+
+    await githubScraper.discoverFiles({
+      ...BASE_OPTIONS,
+      target: "my-user",
+      fetch: wrappedFetch,
+    });
+
+    expect(userEndpointCalled).toBe(true);
+  });
+
+  it("filters files using minimatch pattern", async () => {
+    const mockFetch = createMockFetch([
+      {
+        pattern: /repos\/my-org\/repo\/git\/trees/,
+        response: {
+          sha: "abc",
+          tree: [
+            { path: ".checkstack/systems.yaml", type: "blob" },
+            { path: ".checkstack/deep/nested.yaml", type: "blob" },
+            { path: "other/file.yaml", type: "blob" },
+            { path: ".checkstack/readme.md", type: "blob" },
+          ],
+          truncated: false,
+        },
+      },
+      {
+        pattern: /repos\/my-org\/repo\/contents\//,
+        response: { content: btoa("content"), encoding: "base64" },
+      },
+      {
+        pattern: /repos\/my-org\/repo$/,
+        response: { full_name: "my-org/repo", default_branch: "main" },
+      },
+    ]);
+
+    const files = await githubScraper.discoverFiles({
+      ...BASE_OPTIONS,
+      target: "my-org/repo",
+      fetch: mockFetch,
+    });
+
+    // Should match .checkstack/**/*.yaml only
+    expect(files).toHaveLength(2);
+    expect(files[0].filePath).toBe(".checkstack/systems.yaml");
+    expect(files[1].filePath).toBe(".checkstack/deep/nested.yaml");
+  });
+
+  it("handles pagination via Link header", async () => {
+    const mockFetch: FetchFn = async (input) => {
+      const url = typeof input === "string" ? input : input.toString();
+
+      if (url.includes("orgs/big-org/repos") && !url.includes("page=2")) {
+        return new Response(
+          JSON.stringify([
+            { full_name: "big-org/repo-1", default_branch: "main" },
+          ]),
+          {
+            headers: {
+              "Content-Type": "application/json",
+              Link: '<https://api.github.com/orgs/big-org/repos?per_page=100&page=2>; rel="next"',
+            },
+          },
+        );
+      }
+
+      if (url.includes("page=2")) {
+        return new Response(
+          JSON.stringify([
+            { full_name: "big-org/repo-2", default_branch: "main" },
+          ]),
+          { headers: { "Content-Type": "application/json" } },
+        );
+      }
+
+      if (url.includes("git/trees")) {
+        return new Response(
+          JSON.stringify({ sha: "abc", tree: [], truncated: false }),
+          { headers: { "Content-Type": "application/json" } },
+        );
+      }
+
+      return new Response("Not Found", { status: 404 });
+    };
+
+    const files = await githubScraper.discoverFiles({
+      ...BASE_OPTIONS,
+      target: "big-org",
+      fetch: mockFetch,
+    });
+
+    // Both repos processed but no matching files
+    expect(files).toHaveLength(0);
+  });
+
+  it("continues on individual file fetch errors", async () => {
+    const mockFetch: FetchFn = async (input) => {
+      const url = typeof input === "string" ? input : input.toString();
+
+      // Content requests
+      if (url.includes("contents/") && url.includes("bad.yaml")) {
+        return new Response("Internal Server Error", { status: 500 });
+      }
+      if (url.includes("contents/") && url.includes("good.yaml")) {
+        return new Response(
+          JSON.stringify({ content: btoa("good"), encoding: "base64" }),
+          { headers: { "Content-Type": "application/json" } },
+        );
+      }
+
+      // Tree request
+      if (url.includes("git/trees")) {
+        return new Response(
+          JSON.stringify({
+            sha: "abc",
+            tree: [
+              { path: ".checkstack/good.yaml", type: "blob" },
+              { path: ".checkstack/bad.yaml", type: "blob" },
+            ],
+            truncated: false,
+          }),
+          { headers: { "Content-Type": "application/json" } },
+        );
+      }
+
+      // Repo metadata (must be last — matches broadly)
+      if (url.includes("repos/my-org/repo")) {
+        return new Response(
+          JSON.stringify({ full_name: "my-org/repo", default_branch: "main" }),
+          { headers: { "Content-Type": "application/json" } },
+        );
+      }
+
+      return new Response("Not Found", { status: 404 });
+    };
+
+    const files = await githubScraper.discoverFiles({
+      ...BASE_OPTIONS,
+      target: "my-org/repo",
+      fetch: mockFetch,
+    });
+
+    // Only the good file should be returned
+    expect(files).toHaveLength(1);
+    expect(files[0].filePath).toBe(".checkstack/good.yaml");
+  });
+
+  it("uses custom baseUrl for enterprise installations", async () => {
+    const enterpriseUrl = "https://github.acme.corp/api/v3";
+    const requestedUrls: string[] = [];
+
+    const mockFetch: FetchFn = async (input) => {
+      const url = typeof input === "string" ? input : input.toString();
+      requestedUrls.push(url);
+
+      if (url.includes("git/trees")) {
+        return new Response(
+          JSON.stringify({
+            sha: "abc",
+            tree: [{ path: ".checkstack/sys.yaml", type: "blob" }],
+            truncated: false,
+          }),
+          { headers: { "Content-Type": "application/json" } },
+        );
+      }
+
+      if (url.includes("contents/")) {
+        return new Response(
+          JSON.stringify({ content: btoa("yaml"), encoding: "base64" }),
+          { headers: { "Content-Type": "application/json" } },
+        );
+      }
+
+      if (url.includes("repos/acme/infra")) {
+        return new Response(
+          JSON.stringify({ full_name: "acme/infra", default_branch: "main" }),
+          { headers: { "Content-Type": "application/json" } },
+        );
+      }
+
+      return new Response("Not Found", { status: 404 });
+    };
+
+    const files = await githubScraper.discoverFiles({
+      ...BASE_OPTIONS,
+      target: "acme/infra",
+      baseUrl: enterpriseUrl,
+      fetch: mockFetch,
+    });
+
+    expect(files).toHaveLength(1);
+    // All requests should use the enterprise URL, not api.github.com
+    for (const url of requestedUrls) {
+      expect(url).toStartWith(enterpriseUrl);
+    }
+  });
+});

package/src/scrapers/github-scraper.ts

@@ -0,0 +1,263 @@
+import { minimatch } from "minimatch";
+import type { DiscoveredFile, ScraperOptions, Scraper, FetchFn } from "./types";
+
+const DEFAULT_GITHUB_API_URL = "https://api.github.com";
+
+// ─── GitHub API Types ──────────────────────────────────────────────────────
+
+interface GitHubRepo {
+  full_name: string;
+  default_branch: string;
+}
+
+interface GitHubTreeItem {
+  path: string;
+  type: "blob" | "tree";
+}
+
+interface GitHubTreeResponse {
+  sha: string;
+  tree: GitHubTreeItem[];
+  truncated: boolean;
+}
+
+interface GitHubContentResponse {
+  content: string;
+  encoding: string;
+}
+
+// ─── Helpers ───────────────────────────────────────────────────────────────
+
+/**
+ * Parses the GitHub `Link` header for pagination.
+ * Returns the URL for the next page, or undefined if there is none.
+ */
+function parseNextPageUrl(linkHeader: string | null): string | undefined {
+  if (!linkHeader) return undefined;
+  const match = /<([^>]+)>;\s*rel="next"/.exec(linkHeader);
+  return match?.[1];
+}
+
+/**
+ * Makes an authenticated request to the GitHub API.
+ */
+async function githubFetch(params: {
+  url: string;
+  authToken: string;
+  fetchFn: FetchFn;
+}): Promise<Response> {
+  const { url, authToken, fetchFn } = params;
+  return fetchFn(url, {
+    headers: {
+      Authorization: `Bearer ${authToken}`,
+      Accept: "application/vnd.github+json",
+      "X-GitHub-Api-Version": "2022-11-28",
+    },
+  });
+}
+
+// ─── Core Logic ────────────────────────────────────────────────────────────
+
+/**
+ * Enumerates repositories for an org or user target.
+ * Supports pagination via Link header.
+ */
+async function enumerateRepos(params: {
+  target: string;
+  authToken: string;
+  fetchFn: FetchFn;
+  apiUrl: string;
+}): Promise<GitHubRepo[]> {
+  const { target, authToken, fetchFn, apiUrl } = params;
+  const repos: GitHubRepo[] = [];
+
+  // Try org endpoint first, fall back to user endpoint
+  let url: string | undefined =
+    `${apiUrl}/orgs/${encodeURIComponent(target)}/repos?per_page=100`;
+
+  const orgResponse = await githubFetch({ url, authToken, fetchFn });
+
+  if (orgResponse.status === 404) {
+    // Fall back to user endpoint
+    url = `${apiUrl}/users/${encodeURIComponent(target)}/repos?per_page=100`;
+  } else if (orgResponse.ok) {
+    const data = (await orgResponse.json()) as GitHubRepo[];
+    repos.push(...data);
+    url = parseNextPageUrl(orgResponse.headers.get("Link"));
+  } else {
+    throw new Error(
+      `GitHub API error listing org repos: ${orgResponse.status} ${orgResponse.statusText}`,
+    );
+  }
+
+  // Paginate through remaining pages (or user repos if org 404'd)
+  while (url) {
+    const response = await githubFetch({ url, authToken, fetchFn });
+    if (!response.ok) {
+      throw new Error(
+        `GitHub API error listing repos: ${response.status} ${response.statusText}`,
+      );
+    }
+    const data = (await response.json()) as GitHubRepo[];
+    repos.push(...data);
+    url = parseNextPageUrl(response.headers.get("Link"));
+  }
+
+  return repos;
+}
+
+/**
+ * Gets the file tree for a repository using the Git Trees API.
+ * Returns only blob (file) paths matching the path pattern.
+ */
+async function getMatchingFiles(params: {
+  repo: GitHubRepo;
+  pathPattern: string;
+  authToken: string;
+  fetchFn: FetchFn;
+  apiUrl: string;
+}): Promise<string[]> {
+  const { repo, pathPattern, authToken, fetchFn, apiUrl } = params;
+
+  const url = `${apiUrl}/repos/${repo.full_name}/git/trees/${encodeURIComponent(repo.default_branch)}?recursive=1`;
+  const response = await githubFetch({ url, authToken, fetchFn });
+
+  if (!response.ok) {
+    throw new Error(
+      `GitHub API error getting tree for ${repo.full_name}: ${response.status} ${response.statusText}`,
+    );
+  }
+
+  const tree = (await response.json()) as GitHubTreeResponse;
+
+  return tree.tree
+    .filter((item) => item.type === "blob")
+    .map((item) => item.path)
+    .filter((path) => minimatch(path, pathPattern));
+}
+
+/**
+ * Fetches the content of a single file from a repository.
+ */
+async function fetchFileContent(params: {
+  repoFullName: string;
+  filePath: string;
+  branch: string;
+  authToken: string;
+  fetchFn: FetchFn;
+  apiUrl: string;
+}): Promise<string> {
+  const { repoFullName, filePath, branch, authToken, fetchFn, apiUrl } = params;
+
+  const url = `${apiUrl}/repos/${repoFullName}/contents/${encodeURIComponent(filePath)}?ref=${encodeURIComponent(branch)}`;
+  const response = await githubFetch({ url, authToken, fetchFn });
+
+  if (!response.ok) {
+    throw new Error(
+      `GitHub API error fetching ${filePath} from ${repoFullName}: ${response.status} ${response.statusText}`,
+    );
+  }
+
+  const data = (await response.json()) as GitHubContentResponse;
+
+  if (data.encoding === "base64") {
+    return atob(data.content.replaceAll("\n", ""));
+  }
+
+  return data.content;
+}
+
+// ─── Scraper ───────────────────────────────────────────────────────────────
+
+/**
+ * GitHub scraper implementation.
+ *
+ * Supports:
+ * - Org/user target: enumerates all repos with pagination
+ * - Single repo target: `owner/repo` format
+ * - Default branch resolution per-repo
+ * - Recursive tree walking with minimatch filtering
+ * - Custom base URL for GitHub Enterprise
+ */
+export const githubScraper: Scraper = {
+  async discoverFiles(options: ScraperOptions): Promise<DiscoveredFile[]> {
+    const {
+      target,
+      pathPattern,
+      authToken,
+      baseUrl,
+      logger,
+      fetch: fetchFn = globalThis.fetch,
+    } = options;
+
+    const apiUrl = baseUrl ?? DEFAULT_GITHUB_API_URL;
+    const isSingleRepo = target.includes("/");
+    const files: DiscoveredFile[] = [];
+
+    let repos: GitHubRepo[];
+
+    if (isSingleRepo) {
+      // Single repo mode: fetch repo metadata directly
+      const url = `${apiUrl}/repos/${target}`;
+      const response = await githubFetch({ url, authToken, fetchFn });
+      if (!response.ok) {
+        throw new Error(
+          `GitHub API error fetching repo ${target}: ${response.status} ${response.statusText}`,
+        );
+      }
+      repos = [(await response.json()) as GitHubRepo];
+    } else {
+      repos = await enumerateRepos({ target, authToken, fetchFn, apiUrl });
+    }
+
+    logger.debug(
+      `GitHub scraper: found ${repos.length} repo(s) for target "${target}"`,
+    );
+
+    for (const repo of repos) {
+      try {
+        const matchingPaths = await getMatchingFiles({
+          repo,
+          pathPattern,
+          authToken,
+          fetchFn,
+          apiUrl,
+        });
+
+        logger.debug(
+          `GitHub scraper: ${matchingPaths.length} matching file(s) in ${repo.full_name}`,
+        );
+
+        for (const filePath of matchingPaths) {
+          try {
+            const content = await fetchFileContent({
+              repoFullName: repo.full_name,
+              filePath,
+              branch: repo.default_branch,
+              authToken,
+              fetchFn,
+              apiUrl,
+            });
+
+            files.push({
+              repository: repo.full_name,
+              filePath,
+              content,
+              branch: repo.default_branch,
+            });
+          } catch (error) {
+            logger.error(
+              `GitHub scraper: failed to fetch ${filePath} from ${repo.full_name}: ${error}`,
+            );
+          }
+        }
+      } catch (error) {
+        logger.error(
+          `GitHub scraper: failed to process repo ${repo.full_name}: ${error}`,
+        );
+      }
+    }
+
+    return files;
+  },
+};