@checkstack/gitops-backend 0.1.0

package/src/scrapers/gitlab-scraper.test.ts

@@ -0,0 +1,296 @@
+import { describe, it, expect } from "bun:test";
+import { gitlabScraper } from "./gitlab-scraper";
+import type { ScraperOptions, FetchFn } from "./types";
+import type { Logger } from "@checkstack/backend-api";
+
+const mockLogger: Logger = {
+  info: () => {},
+  error: () => {},
+  warn: () => {},
+  debug: () => {},
+};
+
+const BASE_OPTIONS: Omit<ScraperOptions, "fetch"> = {
+  target: "my-group",
+  pathPattern: ".checkstack/**/*.yaml",
+  authToken: "glpat-test_token",
+  logger: mockLogger,
+};
+
+/**
+ * Creates a mock fetch for GitLab API requests.
+ * Uses if/else URL matching to handle overlapping patterns correctly.
+ */
+function createGitLabMockFetch(config: {
+  projects: Array<{
+    id: number;
+    path_with_namespace: string;
+    default_branch: string;
+    tree: Array<{ id: string; name: string; type: "blob" | "tree"; path: string }>;
+    files: Record<string, string>;
+  }>;
+  /** If true, the target is treated as a group with multiple projects. */
+  groupMode?: boolean;
+}): FetchFn {
+  return async (input: RequestInfo | URL) => {
+    const url = typeof input === "string" ? input : input.toString();
+
+    // Group projects listing
+    if (url.includes("/groups/") && url.includes("/projects")) {
+      return new Response(JSON.stringify(config.projects), {
+        headers: { "Content-Type": "application/json" },
+      });
+    }
+
+    // Check each project
+    for (const project of config.projects) {
+      // File content (raw)
+      const rawFilePattern = `/projects/${project.id}/repository/files/`;
+      if (url.includes(rawFilePattern) && url.includes("/raw")) {
+        // Extract file path from URL
+        const pathMatch = url.match(
+          /\/files\/([^/]+)\/raw/,
+        );
+        if (pathMatch) {
+          const decodedPath = decodeURIComponent(pathMatch[1]);
+          const content = project.files[decodedPath];
+          if (content) {
+            return new Response(content, {
+              headers: { "Content-Type": "text/plain" },
+            });
+          }
+        }
+        return new Response("Not Found", { status: 404 });
+      }
+
+      // Repository tree
+      const treePattern = `/projects/${project.id}/repository/tree`;
+      if (url.includes(treePattern)) {
+        return new Response(JSON.stringify(project.tree), {
+          headers: { "Content-Type": "application/json" },
+        });
+      }
+
+      // Project metadata (single project mode)
+      const encodedPath = encodeURIComponent(project.path_with_namespace);
+      if (url.includes(`/projects/${encodedPath}`)) {
+        return new Response(JSON.stringify(project), {
+          headers: { "Content-Type": "application/json" },
+        });
+      }
+    }
+
+    return new Response("Not Found", { status: 404 });
+  };
+}
+
+describe("gitlabScraper", () => {
+  const sampleProject = {
+    id: 42,
+    path_with_namespace: "my-group/my-project",
+    default_branch: "main",
+    tree: [
+      { id: "a1", name: "systems.yaml", type: "blob" as const, path: ".checkstack/systems.yaml" },
+      { id: "a2", name: "README.md", type: "blob" as const, path: "README.md" },
+      { id: "a3", name: "src", type: "tree" as const, path: "src" },
+    ],
+    files: {
+      ".checkstack/systems.yaml": "apiVersion: checkstack.io/v1alpha1\nkind: System",
+    },
+  };
+
+  it("discovers files from a single project target", async () => {
+    const mockFetch = createGitLabMockFetch({
+      projects: [sampleProject],
+    });
+
+    const files = await gitlabScraper.discoverFiles({
+      ...BASE_OPTIONS,
+      target: "my-group/my-project",
+      fetch: mockFetch,
+    });
+
+    expect(files).toHaveLength(1);
+    expect(files[0].repository).toBe("my-group/my-project");
+    expect(files[0].filePath).toBe(".checkstack/systems.yaml");
+    expect(files[0].content).toContain("checkstack.io/v1alpha1");
+    expect(files[0].branch).toBe("main");
+  });
+
+  it("enumerates projects from a group target", async () => {
+    const mockFetch = createGitLabMockFetch({
+      projects: [
+        sampleProject,
+        {
+          id: 43,
+          path_with_namespace: "my-group/empty-project",
+          default_branch: "develop",
+          tree: [],
+          files: {},
+        },
+      ],
+      groupMode: true,
+    });
+
+    const files = await gitlabScraper.discoverFiles({
+      ...BASE_OPTIONS,
+      fetch: mockFetch,
+    });
+
+    expect(files).toHaveLength(1);
+    expect(files[0].repository).toBe("my-group/my-project");
+  });
+
+  it("filters files using minimatch pattern", async () => {
+    const project = {
+      ...sampleProject,
+      tree: [
+        { id: "a1", name: "systems.yaml", type: "blob" as const, path: ".checkstack/systems.yaml" },
+        { id: "a2", name: "nested.yaml", type: "blob" as const, path: ".checkstack/deep/nested.yaml" },
+        { id: "a3", name: "file.yaml", type: "blob" as const, path: "other/file.yaml" },
+        { id: "a4", name: "readme.md", type: "blob" as const, path: ".checkstack/readme.md" },
+      ],
+      files: {
+        ".checkstack/systems.yaml": "content1",
+        ".checkstack/deep/nested.yaml": "content2",
+      },
+    };
+
+    const mockFetch = createGitLabMockFetch({ projects: [project] });
+
+    const files = await gitlabScraper.discoverFiles({
+      ...BASE_OPTIONS,
+      target: "my-group/my-project",
+      fetch: mockFetch,
+    });
+
+    expect(files).toHaveLength(2);
+    expect(files[0].filePath).toBe(".checkstack/systems.yaml");
+    expect(files[1].filePath).toBe(".checkstack/deep/nested.yaml");
+  });
+
+  it("handles pagination via x-next-page header", async () => {
+    let requestCount = 0;
+
+    const mockFetch: FetchFn = async (input) => {
+      const url = typeof input === "string" ? input : input.toString();
+
+      if (url.includes("/groups/") && url.includes("/projects")) {
+        requestCount++;
+        if (!url.includes("page=2")) {
+          return new Response(
+            JSON.stringify([
+              { id: 1, path_with_namespace: "g/p1", default_branch: "main" },
+            ]),
+            {
+              headers: {
+                "Content-Type": "application/json",
+                "x-next-page": "2",
+              },
+            },
+          );
+        }
+        return new Response(
+          JSON.stringify([
+            { id: 2, path_with_namespace: "g/p2", default_branch: "main" },
+          ]),
+          { headers: { "Content-Type": "application/json" } },
+        );
+      }
+
+      if (url.includes("/repository/tree")) {
+        return new Response(JSON.stringify([]), {
+          headers: { "Content-Type": "application/json" },
+        });
+      }
+
+      return new Response("Not Found", { status: 404 });
+    };
+
+    await gitlabScraper.discoverFiles({
+      ...BASE_OPTIONS,
+      fetch: mockFetch,
+    });
+
+    // Should have made 2 requests for projects (page 1 + page 2)
+    expect(requestCount).toBe(2);
+  });
+
+  it("continues on individual file fetch errors", async () => {
+    const project = {
+      id: 42,
+      path_with_namespace: "my-group/my-project",
+      default_branch: "main",
+      tree: [
+        { id: "a1", name: "good.yaml", type: "blob" as const, path: ".checkstack/good.yaml" },
+        { id: "a2", name: "bad.yaml", type: "blob" as const, path: ".checkstack/bad.yaml" },
+      ],
+      files: {
+        ".checkstack/good.yaml": "good-content",
+        // bad.yaml deliberately missing from files to simulate error
+      },
+    };
+
+    const mockFetch = createGitLabMockFetch({ projects: [project] });
+
+    const files = await gitlabScraper.discoverFiles({
+      ...BASE_OPTIONS,
+      target: "my-group/my-project",
+      fetch: mockFetch,
+    });
+
+    expect(files).toHaveLength(1);
+    expect(files[0].filePath).toBe(".checkstack/good.yaml");
+  });
+
+  it("uses custom baseUrl for self-managed instances", async () => {
+    const selfHostedUrl = "https://gitlab.acme.corp/api/v4";
+    const requestedUrls: string[] = [];
+
+    const mockFetch: FetchFn = async (input) => {
+      const url = typeof input === "string" ? input : input.toString();
+      requestedUrls.push(url);
+
+      if (url.includes("/repository/tree")) {
+        return new Response(
+          JSON.stringify([
+            { id: "a1", name: "sys.yaml", type: "blob", path: ".checkstack/sys.yaml" },
+          ]),
+          { headers: { "Content-Type": "application/json" } },
+        );
+      }
+
+      if (url.includes("/repository/files/") && url.includes("/raw")) {
+        return new Response("yaml-content", {
+          headers: { "Content-Type": "text/plain" },
+        });
+      }
+
+      if (url.includes("/projects/")) {
+        return new Response(
+          JSON.stringify({
+            id: 99,
+            path_with_namespace: "acme/infra",
+            default_branch: "main",
+          }),
+          { headers: { "Content-Type": "application/json" } },
+        );
+      }
+
+      return new Response("Not Found", { status: 404 });
+    };
+
+    const files = await gitlabScraper.discoverFiles({
+      ...BASE_OPTIONS,
+      target: "acme/infra",
+      baseUrl: selfHostedUrl,
+      fetch: mockFetch,
+    });
+
+    expect(files).toHaveLength(1);
+    // All requests should use the self-hosted URL, not gitlab.com
+    for (const url of requestedUrls) {
+      expect(url).toStartWith(selfHostedUrl);
+    }
+  });
+});

package/src/scrapers/gitlab-scraper.ts

@@ -0,0 +1,242 @@
+import { minimatch } from "minimatch";
+import type { DiscoveredFile, ScraperOptions, Scraper, FetchFn } from "./types";
+
+const DEFAULT_GITLAB_API_URL = "https://gitlab.com/api/v4";
+
+// ─── GitLab API Types ──────────────────────────────────────────────────────
+
+interface GitLabProject {
+  id: number;
+  path_with_namespace: string;
+  default_branch: string;
+}
+
+interface GitLabTreeItem {
+  id: string;
+  name: string;
+  type: "blob" | "tree";
+  path: string;
+}
+
+// ─── Helpers ───────────────────────────────────────────────────────────────
+
+/**
+ * Makes an authenticated request to the GitLab API.
+ */
+async function gitlabFetch(params: {
+  url: string;
+  authToken: string;
+  fetchFn: FetchFn;
+}): Promise<Response> {
+  const { url, authToken, fetchFn } = params;
+  return fetchFn(url, {
+    headers: {
+      "PRIVATE-TOKEN": authToken,
+    },
+  });
+}
+
+/**
+ * Paginates through a GitLab API endpoint that uses `x-next-page` header.
+ */
+async function paginateGitLab<T>(params: {
+  initialUrl: string;
+  authToken: string;
+  fetchFn: FetchFn;
+}): Promise<T[]> {
+  const { initialUrl, authToken, fetchFn } = params;
+  const results: T[] = [];
+  let url: string | undefined = initialUrl;
+
+  while (url) {
+    const response = await gitlabFetch({ url, authToken, fetchFn });
+    if (!response.ok) {
+      throw new Error(
+        `GitLab API error: ${response.status} ${response.statusText} (${url})`,
+      );
+    }
+
+    const data = (await response.json()) as T[];
+    results.push(...data);
+
+    const nextPage = response.headers.get("x-next-page");
+    if (nextPage && nextPage.trim() !== "") {
+      // Build next page URL
+      const urlObj: URL = new URL(url);
+      urlObj.searchParams.set("page", nextPage);
+      url = urlObj.toString();
+    } else {
+      url = undefined;
+    }
+  }
+
+  return results;
+}
+
+// ─── Core Logic ────────────────────────────────────────────────────────────
+
+/**
+ * Enumerates projects for a group target (including subgroups).
+ */
+async function enumerateProjects(params: {
+  target: string;
+  authToken: string;
+  fetchFn: FetchFn;
+  apiUrl: string;
+}): Promise<GitLabProject[]> {
+  const { target, authToken, fetchFn, apiUrl } = params;
+  const encodedTarget = encodeURIComponent(target);
+  const url = `${apiUrl}/groups/${encodedTarget}/projects?include_subgroups=true&per_page=100`;
+  return paginateGitLab<GitLabProject>({ initialUrl: url, authToken, fetchFn });
+}
+
+/**
+ * Gets the file tree for a project and filters paths using minimatch.
+ */
+async function getMatchingFiles(params: {
+  project: GitLabProject;
+  pathPattern: string;
+  authToken: string;
+  fetchFn: FetchFn;
+  apiUrl: string;
+}): Promise<string[]> {
+  const { project, pathPattern, authToken, fetchFn, apiUrl } = params;
+
+  const url = `${apiUrl}/projects/${project.id}/repository/tree?recursive=true&ref=${encodeURIComponent(project.default_branch)}&per_page=100`;
+  const items = await paginateGitLab<GitLabTreeItem>({
+    initialUrl: url,
+    authToken,
+    fetchFn,
+  });
+
+  return items
+    .filter((item) => item.type === "blob")
+    .map((item) => item.path)
+    .filter((path) => minimatch(path, pathPattern));
+}
+
+/**
+ * Fetches the raw content of a file from a GitLab project.
+ */
+async function fetchFileContent(params: {
+  projectId: number;
+  filePath: string;
+  branch: string;
+  authToken: string;
+  fetchFn: FetchFn;
+  apiUrl: string;
+}): Promise<string> {
+  const { projectId, filePath, branch, authToken, fetchFn, apiUrl } = params;
+  const encodedPath = encodeURIComponent(filePath);
+  const url = `${apiUrl}/projects/${projectId}/repository/files/${encodedPath}/raw?ref=${encodeURIComponent(branch)}`;
+  const response = await gitlabFetch({ url, authToken, fetchFn });
+
+  if (!response.ok) {
+    throw new Error(
+      `GitLab API error fetching ${filePath} from project ${projectId}: ${response.status} ${response.statusText}`,
+    );
+  }
+
+  return response.text();
+}
+
+// ─── Scraper ───────────────────────────────────────────────────────────────
+
+/**
+ * GitLab scraper implementation.
+ *
+ * Supports:
+ * - Group target: enumerates all projects including subgroups with pagination
+ * - Single project target: `group/project` format
+ * - Default branch resolution per-project
+ * - Recursive tree walking with minimatch filtering
+ * - Custom base URL for self-managed GitLab instances
+ */
+export const gitlabScraper: Scraper = {
+  async discoverFiles(options: ScraperOptions): Promise<DiscoveredFile[]> {
+    const {
+      target,
+      pathPattern,
+      authToken,
+      baseUrl,
+      logger,
+      fetch: fetchFn = globalThis.fetch,
+    } = options;
+
+    const apiUrl = baseUrl ?? DEFAULT_GITLAB_API_URL;
+    const isSingleProject = target.includes("/");
+    const files: DiscoveredFile[] = [];
+
+    let projects: GitLabProject[];
+
+    if (isSingleProject) {
+      // Single project mode: fetch project metadata directly
+      const encodedTarget = encodeURIComponent(target);
+      const url = `${apiUrl}/projects/${encodedTarget}`;
+      const response = await gitlabFetch({ url, authToken, fetchFn });
+      if (!response.ok) {
+        throw new Error(
+          `GitLab API error fetching project ${target}: ${response.status} ${response.statusText}`,
+        );
+      }
+      projects = [(await response.json()) as GitLabProject];
+    } else {
+      projects = await enumerateProjects({
+        target,
+        authToken,
+        fetchFn,
+        apiUrl,
+      });
+    }
+
+    logger.debug(
+      `GitLab scraper: found ${projects.length} project(s) for target "${target}"`,
+    );
+
+    for (const project of projects) {
+      try {
+        const matchingPaths = await getMatchingFiles({
+          project,
+          pathPattern,
+          authToken,
+          fetchFn,
+          apiUrl,
+        });
+
+        logger.debug(
+          `GitLab scraper: ${matchingPaths.length} matching file(s) in ${project.path_with_namespace}`,
+        );
+
+        for (const filePath of matchingPaths) {
+          try {
+            const content = await fetchFileContent({
+              projectId: project.id,
+              filePath,
+              branch: project.default_branch,
+              authToken,
+              fetchFn,
+              apiUrl,
+            });
+
+            files.push({
+              repository: project.path_with_namespace,
+              filePath,
+              content,
+              branch: project.default_branch,
+            });
+          } catch (error) {
+            logger.error(
+              `GitLab scraper: failed to fetch ${filePath} from ${project.path_with_namespace}: ${error}`,
+            );
+          }
+        }
+      } catch (error) {
+        logger.error(
+          `GitLab scraper: failed to process project ${project.path_with_namespace}: ${error}`,
+        );
+      }
+    }
+
+    return files;
+  },
+};

package/src/scrapers/types.ts

@@ -0,0 +1,52 @@
+import type { Logger } from "@checkstack/backend-api";
+
+/**
+ * A fetch function compatible with the standard Fetch API.
+ * We use a custom type alias instead of `typeof globalThis.fetch` because Bun's
+ * native fetch includes a non-standard `preconnect` method that breaks mock implementations.
+ */
+export type FetchFn = (
+  input: RequestInfo | URL,
+  init?: RequestInit,
+) => Promise<Response>;
+
+/**
+ * A file discovered by a scraper from a Git provider.
+ */
+export interface DiscoveredFile {
+  /** Full repository identifier (e.g., "my-org/my-repo"). */
+  repository: string;
+  /** Path to the file within the repository. */
+  filePath: string;
+  /** Raw file content (decoded from base64 if needed). */
+  content: string;
+  /** The branch the file was read from. */
+  branch: string;
+}
+
+/**
+ * Options passed to a scraper's discoverFiles method.
+ */
+export interface ScraperOptions {
+  /** Target: org/user name or "owner/repo" for single-repo mode. */
+  target: string;
+  /** Glob pattern for matching file paths (e.g., ".checkstack/**\/*.yaml"). */
+  pathPattern: string;
+  /** Decrypted auth token for the Git provider API. */
+  authToken: string;
+  /** Custom API base URL for enterprise/on-prem installations. */
+  baseUrl?: string;
+  /** Logger for diagnostic output. */
+  logger: Logger;
+  /** Optional fetch override (defaults to global fetch). Used for testing. */
+  fetch?: FetchFn;
+}
+
+/**
+ * Interface for Git provider scrapers.
+ * Each provider type (GitHub, GitLab) implements this interface.
+ */
+export interface Scraper {
+  /** Discover all YAML descriptor files matching the configured path pattern. */
+  discoverFiles(options: ScraperOptions): Promise<DiscoveredFile[]>;
+}

package/src/secret-resolver.test.ts

@@ -0,0 +1,86 @@
+import { describe, it, expect } from "bun:test";
+import { resolveSecrets } from "./secret-resolver";
+
+const mockSecretStore = {
+  resolve: async (name: string): Promise<string> => {
+    const secrets: Record<string, string> = {
+      "prod-db-password": "s3cret!",
+      "api-key": "key-12345",
+    };
+    const value = secrets[name];
+    if (!value) throw new Error(`Secret not found: ${name}`);
+    return value;
+  },
+};
+
+describe("resolveSecrets", () => {
+  it("resolves a top-level secretRef", async () => {
+    const spec = { password: { secretRef: "prod-db-password" } };
+    const resolved = await resolveSecrets({
+      spec,
+      secretStore: mockSecretStore,
+    });
+    expect(resolved).toEqual({ password: "s3cret!" });
+  });
+
+  it("leaves plain strings untouched", async () => {
+    const spec = { host: "localhost", port: 5432 };
+    const resolved = await resolveSecrets({
+      spec,
+      secretStore: mockSecretStore,
+    });
+    expect(resolved).toEqual({ host: "localhost", port: 5432 });
+  });
+
+  it("resolves nested secretRefs", async () => {
+    const spec = {
+      connection: {
+        host: "db.internal",
+        password: { secretRef: "prod-db-password" },
+      },
+    };
+    const resolved = await resolveSecrets({
+      spec,
+      secretStore: mockSecretStore,
+    });
+    expect(resolved).toEqual({
+      connection: { host: "db.internal", password: "s3cret!" },
+    });
+  });
+
+  it("resolves secretRefs in arrays", async () => {
+    const spec = {
+      credentials: [
+        { name: "db", secret: { secretRef: "prod-db-password" } },
+        { name: "api", secret: { secretRef: "api-key" } },
+      ],
+    };
+    const resolved = await resolveSecrets({
+      spec,
+      secretStore: mockSecretStore,
+    });
+    expect(resolved).toEqual({
+      credentials: [
+        { name: "db", secret: "s3cret!" },
+        { name: "api", secret: "key-12345" },
+      ],
+    });
+  });
+
+  it("throws when a referenced secret is not found", async () => {
+    const spec = { password: { secretRef: "nonexistent" } };
+    await expect(
+      resolveSecrets({ spec, secretStore: mockSecretStore }),
+    ).rejects.toThrow("Secret not found: nonexistent");
+  });
+
+  it("handles null and undefined values gracefully", async () => {
+    const spec = { a: null, b: undefined, c: "value" };
+    const resolved = await resolveSecrets({
+      spec,
+      secretStore: mockSecretStore,
+    });
+    expect(resolved.a).toBeNull();
+    expect(resolved.c).toBe("value");
+  });
+});