@catladder/cli 0.0.0-main-8dbb6c76 → 0.0.0-main-a6b45abe

package/src/apps/cli/commands/project/doctor/checkCloudRun.ts

@@ -0,0 +1,148 @@
+import type { ComponentContext } from "@catladder/pipeline";
+import { isOfDeployType } from "@catladder/pipeline";
+import { CATLADDER_REGISTRY_NAME } from "../../../../../gcloud/artifactsRegistry";
+import { getGcloudServiceAccountNames } from "../../../../../gcloud/serviceAccountNames";
+import { accountExists } from "../../../../../gcloud/serviceAccounts";
+import { mapWithConcurrency } from "../../../../../utils/concurrency";
+import {
+  CLOUD_RUN_DEPLOY_SA_NAME,
+  getCloudRunDeployRoles,
+  getCloudRunRequiredServices,
+} from "../setup/setupCloudRun";
+import type { DoctorReport } from "./DoctorReport";
+import { GcloudProjectInspector } from "./GcloudProjectInspector";
+
+const getCloudRunConfig = (context: ComponentContext) => {
+  const config = context.deploy?.config;
+  if (!isOfDeployType(config, "google-cloudrun")) {
+    throw new Error("not a cloud run context");
+  }
+  return config;
+};
+
+/**
+ * compares the gcloud state `setupCloudRun` would provision for the
+ * CURRENT config against what is actually there: enabled services and
+ * artifacts registry per project, deploy service account existence and
+ * IAM roles per component/env.
+ *
+ * The motivating case (#70): a component gets `cloudSql` after its
+ * setup ran — deploys keep working and only the review teardown hits
+ * the missing `roles/cloudsql.admin` as a 403 in ci.
+ */
+export const checkCloudRun = async (
+  report: DoctorReport,
+  contexts: ComponentContext[],
+) => {
+  const cloudRunContexts = contexts.filter((context) =>
+    isOfDeployType(context.deploy?.config, "google-cloudrun"),
+  );
+  if (cloudRunContexts.length === 0) return;
+
+  report.section("google cloud run infrastructure");
+  const inspector = new GcloudProjectInspector();
+  if (!(await inspector.isAuthenticated())) {
+    report.fail(
+      "gcloud is not authenticated — all google cloud checks skipped",
+      "run: gcloud auth login",
+    );
+    return;
+  }
+
+  // project-level: enabled services + artifacts registry. The required
+  // services are the union over all components deploying to the project
+  // (cloudSql on any of them requires the sql services).
+  const byProject = new Map<string, ComponentContext[]>();
+  for (const context of cloudRunContexts) {
+    const { projectId } = getCloudRunConfig(context);
+    byProject.set(projectId, [...(byProject.get(projectId) ?? []), context]);
+  }
+
+  for (const [projectId, projectContexts] of byProject) {
+    const requiredServices = new Set(
+      projectContexts.flatMap((context) =>
+        getCloudRunRequiredServices(getCloudRunConfig(context)),
+      ),
+    );
+    try {
+      const enabled = await inspector.getEnabledServices(projectId);
+      const missing = [...requiredServices].filter(
+        (service) => !enabled.has(service),
+      );
+      if (missing.length === 0) {
+        report.ok(
+          `${projectId}: all ${requiredServices.size} required services enabled`,
+        );
+      } else {
+        report.fail(
+          `${projectId}: services not enabled: ${missing.join(", ")}`,
+          "run: catladder project setup",
+        );
+      }
+    } catch (e) {
+      report.warn(
+        `${projectId}: could not list enabled services (${e.message})`,
+      );
+    }
+
+    const regions = new Set(
+      projectContexts.map((context) => getCloudRunConfig(context).region),
+    );
+    for (const region of regions) {
+      if (await inspector.hasArtifactsRegistry(projectId, region)) {
+        report.ok(
+          `${projectId}: artifacts registry '${CATLADDER_REGISTRY_NAME}' exists in ${region}`,
+        );
+      } else {
+        report.fail(
+          `${projectId}: artifacts registry '${CATLADDER_REGISTRY_NAME}' missing in ${region}`,
+          "run: catladder project setup",
+        );
+      }
+    }
+  }
+
+  // per component/env: deploy service account exists and carries every
+  // role the current config implies
+  const results = await mapWithConcurrency(
+    cloudRunContexts,
+    5,
+    async (context) => {
+      const config = getCloudRunConfig(context);
+      const { fullIdentifier } = getGcloudServiceAccountNames(context, {
+        name: CLOUD_RUN_DEPLOY_SA_NAME,
+        projectId: config.projectId,
+      });
+      if (!(await accountExists(fullIdentifier))) {
+        return { context, fullIdentifier, exists: false, missingRoles: [] };
+      }
+      const boundRoles = await inspector.getRolesOfMember(
+        config.projectId,
+        `serviceAccount:${fullIdentifier}`,
+      );
+      const missingRoles = getCloudRunDeployRoles(config).filter(
+        (role) => !boundRoles.includes(role),
+      );
+      return { context, fullIdentifier, exists: true, missingRoles };
+    },
+  );
+
+  for (const { context, fullIdentifier, exists, missingRoles } of results) {
+    const label = `${context.env}:${context.name}`;
+    if (!exists) {
+      report.fail(
+        `${label}: deploy service account ${fullIdentifier} does not exist`,
+        `run: catladder project setup ${context.name}`,
+      );
+    } else if (missingRoles.length > 0) {
+      report.fail(
+        `${label}: service account is missing roles: ${missingRoles.join(", ")}`,
+        `run: catladder project setup ${context.name}`,
+      );
+    } else {
+      report.ok(
+        `${label}: service account exists with all config-implied roles`,
+      );
+    }
+  }
+};

package/src/apps/cli/commands/project/doctor/checkGithub.ts

@@ -0,0 +1,211 @@
+import type { Config } from "@catladder/pipeline";
+import {
+  AGGREGATE_CHECK_JOB_NAME,
+  getEnabledPipelineTypes,
+  getPipelineGitRemote,
+} from "@catladder/pipeline";
+import {
+  collectSecretTargets,
+  getConfiguredGithubRepo,
+} from "../../../../../commands/project/commandSecretsSyncGithub";
+import { mapWithConcurrency } from "../../../../../utils/concurrency";
+import {
+  ghApiJson,
+  isGhAuthenticated,
+  listGithubEnvironments,
+  listGithubSecretNames,
+  listGithubVariableNames,
+} from "../../../../../utils/github";
+import type { DoctorReport } from "./DoctorReport";
+
+/** marker for repo-level (environment-less) targets, see collectSecretTargets */
+const REPO_LEVEL = "";
+
+/**
+ * compares the secret/variable NAMES the generated github workflows
+ * reference against what is actually set — per github environment, the
+ * same mapping `project secrets-sync-github` writes. Values cannot be
+ * compared: github secrets are write-only by design (the vault stays
+ * the source of truth).
+ */
+export const checkGithub = async (report: DoctorReport, config: Config) => {
+  if (!getEnabledPipelineTypes(config).includes("github")) return;
+  report.section("github environments & secrets");
+
+  if (!(await isGhAuthenticated())) {
+    report.warn(
+      "github cli (gh) not authenticated — github checks skipped",
+      "run: gh auth login",
+    );
+    return;
+  }
+  const repo = await getConfiguredGithubRepo();
+  if (!repo) {
+    report.fail(
+      `no github repository found on the '${getPipelineGitRemote(config, "github")}' remote`,
+    );
+    return;
+  }
+
+  let targets: Map<string, Set<string>>;
+  try {
+    targets = await collectSecretTargets(config);
+  } catch (e) {
+    report.warn(
+      `could not generate the github workflows to derive secret references (${e.message})`,
+    );
+    return;
+  }
+
+  const referencedNamesOf = (environment: string) =>
+    [...targets.entries()]
+      .filter(([, envs]) => envs.has(environment))
+      .map(([name]) => name);
+
+  const environmentsNeeded = [
+    ...new Set([...targets.values()].flatMap((envs) => [...envs])),
+  ]
+    .filter((environment) => environment !== REPO_LEVEL)
+    .sort();
+
+  let existingEnvironments: Set<string>;
+  try {
+    existingEnvironments = new Set(await listGithubEnvironments(repo));
+  } catch (e) {
+    report.warn(`could not list environments of ${repo} (${e.message})`);
+    return;
+  }
+
+  const perEnvironment = await mapWithConcurrency(
+    environmentsNeeded,
+    8,
+    async (environment) => {
+      const referenced = referencedNamesOf(environment);
+      if (!existingEnvironments.has(environment)) {
+        return { environment, referenced, exists: false as const };
+      }
+      const [secrets, variables] = await Promise.all([
+        listGithubSecretNames(repo, environment),
+        listGithubVariableNames(repo, environment),
+      ]);
+      // github secret names are case-insensitive and the api returns
+      // them uppercased — compare accordingly
+      const present = new Set(
+        [...secrets, ...variables].map((name) => name.toUpperCase()),
+      );
+      const referencedUpper = new Set(
+        referenced.map((name) => name.toUpperCase()),
+      );
+      return {
+        environment,
+        referenced,
+        exists: true as const,
+        missing: referenced.filter((name) => !present.has(name.toUpperCase())),
+        stale: [...present].filter((name) => !referencedUpper.has(name)),
+      };
+    },
+  );
+
+  for (const result of perEnvironment) {
+    const { environment, referenced } = result;
+    if (!result.exists) {
+      report.fail(
+        `environment '${environment}' does not exist (${referenced.length} referenced secrets have nowhere to live)`,
+        "run: catladder project secrets-sync-github",
+      );
+      continue;
+    }
+    if (result.missing.length > 0) {
+      report.fail(
+        `${environment}: missing ${result.missing.length}/${referenced.length} referenced secrets: ${result.missing.join(", ")}`,
+        "run: catladder project secrets-sync-github",
+      );
+    } else {
+      report.ok(
+        `${environment}: all ${referenced.length} referenced secrets/variables present`,
+      );
+    }
+    if (result.stale.length > 0) {
+      report.warn(
+        `${environment}: ${result.stale.length} set but not referenced by any workflow (stale?): ${result.stale.join(", ")}`,
+      );
+    }
+  }
+
+  const repoLevelReferenced = referencedNamesOf(REPO_LEVEL);
+  if (repoLevelReferenced.length > 0) {
+    const [secrets, variables] = await Promise.all([
+      listGithubSecretNames(repo),
+      listGithubVariableNames(repo),
+    ]);
+    const present = new Set(
+      [...secrets, ...variables].map((name) => name.toUpperCase()),
+    );
+    const missing = repoLevelReferenced.filter(
+      (name) => !present.has(name.toUpperCase()),
+    );
+    if (missing.length > 0) {
+      report.fail(
+        `repo level: missing ${missing.length}/${repoLevelReferenced.length} referenced secrets: ${missing.join(", ")}`,
+        "run: catladder project secrets-sync-github",
+      );
+    } else {
+      report.ok(
+        `repo level: all ${repoLevelReferenced.length} referenced secrets/variables present`,
+      );
+    }
+  }
+
+  await checkGithubMergeGating(report, repo);
+};
+
+/**
+ * verifies the merge gating `project setup` configures: auto-merge
+ * enabled and the generated `catladder ✅` aggregate check required on
+ * the default branch. Without it a PR is mergeable while its checks
+ * are still running — gitlab ships this behavior built-in, github
+ * silently does not.
+ */
+const checkGithubMergeGating = async (report: DoctorReport, repo: string) => {
+  let repoInfo: { default_branch: string; allow_auto_merge: boolean };
+  try {
+    repoInfo = await ghApiJson("GET", `repos/${repo}`);
+  } catch (e) {
+    report.warn(`could not read repo settings of ${repo} (${e.message})`);
+    return;
+  }
+  if (repoInfo.allow_auto_merge) {
+    report.ok("auto-merge enabled");
+  } else {
+    report.fail(
+      "auto-merge disabled — the merge-when-pipeline-succeeds button cannot appear",
+      "run: catladder project setup",
+    );
+  }
+
+  const branch = repoInfo.default_branch;
+  const protection = await ghApiJson<any>(
+    "GET",
+    `repos/${repo}/branches/${branch}/protection`,
+  ).catch(() => null);
+  const checks: Array<{ context: string }> =
+    protection?.required_status_checks?.checks ?? [];
+  if (checks.some((c) => c.context === AGGREGATE_CHECK_JOB_NAME)) {
+    report.ok(`'${AGGREGATE_CHECK_JOB_NAME}' required on ${branch}`);
+  } else {
+    report.fail(
+      `'${AGGREGATE_CHECK_JOB_NAME}' is not a required check on ${branch} — PRs can merge while the pipeline is red or still running`,
+      "run: catladder project setup",
+    );
+  }
+  const stale = checks.filter((c) => c.context !== AGGREGATE_CHECK_JOB_NAME);
+  if (stale.length > 0) {
+    report.warn(
+      `${branch} requires ${stale.length} additional named check(s) (${stale
+        .map((c) => c.context)
+        .join(
+          ", ",
+        )}) — job names change with components; prefer only '${AGGREGATE_CHECK_JOB_NAME}'`,
+    );
+  }
+};

package/src/apps/cli/commands/project/doctor/checkGitlab.ts

@@ -0,0 +1,133 @@
+import type { ComponentContext } from "@catladder/pipeline";
+import { isOfDeployType } from "@catladder/pipeline";
+import { differenceInCalendarDays } from "date-fns";
+import type { IO } from "../../../../../core/types";
+import {
+  doGitlabRequest,
+  doGitlabRequestAllPages,
+  getProjectInfo,
+} from "../../../../../utils/gitlab";
+import type { DoctorReport } from "./DoctorReport";
+
+const EXPIRY_WARNING_DAYS = 30;
+
+/**
+ * the gitlab-side outputs of `project setup`: the semantic-release
+ * access token (+ its GL_TOKEN ci variable), the catladder topic and —
+ * for kubernetes deploys — the registry deploy token.
+ *
+ * Kubernetes namespace/rbac checks are deliberately skipped: they would
+ * require connecting kubectl to every cluster, which switches the local
+ * kubeconfig context — doctor must not change anything.
+ */
+export const checkGitlab = async (
+  instance: IO,
+  report: DoctorReport,
+  contexts: ComponentContext[],
+) => {
+  report.section("gitlab project setup");
+
+  let projectId: string;
+  try {
+    projectId = (await getProjectInfo(instance)).id;
+  } catch (e) {
+    report.warn(
+      `could not reach the gitlab api — gitlab checks skipped (${e.message})`,
+    );
+    return;
+  }
+
+  try {
+    const tokens = await doGitlabRequestAllPages(
+      instance,
+      `projects/${projectId}/access_tokens`,
+    );
+    const activeTokens = tokens.filter(
+      (t) => t.name === "semantic-release" && t.active === true,
+    );
+    // the GL_TOKEN ci variable holds exactly one of them — judge by the
+    // one that lives longest
+    const token = activeTokens.sort((a, b) =>
+      String(b.expires_at).localeCompare(String(a.expires_at)),
+    )[0];
+    if (activeTokens.length > 1) {
+      report.warn(
+        `${activeTokens.length} active 'semantic-release' tokens exist — duplicates from earlier setups, consider revoking the old ones`,
+      );
+    }
+    if (!token) {
+      report.fail(
+        "no active 'semantic-release' project access token",
+        "run: catladder project setup",
+      );
+    } else {
+      const daysLeft = differenceInCalendarDays(
+        new Date(token.expires_at),
+        new Date(),
+      );
+      if (daysLeft < 0) {
+        report.fail(
+          `'semantic-release' token expired on ${token.expires_at}`,
+          "run: catladder project setup (rotates the token)",
+        );
+      } else if (daysLeft <= EXPIRY_WARNING_DAYS) {
+        report.warn(
+          `'semantic-release' token expires in ${daysLeft} days (${token.expires_at})`,
+          "run: catladder project setup (rotates the token)",
+        );
+      } else {
+        report.ok(
+          `'semantic-release' token active (expires ${token.expires_at})`,
+        );
+      }
+    }
+
+    try {
+      await doGitlabRequest(
+        instance,
+        `projects/${projectId}/variables/GL_TOKEN`,
+      );
+      report.ok("GL_TOKEN ci variable present");
+    } catch {
+      report.fail(
+        "GL_TOKEN ci variable missing",
+        "run: catladder project setup",
+      );
+    }
+
+    const { topics } = await doGitlabRequest(instance, `projects/${projectId}`);
+    if (topics?.includes("catladder")) {
+      report.ok("'catladder' project topic set");
+    } else {
+      report.fail(
+        "'catladder' project topic missing",
+        "run: catladder project setup",
+      );
+    }
+
+    if (
+      contexts.some((context) =>
+        isOfDeployType(context.deploy?.config, "kubernetes"),
+      )
+    ) {
+      const deployTokens = await doGitlabRequestAllPages(
+        instance,
+        `projects/${projectId}/deploy_tokens`,
+      );
+      if (
+        deployTokens.find(
+          (t: { name: string }) => t.name === "gitlab-deploy-token",
+        )
+      ) {
+        report.ok("'gitlab-deploy-token' registry deploy token present");
+      } else {
+        report.fail(
+          "'gitlab-deploy-token' registry deploy token missing (kubernetes cannot pull images)",
+          "run: catladder project setup",
+        );
+      }
+    }
+  } catch (e) {
+    report.warn(`gitlab checks aborted (${e.message})`);
+  }
+};

package/src/apps/cli/commands/project/doctor/checkStore.ts

@@ -0,0 +1,43 @@
+import { getEnvConfig, readCatladderStore } from "@catladder/pipeline";
+import {
+  getAllComponentsWithAllEnvsFlat,
+  getProjectConfig,
+} from "../../../../../config/getProjectConfig";
+import { getGitRoot } from "../../../../../utils/projects";
+import type { DoctorReport } from "./DoctorReport";
+
+/**
+ * every google-cloudrun project in the config needs its project number
+ * in the committed store — the deterministic cloud run urls
+ * (<service>-<projectNumber>.<region>.run.app) are computed from it at
+ * generation time. This check is fully offline.
+ */
+export const checkStore = async (report: DoctorReport) => {
+  const config = await getProjectConfig();
+  const projectIds = new Set<string>();
+  for (const {
+    componentName,
+    env,
+  } of await getAllComponentsWithAllEnvsFlat()) {
+    const { deploy } = getEnvConfig(config, componentName, env);
+    if (deploy && deploy.type === "google-cloudrun") {
+      projectIds.add(deploy.projectId);
+    }
+  }
+  if (projectIds.size === 0) return;
+
+  report.section("catladder store (.catladder-store/store.yml)");
+  const root = (await getGitRoot()) ?? process.cwd();
+  const store = readCatladderStore(root);
+  for (const projectId of projectIds) {
+    const projectNumber = store.gcloudProjects?.[projectId]?.projectNumber;
+    if (projectNumber) {
+      report.ok(`project number for ${projectId}: ${projectNumber}`);
+    } else {
+      report.fail(
+        `missing project number for ${projectId}`,
+        "run: catladder project setup",
+      );
+    }
+  }
+};

package/src/apps/cli/commands/project/doctor/index.ts

@@ -0,0 +1,59 @@
+import type { ComponentContext } from "@catladder/pipeline";
+import { getEnabledPipelineTypes } from "@catladder/pipeline";
+import {
+  getAllPipelineContexts,
+  getProjectConfig,
+} from "../../../../../config/getProjectConfig";
+import type { IO } from "../../../../../core/types";
+import { checkCloudRun } from "./checkCloudRun";
+import { checkGithub } from "./checkGithub";
+import { checkGitlab } from "./checkGitlab";
+import { checkStore } from "./checkStore";
+import { DoctorReport } from "./DoctorReport";
+
+/**
+ * read-only health check: compares the state `project setup` (and
+ * `project secrets-sync-github`) would provision for the CURRENT config
+ * against what is actually there, and reports drift together with the
+ * command that heals it. `project setup` is a point-in-time
+ * provisioner, so config edits that change required infrastructure
+ * (IAM roles, services, secrets) drift silently until something breaks
+ * in ci — the doctor surfaces that before it does.
+ *
+ * The component filter only narrows the per-component checks; the
+ * cheap project-global checks (store, gitlab, github) always run.
+ */
+export const doctorProject = async (
+  instance: IO,
+  onlyComponents?: string | string[],
+) => {
+  const config = await getProjectConfig();
+  if (!config) {
+    throw new Error("no catladder config found");
+  }
+  instance.log(
+    "🐱 🩺 checking project health — read-only, nothing will be changed",
+  );
+
+  const report = new DoctorReport(instance);
+  await checkStore(report);
+
+  let contexts: ComponentContext[] = [];
+  try {
+    contexts = await getAllPipelineContexts(onlyComponents);
+  } catch (e) {
+    report.section("pipeline contexts");
+    report.fail(
+      `could not create pipeline contexts: ${e.message}`,
+      "run: catladder project setup",
+    );
+  }
+
+  if (getEnabledPipelineTypes(config).includes("gitlab")) {
+    await checkGitlab(instance, report, contexts);
+  }
+  await checkCloudRun(report, contexts);
+  await checkGithub(report, config);
+
+  report.summarize();
+};

package/src/apps/cli/commands/project/setup/ensureGcloudProjectNumbers.ts

@@ -0,0 +1,61 @@
+import { getEnvConfig, readCatladderStore } from "@catladder/pipeline";
+import {
+  getAllComponentsWithAllEnvsFlat,
+  getProjectConfig,
+  invalidateProjectConfig,
+} from "../../../../../config/getProjectConfig";
+import type { IO } from "../../../../../core/types";
+import { fetchGcloudProjectNumber } from "../../../../../gcloud/getProjectNumber";
+import { updateCatladderStore } from "../../../../../store/updateCatladderStore";
+import { getGitRoot } from "../../../../../utils/projects";
+
+/**
+ * fetches the gcloud project numbers of all cloud run deploys into the
+ * catladder store.
+ *
+ * This must run BEFORE any pipeline context is created: context creation
+ * computes the deterministic cloud run urls, which read the project
+ * number from the store and throw when it is missing — on a fresh
+ * project that would make `project setup` fail with the very error it is
+ * supposed to fix (chicken-and-egg).
+ */
+export const ensureGcloudProjectNumbers = async (instance: IO) => {
+  const config = await getProjectConfig();
+  if (!config) return;
+
+  const projectIds = new Set<string>();
+  for (const {
+    componentName,
+    env,
+  } of await getAllComponentsWithAllEnvsFlat()) {
+    const { deploy } = getEnvConfig(config, componentName, env);
+    if (deploy && deploy.type === "google-cloudrun") {
+      projectIds.add(deploy.projectId);
+    }
+  }
+
+  const root = (await getGitRoot()) ?? process.cwd();
+  const store = readCatladderStore(root);
+  const missing = [...projectIds].filter(
+    (projectId) => !store.gcloudProjects?.[projectId]?.projectNumber,
+  );
+
+  for (const projectId of missing) {
+    instance.log(`fetch gcloud project number for ${projectId}...`);
+    const projectNumber = await fetchGcloudProjectNumber(projectId);
+    instance.log(`project number: ${projectNumber}`);
+    await updateCatladderStore((current) => ({
+      ...current,
+      gcloudProjects: {
+        ...current.gcloudProjects,
+        [projectId]: { projectNumber },
+      },
+    }));
+  }
+
+  if (missing.length > 0) {
+    // the store is attached to the config at read time; drop the cached
+    // config so contexts created after this pre-pass see the new values
+    invalidateProjectConfig();
+  }
+};