@catladder/cli 0.0.0-next-71c49b28 → 0.0.0-next-5dc1f907

package/src/apps/cli/commands/project/setup/setupAccessTokens.ts

@@ -1,13 +1,20 @@
 import type { IO } from "../../../../../core/types";
 import { add, format } from "date-fns";
-import { doGitlabRequest, getProjectInfo } from "../../../../../utils/gitlab";
+import {
+  doGitlabRequest,
+  doGitlabRequestAllPages,
+  getProjectInfo,
+} from "../../../../../utils/gitlab";
 
 const TOKEN_NAME = "semantic-release";
 
 export const setupAccessTokens = async (instance: IO) => {
   const { id: projectId } = await getProjectInfo(instance);
 
-  const projectTokens = await doGitlabRequest(
+  // all pages: rotated tokens accumulate one (inactive) entry per
+  // rotation, so the active one quickly moves past the first page —
+  // missing it here created a brand-new duplicate token on every setup
+  const projectTokens = await doGitlabRequestAllPages(
     instance,
     `projects/${projectId}/access_tokens`,
   );

package/src/apps/cli/commands/project/setup/setupCloudRun.ts

@@ -11,6 +11,45 @@ import { fetchGcloudProjectNumber } from "../../../../../gcloud/getProjectNumber
 import { upsertGcloudServiceAccountAndSaveSecret } from "../../../../../gcloud/serviceAccounts";
 import { updateCatladderStore } from "../../../../../store/updateCatladderStore";
 
+export const CLOUD_RUN_DEPLOY_SA_NAME = "cl-d";
+
+/**
+ * the gcloud services the given deploy config requires.
+ *
+ * Shared with `project doctor`: setup enables them, doctor compares
+ * them against what is actually enabled — config edits after setup
+ * (e.g. adding `cloudSql`) would otherwise drift silently.
+ */
+export const getCloudRunRequiredServices = (config: {
+  cloudSql?: unknown;
+}): string[] => [
+  "run.googleapis.com",
+  "artifactregistry.googleapis.com",
+  "cloudscheduler.googleapis.com", // for scheduling jobs
+  "cloudresourcemanager.googleapis.com", // only required to get google cloud project number
+  ...(config.cloudSql
+    ? ["sqladmin.googleapis.com", "sql-component.googleapis.com"]
+    : []),
+];
+
+/**
+ * the IAM roles the deploy service account needs for the given deploy
+ * config. Shared with `project doctor` (same rationale as
+ * {@link getCloudRunRequiredServices}) — the motivating case: a
+ * component gets `cloudSql` after its setup ran, deploys keep working,
+ * and only the review teardown hits the missing `cloudsql.admin`.
+ */
+export const getCloudRunDeployRoles = (config: {
+  cloudSql?: unknown;
+}): string[] => [
+  // TODO: run.admin is a bit much, would be better to follow https://stackoverflow.com/questions/55788714/deploying-to-cloud-run-with-a-custom-service-account-failed-with-iam-serviceacco
+  "roles/artifactregistry.repoAdmin",
+  "roles/run.admin",
+  "roles/iam.serviceAccountUser",
+  "roles/cloudscheduler.admin", // for scheduling
+  ...(config.cloudSql ? ["roles/cloudsql.admin"] : []),
+];
+
 export const setupCloudRun = async (
   instance: IO,
   context: ComponentContext,
@@ -24,18 +63,7 @@ export const setupCloudRun = async (
   // enable services
 
   instance.log("enable required services...");
-  await enableGCloudServices(
-    [
-      "run.googleapis.com",
-      "artifactregistry.googleapis.com",
-      "cloudscheduler.googleapis.com", // for scheduling jobs
-      "cloudresourcemanager.googleapis.com", // only required to get google cloud project number
-      ...(config.cloudSql
-        ? ["sqladmin.googleapis.com", "sql-component.googleapis.com"]
-        : []),
-    ],
-    config,
-  );
+  await enableGCloudServices(getCloudRunRequiredServices(config), config);
   instance.log("upsert artifacts registry...");
   await upsertGcloudArtifactsRegistry(config);
 
@@ -47,15 +75,8 @@ export const setupCloudRun = async (
       projectId: config.projectId,
       displayName: "Catladder gcloud deploy",
       description: "This service account deploys to google cloud",
-      name: "cl-d",
-      // TODO: run.admin is a bit much, would be better to follow https://stackoverflow.com/questions/55788714/deploying-to-cloud-run-with-a-custom-service-account-failed-with-iam-serviceacco
-      roles: [
-        "roles/artifactregistry.repoAdmin",
-        "roles/run.admin",
-        "roles/iam.serviceAccountUser",
-        "roles/cloudscheduler.admin", // for scheduling
-        ...(config.cloudSql ? ["roles/cloudsql.admin"] : []),
-      ],
+      name: CLOUD_RUN_DEPLOY_SA_NAME,
+      roles: getCloudRunDeployRoles(config),
     },
     GCLOUD_DEPLOY_CREDENTIALS_KEY,
   );

package/src/commands/index.ts

@@ -21,6 +21,7 @@ export { commandTriggerCronjobGeneral } from "./project/k8s/commandTriggerCronjo
 export { commandRenewToken } from "./project/commandRenewToken";
 export { commandGetMyTotalWorktime } from "./project/commandGetMyTotalWorktime";
 export { commandSetup } from "./project/commandSetup";
+export { commandDoctor } from "./project/commandDoctor";
 export { commandCiJobOpen, commandCiJobLog } from "./project/commandGitlabCi";
 export { commandProjectRestoreDb } from "./project/commandProjectRestoreDb";
 export { commandSecurityEvaluate } from "./project/commandSecurityEvaluate";

package/src/commands/project/commandDoctor.ts

@@ -0,0 +1,21 @@
+import { doctorProject } from "../../apps/cli/commands/project/doctor";
+import { defineCommand } from "../../core/defineCommand";
+
+export const commandDoctor = defineCommand({
+  name: "project doctor",
+  description:
+    "Checks for drift between the config and the provisioned infrastructure (read-only): service accounts, IAM roles, gitlab tokens, github secrets, store entries.",
+  group: "project",
+  inputs: {
+    component: {
+      type: "string",
+      message: "component name",
+      positional: true,
+      required: false,
+    },
+  },
+  execute: async (ctx) => {
+    const component = await ctx.get("component");
+    await doctorProject(ctx, component);
+  },
+});

package/src/gcloud/__tests__/serviceAccountNames.test.ts

@@ -0,0 +1,63 @@
+import { describe, expect, it } from "vitest";
+import { getGcloudServiceAccountNames } from "../serviceAccountNames";
+
+// the naming is shared between `project setup` (creates accounts) and
+// `project doctor` (computes the expected identifier) — these tests pin
+// it down so a change never silently makes doctor look for accounts
+// under a different name than setup created them
+
+const account = { name: "cl-d", projectId: "some-project" };
+
+describe("getGcloudServiceAccountNames", () => {
+  it("keeps all parts verbatim when they fit into 30 chars", () => {
+    const { fullName, fullIdentifier } = getGcloudServiceAccountNames(
+      {
+        env: "dev",
+        name: "www",
+        fullConfig: { customerName: "acme", appName: "shop" },
+      },
+      account,
+    );
+    expect(fullName).toBe("cl-d-acme-shop-dev-www");
+    expect(fullIdentifier).toBe(
+      "cl-d-acme-shop-dev-www@some-project.iam.gserviceaccount.com",
+    );
+  });
+
+  it("hashes the customer/app middle part when the name would exceed 30 chars", () => {
+    const { fullName } = getGcloudServiceAccountNames(
+      {
+        env: "review",
+        name: "outlet-stats",
+        fullConfig: { customerName: "wea", appName: "food2050" },
+      },
+      account,
+    );
+    expect(fullName).toMatch(/^cl-d-[0-9a-f]{5}-review-outlet-stats$/);
+    expect(fullName.length).toBe(30);
+  });
+
+  it("hashes the env/component suffix too when a 1-char middle still does not fit", () => {
+    const { fullName } = getGcloudServiceAccountNames(
+      {
+        env: "review",
+        name: "a-component-with-a-very-long-name",
+        fullConfig: { customerName: "customer", appName: "app" },
+      },
+      account,
+    );
+    expect(fullName).toMatch(/^cl-d-[0-9a-f]-[0-9a-f]{23}$/);
+    expect(fullName.length).toBe(30);
+  });
+
+  it("is deterministic", () => {
+    const context = {
+      env: "review",
+      name: "outlet-stats",
+      fullConfig: { customerName: "wea", appName: "food2050" },
+    };
+    expect(getGcloudServiceAccountNames(context, account)).toEqual(
+      getGcloudServiceAccountNames(context, account),
+    );
+  });
+});

package/src/gcloud/artifactsRegistry.ts

@@ -1,13 +1,15 @@
 import type { Gcloudregion } from "@catladder/pipeline";
 import { exec } from "child-process-promise";
 
+export const CATLADDER_REGISTRY_NAME = "catladder-deploy";
+
 export const upsertGcloudArtifactsRegistry = async (config: {
   projectId: string;
   region: Gcloudregion;
 }) => {
   try {
     await exec(
-      `gcloud artifacts repositories create catladder-deploy --project="${config.projectId}" --repository-format=docker --location=${config.region}`,
+      `gcloud artifacts repositories create ${CATLADDER_REGISTRY_NAME} --project="${config.projectId}" --repository-format=docker --location=${config.region}`,
     );
   } catch (e) {
     // probably already exists

package/src/gcloud/serviceAccountNames.ts

@@ -0,0 +1,65 @@
+import { createHmac } from "crypto";
+
+/**
+ * the parts of a component context the naming depends on — structural
+ * subset of `ComponentContext` so the naming stays a pure function
+ */
+export type ServiceAccountNamingContext = {
+  env: string;
+  name: string;
+  fullConfig: { customerName: string; appName: string };
+};
+
+/**
+ * the deterministic gcloud service account naming used by
+ * `project setup`. Extracted so `project doctor` can compute the
+ * expected identifier of an account without creating anything.
+ *
+ * gcloud limits account ids to 30 chars: the customer/app middle part
+ * (and, if even that is not enough, the env/component suffix) is
+ * replaced by a hash when the full name would not fit.
+ */
+export const getGcloudServiceAccountNames = (
+  context: ServiceAccountNamingContext,
+  account: { name: string; projectId: string },
+): { fullName: string; fullIdentifier: string } => {
+  const { projectId, name } = account;
+
+  // name has limit of 30
+  const namePrefix = `${name}`;
+  const nameSuffixRaw = `${context.env}-${context.name}`;
+  const nameMiddleRaw = `${context.fullConfig.customerName}-${context.fullConfig.appName}`;
+  const MAX_LENGTH = 30;
+  const NUM_SEPARATORS = 2;
+
+  // we want to first hash middle, then suffix
+  // if for middle we have at least 1 char left, its ok, so we don't hash nameSuffix, otherwise we need to hash that as well
+
+  const middleMaxLength =
+    MAX_LENGTH - namePrefix.length - nameSuffixRaw.length - NUM_SEPARATORS;
+
+  let nameMiddle: string;
+  let nameSuffix: string;
+  if (middleMaxLength < 1) {
+    nameMiddle = hashIfNessecary(nameMiddleRaw, 1);
+    nameSuffix = hashIfNessecary(
+      nameSuffixRaw,
+      MAX_LENGTH - namePrefix.length - 1 - NUM_SEPARATORS,
+    );
+  } else {
+    nameMiddle = hashIfNessecary(nameMiddleRaw, middleMaxLength);
+    nameSuffix = nameSuffixRaw;
+  }
+
+  const fullName = `${namePrefix}-${nameMiddle}-${nameSuffix}`;
+  const fullIdentifier = `${fullName}@${projectId}.iam.gserviceaccount.com`;
+
+  return { fullName, fullIdentifier };
+};
+
+const hashIfNessecary = (str: string, maxLength: number) =>
+  str.length > maxLength ? hash(str, maxLength) : str;
+
+const hash = (str: string, length: number) => {
+  return createHmac("sha256", str).digest("hex").substring(0, length);
+};

package/src/gcloud/serviceAccounts.ts

@@ -1,9 +1,9 @@
 import type { ComponentContext } from "@catladder/pipeline";
 
 import { exec } from "child-process-promise";
-import { createHmac } from "crypto";
 import type { IO } from "../core/types";
 import { upsertAllVariables } from "../utils/gitlab";
+import { getGcloudServiceAccountNames } from "./serviceAccountNames";
 
 export const accountExists = async (fullIdentifier: string) => {
   try {
@@ -25,40 +25,15 @@ const upsertGcloudServiceAccount = async (
   context: ComponentContext,
   account: ServiceAccount,
 ): Promise<string> => {
-  const { projectId, name, displayName, roles, description } = account;
+  const { projectId, displayName, roles, description } = account;
 
-  // name has limit of 30
-  const namePrefix = `${name}`;
-  const nameSuffixRaw = `${context.env}-${context.name}`;
-  const nameMiddleRaw = `${context.fullConfig.customerName}-${context.fullConfig.appName}`;
-  const MAX_LENGTH = 30;
-  const NUM_SEPARATORS = 2;
-
-  // we want to first hash middle, then suffix
-  // if for middle we have at least 1 char left, its ok, so we don't hash nameSuffix, otherwise we need to hash that as well
-
-  const middleMaxLength =
-    MAX_LENGTH - namePrefix.length - nameSuffixRaw.length - NUM_SEPARATORS;
-
-  let nameMiddle: string;
-  let nameSuffix: string;
-  if (middleMaxLength < 1) {
-    nameMiddle = hashIfNessecary(nameMiddleRaw, 1);
-    nameSuffix = hashIfNessecary(
-      nameSuffixRaw,
-      MAX_LENGTH - namePrefix.length - 1 - NUM_SEPARATORS,
-    );
-  } else {
-    nameMiddle = hashIfNessecary(nameMiddleRaw, middleMaxLength);
-    nameSuffix = nameSuffixRaw;
-  }
-
-  const fullName = `${namePrefix}-${nameMiddle}-${nameSuffix}`;
+  const { fullName, fullIdentifier } = getGcloudServiceAccountNames(
+    context,
+    account,
+  );
 
   const fullDisplayName = `${context.fullConfig.customerName}-${context.fullConfig.appName} ${context.env}:${context.name} | ${displayName}`;
 
-  const fullIdentifier = `${fullName}@${projectId}.iam.gserviceaccount.com`;
-
   const existing = await accountExists(fullIdentifier);
 
   if (!existing) {
@@ -111,10 +86,3 @@ export const upsertGcloudServiceAccountAndSaveSecret = async (
   );
   instance.log("done!");
 };
-
-const hashIfNessecary = (str: string, maxLength: number) =>
-  str.length > maxLength ? hash(str, maxLength) : str;
-
-const hash = (str: string, length: number) => {
-  return createHmac("sha256", str).digest("hex").substring(0, length);
-};

package/src/utils/concurrency.ts

@@ -0,0 +1,25 @@
+/**
+ * maps over items with at most `limit` calls in flight, preserving the
+ * input order in the result (unlike reporting-as-you-go, this lets
+ * callers run slow per-item lookups in parallel and still log the
+ * outcomes in a stable order)
+ */
+export const mapWithConcurrency = async <T, R>(
+  items: T[],
+  limit: number,
+  fn: (item: T, index: number) => Promise<R>,
+): Promise<R[]> => {
+  const results = new Array<R>(items.length);
+  let next = 0;
+  const workers = Array.from(
+    { length: Math.max(1, Math.min(limit, items.length)) },
+    async () => {
+      while (next < items.length) {
+        const index = next++;
+        results[index] = await fn(items[index], index);
+      }
+    },
+  );
+  await Promise.all(workers);
+  return results;
+};

package/src/utils/github.ts

@@ -34,6 +34,54 @@ export const getGithubRepoFromRemote = async (
   }
 };
 
+/**
+ * runs a paginated `gh api` query and returns the jq-extracted lines
+ * (one value per line, e.g. names)
+ */
+const ghApiLines = async (path: string, jq: string): Promise<string[]> => {
+  const { stdout } = await execFile(
+    "gh",
+    ["api", "--paginate", path, "--jq", jq],
+    { maxBuffer: 10 * 1024 * 1024 },
+  );
+  return stdout
+    .split("\n")
+    .map((line) => line.trim())
+    .filter(Boolean);
+};
+
+/** names of all deployment environments of the repository */
+export const listGithubEnvironments = (repo: string): Promise<string[]> =>
+  ghApiLines(`repos/${repo}/environments?per_page=100`, ".environments[].name");
+
+/**
+ * names of the actions secrets — of an environment, or of the repo
+ * level when no environment is given. Github secrets are write-only:
+ * names are all that can ever be inspected.
+ */
+export const listGithubSecretNames = (
+  repo: string,
+  environment?: string,
+): Promise<string[]> =>
+  ghApiLines(
+    environment
+      ? `repos/${repo}/environments/${encodeURIComponent(environment)}/secrets?per_page=100`
+      : `repos/${repo}/actions/secrets?per_page=100`,
+    ".secrets[].name",
+  );
+
+/** names of the actions variables (environment or repo level) */
+export const listGithubVariableNames = (
+  repo: string,
+  environment?: string,
+): Promise<string[]> =>
+  ghApiLines(
+    environment
+      ? `repos/${repo}/environments/${encodeURIComponent(environment)}/variables?per_page=100`
+      : `repos/${repo}/actions/variables?per_page=100`,
+    ".variables[].name",
+  );
+
 const ghSetViaStdin = (
   kind: "secret" | "variable",
   repo: string,

package/src/utils/gitlab.ts

@@ -90,6 +90,30 @@ export const doGitlabRequest = async <T = any>(
   );
 };
 
+/**
+ * GET a list endpoint across ALL pages (gitlab returns 20 items per
+ * page by default — long-lived projects easily exceed that, e.g.
+ * rotated access tokens accumulate one entry per rotation)
+ */
+export const doGitlabRequestAllPages = async <T = any>(
+  io: IO | null,
+  path: string,
+): Promise<T[]> => {
+  const PER_PAGE = 100;
+  const separator = path.includes("?") ? "&" : "?";
+  const all: T[] = [];
+  for (let page = 1; ; page++) {
+    const items = await doGitlabRequest<T[]>(
+      io,
+      `${path}${separator}per_page=${PER_PAGE}&page=${page}`,
+    );
+    if (!items?.length) break;
+    all.push(...items);
+    if (items.length < PER_PAGE) break;
+  }
+  return all;
+};
+
 export const getProjectInfo = async (
   io: IO | null,
 ): Promise<{ id: string; web_url: string }> => {

package/tsconfig.json

@@ -16,5 +16,6 @@
       "@catladder/pipeline": ["../pipeline/src/index.ts"]
     }
   },
-  "include": ["./src/**/*"]
+  "include": ["./src/**/*"],
+  "exclude": ["**/__tests__", "**/*.test.ts", "**/*.test.tsx"]
 }