@catladder/cli 0.0.0-main-2355fd7c → 0.0.0-main-018e2dee

package/package.json

@@ -53,7 +53,7 @@
     }
   ],
   "license": "MIT",
-  "version": "0.0.0-main-2355fd7c",
+  "version": "0.0.0-main-018e2dee",
   "scripts": {
     "lint": "eslint \"src/**/*.ts\"",
     "lint:fix": "eslint \"src/**/*.ts\" --fix",

package/src/apps/catenv/catenv.ts

@@ -1,4 +1,5 @@
 import { getProjectConfig } from "../../config/getProjectConfig";
+import { writeLocalAccessKeyFiles } from "../../localAccess/keyFiles";
 import { printVariables } from "./printVariables";
 import type { Choice } from "./types";
 import { printVerboseBanner } from "./verboseBanner";
@@ -42,6 +43,9 @@ export default async (choice?: Choice, options?: Options) => {
     printVerboseBanner();
   }
 
+  // before the env vars are resolved: they point at the key files
+  await writeLocalAccessKeyFiles(io, config);
+
   await Promise.all([
     generatePipelineFiles(context),
     writeDotEnvFiles(context, choice),

package/src/apps/cli/commands/project/doctor/GcloudProjectInspector.ts

@@ -1,7 +1,11 @@
 import { exec } from "child-process-promise";
 import { CATLADDER_REGISTRY_NAME } from "../../../../../gcloud/artifactsRegistry";
 
-type IamBinding = { role: string; members?: string[] };
+type IamBinding = {
+  role: string;
+  members?: string[];
+  condition?: unknown;
+};
 
 /**
  * read-only gcloud queries for the doctor, cached per project: checking
@@ -41,6 +45,26 @@ export class GcloudProjectInspector {
       .map((binding) => binding.role);
   }
 
+  /** roles bound to the member without an iam condition */
+  async getUnconditionalRolesOfMember(
+    projectId: string,
+    member: string,
+  ): Promise<string[]> {
+    return (await this.getIamBindings(projectId))
+      .filter((b) => !b.condition && b.members?.includes(member))
+      .map((b) => b.role);
+  }
+
+  /** roles bound to the member under an iam condition */
+  async getConditionalRolesOfMember(
+    projectId: string,
+    member: string,
+  ): Promise<string[]> {
+    return (await this.getIamBindings(projectId))
+      .filter((b) => b.condition && b.members?.includes(member))
+      .map((b) => b.role);
+  }
+
   getEnabledServices(projectId: string): Promise<Set<string>> {
     if (!this.enabledServices.has(projectId)) {
       this.enabledServices.set(

package/src/apps/cli/commands/project/doctor/__tests__/checkLocalAccess.test.ts

@@ -0,0 +1,127 @@
+import type { ComponentContext, Config } from "@catladder/pipeline";
+import { beforeEach, describe, expect, it, vi } from "vitest";
+
+const state = { exists: true, keys: ["k1"] as string[] };
+vi.mock("../../../../../../gcloud/serviceAccounts", () => ({
+  accountExists: async () => state.exists,
+}));
+vi.mock("../../../../../../gcloud/localAccess", async (orig) => ({
+  ...(await orig<object>()),
+  listUserManagedKeyIds: async () => state.keys,
+}));
+
+import { checkLocalAccess } from "../checkLocalAccess";
+
+const cloudRun = { type: "google-cloudrun", projectId: "p", region: "r" };
+
+const context = {
+  env: "dev",
+  name: "web",
+  fullConfig: { customerName: "acme", appName: "shop" },
+  deploy: { config: cloudRun },
+} as unknown as ComponentContext;
+
+const config = (extra: Partial<Config> = {}) =>
+  ({
+    appName: "shop",
+    customerName: "acme",
+    components: { web: { dir: ".", build: false, deploy: cloudRun } },
+    localAccess: { gcp: { presets: ["logs"] } },
+    secrets: { vault: { type: "bitwarden" } },
+    ...extra,
+  }) as Config;
+
+const run = async (
+  cfg: Config,
+  bound: string[] = ["roles/logging.viewer"],
+  conditional: string[] = [],
+) => {
+  const out: string[] = [];
+  const report = {
+    section: (t: string) => out.push(`section:${t}`),
+    ok: (m: string) => out.push(`ok:${m}`),
+    warn: (m: string) => out.push(`warn:${m}`),
+    fail: (m: string) => out.push(`fail:${m}`),
+  } as any;
+  const inspector = {
+    getUnconditionalRolesOfMember: async () => bound,
+    getConditionalRolesOfMember: async () => conditional,
+  } as any;
+  await checkLocalAccess(report, cfg, [context], inspector);
+  return out;
+};
+
+beforeEach(() => {
+  state.exists = true;
+  state.keys = ["k1"];
+});
+
+describe("checkLocalAccess", () => {
+  it("is silent without a declaration and without an account", async () => {
+    state.exists = false;
+    expect(await run(config({ localAccess: undefined }))).toEqual([]);
+  });
+
+  it("is ok when roles and key match", async () => {
+    const out = await run(config());
+    expect(out.some((l) => l.startsWith("ok:"))).toBe(true);
+    expect(out.some((l) => /^(fail|warn):/.test(l))).toBe(false);
+  });
+
+  it("reports missing and extra roles", async () => {
+    const out = await run(config(), ["roles/run.admin"]);
+    expect(out.join("\n")).toMatch(
+      /fail:.*missing roles: roles\/logging.viewer/,
+    );
+    expect(out.join("\n")).toMatch(/fail:.*undeclared roles: roles\/run.admin/);
+  });
+
+  it("reports conditional roles", async () => {
+    const out = await run(config(), ["roles/logging.viewer"], ["roles/x.y"]);
+    expect(out.join("\n")).toMatch(/fail:.*conditional roles/);
+  });
+
+  it("reports a missing account", async () => {
+    state.exists = false;
+    expect((await run(config())).join("\n")).toMatch(/fail:.*does not exist/);
+  });
+
+  it("warns about an account that is not declared any more", async () => {
+    expect((await run(config({ localAccess: undefined }))).join("\n")).toMatch(
+      /warn:.*exists although no local access is declared/,
+    );
+  });
+
+  it.each([[[]], [["k1", "k2"]]])("reports %j keys", async (keys) => {
+    state.keys = keys;
+    expect((await run(config())).join("\n")).toMatch(
+      new RegExp(`fail:.*${keys.length} user-managed keys`),
+    );
+  });
+
+  it("refuses the gitlab vault", async () => {
+    const out = await run(config({ secrets: { vault: "gitlab" } } as any));
+    expect(out.join("\n")).toMatch(/fail:.*not a CI variable store/);
+  });
+
+  it("reports an invalid declaration instead of throwing", async () => {
+    const out = await run(
+      config({ localAccess: { gcp: { roles: ["roles/owner"] } } }),
+    );
+    expect(out.join("\n")).toMatch(/fail:.*not allowed/);
+  });
+
+  it("reports a credentialsEnvVar exported by several envs", async () => {
+    const out = await run(
+      config({
+        localAccess: {
+          gcp: {
+            presets: ["logs"],
+            credentialsEnvVar: "GOOGLE_APPLICATION_CREDENTIALS",
+          },
+        },
+      }),
+    );
+    expect(out.join("\n")).toMatch(/fail:.*declared by both/);
+  });
+});

package/src/apps/cli/commands/project/doctor/checkCloudRun.ts

@@ -1,4 +1,4 @@
-import type { ComponentContext } from "@catladder/pipeline";
+import type { ComponentContext, Config } from "@catladder/pipeline";
 import { isOfDeployType } from "@catladder/pipeline";
 import { CATLADDER_REGISTRY_NAME } from "../../../../../gcloud/artifactsRegistry";
 import { getGcloudServiceAccountNames } from "../../../../../gcloud/serviceAccountNames";
@@ -9,6 +9,7 @@ import {
   getCloudRunDeployRoles,
   getCloudRunRequiredServices,
 } from "../setup/setupCloudRun";
+import { checkLocalAccess } from "./checkLocalAccess";
 import type { DoctorReport } from "./DoctorReport";
 import { GcloudProjectInspector } from "./GcloudProjectInspector";
 
@@ -33,6 +34,7 @@ const getCloudRunConfig = (context: ComponentContext) => {
 export const checkCloudRun = async (
   report: DoctorReport,
   contexts: ComponentContext[],
+  config: Config,
 ) => {
   const cloudRunContexts = contexts.filter((context) =>
     isOfDeployType(context.deploy?.config, "google-cloudrun"),
@@ -145,4 +147,6 @@ export const checkCloudRun = async (
       );
     }
   }
+
+  await checkLocalAccess(report, config, cloudRunContexts, inspector);
 };

package/src/apps/cli/commands/project/doctor/checkLocalAccess.ts

@@ -0,0 +1,146 @@
+import type { ComponentContext, Config } from "@catladder/pipeline";
+import {
+  diffRoleBindings,
+  getLocalAccessGcpCredentialsEnvVars,
+  getLocalAccessGcpRoles,
+  getVaultConfig,
+  isOfDeployType,
+} from "@catladder/pipeline";
+import {
+  getLocalAccessServiceAccountNames,
+  listUserManagedKeyIds,
+} from "../../../../../gcloud/localAccess";
+import { accountExists } from "../../../../../gcloud/serviceAccounts";
+import { mapWithConcurrency } from "../../../../../utils/concurrency";
+import type { DoctorReport } from "./DoctorReport";
+import type { GcloudProjectInspector } from "./GcloudProjectInspector";
+
+/**
+ * compares the read-only local access service accounts (`localAccess`
+ * in catladder.ts) with their declaration: missing account, missing and
+ * extra bindings, and an account that exists although nothing is
+ * declared any more. Expects an authenticated inspector.
+ */
+export const checkLocalAccess = async (
+  report: DoctorReport,
+  config: Config,
+  contexts: ComponentContext[],
+  inspector: GcloudProjectInspector,
+) => {
+  const results = await mapWithConcurrency(contexts, 5, async (context) => {
+    const deployConfig = context.deploy?.config;
+    if (!isOfDeployType(deployConfig, "google-cloudrun")) return null;
+    const label = `${context.env}:${context.name}`;
+    let declared: string[];
+    try {
+      declared = getLocalAccessGcpRoles(config, context.env);
+      getLocalAccessGcpCredentialsEnvVars(config, context.name);
+    } catch (e) {
+      return { label, invalid: e.message as string };
+    }
+    const { fullIdentifier } = getLocalAccessServiceAccountNames(
+      context,
+      deployConfig.projectId,
+    );
+    const exists = await accountExists(fullIdentifier);
+    const member = `serviceAccount:${fullIdentifier}`;
+    const bound = exists
+      ? await inspector.getUnconditionalRolesOfMember(
+          deployConfig.projectId,
+          member,
+        )
+      : [];
+    const conditional = exists
+      ? await inspector.getConditionalRolesOfMember(
+          deployConfig.projectId,
+          member,
+        )
+      : [];
+    const keyCount = exists
+      ? (await listUserManagedKeyIds(fullIdentifier)).length
+      : 0;
+    return {
+      label,
+      declared,
+      fullIdentifier,
+      exists,
+      bound,
+      conditional,
+      keyCount,
+    };
+  });
+
+  const relevant = results.filter(
+    (r) => r && ("invalid" in r || r.declared.length > 0 || r.exists),
+  );
+  if (relevant.length === 0) return;
+
+  report.section("local access (read-only service accounts)");
+  for (const result of relevant) {
+    if (!result) continue;
+    if ("invalid" in result) {
+      report.fail(`${result.label}: ${result.invalid}`);
+      continue;
+    }
+    const {
+      label,
+      declared,
+      fullIdentifier,
+      exists,
+      bound,
+      conditional,
+      keyCount,
+    } = result;
+    if (declared.length > 0 && getVaultConfig(config).type === "gitlab") {
+      report.fail(
+        `${label}: localAccess needs a vault that is not a CI variable store (the gitlab vault would expose the key to every CI job)`,
+      );
+      continue;
+    }
+    const heal = `run: catladder project setup`;
+    if (declared.length === 0) {
+      report.warn(
+        `${label}: ${fullIdentifier} exists although no local access is declared`,
+        heal,
+      );
+    } else if (!exists) {
+      report.fail(`${label}: ${fullIdentifier} does not exist`, heal);
+    } else {
+      const { missing, extra } = diffRoleBindings(declared, bound);
+      if (missing.length > 0) {
+        report.fail(
+          `${label}: local access account is missing roles: ${missing.join(", ")}`,
+          heal,
+        );
+      }
+      if (extra.length > 0) {
+        report.fail(
+          `${label}: local access account has undeclared roles: ${extra.join(", ")}`,
+          heal,
+        );
+      }
+      if (conditional.length > 0) {
+        report.fail(
+          `${label}: local access account has conditional roles (${conditional.join(", ")}) that setup does not remove`,
+          "remove them by hand",
+        );
+      }
+      if (keyCount !== 1) {
+        report.fail(
+          `${label}: local access account has ${keyCount} user-managed keys, expected exactly 1`,
+          heal,
+        );
+      }
+      if (
+        missing.length === 0 &&
+        extra.length === 0 &&
+        conditional.length === 0 &&
+        keyCount === 1
+      ) {
+        report.ok(
+          `${label}: local access account has exactly the declared roles`,
+        );
+      }
+    }
+  }
+};

package/src/apps/cli/commands/project/doctor/index.ts

@@ -54,7 +54,7 @@ export const doctorProject = async (
   if (getEnabledPipelineTypes(config).includes("gitlab")) {
     await checkGitlab(instance, report, contexts);
   }
-  await checkCloudRun(report, contexts);
+  await checkCloudRun(report, contexts, config);
   await checkGithubImageCasing(report, config);
   await checkGithubReleaseWorkflow(report, config);
   await checkGithub(report, config);

package/src/apps/cli/commands/project/setup/setupContext.ts

@@ -2,6 +2,7 @@ import type { ComponentContext } from "@catladder/pipeline";
 import { isOfDeployType } from "@catladder/pipeline";
 import type { IO } from "../../../../../core/types";
 import { setupCloudRun } from "./setupCloudRun";
+import { setupLocalAccess } from "./setupLocalAccess";
 
 import { setupKubernetes } from "./setupKubernetes";
 import { logSection } from "./logSection";
@@ -13,6 +14,7 @@ export const setupContext = async (instance: IO, context: ComponentContext) => {
     async () => {
       if (isOfDeployType(context.deploy?.config, "google-cloudrun")) {
         await setupCloudRun(instance, context);
+        await setupLocalAccess(instance, context);
       }
 
       const deployConfig = context.deploy?.config;

package/src/apps/cli/commands/project/setup/setupLocalAccess.ts

@@ -0,0 +1,44 @@
+import type { ComponentContext } from "@catladder/pipeline";
+import {
+  getLocalAccessGcpCredentialsEnvVars,
+  getLocalAccessGcpRoles,
+  getVaultConfig,
+  isOfDeployType,
+} from "@catladder/pipeline";
+import { getProjectConfig } from "../../../../../config/getProjectConfig";
+import type { IO } from "../../../../../core/types";
+import { syncLocalAccessServiceAccount } from "../../../../../gcloud/localAccess";
+
+/**
+ * provisions the local access (`localAccess` in catladder.ts)
+ * of a google cloud run component/env: a dedicated service account with
+ * exactly the declared bindings and its key in the secret store. Also
+ * removes what is no longer declared.
+ */
+export const setupLocalAccess = async (
+  instance: IO,
+  context: ComponentContext,
+) => {
+  const config = context.deploy?.config;
+  if (!isOfDeployType(config, "google-cloudrun")) return;
+  const projectConfig = await getProjectConfig();
+  const roles = getLocalAccessGcpRoles(projectConfig, context.env);
+  // validates `credentialsEnvVar` (catenv silently skips invalid ones)
+  getLocalAccessGcpCredentialsEnvVars(projectConfig, context.name);
+
+  if (roles.length > 0 && getVaultConfig(projectConfig).type === "gitlab") {
+    // the gitlab vault stores secrets as CI variables available to every
+    // job on every branch: the key would leak to anyone who can push
+    throw new Error(
+      `localAccess (${context.env}:${context.name}) needs a vault that is not a CI variable store (e.g. bitwarden): the gitlab vault would expose the service account key to every CI job. Configure \`vault\` or remove the localAccess declaration.`,
+    );
+  }
+
+  instance.log("sync local access service account...");
+  await syncLocalAccessServiceAccount(
+    instance,
+    context,
+    config.projectId,
+    roles,
+  );
+};

package/src/config/__tests__/getEnvVarsResolved.test.ts

@@ -0,0 +1,45 @@
+import { beforeEach, describe, expect, it, vi } from "vitest";
+
+const keyFileVars = vi.fn();
+vi.mock("../../localAccess/keyFiles", () => ({
+  getLocalAccessKeyFileVars: (...args: unknown[]) => keyFileVars(...args),
+}));
+vi.mock("../../utils/projects", () => ({ getGitRoot: async () => "/repo" }));
+vi.mock("@catladder/pipeline", async (orig) => ({
+  ...(await orig<object>()),
+  readConfig: async () => ({ config: { components: { web: {} } } }),
+  getEnvironment: async () => ({
+    envVars: { PLAIN: "value" },
+    secretEnvVarKeys: [],
+  }),
+}));
+
+import { getEnvVarsResolved } from "../getProjectConfig";
+
+const io = {
+  getVaultManager: async () => ({ readSecrets: async () => ({}) }),
+} as any;
+
+beforeEach(() => {
+  keyFileVars.mockReset();
+  keyFileVars.mockResolvedValue({
+    CATLADDER_LOCAL_ACCESS_GCP_DEV_KEY_FILE: "/k",
+  });
+});
+
+describe("getEnvVarsResolved", () => {
+  it("adds the local access key file paths for the local env", async () => {
+    expect(await getEnvVarsResolved(io, "local", "web")).toEqual({
+      PLAIN: "value",
+      CATLADDER_LOCAL_ACCESS_GCP_DEV_KEY_FILE: "/k",
+    });
+    expect(keyFileVars).toHaveBeenCalledWith(expect.anything(), "web");
+  });
+
+  it("leaves other envs untouched", async () => {
+    expect(await getEnvVarsResolved(io, "dev", "web")).toEqual({
+      PLAIN: "value",
+    });
+    expect(keyFileVars).not.toHaveBeenCalled();
+  });
+});

package/src/config/getProjectConfig.ts

@@ -13,6 +13,7 @@ import {
 } from "@catladder/pipeline";
 
 import type { IO } from "../core/types";
+import { getLocalAccessKeyFileVars } from "../localAccess/keyFiles";
 import { getVariableValueByRawName } from "../utils/gitlab";
 
 import { getGitRoot } from "../utils/projects";
@@ -191,12 +192,21 @@ export const getEnvVarsResolved = async (
 
   // in the pipeline the secrets alreadyy exists  and bash will expand them
   // but here we need to manually load them
-  return resolveSecrets(io, [
+  const resolved = await resolveSecrets(io, [
     {
       envVars: envionment.envVars,
       secretEnvVarKeys: envionment.secretEnvVarKeys,
     },
   ]);
+  if (env !== "local") return resolved;
+  // paths of the local access key files (written by catenv)
+  return {
+    ...resolved,
+    ...(await getLocalAccessKeyFileVars(
+      await getProjectConfig(),
+      componentName,
+    )),
+  };
 };
 
 /**

package/src/gcloud/__tests__/localAccess.test.ts

@@ -0,0 +1,82 @@
+import { describe, expect, it } from "vitest";
+import type { ComponentContext } from "@catladder/pipeline";
+import {
+  getLocalAccessServiceAccountNames,
+  getConditionalRolesOfMemberInBindings,
+  getRolesOfMemberInBindings,
+  parseServiceAccountKey,
+} from "../localAccess";
+
+const context = {
+  env: "review",
+  name: "web",
+  fullConfig: { customerName: "acme", appName: "shop" },
+} as ComponentContext;
+
+describe("getLocalAccessServiceAccountNames", () => {
+  it("is deterministic, within the 30 char limit and distinct from the deploy account", () => {
+    const a = getLocalAccessServiceAccountNames(context, "some-project");
+    const b = getLocalAccessServiceAccountNames(context, "some-project");
+    expect(a).toEqual(b);
+    expect(a.fullName.length).toBeLessThanOrEqual(30);
+    expect(a.fullName.startsWith("local-diag-")).toBe(true);
+    expect(a.fullIdentifier).toBe(
+      `${a.fullName}@some-project.iam.gserviceaccount.com`,
+    );
+  });
+});
+
+describe("getRolesOfMemberInBindings", () => {
+  it("collects the sorted roles of exactly the given member", () => {
+    const m = "serviceAccount:a@p.iam.gserviceaccount.com";
+    expect(
+      getRolesOfMemberInBindings(
+        [
+          { role: "roles/logging.viewer", members: [m, "user:x@y.z"] },
+          { role: "roles/b.viewer", members: [m] },
+          { role: "roles/run.admin", members: ["user:x@y.z"] },
+          { role: "roles/other" },
+        ],
+        m,
+      ),
+    ).toEqual(["roles/b.viewer", "roles/logging.viewer"]);
+  });
+});
+
+describe("conditional bindings", () => {
+  const m = "serviceAccount:a@p.iam.gserviceaccount.com";
+  const bindings = [
+    { role: "roles/logging.viewer", members: [m] },
+    {
+      role: "roles/storage.objectViewer",
+      members: [m],
+      condition: { title: "t", expression: "true" },
+    },
+  ];
+  it("keeps them out of the plain roles and reports them separately", () => {
+    expect(getRolesOfMemberInBindings(bindings, m)).toEqual([
+      "roles/logging.viewer",
+    ]);
+    expect(getConditionalRolesOfMemberInBindings(bindings, m)).toEqual([
+      "roles/storage.objectViewer",
+    ]);
+  });
+});
+
+describe("parseServiceAccountKey", () => {
+  it("accepts a key and returns its id", () => {
+    const raw = JSON.stringify({ private_key_id: "abc", private_key: "x" });
+    expect(parseServiceAccountKey(raw)).toEqual({
+      key: raw,
+      privateKeyId: "abc",
+    });
+  });
+  it.each(["", "not json", "{}", '{"private_key_id":"a"}'])(
+    "rejects %j instead of storing it",
+    (raw) => {
+      expect(() => parseServiceAccountKey(raw)).toThrow(
+        /valid service account key/,
+      );
+    },
+  );
+});