@catladder/cli 5.0.1 → 5.1.0

package/src/gcloud/serviceAccounts.ts

@@ -2,7 +2,7 @@ import type { ComponentContext } from "@catladder/pipeline";
 
 import { exec } from "child-process-promise";
 import type { IO } from "../core/types";
-import { upsertAllVariables } from "../utils/gitlab";
+import { writeSecretsAndMirror } from "../secrets";
 import { getGcloudServiceAccountNames } from "./serviceAccountNames";
 
 export const accountExists = async (fullIdentifier: string) => {
@@ -76,13 +76,22 @@ export const upsertGcloudServiceAccountAndSaveSecret = async (
   instance.log("upserting service account " + account.name + "...");
   const key = await upsertGcloudServiceAccount(context, account);
 
-  await upsertAllVariables(
+  // through the vault (and from there to every enabled ci backend) —
+  // the key is a secret like any other, the deploy jobs read it under
+  // its declared name. Writing it to gitlab directly would break every
+  // project that has no gitlab.
+  await writeSecretsAndMirror(
     instance,
-    {
-      [secretName]: key,
-    },
-    context.env,
-    context.name,
+    [
+      {
+        env: context.env,
+        componentName: context.name,
+        secrets: { [secretName]: key },
+      },
+    ],
+    // a fresh key is created on every run, the previous one is deleted
+    // at google — a backup copy of it would only be a dead secret
+    { backup: false },
   );
   instance.log("done!");
 };

package/src/secrets/__tests__/writeSecrets.test.ts

@@ -0,0 +1,147 @@
+import type { Config } from "@catladder/pipeline";
+import { beforeEach, describe, expect, it, vi } from "vitest";
+import type { IO } from "../../core/types";
+import { writeSecretsAndMirror } from "../writeSecrets";
+
+const { getProjectConfig, getEnvironment } = vi.hoisted(() => ({
+  getProjectConfig: vi.fn(),
+  getEnvironment: vi.fn(),
+}));
+const { upsertAllVariables } = vi.hoisted(() => ({
+  upsertAllVariables: vi.fn(),
+}));
+const { isGhAuthenticated } = vi.hoisted(() => ({
+  isGhAuthenticated: vi.fn(),
+}));
+const { collectSecretTargets, getConfiguredGithubRepo, pushSecretsToGithub } =
+  vi.hoisted(() => ({
+    collectSecretTargets: vi.fn(),
+    getConfiguredGithubRepo: vi.fn(),
+    pushSecretsToGithub: vi.fn(),
+  }));
+
+vi.mock("../../config/getProjectConfig", () => ({
+  getProjectConfig,
+  getEnvironment,
+}));
+vi.mock("../../utils/gitlab", () => ({ upsertAllVariables }));
+vi.mock("../../utils/github", () => ({ isGhAuthenticated }));
+vi.mock("../github", () => ({
+  collectSecretTargets,
+  getConfiguredGithubRepo,
+  pushSecretsToGithub,
+}));
+
+const vaultWriteSecrets = vi.fn();
+const io = {
+  log: vi.fn(),
+  getVaultManager: async () => ({ writeSecrets: vaultWriteSecrets }),
+} as unknown as IO;
+
+const config = (partial: Partial<Config>): Config =>
+  ({
+    customerName: "pan",
+    appName: "app",
+    components: {},
+    ...partial,
+  }) as Config;
+
+const writes = [
+  { env: "dev", componentName: "www", secrets: { API_KEY: "sesame" } },
+];
+
+beforeEach(() => {
+  vi.clearAllMocks();
+  getEnvironment.mockResolvedValue({
+    secretEnvVarKeys: [{ key: "API_KEY" }],
+    jobOnlyVars: {
+      build: { secretEnvVarKeys: [] },
+      deploy: { secretEnvVarKeys: [] },
+    },
+  });
+  isGhAuthenticated.mockResolvedValue(true);
+  getConfiguredGithubRepo.mockResolvedValue("panter/app");
+  collectSecretTargets.mockResolvedValue(
+    new Map([["CL_dev_www_API_KEY", new Set(["dev"])]]),
+  );
+});
+
+describe("writeSecretsAndMirror", () => {
+  it("never touches the gitlab api on a project without a gitlab pipeline", async () => {
+    getProjectConfig.mockResolvedValue(
+      config({
+        pipelines: { github: true },
+        secrets: { vault: { type: "bitwarden" } },
+      }),
+    );
+
+    await writeSecretsAndMirror(io, writes);
+
+    expect(vaultWriteSecrets).toHaveBeenCalledWith(
+      "dev",
+      "www",
+      { API_KEY: "sesame" },
+      io,
+      {},
+    );
+    expect(upsertAllVariables).not.toHaveBeenCalled();
+    expect(pushSecretsToGithub).toHaveBeenCalledWith(
+      io,
+      "panter/app",
+      [{ name: "CL_dev_www_API_KEY", value: "sesame", kind: "secret" }],
+      expect.any(Map),
+    );
+  });
+
+  it("does not mirror to gitlab when gitlab IS the vault (already written)", async () => {
+    getProjectConfig.mockResolvedValue(config({ pipelines: { gitlab: true } }));
+
+    await writeSecretsAndMirror(io, writes);
+
+    expect(vaultWriteSecrets).toHaveBeenCalledTimes(1);
+    expect(upsertAllVariables).not.toHaveBeenCalled();
+    expect(pushSecretsToGithub).not.toHaveBeenCalled();
+  });
+
+  it("mirrors to gitlab when it is only a backend, and forwards the backup option", async () => {
+    getProjectConfig.mockResolvedValue(
+      config({
+        pipelines: { gitlab: true, github: true },
+        secrets: { vault: "bitwarden" },
+      }),
+    );
+
+    await writeSecretsAndMirror(io, writes, { backup: false });
+
+    expect(vaultWriteSecrets).toHaveBeenCalledWith(
+      "dev",
+      "www",
+      { API_KEY: "sesame" },
+      io,
+      { backup: false },
+    );
+    expect(upsertAllVariables).toHaveBeenCalledWith(
+      io,
+      { API_KEY: "sesame" },
+      "dev",
+      "www",
+      false,
+    );
+    expect(pushSecretsToGithub).toHaveBeenCalled();
+  });
+
+  it("keeps the secrets in the vault when the github mirror cannot run", async () => {
+    getProjectConfig.mockResolvedValue(
+      config({
+        pipelines: { github: true },
+        secrets: { vault: "bitwarden" },
+      }),
+    );
+    isGhAuthenticated.mockResolvedValue(false);
+
+    await writeSecretsAndMirror(io, writes);
+
+    expect(vaultWriteSecrets).toHaveBeenCalledTimes(1);
+    expect(pushSecretsToGithub).not.toHaveBeenCalled();
+  });
+});

package/src/secrets/github.ts

@@ -0,0 +1,157 @@
+import type { Config } from "@catladder/pipeline";
+import {
+  getPipelineGitRemote,
+  getSecretVarName,
+  GithubBackend,
+} from "@catladder/pipeline";
+import {
+  getAllComponentsWithAllEnvsHierarchical,
+  getEnvironment,
+  getProjectConfig,
+} from "../config/getProjectConfig";
+import type { IO } from "../core/types";
+import {
+  ensureGithubEnvironment,
+  getGithubRepoFromRemote,
+  setGithubSecret,
+  setGithubVariable,
+} from "../utils/github";
+
+/**
+ * all secrets declared in the config, with their values read from the
+ * vault (github secrets are write-only and can never act as a source)
+ */
+export const collectSecretsFromVault = async (io: IO, envFilter?: string[]) => {
+  const envAndComponents = await getAllComponentsWithAllEnvsHierarchical();
+
+  const kinds = new Map<string, "secret" | "variable">();
+  for (const [componentName, envs] of Object.entries(envAndComponents)) {
+    for (const env of envs) {
+      if (envFilter && !envFilter.includes(env)) {
+        continue;
+      }
+      const { secretEnvVarKeys, jobOnlyVars } = await getEnvironment(
+        env,
+        componentName,
+      );
+      const keys = [
+        ...secretEnvVarKeys,
+        ...jobOnlyVars.build.secretEnvVarKeys,
+        ...jobOnlyVars.deploy.secretEnvVarKeys,
+      ];
+      keys.forEach(({ key, kind }) =>
+        kinds.set(getSecretVarName(env, componentName, key), kind ?? "secret"),
+      );
+    }
+  }
+
+  const names = [...kinds.keys()];
+  const values = await (await io.getVaultManager()).readSecrets(names, io);
+  const secrets = names
+    .filter((name) => values[name] !== undefined)
+    .map((name) => ({
+      name,
+      value: values[name],
+      kind: kinds.get(name) ?? ("secret" as const),
+    }));
+  const missing = names.filter((name) => values[name] === undefined);
+  return { secrets, missing };
+};
+
+/** marker for repo-level (environment-less) targets */
+const REPO_LEVEL = "";
+
+/**
+ * derives which github environment(s) each secret/variable must live in
+ * by scanning the GENERATED workflows: a `${{ secrets.X }}` / `${{
+ * vars.X }}` reference in a job's env targets that job's environment
+ * (or the repo level when the job has none). Deriving from the real
+ * output means the sync can never drift from what the workflows expect.
+ *
+ * Environment secrets are the answer to github's cap of 100 secrets
+ * per REPO (a multi-env project easily exceeds it): the cap applies
+ * per environment instead.
+ */
+export const collectSecretTargets = async (
+  config: Config,
+): Promise<Map<string, Set<string>>> => {
+  const workflows = await new GithubBackend().createWorkflows(config);
+  const targets = new Map<string, Set<string>>();
+  for (const workflow of Object.values(workflows)) {
+    for (const job of Object.values(workflow.jobs)) {
+      const environment =
+        typeof job.environment === "object"
+          ? job.environment.name
+          : job.environment;
+      for (const value of Object.values(job.env ?? {})) {
+        if (typeof value !== "string") continue;
+        for (const match of value.matchAll(
+          /\$\{\{\s*(?:secrets|vars)\.([A-Za-z0-9_]+)\s*\}\}/g,
+        )) {
+          const name = match[1];
+          if (!targets.has(name)) {
+            targets.set(name, new Set());
+          }
+          targets.get(name)!.add(environment ?? REPO_LEVEL);
+        }
+      }
+    }
+  }
+  return targets;
+};
+
+/**
+ * pushes vault values to github via the gh cli: sensitive ones as
+ * secrets (masked), "variable"-kind ones as variables (plain) — each
+ * into the environment(s) whose jobs reference them
+ */
+export const pushSecretsToGithub = async (
+  io: IO,
+  repo: string,
+  secrets: { name: string; value: string; kind: "secret" | "variable" }[],
+  targets: Map<string, Set<string>>,
+) => {
+  const environments = [
+    ...new Set(
+      [...targets.values()].flatMap((envs) =>
+        [...envs].filter((env) => env !== REPO_LEVEL),
+      ),
+    ),
+  ];
+  for (const environment of environments) {
+    await ensureGithubEnvironment(repo, environment);
+    io.log(`🌍 environment ${environment}`);
+  }
+
+  for (const { name, value, kind } of secrets) {
+    const envs = targets.get(name);
+    if (!envs) {
+      io.log(`⏭️ ${name} (not referenced by any generated workflow)`);
+      continue;
+    }
+    for (const environment of envs) {
+      const suffix = environment === REPO_LEVEL ? "" : ` → ${environment}`;
+      if (kind === "variable") {
+        await setGithubVariable(repo, name, value, environment || undefined);
+        io.log(`📖 ${name} (variable)${suffix}`);
+      } else {
+        await setGithubSecret(repo, name, value, environment || undefined);
+        io.log(`✅ ${name}${suffix}`);
+      }
+    }
+  }
+};
+
+/**
+ * resolves the github repository from the configured git remote
+ * (`pipelines.github.gitRemote`, defaults to origin)
+ */
+export const getConfiguredGithubRepo = async (): Promise<
+  string | undefined
+> => {
+  const config = await getProjectConfig();
+  if (!config) {
+    return undefined;
+  }
+  return getGithubRepoFromRemote(getPipelineGitRemote(config, "github"));
+};

package/src/secrets/index.ts

@@ -0,0 +1,2 @@
+export * from "./github";
+export * from "./writeSecrets";

package/src/secrets/writeSecrets.ts

@@ -0,0 +1,138 @@
+import type { Config } from "@catladder/pipeline";
+import {
+  getEnabledPipelineTypes,
+  getSecretVarName,
+  getVaultConfig,
+} from "@catladder/pipeline";
+import { getEnvironment, getProjectConfig } from "../config/getProjectConfig";
+import type { IO } from "../core/types";
+import { isGhAuthenticated } from "../utils/github";
+import { upsertAllVariables } from "../utils/gitlab";
+import type { WriteSecretsOptions } from "../vault/types";
+import {
+  collectSecretTargets,
+  getConfiguredGithubRepo,
+  pushSecretsToGithub,
+} from "./github";
+
+/** secrets of one env of one component, ready to be written */
+export type SecretsWrite = {
+  env: string;
+  componentName: string;
+  secrets: Record<string, unknown>;
+};
+
+/**
+ * the one door for writing secret values: they go into the vault (the
+ * source of truth) and are mirrored to every enabled CI backend —
+ * which vault and which backends exist is invisible to the caller.
+ *
+ * Everything that produces a secret value must come through here: the
+ * secrets commands, but also the credentials `project setup`
+ * provisions (gcloud service account keys, kubernetes tokens).
+ */
+export const writeSecretsAndMirror = async (
+  io: IO,
+  writes: SecretsWrite[],
+  options: WriteSecretsOptions = {},
+): Promise<void> => {
+  const config = await getProjectConfig();
+  if (!config) {
+    throw new Error("no catladder config found");
+  }
+
+  io.log("");
+  io.log("writing secrets to the vault, please wait...");
+  io.log("");
+  for (const { env, componentName, secrets } of writes) {
+    io.log("writing " + env + ":" + componentName + "...\n");
+    await (
+      await io.getVaultManager()
+    ).writeSecrets(env, componentName, secrets, io, options);
+    io.log("");
+    io.log("✅ " + env + ":" + componentName);
+    io.log("--------------------------------\n");
+  }
+
+  await mirrorSecretsToCiBackends(io, config, writes, options);
+};
+
+/**
+ * the vault is the source of truth; every enabled CI backend receives
+ * a mirrored copy of the written secrets (gitlab variables, github
+ * secrets), so all CI systems stay in sync
+ */
+const mirrorSecretsToCiBackends = async (
+  io: IO,
+  config: Config,
+  writes: SecretsWrite[],
+  options: WriteSecretsOptions,
+) => {
+  const enabled = getEnabledPipelineTypes(config);
+
+  // gitlab mirror (unless gitlab is the vault itself, then it is
+  // already written)
+  if (getVaultConfig(config).type !== "gitlab" && enabled.includes("gitlab")) {
+    for (const { env, componentName, secrets } of writes) {
+      await upsertAllVariables(
+        io,
+        secrets,
+        env,
+        componentName,
+        options.backup ?? true,
+      );
+    }
+    io.log("✅ mirrored to gitlab");
+  }
+
+  if (!enabled.includes("github")) {
+    return;
+  }
+  const kinds = new Map<string, "secret" | "variable">();
+  for (const { env, componentName } of writes) {
+    const { secretEnvVarKeys, jobOnlyVars } = await getEnvironment(
+      env,
+      componentName,
+    );
+    [
+      ...secretEnvVarKeys,
+      ...jobOnlyVars.build.secretEnvVarKeys,
+      ...jobOnlyVars.deploy.secretEnvVarKeys,
+    ].forEach(({ key, kind }) =>
+      kinds.set(getSecretVarName(env, componentName, key), kind ?? "secret"),
+    );
+  }
+  const secrets = writes.flatMap(({ env, componentName, secrets }) =>
+    Object.entries(secrets).map(([key, value]) => {
+      const name = getSecretVarName(env, componentName, key);
+      return {
+        name,
+        value: typeof value === "string" ? value : JSON.stringify(value ?? ""),
+        kind: kinds.get(name) ?? ("secret" as const),
+      };
+    }),
+  );
+  if (secrets.length === 0) {
+    return;
+  }
+  if (!(await isGhAuthenticated())) {
+    io.log(
+      "⚠️ github pipeline is enabled, but the github cli (gh) is not authenticated — run `project secrets-sync-github` later to mirror the secrets",
+    );
+    return;
+  }
+  const repo = await getConfiguredGithubRepo();
+  if (!repo) {
+    io.log(
+      "⚠️ github pipeline is enabled, but no github repository was found — run `project secrets-sync-github` later to mirror the secrets",
+    );
+    return;
+  }
+  io.log(`mirroring ${secrets.length} secrets to github ('${repo}')...`);
+  await pushSecretsToGithub(
+    io,
+    repo,
+    secrets,
+    await collectSecretTargets(config),
+  );
+};

package/src/utils/gitlab.ts

@@ -57,6 +57,21 @@ export const getGitlabToken = async (io: IO | null) => {
   return getPreference(TOKEN_KEY);
 };
 
+/**
+ * a gitlab api call that came back 404. Carries the path so a caller
+ * that wants to treat "does not exist yet" as normal can say so
+ * (`instanceof`) instead of matching on a message.
+ */
+export class GitlabNotFoundError extends Error {
+  constructor(
+    readonly path: string,
+    readonly host: string,
+  ) {
+    super(`gitlab api: ${path} not found on ${host}`);
+    this.name = "GitlabNotFoundError";
+  }
+}
+
 type Method = "GET" | "PUT" | "POST" | "DELETE";
 export const doGitlabRequest = async <T = any>(
   io: IO | null,
@@ -87,7 +102,7 @@ export const doGitlabRequest = async <T = any>(
     return null;
   }
   if (result.status === 404) {
-    throw new Error("not found");
+    throw new GitlabNotFoundError(path, gitRemoteHost);
   }
 
   throw new Error(
@@ -124,12 +139,24 @@ export const doGitlabRequestAllPages = async <T = any>(
 export const getProjectInfo = async (
   io: IO | null,
 ): Promise<{ id: string; web_url: string }> => {
-  const { gitRemotePath } = await getGitRemoteHostAndPath();
-  const project = await doGitlabRequest(
-    io,
-    `projects/${encodeURIComponent(gitRemotePath)}`,
-  );
-  return project;
+  const { gitRemoteHost, gitRemotePath } = await getGitRemoteHostAndPath();
+  try {
+    return await doGitlabRequest(
+      io,
+      `projects/${encodeURIComponent(gitRemotePath)}`,
+    );
+  } catch (e) {
+    if (e instanceof GitlabNotFoundError) {
+      // the classic: the remote is not a gitlab at all (a github one,
+      // say), so the request goes to a host without /api/v4 and every
+      // path answers 404
+      throw new Error(
+        `no gitlab project '${gitRemotePath}' on '${gitRemoteHost}'. ` +
+          "Check the git remote and `pipelines` in catladder.ts — this command talks to the gitlab api.",
+      );
+    }
+    throw e;
+  }
 };
 
 type GitlabVariable = {

package/src/vault/BitwardenVault.ts

@@ -108,6 +108,12 @@ export class BitwardenVault implements SecretsVault {
     return secrets;
   }
 
+  /**
+   * upserts into the item's yaml note. The note is rewritten as a
+   * whole, so the values that are not part of this write have to be
+   * carried over — a single-key write (`secrets-set`, a credential
+   * provisioned by `project setup`) must not drop the rest.
+   */
   async writeSecrets(
     io: IO,
     env: string,
@@ -130,7 +136,10 @@ export class BitwardenVault implements SecretsVault {
       ])
     ).find((item) => item.name === name);
 
-    const notes = stringify(secrets);
+    const previous = existing?.notes
+      ? ((parse(existing.notes) as Record<string, unknown> | null) ?? {})
+      : {};
+    const notes = stringify({ ...previous, ...secrets });
 
     if (existing) {
       await runBw(bwSession, [

package/src/vault/GitlabVault.ts

@@ -1,6 +1,6 @@
 import type { IO } from "../core/types";
 import { getAllVariables, upsertAllVariables } from "../utils/gitlab";
-import type { SecretsVault } from "./types";
+import type { SecretsVault, WriteSecretsOptions } from "./types";
 
 /**
  * the legacy (and default) vault: the gitlab project variables double
@@ -19,7 +19,14 @@ export class GitlabVault implements SecretsVault {
     env: string,
     componentName: string,
     secrets: Record<string, unknown>,
+    options: WriteSecretsOptions = {},
   ): Promise<void> {
-    await upsertAllVariables(io, secrets, env, componentName);
+    await upsertAllVariables(
+      io,
+      secrets,
+      env,
+      componentName,
+      options.backup ?? true,
+    );
   }
 }

package/src/vault/VaultManager.ts

@@ -13,7 +13,7 @@ import {
   writeSecretsCache,
 } from "./cache";
 import { GitlabVault } from "./GitlabVault";
-import type { SecretsMode, SecretsVault } from "./types";
+import type { SecretsMode, SecretsVault, WriteSecretsOptions } from "./types";
 
 /**
  * only secrets of this env are cached locally — the other envs'
@@ -119,11 +119,12 @@ export class VaultManager {
     componentName: string,
     secrets: Record<string, unknown>,
     io: IO | null = this.io,
+    options: WriteSecretsOptions = {},
   ): Promise<void> {
     if (!io) {
       throw new Error("writing to the vault requires an interactive context");
     }
-    await this.vault.writeSecrets(io, env, componentName, secrets);
+    await this.vault.writeSecrets(io, env, componentName, secrets, options);
     if (env === CACHED_ENV) {
       await updateSecretsCache(
         this.vault.id,