@catladder/cli 5.0.1 → 5.1.1

package/src/commands/project/secrets/document.ts

@@ -1,21 +1,6 @@
-import type { Config } from "@catladder/pipeline";
-import {
-  getEnabledPipelineTypes,
-  getSecretVarName,
-  getVaultConfig,
-} from "@catladder/pipeline";
-import {
-  getEnvironment,
-  getProjectConfig,
-} from "../../../config/getProjectConfig";
+import { getSecretVarName } from "@catladder/pipeline";
 import type { IO } from "../../../core/types";
-import { isGhAuthenticated } from "../../../utils/github";
-import { upsertAllVariables } from "../../../utils/gitlab";
-import {
-  collectSecretTargets,
-  getConfiguredGithubRepo,
-  pushSecretsToGithub,
-} from "../commandSecretsSyncGithub";
+import type { SecretsWrite } from "../../../secrets";
 import type { ScopedSecretKeys } from "./scope";
 
 /** placeholder for secrets that have no value in the vault yet */
@@ -31,13 +16,6 @@ export type SecretsDocument = {
   };
 };
 
-/** secrets of one env of one component, ready to be written */
-export type SecretsWrite = {
-  env: string;
-  componentName: string;
-  secrets: Record<string, unknown>;
-};
-
 /* for convenience, parse json values. that makes it easier to edit secrets that are objects */
 const parseJsonValue = (value: string): unknown => {
   try {
@@ -164,106 +142,3 @@ export const collectSecretsWrites = (
   }
   return { writes, skipped };
 };
-
-/**
- * writes secrets to the vault (the source of truth) and mirrors them
- * to every enabled CI backend — which backends exist is invisible to
- * the caller
- */
-export const writeSecretsAndMirror = async (
-  io: IO,
-  writes: SecretsWrite[],
-): Promise<void> => {
-  const config = await getProjectConfig();
-  if (!config) {
-    throw new Error("no catladder config found");
-  }
-
-  io.log("");
-  io.log("writing secrets to the vault, please wait...");
-  io.log("");
-  for (const { env, componentName, secrets } of writes) {
-    io.log("writing " + env + ":" + componentName + "...\n");
-    await (
-      await io.getVaultManager()
-    ).writeSecrets(env, componentName, secrets);
-    io.log("");
-    io.log("✅ " + env + ":" + componentName);
-    io.log("--------------------------------\n");
-  }
-
-  await mirrorSecretsToCiBackends(io, config, writes);
-};
-
-/**
- * the vault is the source of truth; every enabled CI backend receives
- * a mirrored copy of the written secrets (gitlab variables, github
- * secrets), so all CI systems stay in sync
- */
-const mirrorSecretsToCiBackends = async (
-  io: IO,
-  config: Config,
-  writes: SecretsWrite[],
-) => {
-  const enabled = getEnabledPipelineTypes(config);
-
-  // gitlab mirror (unless gitlab is the vault itself, then it is
-  // already written)
-  if (getVaultConfig(config).type !== "gitlab" && enabled.includes("gitlab")) {
-    for (const { env, componentName, secrets } of writes) {
-      await upsertAllVariables(io, secrets, env, componentName);
-    }
-    io.log("✅ mirrored to gitlab");
-  }
-
-  if (!enabled.includes("github")) {
-    return;
-  }
-  const kinds = new Map<string, "secret" | "variable">();
-  for (const { env, componentName } of writes) {
-    const { secretEnvVarKeys, jobOnlyVars } = await getEnvironment(
-      env,
-      componentName,
-    );
-    [
-      ...secretEnvVarKeys,
-      ...jobOnlyVars.build.secretEnvVarKeys,
-      ...jobOnlyVars.deploy.secretEnvVarKeys,
-    ].forEach(({ key, kind }) =>
-      kinds.set(getSecretVarName(env, componentName, key), kind ?? "secret"),
-    );
-  }
-  const secrets = writes.flatMap(({ env, componentName, secrets }) =>
-    Object.entries(secrets).map(([key, value]) => {
-      const name = getSecretVarName(env, componentName, key);
-      return {
-        name,
-        value: typeof value === "string" ? value : JSON.stringify(value ?? ""),
-        kind: kinds.get(name) ?? ("secret" as const),
-      };
-    }),
-  );
-  if (secrets.length === 0) {
-    return;
-  }
-  if (!(await isGhAuthenticated())) {
-    io.log(
-      "⚠️ github pipeline is enabled, but the github cli (gh) is not authenticated — run `project secrets-sync-github` later to mirror the secrets",
-    );
-    return;
-  }
-  const repo = await getConfiguredGithubRepo();
-  if (!repo) {
-    io.log(
-      "⚠️ github pipeline is enabled, but no github repository was found — run `project secrets-sync-github` later to mirror the secrets",
-    );
-    return;
-  }
-  io.log(`mirroring ${secrets.length} secrets to github ('${repo}')...`);
-  await pushSecretsToGithub(
-    io,
-    repo,
-    secrets,
-    await collectSecretTargets(config),
-  );
-};

package/src/gcloud/serviceAccounts.ts

@@ -2,7 +2,7 @@ import type { ComponentContext } from "@catladder/pipeline";
 
 import { exec } from "child-process-promise";
 import type { IO } from "../core/types";
-import { upsertAllVariables } from "../utils/gitlab";
+import { writeSecretsAndMirror } from "../secrets";
 import { getGcloudServiceAccountNames } from "./serviceAccountNames";
 
 export const accountExists = async (fullIdentifier: string) => {
@@ -76,13 +76,22 @@ export const upsertGcloudServiceAccountAndSaveSecret = async (
   instance.log("upserting service account " + account.name + "...");
   const key = await upsertGcloudServiceAccount(context, account);
 
-  await upsertAllVariables(
+  // through the vault (and from there to every enabled ci backend) —
+  // the key is a secret like any other, the deploy jobs read it under
+  // its declared name. Writing it to gitlab directly would break every
+  // project that has no gitlab.
+  await writeSecretsAndMirror(
     instance,
-    {
-      [secretName]: key,
-    },
-    context.env,
-    context.name,
+    [
+      {
+        env: context.env,
+        componentName: context.name,
+        secrets: { [secretName]: key },
+      },
+    ],
+    // a fresh key is created on every run, the previous one is deleted
+    // at google — a backup copy of it would only be a dead secret
+    { backup: false },
   );
   instance.log("done!");
 };

package/src/secrets/__tests__/writeSecrets.test.ts

@@ -0,0 +1,147 @@
+import type { Config } from "@catladder/pipeline";
+import { beforeEach, describe, expect, it, vi } from "vitest";
+import type { IO } from "../../core/types";
+import { writeSecretsAndMirror } from "../writeSecrets";
+
+const { getProjectConfig, getEnvironment } = vi.hoisted(() => ({
+  getProjectConfig: vi.fn(),
+  getEnvironment: vi.fn(),
+}));
+const { upsertAllVariables } = vi.hoisted(() => ({
+  upsertAllVariables: vi.fn(),
+}));
+const { isGhAuthenticated } = vi.hoisted(() => ({
+  isGhAuthenticated: vi.fn(),
+}));
+const { collectSecretTargets, getConfiguredGithubRepo, pushSecretsToGithub } =
+  vi.hoisted(() => ({
+    collectSecretTargets: vi.fn(),
+    getConfiguredGithubRepo: vi.fn(),
+    pushSecretsToGithub: vi.fn(),
+  }));
+
+vi.mock("../../config/getProjectConfig", () => ({
+  getProjectConfig,
+  getEnvironment,
+}));
+vi.mock("../../utils/gitlab", () => ({ upsertAllVariables }));
+vi.mock("../../utils/github", () => ({ isGhAuthenticated }));
+vi.mock("../github", () => ({
+  collectSecretTargets,
+  getConfiguredGithubRepo,
+  pushSecretsToGithub,
+}));
+
+const vaultWriteSecrets = vi.fn();
+const io = {
+  log: vi.fn(),
+  getVaultManager: async () => ({ writeSecrets: vaultWriteSecrets }),
+} as unknown as IO;
+
+const config = (partial: Partial<Config>): Config =>
+  ({
+    customerName: "pan",
+    appName: "app",
+    components: {},
+    ...partial,
+  }) as Config;
+
+const writes = [
+  { env: "dev", componentName: "www", secrets: { API_KEY: "sesame" } },
+];
+
+beforeEach(() => {
+  vi.clearAllMocks();
+  getEnvironment.mockResolvedValue({
+    secretEnvVarKeys: [{ key: "API_KEY" }],
+    jobOnlyVars: {
+      build: { secretEnvVarKeys: [] },
+      deploy: { secretEnvVarKeys: [] },
+    },
+  });
+  isGhAuthenticated.mockResolvedValue(true);
+  getConfiguredGithubRepo.mockResolvedValue("panter/app");
+  collectSecretTargets.mockResolvedValue(
+    new Map([["CL_dev_www_API_KEY", new Set(["dev"])]]),
+  );
+});
+
+describe("writeSecretsAndMirror", () => {
+  it("never touches the gitlab api on a project without a gitlab pipeline", async () => {
+    getProjectConfig.mockResolvedValue(
+      config({
+        pipelines: { github: true },
+        secrets: { vault: { type: "bitwarden" } },
+      }),
+    );
+
+    await writeSecretsAndMirror(io, writes);
+
+    expect(vaultWriteSecrets).toHaveBeenCalledWith(
+      "dev",
+      "www",
+      { API_KEY: "sesame" },
+      io,
+      {},
+    );
+    expect(upsertAllVariables).not.toHaveBeenCalled();
+    expect(pushSecretsToGithub).toHaveBeenCalledWith(
+      io,
+      "panter/app",
+      [{ name: "CL_dev_www_API_KEY", value: "sesame", kind: "secret" }],
+      expect.any(Map),
+    );
+  });
+
+  it("does not mirror to gitlab when gitlab IS the vault (already written)", async () => {
+    getProjectConfig.mockResolvedValue(config({ pipelines: { gitlab: true } }));
+
+    await writeSecretsAndMirror(io, writes);
+
+    expect(vaultWriteSecrets).toHaveBeenCalledTimes(1);
+    expect(upsertAllVariables).not.toHaveBeenCalled();
+    expect(pushSecretsToGithub).not.toHaveBeenCalled();
+  });
+
+  it("mirrors to gitlab when it is only a backend, and forwards the backup option", async () => {
+    getProjectConfig.mockResolvedValue(
+      config({
+        pipelines: { gitlab: true, github: true },
+        secrets: { vault: "bitwarden" },
+      }),
+    );
+
+    await writeSecretsAndMirror(io, writes, { backup: false });
+
+    expect(vaultWriteSecrets).toHaveBeenCalledWith(
+      "dev",
+      "www",
+      { API_KEY: "sesame" },
+      io,
+      { backup: false },
+    );
+    expect(upsertAllVariables).toHaveBeenCalledWith(
+      io,
+      { API_KEY: "sesame" },
+      "dev",
+      "www",
+      false,
+    );
+    expect(pushSecretsToGithub).toHaveBeenCalled();
+  });
+
+  it("keeps the secrets in the vault when the github mirror cannot run", async () => {
+    getProjectConfig.mockResolvedValue(
+      config({
+        pipelines: { github: true },
+        secrets: { vault: "bitwarden" },
+      }),
+    );
+    isGhAuthenticated.mockResolvedValue(false);
+
+    await writeSecretsAndMirror(io, writes);
+
+    expect(vaultWriteSecrets).toHaveBeenCalledTimes(1);
+    expect(pushSecretsToGithub).not.toHaveBeenCalled();
+  });
+});

package/src/secrets/github.ts

@@ -0,0 +1,157 @@
+import type { Config } from "@catladder/pipeline";
+import {
+  getPipelineGitRemote,
+  getSecretVarName,
+  GithubBackend,
+} from "@catladder/pipeline";
+import {
+  getAllComponentsWithAllEnvsHierarchical,
+  getEnvironment,
+  getProjectConfig,
+} from "../config/getProjectConfig";
+import type { IO } from "../core/types";
+import {
+  ensureGithubEnvironment,
+  getGithubRepoFromRemote,
+  setGithubSecret,
+  setGithubVariable,
+} from "../utils/github";
+
+/**
+ * all secrets declared in the config, with their values read from the
+ * vault (github secrets are write-only and can never act as a source)
+ */
+export const collectSecretsFromVault = async (io: IO, envFilter?: string[]) => {
+  const envAndComponents = await getAllComponentsWithAllEnvsHierarchical();
+
+  const kinds = new Map<string, "secret" | "variable">();
+  for (const [componentName, envs] of Object.entries(envAndComponents)) {
+    for (const env of envs) {
+      if (envFilter && !envFilter.includes(env)) {
+        continue;
+      }
+      const { secretEnvVarKeys, jobOnlyVars } = await getEnvironment(
+        env,
+        componentName,
+      );
+      const keys = [
+        ...secretEnvVarKeys,
+        ...jobOnlyVars.build.secretEnvVarKeys,
+        ...jobOnlyVars.deploy.secretEnvVarKeys,
+      ];
+      keys.forEach(({ key, kind }) =>
+        kinds.set(getSecretVarName(env, componentName, key), kind ?? "secret"),
+      );
+    }
+  }
+
+  const names = [...kinds.keys()];
+  const values = await (await io.getVaultManager()).readSecrets(names, io);
+  const secrets = names
+    .filter((name) => values[name] !== undefined)
+    .map((name) => ({
+      name,
+      value: values[name],
+      kind: kinds.get(name) ?? ("secret" as const),
+    }));
+  const missing = names.filter((name) => values[name] === undefined);
+  return { secrets, missing };
+};
+
+/** marker for repo-level (environment-less) targets */
+const REPO_LEVEL = "";
+
+/**
+ * derives which github environment(s) each secret/variable must live in
+ * by scanning the GENERATED workflows: a `${{ secrets.X }}` / `${{
+ * vars.X }}` reference in a job's env targets that job's environment
+ * (or the repo level when the job has none). Deriving from the real
+ * output means the sync can never drift from what the workflows expect.
+ *
+ * Environment secrets are the answer to github's cap of 100 secrets
+ * per REPO (a multi-env project easily exceeds it): the cap applies
+ * per environment instead.
+ */
+export const collectSecretTargets = async (
+  config: Config,
+): Promise<Map<string, Set<string>>> => {
+  const workflows = await new GithubBackend().createWorkflows(config);
+  const targets = new Map<string, Set<string>>();
+  for (const workflow of Object.values(workflows)) {
+    for (const job of Object.values(workflow.jobs)) {
+      const environment =
+        typeof job.environment === "object"
+          ? job.environment.name
+          : job.environment;
+      for (const value of Object.values(job.env ?? {})) {
+        if (typeof value !== "string") continue;
+        for (const match of value.matchAll(
+          /\$\{\{\s*(?:secrets|vars)\.([A-Za-z0-9_]+)\s*\}\}/g,
+        )) {
+          const name = match[1];
+          if (!targets.has(name)) {
+            targets.set(name, new Set());
+          }
+          targets.get(name)!.add(environment ?? REPO_LEVEL);
+        }
+      }
+    }
+  }
+  return targets;
+};
+
+/**
+ * pushes vault values to github via the gh cli: sensitive ones as
+ * secrets (masked), "variable"-kind ones as variables (plain) — each
+ * into the environment(s) whose jobs reference them
+ */
+export const pushSecretsToGithub = async (
+  io: IO,
+  repo: string,
+  secrets: { name: string; value: string; kind: "secret" | "variable" }[],
+  targets: Map<string, Set<string>>,
+) => {
+  const environments = [
+    ...new Set(
+      [...targets.values()].flatMap((envs) =>
+        [...envs].filter((env) => env !== REPO_LEVEL),
+      ),
+    ),
+  ];
+  for (const environment of environments) {
+    await ensureGithubEnvironment(repo, environment);
+    io.log(`🌍 environment ${environment}`);
+  }
+
+  for (const { name, value, kind } of secrets) {
+    const envs = targets.get(name);
+    if (!envs) {
+      io.log(`⏭️ ${name} (not referenced by any generated workflow)`);
+      continue;
+    }
+    for (const environment of envs) {
+      const suffix = environment === REPO_LEVEL ? "" : ` → ${environment}`;
+      if (kind === "variable") {
+        await setGithubVariable(repo, name, value, environment || undefined);
+        io.log(`📖 ${name} (variable)${suffix}`);
+      } else {
+        await setGithubSecret(repo, name, value, environment || undefined);
+        io.log(`✅ ${name}${suffix}`);
+      }
+    }
+  }
+};
+
+/**
+ * resolves the github repository from the configured git remote
+ * (`pipelines.github.gitRemote`, defaults to origin)
+ */
+export const getConfiguredGithubRepo = async (): Promise<
+  string | undefined
+> => {
+  const config = await getProjectConfig();
+  if (!config) {
+    return undefined;
+  }
+  return getGithubRepoFromRemote(getPipelineGitRemote(config, "github"));
+};

package/src/secrets/index.ts

@@ -0,0 +1,2 @@
+export * from "./github";
+export * from "./writeSecrets";

package/src/secrets/writeSecrets.ts

@@ -0,0 +1,138 @@
+import type { Config } from "@catladder/pipeline";
+import {
+  getEnabledPipelineTypes,
+  getSecretVarName,
+  getVaultConfig,
+} from "@catladder/pipeline";
+import { getEnvironment, getProjectConfig } from "../config/getProjectConfig";
+import type { IO } from "../core/types";
+import { isGhAuthenticated } from "../utils/github";
+import { upsertAllVariables } from "../utils/gitlab";
+import type { WriteSecretsOptions } from "../vault/types";
+import {
+  collectSecretTargets,
+  getConfiguredGithubRepo,
+  pushSecretsToGithub,
+} from "./github";
+
+/** secrets of one env of one component, ready to be written */
+export type SecretsWrite = {
+  env: string;
+  componentName: string;
+  secrets: Record<string, unknown>;
+};
+
+/**
+ * the one door for writing secret values: they go into the vault (the
+ * source of truth) and are mirrored to every enabled CI backend —
+ * which vault and which backends exist is invisible to the caller.
+ *
+ * Everything that produces a secret value must come through here: the
+ * secrets commands, but also the credentials `project setup`
+ * provisions (gcloud service account keys, kubernetes tokens).
+ */
+export const writeSecretsAndMirror = async (
+  io: IO,
+  writes: SecretsWrite[],
+  options: WriteSecretsOptions = {},
+): Promise<void> => {
+  const config = await getProjectConfig();
+  if (!config) {
+    throw new Error("no catladder config found");
+  }
+
+  io.log("");
+  io.log("writing secrets to the vault, please wait...");
+  io.log("");
+  for (const { env, componentName, secrets } of writes) {
+    io.log("writing " + env + ":" + componentName + "...\n");
+    await (
+      await io.getVaultManager()
+    ).writeSecrets(env, componentName, secrets, io, options);
+    io.log("");
+    io.log("✅ " + env + ":" + componentName);
+    io.log("--------------------------------\n");
+  }
+
+  await mirrorSecretsToCiBackends(io, config, writes, options);
+};
+
+/**
+ * the vault is the source of truth; every enabled CI backend receives
+ * a mirrored copy of the written secrets (gitlab variables, github
+ * secrets), so all CI systems stay in sync
+ */
+const mirrorSecretsToCiBackends = async (
+  io: IO,
+  config: Config,
+  writes: SecretsWrite[],
+  options: WriteSecretsOptions,
+) => {
+  const enabled = getEnabledPipelineTypes(config);
+
+  // gitlab mirror (unless gitlab is the vault itself, then it is
+  // already written)
+  if (getVaultConfig(config).type !== "gitlab" && enabled.includes("gitlab")) {
+    for (const { env, componentName, secrets } of writes) {
+      await upsertAllVariables(
+        io,
+        secrets,
+        env,
+        componentName,
+        options.backup ?? true,
+      );
+    }
+    io.log("✅ mirrored to gitlab");
+  }
+
+  if (!enabled.includes("github")) {
+    return;
+  }
+  const kinds = new Map<string, "secret" | "variable">();
+  for (const { env, componentName } of writes) {
+    const { secretEnvVarKeys, jobOnlyVars } = await getEnvironment(
+      env,
+      componentName,
+    );
+    [
+      ...secretEnvVarKeys,
+      ...jobOnlyVars.build.secretEnvVarKeys,
+      ...jobOnlyVars.deploy.secretEnvVarKeys,
+    ].forEach(({ key, kind }) =>
+      kinds.set(getSecretVarName(env, componentName, key), kind ?? "secret"),
+    );
+  }
+  const secrets = writes.flatMap(({ env, componentName, secrets }) =>
+    Object.entries(secrets).map(([key, value]) => {
+      const name = getSecretVarName(env, componentName, key);
+      return {
+        name,
+        value: typeof value === "string" ? value : JSON.stringify(value ?? ""),
+        kind: kinds.get(name) ?? ("secret" as const),
+      };
+    }),
+  );
+  if (secrets.length === 0) {
+    return;
+  }
+  if (!(await isGhAuthenticated())) {
+    io.log(
+      "⚠️ github pipeline is enabled, but the github cli (gh) is not authenticated — run `project secrets-sync-github` later to mirror the secrets",
+    );
+    return;
+  }
+  const repo = await getConfiguredGithubRepo();
+  if (!repo) {
+    io.log(
+      "⚠️ github pipeline is enabled, but no github repository was found — run `project secrets-sync-github` later to mirror the secrets",
+    );
+    return;
+  }
+  io.log(`mirroring ${secrets.length} secrets to github ('${repo}')...`);
+  await pushSecretsToGithub(
+    io,
+    repo,
+    secrets,
+    await collectSecretTargets(config),
+  );
+};

package/src/utils/github.ts

@@ -1,5 +1,6 @@
 import { execFile as execFileCb, spawn } from "child_process";
 import { promisify } from "util";
+import { parseGithubRepoFromRemoteUrl } from "@catladder/pipeline";
 
 const execFile = promisify(execFileCb);
 
@@ -25,10 +26,7 @@ export const getGithubRepoFromRemote = async (
 ): Promise<string | undefined> => {
   try {
     const { stdout } = await execFile("git", ["remote", "get-url", remoteName]);
-    const match = stdout.match(
-      /github\.com[:/]([\w.-]+\/[\w.-]+?)(\.git)?\s*$/,
-    );
-    return match?.[1];
+    return parseGithubRepoFromRemoteUrl(stdout);
   } catch {
     return undefined;
   }