@catladder/cli 5.0.0 → 5.1.0

package/src/gcloud/serviceAccounts.ts

@@ -2,7 +2,7 @@ import type { ComponentContext } from "@catladder/pipeline";
 
 import { exec } from "child-process-promise";
 import type { IO } from "../core/types";
-import { upsertAllVariables } from "../utils/gitlab";
+import { writeSecretsAndMirror } from "../secrets";
 import { getGcloudServiceAccountNames } from "./serviceAccountNames";
 
 export const accountExists = async (fullIdentifier: string) => {
@@ -76,13 +76,22 @@ export const upsertGcloudServiceAccountAndSaveSecret = async (
   instance.log("upserting service account " + account.name + "...");
   const key = await upsertGcloudServiceAccount(context, account);
 
-  await upsertAllVariables(
+  // through the vault (and from there to every enabled ci backend) —
+  // the key is a secret like any other, the deploy jobs read it under
+  // its declared name. Writing it to gitlab directly would break every
+  // project that has no gitlab.
+  await writeSecretsAndMirror(
     instance,
-    {
-      [secretName]: key,
-    },
-    context.env,
-    context.name,
+    [
+      {
+        env: context.env,
+        componentName: context.name,
+        secrets: { [secretName]: key },
+      },
+    ],
+    // a fresh key is created on every run, the previous one is deleted
+    // at google — a backup copy of it would only be a dead secret
+    { backup: false },
   );
   instance.log("done!");
 };

package/src/release/__tests__/releaseEntry.test.ts

@@ -0,0 +1,88 @@
+import { afterEach, describe, expect, it, vi } from "vitest";
+import { createReleaseEntry } from "../releaseEntry";
+
+const gitlabEnv = {
+  GITHUB_ACTIONS: undefined,
+  GL_TOKEN: "glpat-token",
+  CI_API_V4_URL: "https://git.example.com/api/v4",
+  CI_PROJECT_ID: "42",
+};
+
+const githubEnv = {
+  GITHUB_ACTIONS: "true",
+  GITHUB_TOKEN: "gh-token",
+  GITHUB_REPOSITORY: "panter/catladder",
+};
+
+const stubEnv = (env: Record<string, string | undefined>) => {
+  Object.entries(env).forEach(([key, value]) => vi.stubEnv(key, value as any));
+};
+
+const stubFetch = (response: Partial<Response> = {}) => {
+  const fetchMock = vi.fn().mockResolvedValue({
+    ok: true,
+    status: 201,
+    text: async () => "",
+    ...response,
+  });
+  vi.stubGlobal("fetch", fetchMock);
+  return fetchMock;
+};
+
+afterEach(() => {
+  vi.unstubAllEnvs();
+  vi.unstubAllGlobals();
+});
+
+describe("createReleaseEntry", () => {
+  it("creates the gitlab release for the pushed tag", async () => {
+    stubEnv(gitlabEnv);
+    const fetchMock = stubFetch();
+
+    await createReleaseEntry("v1.2.0", "- Add the export endpoint");
+
+    expect(fetchMock).toHaveBeenCalledTimes(1);
+    const [url, init] = fetchMock.mock.calls[0];
+    expect(url).toBe("https://git.example.com/api/v4/projects/42/releases");
+    expect(init.method).toBe("POST");
+    expect(init.headers["PRIVATE-TOKEN"]).toBe("glpat-token");
+    expect(JSON.parse(init.body)).toEqual({
+      tag_name: "v1.2.0",
+      description: "- Add the export endpoint",
+    });
+  });
+
+  it("creates the github release for the pushed tag", async () => {
+    stubEnv(githubEnv);
+    const fetchMock = stubFetch();
+
+    await createReleaseEntry("v1.2.0", "- Add the export endpoint");
+
+    const [url, init] = fetchMock.mock.calls[0];
+    expect(url).toBe("https://api.github.com/repos/panter/catladder/releases");
+    expect(init.headers.authorization).toBe("Bearer gh-token");
+    expect(JSON.parse(init.body)).toEqual({
+      tag_name: "v1.2.0",
+      name: "v1.2.0",
+      body: "- Add the export endpoint",
+    });
+  });
+
+  it("does not throw when the api call fails — the tag is already pushed", async () => {
+    stubEnv(gitlabEnv);
+    stubFetch({ ok: false, status: 403, text: async () => "forbidden" });
+
+    await expect(
+      createReleaseEntry("v1.2.0", "- notes"),
+    ).resolves.toBeUndefined();
+  });
+
+  it("skips without api context instead of failing the release", async () => {
+    stubEnv({ ...gitlabEnv, GL_TOKEN: undefined });
+    const fetchMock = stubFetch();
+
+    await createReleaseEntry("v1.2.0", "- notes");
+
+    expect(fetchMock).not.toHaveBeenCalled();
+  });
+});

package/src/release/changesetsReleaseJob.ts

@@ -23,6 +23,7 @@ import {
   git,
   gitWithEnv,
 } from "./releaseGit";
+import { createReleaseEntry } from "./releaseEntry";
 import { appendStepSummary } from "./stepSummary";
 
 const CHANGESET_DIR = ".changeset";
@@ -276,6 +277,8 @@ export const changesetsReleaseJob = async () => {
   await pushCommitAndTag(tag);
   console.log(`released ${tag}`);
   appendStepSummary(`🚀 **Released ${tag}**\n\n${entries}`);
+  // the tag alone creates no entry on the releases page of the git host
+  await createReleaseEntry(tag, entries);
 
   if (process.env.GITHUB_ACTIONS === "true") {
     await dispatchTaggedReleaseWorkflow(tag);

package/src/release/releaseEntry.ts

@@ -0,0 +1,86 @@
+/**
+ * the release entry on the git host — gitlab's `/-/releases`, github's
+ * `/releases`. Pushing a tag does NOT create one: on the
+ * semantic-release path `@semantic-release/gitlab` used to do it, the
+ * changesets path creates it here (on both backends).
+ *
+ * Never fatal: when this runs, the release commit and tag are already
+ * pushed and the taggedRelease pipeline is on its way — a failing api
+ * call must not turn a done release into a red job (and a rerun would
+ * fail on the existing tag).
+ */
+
+const post = async (
+  url: string,
+  headers: Record<string, string>,
+  body: unknown,
+) => {
+  const response = await fetch(url, {
+    method: "POST",
+    headers: { "content-type": "application/json", ...headers },
+    body: JSON.stringify(body),
+  });
+  if (!response.ok) {
+    throw new Error(
+      `POST ${url} failed: ${response.status} ${await response.text()}`,
+    );
+  }
+};
+
+/**
+ * GL_TOKEN is the project access token the release push already used,
+ * so its api scope is available here too
+ */
+const createGitlabReleaseEntry = async (tag: string, notes: string) => {
+  const token = process.env.GL_TOKEN;
+  const apiUrl = process.env.CI_API_V4_URL;
+  const projectId = process.env.CI_PROJECT_ID;
+  if (!token || !apiUrl || !projectId) {
+    console.warn(
+      "no GL_TOKEN / gitlab api context — skipping the release entry (the tag is pushed)",
+    );
+    return;
+  }
+  await post(
+    `${apiUrl}/projects/${projectId}/releases`,
+    { "PRIVATE-TOKEN": token },
+    // name defaults to the tag on gitlab
+    { tag_name: tag, description: notes },
+  );
+  console.log(`created the gitlab release entry for ${tag}`);
+};
+
+/** the workflow token, granted `contents: write` by the release job */
+const createGithubReleaseEntry = async (tag: string, notes: string) => {
+  const token = process.env.GITHUB_TOKEN;
+  const apiUrl = process.env.GITHUB_API_URL ?? "https://api.github.com";
+  const repository = process.env.GITHUB_REPOSITORY;
+  if (!token || !repository) {
+    console.warn(
+      "no GITHUB_TOKEN / repository context — skipping the release entry (the tag is pushed)",
+    );
+    return;
+  }
+  await post(
+    `${apiUrl}/repos/${repository}/releases`,
+    { authorization: `Bearer ${token}`, accept: "application/vnd.github+json" },
+    { tag_name: tag, name: tag, body: notes },
+  );
+  console.log(`created the github release entry for ${tag}`);
+};
+
+export const createReleaseEntry = async (tag: string, notes: string) => {
+  try {
+    if (process.env.GITHUB_ACTIONS === "true") {
+      await createGithubReleaseEntry(tag, notes);
+    } else {
+      await createGitlabReleaseEntry(tag, notes);
+    }
+  } catch (error) {
+    console.warn(
+      `⚠️ could not create the release entry for ${tag}: ${error}\n` +
+        "the release itself is done (commit, tag and changelog are pushed) — " +
+        "create the entry by hand if you need it",
+    );
+  }
+};

package/src/secrets/__tests__/writeSecrets.test.ts

@@ -0,0 +1,147 @@
+import type { Config } from "@catladder/pipeline";
+import { beforeEach, describe, expect, it, vi } from "vitest";
+import type { IO } from "../../core/types";
+import { writeSecretsAndMirror } from "../writeSecrets";
+
+const { getProjectConfig, getEnvironment } = vi.hoisted(() => ({
+  getProjectConfig: vi.fn(),
+  getEnvironment: vi.fn(),
+}));
+const { upsertAllVariables } = vi.hoisted(() => ({
+  upsertAllVariables: vi.fn(),
+}));
+const { isGhAuthenticated } = vi.hoisted(() => ({
+  isGhAuthenticated: vi.fn(),
+}));
+const { collectSecretTargets, getConfiguredGithubRepo, pushSecretsToGithub } =
+  vi.hoisted(() => ({
+    collectSecretTargets: vi.fn(),
+    getConfiguredGithubRepo: vi.fn(),
+    pushSecretsToGithub: vi.fn(),
+  }));
+
+vi.mock("../../config/getProjectConfig", () => ({
+  getProjectConfig,
+  getEnvironment,
+}));
+vi.mock("../../utils/gitlab", () => ({ upsertAllVariables }));
+vi.mock("../../utils/github", () => ({ isGhAuthenticated }));
+vi.mock("../github", () => ({
+  collectSecretTargets,
+  getConfiguredGithubRepo,
+  pushSecretsToGithub,
+}));
+
+const vaultWriteSecrets = vi.fn();
+const io = {
+  log: vi.fn(),
+  getVaultManager: async () => ({ writeSecrets: vaultWriteSecrets }),
+} as unknown as IO;
+
+const config = (partial: Partial<Config>): Config =>
+  ({
+    customerName: "pan",
+    appName: "app",
+    components: {},
+    ...partial,
+  }) as Config;
+
+const writes = [
+  { env: "dev", componentName: "www", secrets: { API_KEY: "sesame" } },
+];
+
+beforeEach(() => {
+  vi.clearAllMocks();
+  getEnvironment.mockResolvedValue({
+    secretEnvVarKeys: [{ key: "API_KEY" }],
+    jobOnlyVars: {
+      build: { secretEnvVarKeys: [] },
+      deploy: { secretEnvVarKeys: [] },
+    },
+  });
+  isGhAuthenticated.mockResolvedValue(true);
+  getConfiguredGithubRepo.mockResolvedValue("panter/app");
+  collectSecretTargets.mockResolvedValue(
+    new Map([["CL_dev_www_API_KEY", new Set(["dev"])]]),
+  );
+});
+
+describe("writeSecretsAndMirror", () => {
+  it("never touches the gitlab api on a project without a gitlab pipeline", async () => {
+    getProjectConfig.mockResolvedValue(
+      config({
+        pipelines: { github: true },
+        secrets: { vault: { type: "bitwarden" } },
+      }),
+    );
+
+    await writeSecretsAndMirror(io, writes);
+
+    expect(vaultWriteSecrets).toHaveBeenCalledWith(
+      "dev",
+      "www",
+      { API_KEY: "sesame" },
+      io,
+      {},
+    );
+    expect(upsertAllVariables).not.toHaveBeenCalled();
+    expect(pushSecretsToGithub).toHaveBeenCalledWith(
+      io,
+      "panter/app",
+      [{ name: "CL_dev_www_API_KEY", value: "sesame", kind: "secret" }],
+      expect.any(Map),
+    );
+  });
+
+  it("does not mirror to gitlab when gitlab IS the vault (already written)", async () => {
+    getProjectConfig.mockResolvedValue(config({ pipelines: { gitlab: true } }));
+
+    await writeSecretsAndMirror(io, writes);
+
+    expect(vaultWriteSecrets).toHaveBeenCalledTimes(1);
+    expect(upsertAllVariables).not.toHaveBeenCalled();
+    expect(pushSecretsToGithub).not.toHaveBeenCalled();
+  });
+
+  it("mirrors to gitlab when it is only a backend, and forwards the backup option", async () => {
+    getProjectConfig.mockResolvedValue(
+      config({
+        pipelines: { gitlab: true, github: true },
+        secrets: { vault: "bitwarden" },
+      }),
+    );
+
+    await writeSecretsAndMirror(io, writes, { backup: false });
+
+    expect(vaultWriteSecrets).toHaveBeenCalledWith(
+      "dev",
+      "www",
+      { API_KEY: "sesame" },
+      io,
+      { backup: false },
+    );
+    expect(upsertAllVariables).toHaveBeenCalledWith(
+      io,
+      { API_KEY: "sesame" },
+      "dev",
+      "www",
+      false,
+    );
+    expect(pushSecretsToGithub).toHaveBeenCalled();
+  });
+
+  it("keeps the secrets in the vault when the github mirror cannot run", async () => {
+    getProjectConfig.mockResolvedValue(
+      config({
+        pipelines: { github: true },
+        secrets: { vault: "bitwarden" },
+      }),
+    );
+    isGhAuthenticated.mockResolvedValue(false);
+
+    await writeSecretsAndMirror(io, writes);
+
+    expect(vaultWriteSecrets).toHaveBeenCalledTimes(1);
+    expect(pushSecretsToGithub).not.toHaveBeenCalled();
+  });
+});

package/src/secrets/github.ts

@@ -0,0 +1,157 @@
+import type { Config } from "@catladder/pipeline";
+import {
+  getPipelineGitRemote,
+  getSecretVarName,
+  GithubBackend,
+} from "@catladder/pipeline";
+import {
+  getAllComponentsWithAllEnvsHierarchical,
+  getEnvironment,
+  getProjectConfig,
+} from "../config/getProjectConfig";
+import type { IO } from "../core/types";
+import {
+  ensureGithubEnvironment,
+  getGithubRepoFromRemote,
+  setGithubSecret,
+  setGithubVariable,
+} from "../utils/github";
+
+/**
+ * all secrets declared in the config, with their values read from the
+ * vault (github secrets are write-only and can never act as a source)
+ */
+export const collectSecretsFromVault = async (io: IO, envFilter?: string[]) => {
+  const envAndComponents = await getAllComponentsWithAllEnvsHierarchical();
+
+  const kinds = new Map<string, "secret" | "variable">();
+  for (const [componentName, envs] of Object.entries(envAndComponents)) {
+    for (const env of envs) {
+      if (envFilter && !envFilter.includes(env)) {
+        continue;
+      }
+      const { secretEnvVarKeys, jobOnlyVars } = await getEnvironment(
+        env,
+        componentName,
+      );
+      const keys = [
+        ...secretEnvVarKeys,
+        ...jobOnlyVars.build.secretEnvVarKeys,
+        ...jobOnlyVars.deploy.secretEnvVarKeys,
+      ];
+      keys.forEach(({ key, kind }) =>
+        kinds.set(getSecretVarName(env, componentName, key), kind ?? "secret"),
+      );
+    }
+  }
+
+  const names = [...kinds.keys()];
+  const values = await (await io.getVaultManager()).readSecrets(names, io);
+  const secrets = names
+    .filter((name) => values[name] !== undefined)
+    .map((name) => ({
+      name,
+      value: values[name],
+      kind: kinds.get(name) ?? ("secret" as const),
+    }));
+  const missing = names.filter((name) => values[name] === undefined);
+  return { secrets, missing };
+};
+
+/** marker for repo-level (environment-less) targets */
+const REPO_LEVEL = "";
+
+/**
+ * derives which github environment(s) each secret/variable must live in
+ * by scanning the GENERATED workflows: a `${{ secrets.X }}` / `${{
+ * vars.X }}` reference in a job's env targets that job's environment
+ * (or the repo level when the job has none). Deriving from the real
+ * output means the sync can never drift from what the workflows expect.
+ *
+ * Environment secrets are the answer to github's cap of 100 secrets
+ * per REPO (a multi-env project easily exceeds it): the cap applies
+ * per environment instead.
+ */
+export const collectSecretTargets = async (
+  config: Config,
+): Promise<Map<string, Set<string>>> => {
+  const workflows = await new GithubBackend().createWorkflows(config);
+  const targets = new Map<string, Set<string>>();
+  for (const workflow of Object.values(workflows)) {
+    for (const job of Object.values(workflow.jobs)) {
+      const environment =
+        typeof job.environment === "object"
+          ? job.environment.name
+          : job.environment;
+      for (const value of Object.values(job.env ?? {})) {
+        if (typeof value !== "string") continue;
+        for (const match of value.matchAll(
+          /\$\{\{\s*(?:secrets|vars)\.([A-Za-z0-9_]+)\s*\}\}/g,
+        )) {
+          const name = match[1];
+          if (!targets.has(name)) {
+            targets.set(name, new Set());
+          }
+          targets.get(name)!.add(environment ?? REPO_LEVEL);
+        }
+      }
+    }
+  }
+  return targets;
+};
+
+/**
+ * pushes vault values to github via the gh cli: sensitive ones as
+ * secrets (masked), "variable"-kind ones as variables (plain) — each
+ * into the environment(s) whose jobs reference them
+ */
+export const pushSecretsToGithub = async (
+  io: IO,
+  repo: string,
+  secrets: { name: string; value: string; kind: "secret" | "variable" }[],
+  targets: Map<string, Set<string>>,
+) => {
+  const environments = [
+    ...new Set(
+      [...targets.values()].flatMap((envs) =>
+        [...envs].filter((env) => env !== REPO_LEVEL),
+      ),
+    ),
+  ];
+  for (const environment of environments) {
+    await ensureGithubEnvironment(repo, environment);
+    io.log(`🌍 environment ${environment}`);
+  }
+
+  for (const { name, value, kind } of secrets) {
+    const envs = targets.get(name);
+    if (!envs) {
+      io.log(`⏭️ ${name} (not referenced by any generated workflow)`);
+      continue;
+    }
+    for (const environment of envs) {
+      const suffix = environment === REPO_LEVEL ? "" : ` → ${environment}`;
+      if (kind === "variable") {
+        await setGithubVariable(repo, name, value, environment || undefined);
+        io.log(`📖 ${name} (variable)${suffix}`);
+      } else {
+        await setGithubSecret(repo, name, value, environment || undefined);
+        io.log(`✅ ${name}${suffix}`);
+      }
+    }
+  }
+};
+
+/**
+ * resolves the github repository from the configured git remote
+ * (`pipelines.github.gitRemote`, defaults to origin)
+ */
+export const getConfiguredGithubRepo = async (): Promise<
+  string | undefined
+> => {
+  const config = await getProjectConfig();
+  if (!config) {
+    return undefined;
+  }
+  return getGithubRepoFromRemote(getPipelineGitRemote(config, "github"));
+};

package/src/secrets/index.ts

@@ -0,0 +1,2 @@
+export * from "./github";
+export * from "./writeSecrets";

package/src/secrets/writeSecrets.ts

@@ -0,0 +1,138 @@
+import type { Config } from "@catladder/pipeline";
+import {
+  getEnabledPipelineTypes,
+  getSecretVarName,
+  getVaultConfig,
+} from "@catladder/pipeline";
+import { getEnvironment, getProjectConfig } from "../config/getProjectConfig";
+import type { IO } from "../core/types";
+import { isGhAuthenticated } from "../utils/github";
+import { upsertAllVariables } from "../utils/gitlab";
+import type { WriteSecretsOptions } from "../vault/types";
+import {
+  collectSecretTargets,
+  getConfiguredGithubRepo,
+  pushSecretsToGithub,
+} from "./github";
+
+/** secrets of one env of one component, ready to be written */
+export type SecretsWrite = {
+  env: string;
+  componentName: string;
+  secrets: Record<string, unknown>;
+};
+
+/**
+ * the one door for writing secret values: they go into the vault (the
+ * source of truth) and are mirrored to every enabled CI backend —
+ * which vault and which backends exist is invisible to the caller.
+ *
+ * Everything that produces a secret value must come through here: the
+ * secrets commands, but also the credentials `project setup`
+ * provisions (gcloud service account keys, kubernetes tokens).
+ */
+export const writeSecretsAndMirror = async (
+  io: IO,
+  writes: SecretsWrite[],
+  options: WriteSecretsOptions = {},
+): Promise<void> => {
+  const config = await getProjectConfig();
+  if (!config) {
+    throw new Error("no catladder config found");
+  }
+
+  io.log("");
+  io.log("writing secrets to the vault, please wait...");
+  io.log("");
+  for (const { env, componentName, secrets } of writes) {
+    io.log("writing " + env + ":" + componentName + "...\n");
+    await (
+      await io.getVaultManager()
+    ).writeSecrets(env, componentName, secrets, io, options);
+    io.log("");
+    io.log("✅ " + env + ":" + componentName);
+    io.log("--------------------------------\n");
+  }
+
+  await mirrorSecretsToCiBackends(io, config, writes, options);
+};
+
+/**
+ * the vault is the source of truth; every enabled CI backend receives
+ * a mirrored copy of the written secrets (gitlab variables, github
+ * secrets), so all CI systems stay in sync
+ */
+const mirrorSecretsToCiBackends = async (
+  io: IO,
+  config: Config,
+  writes: SecretsWrite[],
+  options: WriteSecretsOptions,
+) => {
+  const enabled = getEnabledPipelineTypes(config);
+
+  // gitlab mirror (unless gitlab is the vault itself, then it is
+  // already written)
+  if (getVaultConfig(config).type !== "gitlab" && enabled.includes("gitlab")) {
+    for (const { env, componentName, secrets } of writes) {
+      await upsertAllVariables(
+        io,
+        secrets,
+        env,
+        componentName,
+        options.backup ?? true,
+      );
+    }
+    io.log("✅ mirrored to gitlab");
+  }
+
+  if (!enabled.includes("github")) {
+    return;
+  }
+  const kinds = new Map<string, "secret" | "variable">();
+  for (const { env, componentName } of writes) {
+    const { secretEnvVarKeys, jobOnlyVars } = await getEnvironment(
+      env,
+      componentName,
+    );
+    [
+      ...secretEnvVarKeys,
+      ...jobOnlyVars.build.secretEnvVarKeys,
+      ...jobOnlyVars.deploy.secretEnvVarKeys,
+    ].forEach(({ key, kind }) =>
+      kinds.set(getSecretVarName(env, componentName, key), kind ?? "secret"),
+    );
+  }
+  const secrets = writes.flatMap(({ env, componentName, secrets }) =>
+    Object.entries(secrets).map(([key, value]) => {
+      const name = getSecretVarName(env, componentName, key);
+      return {
+        name,
+        value: typeof value === "string" ? value : JSON.stringify(value ?? ""),
+        kind: kinds.get(name) ?? ("secret" as const),
+      };
+    }),
+  );
+  if (secrets.length === 0) {
+    return;
+  }
+  if (!(await isGhAuthenticated())) {
+    io.log(
+      "⚠️ github pipeline is enabled, but the github cli (gh) is not authenticated — run `project secrets-sync-github` later to mirror the secrets",
+    );
+    return;
+  }
+  const repo = await getConfiguredGithubRepo();
+  if (!repo) {
+    io.log(
+      "⚠️ github pipeline is enabled, but no github repository was found — run `project secrets-sync-github` later to mirror the secrets",
+    );
+    return;
+  }
+  io.log(`mirroring ${secrets.length} secrets to github ('${repo}')...`);
+  await pushSecretsToGithub(
+    io,
+    repo,
+    secrets,
+    await collectSecretTargets(config),
+  );
+};