@catladder/cli 0.0.0-fix-github-release-deploy-key-8a37ce13 → 0.0.0-fix-ghcr-lowercase-image-names-c09dd5a2

package/src/secrets/writeSecrets.ts

@@ -0,0 +1,138 @@
+import type { Config } from "@catladder/pipeline";
+import {
+  getEnabledPipelineTypes,
+  getSecretVarName,
+  getVaultConfig,
+} from "@catladder/pipeline";
+import { getEnvironment, getProjectConfig } from "../config/getProjectConfig";
+import type { IO } from "../core/types";
+import { isGhAuthenticated } from "../utils/github";
+import { upsertAllVariables } from "../utils/gitlab";
+import type { WriteSecretsOptions } from "../vault/types";
+import {
+  collectSecretTargets,
+  getConfiguredGithubRepo,
+  pushSecretsToGithub,
+} from "./github";
+
+/** secrets of one env of one component, ready to be written */
+export type SecretsWrite = {
+  env: string;
+  componentName: string;
+  secrets: Record<string, unknown>;
+};
+
+/**
+ * the one door for writing secret values: they go into the vault (the
+ * source of truth) and are mirrored to every enabled CI backend —
+ * which vault and which backends exist is invisible to the caller.
+ *
+ * Everything that produces a secret value must come through here: the
+ * secrets commands, but also the credentials `project setup`
+ * provisions (gcloud service account keys, kubernetes tokens).
+ */
+export const writeSecretsAndMirror = async (
+  io: IO,
+  writes: SecretsWrite[],
+  options: WriteSecretsOptions = {},
+): Promise<void> => {
+  const config = await getProjectConfig();
+  if (!config) {
+    throw new Error("no catladder config found");
+  }
+
+  io.log("");
+  io.log("writing secrets to the vault, please wait...");
+  io.log("");
+  for (const { env, componentName, secrets } of writes) {
+    io.log("writing " + env + ":" + componentName + "...\n");
+    await (
+      await io.getVaultManager()
+    ).writeSecrets(env, componentName, secrets, io, options);
+    io.log("");
+    io.log("✅ " + env + ":" + componentName);
+    io.log("--------------------------------\n");
+  }
+
+  await mirrorSecretsToCiBackends(io, config, writes, options);
+};
+
+/**
+ * the vault is the source of truth; every enabled CI backend receives
+ * a mirrored copy of the written secrets (gitlab variables, github
+ * secrets), so all CI systems stay in sync
+ */
+const mirrorSecretsToCiBackends = async (
+  io: IO,
+  config: Config,
+  writes: SecretsWrite[],
+  options: WriteSecretsOptions,
+) => {
+  const enabled = getEnabledPipelineTypes(config);
+
+  // gitlab mirror (unless gitlab is the vault itself, then it is
+  // already written)
+  if (getVaultConfig(config).type !== "gitlab" && enabled.includes("gitlab")) {
+    for (const { env, componentName, secrets } of writes) {
+      await upsertAllVariables(
+        io,
+        secrets,
+        env,
+        componentName,
+        options.backup ?? true,
+      );
+    }
+    io.log("✅ mirrored to gitlab");
+  }
+
+  if (!enabled.includes("github")) {
+    return;
+  }
+  const kinds = new Map<string, "secret" | "variable">();
+  for (const { env, componentName } of writes) {
+    const { secretEnvVarKeys, jobOnlyVars } = await getEnvironment(
+      env,
+      componentName,
+    );
+    [
+      ...secretEnvVarKeys,
+      ...jobOnlyVars.build.secretEnvVarKeys,
+      ...jobOnlyVars.deploy.secretEnvVarKeys,
+    ].forEach(({ key, kind }) =>
+      kinds.set(getSecretVarName(env, componentName, key), kind ?? "secret"),
+    );
+  }
+  const secrets = writes.flatMap(({ env, componentName, secrets }) =>
+    Object.entries(secrets).map(([key, value]) => {
+      const name = getSecretVarName(env, componentName, key);
+      return {
+        name,
+        value: typeof value === "string" ? value : JSON.stringify(value ?? ""),
+        kind: kinds.get(name) ?? ("secret" as const),
+      };
+    }),
+  );
+  if (secrets.length === 0) {
+    return;
+  }
+  if (!(await isGhAuthenticated())) {
+    io.log(
+      "⚠️ github pipeline is enabled, but the github cli (gh) is not authenticated — run `project secrets-sync-github` later to mirror the secrets",
+    );
+    return;
+  }
+  const repo = await getConfiguredGithubRepo();
+  if (!repo) {
+    io.log(
+      "⚠️ github pipeline is enabled, but no github repository was found — run `project secrets-sync-github` later to mirror the secrets",
+    );
+    return;
+  }
+  io.log(`mirroring ${secrets.length} secrets to github ('${repo}')...`);
+  await pushSecretsToGithub(
+    io,
+    repo,
+    secrets,
+    await collectSecretTargets(config),
+  );
+};

package/src/utils/github.ts

@@ -1,5 +1,6 @@
 import { execFile as execFileCb, spawn } from "child_process";
 import { promisify } from "util";
+import { parseGithubRepoFromRemoteUrl } from "@catladder/pipeline";
 
 const execFile = promisify(execFileCb);
 
@@ -25,10 +26,7 @@ export const getGithubRepoFromRemote = async (
 ): Promise<string | undefined> => {
   try {
     const { stdout } = await execFile("git", ["remote", "get-url", remoteName]);
-    const match = stdout.match(
-      /github\.com[:/]([\w.-]+\/[\w.-]+?)(\.git)?\s*$/,
-    );
-    return match?.[1];
+    return parseGithubRepoFromRemoteUrl(stdout);
   } catch {
     return undefined;
   }

package/src/utils/gitlab.ts

@@ -57,6 +57,21 @@ export const getGitlabToken = async (io: IO | null) => {
   return getPreference(TOKEN_KEY);
 };
 
+/**
+ * a gitlab api call that came back 404. Carries the path so a caller
+ * that wants to treat "does not exist yet" as normal can say so
+ * (`instanceof`) instead of matching on a message.
+ */
+export class GitlabNotFoundError extends Error {
+  constructor(
+    readonly path: string,
+    readonly host: string,
+  ) {
+    super(`gitlab api: ${path} not found on ${host}`);
+    this.name = "GitlabNotFoundError";
+  }
+}
+
 type Method = "GET" | "PUT" | "POST" | "DELETE";
 export const doGitlabRequest = async <T = any>(
   io: IO | null,
@@ -87,7 +102,7 @@ export const doGitlabRequest = async <T = any>(
     return null;
   }
   if (result.status === 404) {
-    throw new Error("not found");
+    throw new GitlabNotFoundError(path, gitRemoteHost);
   }
 
   throw new Error(
@@ -124,12 +139,24 @@ export const doGitlabRequestAllPages = async <T = any>(
 export const getProjectInfo = async (
   io: IO | null,
 ): Promise<{ id: string; web_url: string }> => {
-  const { gitRemotePath } = await getGitRemoteHostAndPath();
-  const project = await doGitlabRequest(
-    io,
-    `projects/${encodeURIComponent(gitRemotePath)}`,
-  );
-  return project;
+  const { gitRemoteHost, gitRemotePath } = await getGitRemoteHostAndPath();
+  try {
+    return await doGitlabRequest(
+      io,
+      `projects/${encodeURIComponent(gitRemotePath)}`,
+    );
+  } catch (e) {
+    if (e instanceof GitlabNotFoundError) {
+      // the classic: the remote is not a gitlab at all (a github one,
+      // say), so the request goes to a host without /api/v4 and every
+      // path answers 404
+      throw new Error(
+        `no gitlab project '${gitRemotePath}' on '${gitRemoteHost}'. ` +
+          "Check the git remote and `pipelines` in catladder.ts — this command talks to the gitlab api.",
+      );
+    }
+    throw e;
+  }
 };
 
 type GitlabVariable = {

package/src/vault/BitwardenVault.ts

@@ -108,6 +108,12 @@ export class BitwardenVault implements SecretsVault {
     return secrets;
   }
 
+  /**
+   * upserts into the item's yaml note. The note is rewritten as a
+   * whole, so the values that are not part of this write have to be
+   * carried over — a single-key write (`secrets-set`, a credential
+   * provisioned by `project setup`) must not drop the rest.
+   */
   async writeSecrets(
     io: IO,
     env: string,
@@ -130,7 +136,10 @@ export class BitwardenVault implements SecretsVault {
       ])
     ).find((item) => item.name === name);
 
-    const notes = stringify(secrets);
+    const previous = existing?.notes
+      ? ((parse(existing.notes) as Record<string, unknown> | null) ?? {})
+      : {};
+    const notes = stringify({ ...previous, ...secrets });
 
     if (existing) {
       await runBw(bwSession, [

package/src/vault/GitlabVault.ts

@@ -1,6 +1,6 @@
 import type { IO } from "../core/types";
 import { getAllVariables, upsertAllVariables } from "../utils/gitlab";
-import type { SecretsVault } from "./types";
+import type { SecretsVault, WriteSecretsOptions } from "./types";
 
 /**
  * the legacy (and default) vault: the gitlab project variables double
@@ -19,7 +19,14 @@ export class GitlabVault implements SecretsVault {
     env: string,
     componentName: string,
     secrets: Record<string, unknown>,
+    options: WriteSecretsOptions = {},
   ): Promise<void> {
-    await upsertAllVariables(io, secrets, env, componentName);
+    await upsertAllVariables(
+      io,
+      secrets,
+      env,
+      componentName,
+      options.backup ?? true,
+    );
   }
 }

package/src/vault/VaultManager.ts

@@ -13,7 +13,7 @@ import {
   writeSecretsCache,
 } from "./cache";
 import { GitlabVault } from "./GitlabVault";
-import type { SecretsMode, SecretsVault } from "./types";
+import type { SecretsMode, SecretsVault, WriteSecretsOptions } from "./types";
 
 /**
  * only secrets of this env are cached locally — the other envs'
@@ -119,11 +119,12 @@ export class VaultManager {
     componentName: string,
     secrets: Record<string, unknown>,
     io: IO | null = this.io,
+    options: WriteSecretsOptions = {},
   ): Promise<void> {
     if (!io) {
       throw new Error("writing to the vault requires an interactive context");
     }
-    await this.vault.writeSecrets(io, env, componentName, secrets);
+    await this.vault.writeSecrets(io, env, componentName, secrets, options);
     if (env === CACHED_ENV) {
       await updateSecretsCache(
         this.vault.id,

package/src/vault/__tests__/BitwardenVault.test.ts

@@ -0,0 +1,88 @@
+import type { Config } from "@catladder/pipeline";
+import { beforeEach, describe, expect, it, vi } from "vitest";
+import { parse } from "yaml";
+import type { IO } from "../../core/types";
+import { BitwardenVault } from "../BitwardenVault";
+
+const { getBwSession, runBw, runBwJson } = vi.hoisted(() => ({
+  getBwSession: vi.fn(),
+  runBw: vi.fn(),
+  runBwJson: vi.fn(),
+}));
+vi.mock("../bw", () => ({ getBwSession, runBw, runBwJson }));
+
+const io = { log: vi.fn() } as unknown as IO;
+const config = { customerName: "pan", appName: "app" } as Config;
+const ITEM_NAME = "pan/app/dev/www/secrets.yml";
+
+/** the item the vault would edit, or none at all */
+const bwContains = (item?: Record<string, unknown>) =>
+  runBwJson.mockImplementation(async (_session: string, args: string[]) =>
+    args[1] === "collections"
+      ? [{ id: "collection-id", name: "catladder", organizationId: "org-id" }]
+      : item
+        ? [item]
+        : [],
+  );
+
+/** the notes of the `bw edit|create item` payload, parsed back */
+const writtenNotes = () => {
+  const [, args] = runBw.mock.calls[0];
+  const payload = JSON.parse(
+    Buffer.from(args[args.length - 1], "base64").toString("utf-8"),
+  );
+  return parse(payload.notes);
+};
+
+beforeEach(() => {
+  vi.clearAllMocks();
+  getBwSession.mockResolvedValue("session");
+});
+
+describe("BitwardenVault.writeSecrets", () => {
+  it("merges into the existing note instead of replacing it", async () => {
+    bwContains({
+      id: "item-id",
+      name: ITEM_NAME,
+      notes: "API_KEY: sesame\nSMTP_PASSWORD: hunter2\n",
+    });
+
+    await new BitwardenVault(config, {}, false).writeSecrets(io, "dev", "www", {
+      // the shape `project setup` writes: one provisioned credential,
+      // nothing else — the other secrets must survive it
+      GCLOUD_DEPLOY_credentialsKey: '{"type":"service_account"}',
+    });
+
+    expect(runBw.mock.calls[0][1].slice(0, 3)).toEqual([
+      "edit",
+      "item",
+      "item-id",
+    ]);
+    expect(writtenNotes()).toEqual({
+      API_KEY: "sesame",
+      SMTP_PASSWORD: "hunter2",
+      GCLOUD_DEPLOY_credentialsKey: '{"type":"service_account"}',
+    });
+  });
+
+  it("overwrites the values it is given", async () => {
+    bwContains({ id: "item-id", name: ITEM_NAME, notes: "API_KEY: old\n" });
+
+    await new BitwardenVault(config, {}, false).writeSecrets(io, "dev", "www", {
+      API_KEY: "rotated",
+    });
+
+    expect(writtenNotes()).toEqual({ API_KEY: "rotated" });
+  });
+
+  it("creates the item when the env/component has no note yet", async () => {
+    bwContains();
+
+    await new BitwardenVault(config, {}, false).writeSecrets(io, "dev", "www", {
+      API_KEY: "sesame",
+    });
+
+    expect(runBw.mock.calls[0][1].slice(0, 2)).toEqual(["create", "item"]);
+    expect(writtenNotes()).toEqual({ API_KEY: "sesame" });
+  });
+});

package/src/vault/types.ts

@@ -20,16 +20,32 @@ export type SecretsVault = {
 
   /**
    * upserts the secrets of one env and component
-   * (keys as declared in the config, not the raw variable names)
+   * (keys as declared in the config, not the raw variable names).
+   * Upsert, not replace: values that are not part of `secrets` stay
+   * untouched.
    */
   writeSecrets(
     io: IO,
     env: string,
     componentName: string,
     secrets: Record<string, unknown>,
+    options?: WriteSecretsOptions,
   ): Promise<void>;
 };
 
+export type WriteSecretsOptions = {
+  /**
+   * keep the previous value around where the store supports it (the
+   * gitlab store writes `<KEY>_backup_<timestamp>` variables) —
+   * a hand-written secret's only undo.
+   *
+   * Machine-provisioned credentials (service account keys, kubernetes
+   * tokens) are regenerated on demand and their old value is worthless,
+   * so `project setup` writes them with `backup: false`.
+   */
+  backup?: boolean;
+};
+
 /**
  * how secret resolution may interact with the vault:
  * - auto (default): contact the vault when the local cache is