@catladder/cli 5.0.1 → 5.1.0

package/package.json

@@ -53,7 +53,7 @@
     }
   ],
   "license": "MIT",
-  "version": "5.0.1",
+  "version": "5.1.0",
   "scripts": {
     "lint": "eslint \"src/**/*.ts\"",
     "lint:fix": "eslint \"src/**/*.ts\" --fix",

package/src/apps/cli/commands/project/doctor/checkGithub.ts

@@ -7,7 +7,7 @@ import {
 import {
   collectSecretTargets,
   getConfiguredGithubRepo,
-} from "../../../../../commands/project/commandSecretsSyncGithub";
+} from "../../../../../secrets";
 import { mapWithConcurrency } from "../../../../../utils/concurrency";
 import {
   ghApiJson,

package/src/apps/cli/commands/project/setup/setupAccessTokens.ts

@@ -4,6 +4,7 @@ import {
   doGitlabRequest,
   doGitlabRequestAllPages,
   getProjectInfo,
+  GitlabNotFoundError,
 } from "../../../../../utils/gitlab";
 
 const TOKEN_NAME = "semantic-release";
@@ -55,10 +56,10 @@ export const setupAccessTokens = async (instance: IO) => {
       "PUT",
     );
   } catch (e) {
-    if (e.message !== "not found") {
+    if (!(e instanceof GitlabNotFoundError)) {
       throw e;
     }
-    // not found
+    // the variable does not exist yet: create it
     await doGitlabRequest(
       instance,
       `projects/${projectId}/variables`,

package/src/apps/cli/commands/project/setup/setupGithub.ts

@@ -9,7 +9,7 @@ import { mkdtemp, readFile, rm } from "fs/promises";
 import { tmpdir } from "os";
 import { join } from "path";
 import { promisify } from "util";
-import { getConfiguredGithubRepo } from "../../../../../commands/project/commandSecretsSyncGithub";
+import { getConfiguredGithubRepo } from "../../../../../secrets";
 import type { IO } from "../../../../../core/types";
 import {
   ghApiJson,

package/src/apps/cli/commands/project/setup/setupKubernetes.ts

@@ -1,27 +1,26 @@
-import type { ComponentContext } from "@catladder/pipeline";
+import type { ComponentContext, Config } from "@catladder/pipeline";
 import {
+  getEnabledPipelineTypes,
   getFullKubernetesClusterName,
   isOfDeployType,
 } from "@catladder/pipeline";
+import { getProjectConfig } from "../../../../../config/getProjectConfig";
 import type { IO } from "../../../../../core/types";
 import { exec } from "child-process-promise";
+import { writeSecretsAndMirror } from "../../../../../secrets";
 import { connectToCluster } from "../../../../../utils/cluster";
-import {
-  doGitlabRequest,
-  getProjectInfo,
-  upsertAllVariables,
-} from "../../../../../utils/gitlab";
+import { doGitlabRequest, getProjectInfo } from "../../../../../utils/gitlab";
 import ensureNamespace from "../utils/ensureNamespace";
 
-export const setupKubernetes = async (
-  instance: IO,
-  context: ComponentContext,
-) => {
-  const deployConfig = context.deploy?.config;
-  if (!isOfDeployType(deployConfig, "kubernetes")) {
-    throw new Error("cannot run setupKubernetes on non-kubernetes deployments");
+/**
+ * the cluster pulls the images from the gitlab container registry, and
+ * a deploy token is how it authenticates there. Nothing to do when the
+ * project has no gitlab pipeline (the images live elsewhere then).
+ */
+const ensureGitlabDeployToken = async (instance: IO, config: Config | null) => {
+  if (!config || !getEnabledPipelineTypes(config).includes("gitlab")) {
+    return;
   }
-
   const { id: projectId } = await getProjectInfo(instance);
   const deploy_tokens = await doGitlabRequest(
     instance,
@@ -48,6 +47,18 @@ export const setupKubernetes = async (
       "POST",
     );
   }
+};
+
+export const setupKubernetes = async (
+  instance: IO,
+  context: ComponentContext,
+) => {
+  const deployConfig = context.deploy?.config;
+  if (!isOfDeployType(deployConfig, "kubernetes")) {
+    throw new Error("cannot run setupKubernetes on non-kubernetes deployments");
+  }
+
+  await ensureGitlabDeployToken(instance, await getProjectConfig());
 
   const fullName = getFullKubernetesClusterName(deployConfig.cluster);
   instance.log(`cluster: ${fullName}`);
@@ -158,14 +169,14 @@ EOF
   instance.log("service accounts created / updated!");
 
   instance.log("");
-  instance.log("pusing secrets to gitlab...");
+  instance.log("storing the cluster credentials...");
 
-  await upsertAllVariables(
+  await writeSecretsAndMirror(
     instance,
-    vars,
-    context.env,
-    context.name,
-    false, // no backup
+    [{ env: context.env, componentName: context.name, secrets: vars }],
+    // the service account token is recreated on every run — the
+    // previous one is worth nothing
+    { backup: false },
   );
   instance.log("done!");
 };

package/src/commands/index.ts

@@ -29,6 +29,7 @@ export { commandDoctor } from "./project/commandDoctor";
 export { commandCiJobOpen, commandCiJobLog } from "./project/commandGitlabCi";
 export { commandProjectRestoreDb } from "./project/commandProjectRestoreDb";
 export { commandSecurityEvaluate } from "./project/commandSecurityEvaluate";
+export { commandMrPin, commandMrUnpin } from "./project/commandMrPin";
 
 // Project k8s commands
 export { commandNamespace } from "./project/k8s/commandNamespace";

package/src/commands/project/commandConfigSecrets.ts

@@ -2,11 +2,11 @@ import { stripIndents } from "common-tags";
 import { defineCommand } from "../../core/defineCommand";
 import { editAsFile } from "../../utils/editAsFile";
 import { delay } from "../../utils/promise";
+import { writeSecretsAndMirror } from "../../secrets";
 import {
   buildSecretsDocument,
   checkSecretsDocument,
   collectSecretsWrites,
-  writeSecretsAndMirror,
 } from "./secrets/document";
 import {
   formatScope,

package/src/commands/project/commandMrPin.ts

@@ -0,0 +1,184 @@
+import { exec } from "child-process-promise";
+import {
+  getAutoStopConfig,
+  getEnabledPipelineTypes,
+} from "@catladder/pipeline";
+import { getProjectConfig } from "../../config/getProjectConfig";
+import { defineCommand } from "../../core/defineCommand";
+import type { IO } from "../../core/types";
+import { doGitlabRequest, getProjectInfo } from "../../utils/gitlab";
+
+const PIN_LABEL_COLOR = "#428fdc";
+
+type GitlabMr = {
+  iid: number;
+  title: string;
+  web_url: string;
+  labels: string[];
+};
+
+/**
+ * the configured pin label — errors when the project cannot pin
+ * (github-only, or pinning disabled)
+ */
+const getPinLabel = async (): Promise<string> => {
+  const config = await getProjectConfig();
+  if (!config) {
+    throw new Error("no catladder config found");
+  }
+  if (!getEnabledPipelineTypes(config).includes("gitlab")) {
+    throw new Error(
+      "pinning review apps needs the gitlab pipeline — github has no auto-stop, review apps live until the pull request is closed",
+    );
+  }
+  const { pinLabel } = getAutoStopConfig(config);
+  if (pinLabel === false) {
+    throw new Error(
+      "review-app pinning is disabled in this project (autoStop.pinLabel: false)",
+    );
+  }
+  return pinLabel;
+};
+
+const findMr = async (
+  io: IO,
+  projectId: string,
+  mrIid: number | undefined,
+): Promise<GitlabMr> => {
+  if (mrIid) {
+    return doGitlabRequest(io, `projects/${projectId}/merge_requests/${mrIid}`);
+  }
+  const branch = (await exec("git rev-parse --abbrev-ref HEAD")).stdout.trim();
+  const mrs = await doGitlabRequest<GitlabMr[]>(
+    io,
+    `projects/${projectId}/merge_requests?state=opened&source_branch=${encodeURIComponent(branch)}`,
+  );
+  if (!mrs?.length) {
+    throw new Error(
+      `no open merge request found for branch '${branch}' — pass one with --mr <iid>`,
+    );
+  }
+  return mrs[0];
+};
+
+const ensurePinLabelExists = async (
+  io: IO,
+  projectId: string,
+  pinLabel: string,
+) => {
+  const existing = await doGitlabRequest<Array<{ name: string }>>(
+    io,
+    `projects/${projectId}/labels?search=${encodeURIComponent(pinLabel)}`,
+  );
+  if (existing?.some((label) => label.name === pinLabel)) {
+    return;
+  }
+  await doGitlabRequest(
+    io,
+    `projects/${projectId}/labels`,
+    {
+      name: pinLabel,
+      color: PIN_LABEL_COLOR,
+      description:
+        "review apps of this MR are pinned — deploys use auto_stop_in: never (managed by catladder)",
+    },
+    "POST",
+  );
+  io.log(`created the '${pinLabel}' label in the project`);
+};
+
+export const commandMrPin = defineCommand({
+  name: "mr pin",
+  description:
+    "pin the review apps of a merge request (disable their auto-stop) by adding the pin label; merging or closing the MR still stops them",
+  group: "mr",
+  inputs: {
+    mr: {
+      type: "number",
+      message:
+        "merge request IID (defaults to the open MR of the current branch)",
+      positional: true,
+      required: false,
+    },
+    pipeline: {
+      type: "boolean",
+      message:
+        "trigger a merge-request pipeline so the pin takes effect immediately",
+      required: false,
+    },
+  },
+  execute: async (ctx) => {
+    const pinLabel = await getPinLabel();
+    const { id: projectId } = await getProjectInfo(ctx);
+    const mr = await findMr(ctx, projectId, await ctx.get("mr"));
+
+    await ensurePinLabelExists(ctx, projectId, pinLabel);
+
+    if (mr.labels?.includes(pinLabel)) {
+      ctx.log(`already pinned: mr!${mr.iid} carries the '${pinLabel}' label`);
+    } else {
+      await doGitlabRequest(
+        ctx,
+        `projects/${projectId}/merge_requests/${mr.iid}`,
+        { add_labels: pinLabel },
+        "PUT",
+      );
+      ctx.log(
+        `📌 pinned mr!${mr.iid} "${mr.title}" — label '${pinLabel}' added`,
+      );
+    }
+
+    // the label only affects pipelines created after it was set, so
+    // trigger one — otherwise the currently deployed review apps keep
+    // their old auto-stop timer until the next push
+    if ((await ctx.get("pipeline")) ?? true) {
+      const pipeline = await doGitlabRequest<{ web_url: string }>(
+        ctx,
+        `projects/${projectId}/merge_requests/${mr.iid}/pipelines`,
+        {},
+        "POST",
+      );
+      ctx.log(
+        `🚀 triggered a pipeline so the pin takes effect now: ${pipeline.web_url}`,
+      );
+    } else {
+      ctx.log("the pin takes effect with the next pipeline of the MR");
+    }
+  },
+});
+
+export const commandMrUnpin = defineCommand({
+  name: "mr unpin",
+  description:
+    "unpin the review apps of a merge request — the auto-stop timer re-arms with the next deploy",
+  group: "mr",
+  inputs: {
+    mr: {
+      type: "number",
+      message:
+        "merge request IID (defaults to the open MR of the current branch)",
+      positional: true,
+      required: false,
+    },
+  },
+  execute: async (ctx) => {
+    const pinLabel = await getPinLabel();
+    const { id: projectId } = await getProjectInfo(ctx);
+    const mr = await findMr(ctx, projectId, await ctx.get("mr"));
+
+    if (!mr.labels?.includes(pinLabel)) {
+      ctx.log(`mr!${mr.iid} is not pinned (no '${pinLabel}' label)`);
+      return;
+    }
+    await doGitlabRequest(
+      ctx,
+      `projects/${projectId}/merge_requests/${mr.iid}`,
+      { remove_labels: pinLabel },
+      "PUT",
+    );
+    ctx.log(`unpinned mr!${mr.iid} — label '${pinLabel}' removed`);
+    ctx.log(
+      "the deployed review apps stay pinned until the next deploy re-arms their auto-stop timer",
+    );
+  },
+});

package/src/commands/project/commandSecretsSyncGithub.ts

@@ -1,163 +1,16 @@
-import type { Config } from "@catladder/pipeline";
 import {
   getEnabledPipelineTypes,
   getPipelineGitRemote,
-  getSecretVarName,
-  GithubBackend,
 } from "@catladder/pipeline";
-import {
-  getAllComponentsWithAllEnvsHierarchical,
-  getEnvironment,
-  getProjectConfig,
-} from "../../config/getProjectConfig";
+import { getProjectConfig } from "../../config/getProjectConfig";
 import { defineCommand } from "../../core/defineCommand";
-import type { IO } from "../../core/types";
 import {
-  ensureGithubEnvironment,
-  getGithubRepoFromRemote,
-  isGhAuthenticated,
-  setGithubSecret,
-  setGithubVariable,
-} from "../../utils/github";
-
-/**
- * all secrets declared in the config, with their values read from the
- * vault (github secrets are write-only and can never act as a source)
- */
-export const collectSecretsFromVault = async (io: IO, envFilter?: string[]) => {
-  const envAndComponents = await getAllComponentsWithAllEnvsHierarchical();
-
-  const kinds = new Map<string, "secret" | "variable">();
-  for (const [componentName, envs] of Object.entries(envAndComponents)) {
-    for (const env of envs) {
-      if (envFilter && !envFilter.includes(env)) {
-        continue;
-      }
-      const { secretEnvVarKeys, jobOnlyVars } = await getEnvironment(
-        env,
-        componentName,
-      );
-      const keys = [
-        ...secretEnvVarKeys,
-        ...jobOnlyVars.build.secretEnvVarKeys,
-        ...jobOnlyVars.deploy.secretEnvVarKeys,
-      ];
-      keys.forEach(({ key, kind }) =>
-        kinds.set(getSecretVarName(env, componentName, key), kind ?? "secret"),
-      );
-    }
-  }
-
-  const names = [...kinds.keys()];
-  const values = await (await io.getVaultManager()).readSecrets(names, io);
-  const secrets = names
-    .filter((name) => values[name] !== undefined)
-    .map((name) => ({
-      name,
-      value: values[name],
-      kind: kinds.get(name) ?? ("secret" as const),
-    }));
-  const missing = names.filter((name) => values[name] === undefined);
-  return { secrets, missing };
-};
-
-/** marker for repo-level (environment-less) targets */
-const REPO_LEVEL = "";
-
-/**
- * derives which github environment(s) each secret/variable must live in
- * by scanning the GENERATED workflows: a `${{ secrets.X }}` / `${{
- * vars.X }}` reference in a job's env targets that job's environment
- * (or the repo level when the job has none). Deriving from the real
- * output means the sync can never drift from what the workflows expect.
- *
- * Environment secrets are the answer to github's cap of 100 secrets
- * per REPO (a multi-env project easily exceeds it): the cap applies
- * per environment instead.
- */
-export const collectSecretTargets = async (
-  config: Config,
-): Promise<Map<string, Set<string>>> => {
-  const workflows = await new GithubBackend().createWorkflows(config);
-  const targets = new Map<string, Set<string>>();
-  for (const workflow of Object.values(workflows)) {
-    for (const job of Object.values(workflow.jobs)) {
-      const environment =
-        typeof job.environment === "object"
-          ? job.environment.name
-          : job.environment;
-      for (const value of Object.values(job.env ?? {})) {
-        if (typeof value !== "string") continue;
-        for (const match of value.matchAll(
-          /\$\{\{\s*(?:secrets|vars)\.([A-Za-z0-9_]+)\s*\}\}/g,
-        )) {
-          const name = match[1];
-          if (!targets.has(name)) {
-            targets.set(name, new Set());
-          }
-          targets.get(name)!.add(environment ?? REPO_LEVEL);
-        }
-      }
-    }
-  }
-  return targets;
-};
-
-/**
- * pushes vault values to github via the gh cli: sensitive ones as
- * secrets (masked), "variable"-kind ones as variables (plain) — each
- * into the environment(s) whose jobs reference them
- */
-export const pushSecretsToGithub = async (
-  io: IO,
-  repo: string,
-  secrets: { name: string; value: string; kind: "secret" | "variable" }[],
-  targets: Map<string, Set<string>>,
-) => {
-  const environments = [
-    ...new Set(
-      [...targets.values()].flatMap((envs) =>
-        [...envs].filter((env) => env !== REPO_LEVEL),
-      ),
-    ),
-  ];
-  for (const environment of environments) {
-    await ensureGithubEnvironment(repo, environment);
-    io.log(`🌍 environment ${environment}`);
-  }
-
-  for (const { name, value, kind } of secrets) {
-    const envs = targets.get(name);
-    if (!envs) {
-      io.log(`⏭️ ${name} (not referenced by any generated workflow)`);
-      continue;
-    }
-    for (const environment of envs) {
-      const suffix = environment === REPO_LEVEL ? "" : ` → ${environment}`;
-      if (kind === "variable") {
-        await setGithubVariable(repo, name, value, environment || undefined);
-        io.log(`📖 ${name} (variable)${suffix}`);
-      } else {
-        await setGithubSecret(repo, name, value, environment || undefined);
-        io.log(`✅ ${name}${suffix}`);
-      }
-    }
-  }
-};
-
-/**
- * resolves the github repository from the configured git remote
- * (`pipelines.github.gitRemote`, defaults to origin)
- */
-export const getConfiguredGithubRepo = async (): Promise<
-  string | undefined
-> => {
-  const config = await getProjectConfig();
-  if (!config) {
-    return undefined;
-  }
-  return getGithubRepoFromRemote(getPipelineGitRemote(config, "github"));
-};
+  collectSecretsFromVault,
+  collectSecretTargets,
+  getConfiguredGithubRepo,
+  pushSecretsToGithub,
+} from "../../secrets";
+import { isGhAuthenticated } from "../../utils/github";
 
 /**
  * copies the catladder secrets from the vault to github actions

package/src/commands/project/secrets/commandSecretsPush.ts

@@ -2,12 +2,9 @@ import { readFile } from "fs-extra";
 import { parse } from "yaml";
 import { defineCommand } from "../../../core/defineCommand";
 import { readStdin } from "../../../utils/readStdin";
+import { writeSecretsAndMirror } from "../../../secrets";
 import type { SecretsDocument } from "./document";
-import {
-  checkSecretsDocument,
-  collectSecretsWrites,
-  writeSecretsAndMirror,
-} from "./document";
+import { checkSecretsDocument, collectSecretsWrites } from "./document";
 import {
   getScopedSecretKeys,
   resolveSecretsScope,

package/src/commands/project/secrets/commandSecretsSet.ts

@@ -2,8 +2,8 @@ import { readFile } from "fs-extra";
 import { defineCommand } from "../../../core/defineCommand";
 import type { IO } from "../../../core/types";
 import { readStdin } from "../../../utils/readStdin";
-import type { SecretsWrite } from "./document";
-import { writeSecretsAndMirror } from "./document";
+import type { SecretsWrite } from "../../../secrets";
+import { writeSecretsAndMirror } from "../../../secrets";
 import {
   getScopedSecretKeys,
   resolveSecretsScope,

package/src/commands/project/secrets/document.ts

@@ -1,21 +1,6 @@
-import type { Config } from "@catladder/pipeline";
-import {
-  getEnabledPipelineTypes,
-  getSecretVarName,
-  getVaultConfig,
-} from "@catladder/pipeline";
-import {
-  getEnvironment,
-  getProjectConfig,
-} from "../../../config/getProjectConfig";
+import { getSecretVarName } from "@catladder/pipeline";
 import type { IO } from "../../../core/types";
-import { isGhAuthenticated } from "../../../utils/github";
-import { upsertAllVariables } from "../../../utils/gitlab";
-import {
-  collectSecretTargets,
-  getConfiguredGithubRepo,
-  pushSecretsToGithub,
-} from "../commandSecretsSyncGithub";
+import type { SecretsWrite } from "../../../secrets";
 import type { ScopedSecretKeys } from "./scope";
 
 /** placeholder for secrets that have no value in the vault yet */
@@ -31,13 +16,6 @@ export type SecretsDocument = {
   };
 };
 
-/** secrets of one env of one component, ready to be written */
-export type SecretsWrite = {
-  env: string;
-  componentName: string;
-  secrets: Record<string, unknown>;
-};
-
 /* for convenience, parse json values. that makes it easier to edit secrets that are objects */
 const parseJsonValue = (value: string): unknown => {
   try {
@@ -164,106 +142,3 @@ export const collectSecretsWrites = (
   }
   return { writes, skipped };
 };
-
-/**
- * writes secrets to the vault (the source of truth) and mirrors them
- * to every enabled CI backend — which backends exist is invisible to
- * the caller
- */
-export const writeSecretsAndMirror = async (
-  io: IO,
-  writes: SecretsWrite[],
-): Promise<void> => {
-  const config = await getProjectConfig();
-  if (!config) {
-    throw new Error("no catladder config found");
-  }
-
-  io.log("");
-  io.log("writing secrets to the vault, please wait...");
-  io.log("");
-  for (const { env, componentName, secrets } of writes) {
-    io.log("writing " + env + ":" + componentName + "...\n");
-    await (
-      await io.getVaultManager()
-    ).writeSecrets(env, componentName, secrets);
-    io.log("");
-    io.log("✅ " + env + ":" + componentName);
-    io.log("--------------------------------\n");
-  }
-
-  await mirrorSecretsToCiBackends(io, config, writes);
-};
-
-/**
- * the vault is the source of truth; every enabled CI backend receives
- * a mirrored copy of the written secrets (gitlab variables, github
- * secrets), so all CI systems stay in sync
- */
-const mirrorSecretsToCiBackends = async (
-  io: IO,
-  config: Config,
-  writes: SecretsWrite[],
-) => {
-  const enabled = getEnabledPipelineTypes(config);
-
-  // gitlab mirror (unless gitlab is the vault itself, then it is
-  // already written)
-  if (getVaultConfig(config).type !== "gitlab" && enabled.includes("gitlab")) {
-    for (const { env, componentName, secrets } of writes) {
-      await upsertAllVariables(io, secrets, env, componentName);
-    }
-    io.log("✅ mirrored to gitlab");
-  }
-
-  if (!enabled.includes("github")) {
-    return;
-  }
-  const kinds = new Map<string, "secret" | "variable">();
-  for (const { env, componentName } of writes) {
-    const { secretEnvVarKeys, jobOnlyVars } = await getEnvironment(
-      env,
-      componentName,
-    );
-    [
-      ...secretEnvVarKeys,
-      ...jobOnlyVars.build.secretEnvVarKeys,
-      ...jobOnlyVars.deploy.secretEnvVarKeys,
-    ].forEach(({ key, kind }) =>
-      kinds.set(getSecretVarName(env, componentName, key), kind ?? "secret"),
-    );
-  }
-  const secrets = writes.flatMap(({ env, componentName, secrets }) =>
-    Object.entries(secrets).map(([key, value]) => {
-      const name = getSecretVarName(env, componentName, key);
-      return {
-        name,
-        value: typeof value === "string" ? value : JSON.stringify(value ?? ""),
-        kind: kinds.get(name) ?? ("secret" as const),
-      };
-    }),
-  );
-  if (secrets.length === 0) {
-    return;
-  }
-  if (!(await isGhAuthenticated())) {
-    io.log(
-      "⚠️ github pipeline is enabled, but the github cli (gh) is not authenticated — run `project secrets-sync-github` later to mirror the secrets",
-    );
-    return;
-  }
-  const repo = await getConfiguredGithubRepo();
-  if (!repo) {
-    io.log(
-      "⚠️ github pipeline is enabled, but no github repository was found — run `project secrets-sync-github` later to mirror the secrets",
-    );
-    return;
-  }
-  io.log(`mirroring ${secrets.length} secrets to github ('${repo}')...`);
-  await pushSecretsToGithub(
-    io,
-    repo,
-    secrets,
-    await collectSecretTargets(config),
-  );
-};