@catladder/cli 0.0.0-main-8dbb6c76 → 0.0.0-main-a6b45abe

package/src/commands/project/secrets/commandSecretsPush.ts

@@ -0,0 +1,100 @@
+import { readFile } from "fs-extra";
+import { parse } from "yaml";
+import { defineCommand } from "../../../core/defineCommand";
+import { readStdin } from "../../../utils/readStdin";
+import type { SecretsDocument } from "./document";
+import {
+  checkSecretsDocument,
+  collectSecretsWrites,
+  writeSecretsAndMirror,
+} from "./document";
+import {
+  getScopedSecretKeys,
+  resolveSecretsScope,
+  secretsScopeChoices,
+} from "./scope";
+import { parseKeysInput } from "./parseKeysInput";
+
+export const commandSecretsPush = defineCommand({
+  name: "project secrets-push",
+  description:
+    "writes secrets from a yaml document (secrets-pull format, --file or stdin) to the vault; may be partial",
+  group: "project",
+  inputs: {
+    envComponent: {
+      type: "string",
+      message: "scope: env:component, env:, :component or empty for everything",
+      positional: true,
+      required: false,
+      choices: async () => secretsScopeChoices(),
+    },
+    key: {
+      type: "string",
+      message: "only write these secrets (comma-separated keys)",
+      required: false,
+    },
+    file: {
+      type: "string",
+      message:
+        "yaml file to push (component → env → key: value); stdin when omitted",
+      required: false,
+    },
+  },
+  execute: async (ctx) => {
+    const scope = await resolveSecretsScope(await ctx.get("envComponent"));
+    const scoped = await getScopedSecretKeys(
+      scope,
+      parseKeysInput(await ctx.get("key")),
+    );
+
+    const file = await ctx.get("file");
+    if (!file && process.stdin.isTTY) {
+      throw new Error(
+        "nothing to push — pass --file <secrets.yml> or pipe yaml (secrets-pull format) to stdin",
+      );
+    }
+    const raw = file
+      ? (await readFile(file)).toString("utf-8")
+      : await readStdin();
+    const document = parse(raw) as SecretsDocument;
+    if (!document || typeof document !== "object") {
+      throw new Error(
+        "the document must be a yaml object: component → env → key: value (see secrets-pull)",
+      );
+    }
+
+    const problems = checkSecretsDocument(document, scoped, {
+      requireComplete: false,
+    });
+    if (problems.length > 0) {
+      throw new Error(
+        "😿 the document does not match the config:\n" +
+          problems.map((problem) => `  - ${problem}`).join("\n"),
+      );
+    }
+
+    const { writes, skipped } = collectSecretsWrites(document, scoped);
+    if (skipped.length > 0) {
+      ctx.log("⏭️ skipping placeholder/empty values:");
+      skipped.forEach((entry) => ctx.log(`  - ${entry}`));
+    }
+    if (writes.length === 0) {
+      ctx.log("no secrets to write 🤷");
+      return;
+    }
+
+    ctx.log("about to write:");
+    writes.forEach(({ env, componentName, secrets }) =>
+      ctx.log(
+        `  - ${env}:${componentName}: ${Object.keys(secrets).join(", ")}`,
+      ),
+    );
+    const confirmed = await ctx.confirm("continue? 🤔");
+    if (!confirmed) {
+      throw new Error("abort");
+    }
+
+    await writeSecretsAndMirror(ctx, writes);
+    ctx.log("done! 😻");
+  },
+});

package/src/commands/project/secrets/commandSecretsSet.ts

@@ -0,0 +1,102 @@
+import { readFile } from "fs-extra";
+import { defineCommand } from "../../../core/defineCommand";
+import type { IO } from "../../../core/types";
+import { readStdin } from "../../../utils/readStdin";
+import type { SecretsWrite } from "./document";
+import { writeSecretsAndMirror } from "./document";
+import {
+  getScopedSecretKeys,
+  resolveSecretsScope,
+  secretsScopeChoices,
+} from "./scope";
+
+const resolveValue = async (
+  io: IO,
+  value: string | undefined,
+  valueFile: string | undefined,
+): Promise<string> => {
+  if (valueFile) {
+    // a trailing newline is almost never part of the secret
+    return (await readFile(valueFile)).toString("utf-8").replace(/\n$/, "");
+  }
+  if (value !== undefined) {
+    return value;
+  }
+  if (!process.stdin.isTTY) {
+    return (await readStdin()).replace(/\n$/, "");
+  }
+  return io.promptDirect({
+    name: "value",
+    type: "string",
+    message: "value",
+  });
+};
+
+export const commandSecretsSet = defineCommand({
+  name: "project secrets-set",
+  description:
+    "sets one secret — in one env (dev:web), one env everywhere (dev:), or all envs of a component (:web); value via --value, --value-file or stdin",
+  group: "project",
+  inputs: {
+    envComponent: {
+      type: "string",
+      message: "scope: env:component, env:, or :component",
+      positional: true,
+      choices: async () => secretsScopeChoices(),
+    },
+    key: {
+      type: "string",
+      message: "which secret?",
+      positional: true,
+      choices: async (ctx) => {
+        const scope = await resolveSecretsScope(await ctx.get("envComponent"));
+        const scoped = await getScopedSecretKeys(scope);
+        return [...new Set(scoped.flatMap(({ keys }) => keys))];
+      },
+    },
+    value: {
+      type: "string",
+      message:
+        "the secret value (prefer --value-file or stdin over shell history)",
+      required: false,
+    },
+    valueFile: {
+      type: "string",
+      message: "read the secret value from this file",
+      required: false,
+    },
+  },
+  execute: async (ctx) => {
+    const scope = await resolveSecretsScope(await ctx.get("envComponent"));
+    const key = await ctx.get("key");
+    // drops env/component pairs where the key is not declared and
+    // throws when it is declared nowhere in the scope
+    const scoped = await getScopedSecretKeys(scope, [key]);
+    const value = await resolveValue(
+      ctx,
+      await ctx.get("value"),
+      await ctx.get("valueFile"),
+    );
+
+    const targets = scoped.map(
+      ({ env, componentName }) => `${env}:${componentName}`,
+    );
+    ctx.log(`setting ${key} in: ${targets.join(", ")}`);
+    if (targets.length > 1) {
+      const confirmed = await ctx.confirm(
+        `write the same value to ${targets.length} environments? 🤔`,
+      );
+      if (!confirmed) {
+        throw new Error("abort");
+      }
+    }
+
+    const writes: SecretsWrite[] = scoped.map(({ env, componentName }) => ({
+      env,
+      componentName,
+      secrets: { [key]: value },
+    }));
+    await writeSecretsAndMirror(ctx, writes);
+    ctx.log("done! 😻");
+  },
+});

package/src/commands/project/secrets/document.ts

@@ -0,0 +1,269 @@
+import type { Config } from "@catladder/pipeline";
+import {
+  getEnabledPipelineTypes,
+  getSecretVarName,
+  getVaultConfig,
+} from "@catladder/pipeline";
+import {
+  getEnvironment,
+  getProjectConfig,
+} from "../../../config/getProjectConfig";
+import type { IO } from "../../../core/types";
+import { isGhAuthenticated } from "../../../utils/github";
+import { upsertAllVariables } from "../../../utils/gitlab";
+import {
+  collectSecretTargets,
+  getConfiguredGithubRepo,
+  pushSecretsToGithub,
+} from "../commandSecretsSyncGithub";
+import type { ScopedSecretKeys } from "./scope";
+
+/** placeholder for secrets that have no value in the vault yet */
+export const FILL_ME = "🚨 FILL ME";
+
+/**
+ * component → env → key → value: the document shape shared by the
+ * editor, `secrets-pull` and `secrets-push`
+ */
+export type SecretsDocument = {
+  [componentName: string]: {
+    [env: string]: Record<string, unknown>;
+  };
+};
+
+/** secrets of one env of one component, ready to be written */
+export type SecretsWrite = {
+  env: string;
+  componentName: string;
+  secrets: Record<string, unknown>;
+};
+
+/* for convenience, parse json values. that makes it easier to edit secrets that are objects */
+const parseJsonValue = (value: string): unknown => {
+  try {
+    return JSON.parse(value);
+  } catch (e) {
+    return value;
+  }
+};
+
+/**
+ * builds the document for the given scope with the current values from
+ * the vault; unset secrets appear as the FILL_ME placeholder
+ */
+export const buildSecretsDocument = async (
+  io: IO,
+  scoped: ScopedSecretKeys,
+): Promise<SecretsDocument> => {
+  const names = scoped.flatMap(({ env, componentName, keys }) =>
+    keys.map((key) => getSecretVarName(env, componentName, key)),
+  );
+  const values = await (await io.getVaultManager()).readSecrets(names, io);
+
+  const document: SecretsDocument = {};
+  for (const { env, componentName, keys } of scoped) {
+    document[componentName] ??= {};
+    document[componentName][env] = Object.fromEntries(
+      keys.map((key) => {
+        const value = values[getSecretVarName(env, componentName, key)];
+        return [key, value === undefined ? FILL_ME : parseJsonValue(value)];
+      }),
+    );
+  }
+  return document;
+};
+
+/**
+ * validates a document against the scope. Undeclared components, envs
+ * or keys are always problems; missing keys only when `requireComplete`
+ * (the guided editor flow) — pushes may be partial.
+ */
+export const checkSecretsDocument = (
+  document: SecretsDocument,
+  scoped: ScopedSecretKeys,
+  { requireComplete }: { requireComplete: boolean },
+): string[] => {
+  const problems: string[] = [];
+
+  const scopedKeys = new Map<string, string[]>();
+  scoped.forEach(({ env, componentName, keys }) =>
+    scopedKeys.set(`${env}:${componentName}`, keys),
+  );
+  const scopedComponents = new Set(scoped.map((s) => s.componentName));
+
+  for (const [componentName, envs] of Object.entries(document ?? {})) {
+    if (!scopedComponents.has(componentName)) {
+      problems.push(
+        `"${componentName}" is not a component in this scope — remove it or widen the scope`,
+      );
+      continue;
+    }
+    for (const [env, secrets] of Object.entries(envs ?? {})) {
+      const keys = scopedKeys.get(`${env}:${componentName}`);
+      if (!keys) {
+        problems.push(
+          `"${env}:${componentName}" is not in this scope — remove it or widen the scope`,
+        );
+        continue;
+      }
+      const extraneous = Object.keys(secrets ?? {}).filter(
+        (key) => !keys.includes(key),
+      );
+      extraneous.forEach((key) =>
+        problems.push(
+          `"${env}:${componentName}": "${key}" is not declared in the config (or filtered out by --key)`,
+        ),
+      );
+    }
+  }
+
+  if (requireComplete) {
+    for (const { env, componentName, keys } of scoped) {
+      const provided = Object.keys(document?.[componentName]?.[env] ?? {});
+      keys
+        .filter((key) => !provided.includes(key))
+        .forEach((key) =>
+          problems.push(
+            `"${env}:${componentName}": "${key}" has not been provided`,
+          ),
+        );
+    }
+  }
+
+  return problems;
+};
+
+/**
+ * turns a (validated) document into writes. FILL_ME placeholders and
+ * empty (null) values are skipped, so an untouched placeholder can
+ * never end up as a live secret value.
+ */
+export const collectSecretsWrites = (
+  document: SecretsDocument,
+  scoped: ScopedSecretKeys,
+): { writes: SecretsWrite[]; skipped: string[] } => {
+  const writes: SecretsWrite[] = [];
+  const skipped: string[] = [];
+  for (const { env, componentName, keys } of scoped) {
+    const provided = document?.[componentName]?.[env] ?? {};
+    const secrets: Record<string, unknown> = {};
+    for (const key of keys) {
+      if (!(key in provided)) {
+        continue;
+      }
+      const value = provided[key];
+      if (value === FILL_ME || value === null || value === undefined) {
+        skipped.push(`${env}:${componentName} ${key}`);
+        continue;
+      }
+      secrets[key] = value;
+    }
+    if (Object.keys(secrets).length > 0) {
+      writes.push({ env, componentName, secrets });
+    }
+  }
+  return { writes, skipped };
+};
+
+/**
+ * writes secrets to the vault (the source of truth) and mirrors them
+ * to every enabled CI backend — which backends exist is invisible to
+ * the caller
+ */
+export const writeSecretsAndMirror = async (
+  io: IO,
+  writes: SecretsWrite[],
+): Promise<void> => {
+  const config = await getProjectConfig();
+  if (!config) {
+    throw new Error("no catladder config found");
+  }
+
+  io.log("");
+  io.log("writing secrets to the vault, please wait...");
+  io.log("");
+  for (const { env, componentName, secrets } of writes) {
+    io.log("writing " + env + ":" + componentName + "...\n");
+    await (
+      await io.getVaultManager()
+    ).writeSecrets(env, componentName, secrets);
+    io.log("");
+    io.log("✅ " + env + ":" + componentName);
+    io.log("--------------------------------\n");
+  }
+
+  await mirrorSecretsToCiBackends(io, config, writes);
+};
+
+/**
+ * the vault is the source of truth; every enabled CI backend receives
+ * a mirrored copy of the written secrets (gitlab variables, github
+ * secrets), so all CI systems stay in sync
+ */
+const mirrorSecretsToCiBackends = async (
+  io: IO,
+  config: Config,
+  writes: SecretsWrite[],
+) => {
+  const enabled = getEnabledPipelineTypes(config);
+
+  // gitlab mirror (unless gitlab is the vault itself, then it is
+  // already written)
+  if (getVaultConfig(config).type !== "gitlab" && enabled.includes("gitlab")) {
+    for (const { env, componentName, secrets } of writes) {
+      await upsertAllVariables(io, secrets, env, componentName);
+    }
+    io.log("✅ mirrored to gitlab");
+  }
+
+  if (!enabled.includes("github")) {
+    return;
+  }
+  const kinds = new Map<string, "secret" | "variable">();
+  for (const { env, componentName } of writes) {
+    const { secretEnvVarKeys, jobOnlyVars } = await getEnvironment(
+      env,
+      componentName,
+    );
+    [
+      ...secretEnvVarKeys,
+      ...jobOnlyVars.build.secretEnvVarKeys,
+      ...jobOnlyVars.deploy.secretEnvVarKeys,
+    ].forEach(({ key, kind }) =>
+      kinds.set(getSecretVarName(env, componentName, key), kind ?? "secret"),
+    );
+  }
+  const secrets = writes.flatMap(({ env, componentName, secrets }) =>
+    Object.entries(secrets).map(([key, value]) => {
+      const name = getSecretVarName(env, componentName, key);
+      return {
+        name,
+        value: typeof value === "string" ? value : JSON.stringify(value ?? ""),
+        kind: kinds.get(name) ?? ("secret" as const),
+      };
+    }),
+  );
+  if (secrets.length === 0) {
+    return;
+  }
+  if (!(await isGhAuthenticated())) {
+    io.log(
+      "⚠️ github pipeline is enabled, but the github cli (gh) is not authenticated — run `project secrets-sync-github` later to mirror the secrets",
+    );
+    return;
+  }
+  const repo = await getConfiguredGithubRepo();
+  if (!repo) {
+    io.log(
+      "⚠️ github pipeline is enabled, but no github repository was found — run `project secrets-sync-github` later to mirror the secrets",
+    );
+    return;
+  }
+  io.log(`mirroring ${secrets.length} secrets to github ('${repo}')...`);
+  await pushSecretsToGithub(
+    io,
+    repo,
+    secrets,
+    await collectSecretTargets(config),
+  );
+};

package/src/commands/project/secrets/parseKeysInput.ts

@@ -0,0 +1,8 @@
+/** parses a comma-separated `--key` input into a list of keys */
+export const parseKeysInput = (input?: string): string[] | undefined => {
+  const keys = input
+    ?.split(",")
+    .map((key) => key.trim())
+    .filter(Boolean);
+  return keys && keys.length > 0 ? keys : undefined;
+};

package/src/commands/project/secrets/scope.ts

@@ -0,0 +1,151 @@
+import {
+  getAllComponentsWithAllEnvsHierarchical,
+  getEnvironment,
+} from "../../../config/getProjectConfig";
+
+/** which envs of which components a secrets command operates on */
+export type SecretsScope = {
+  [componentName: string]: string[];
+};
+
+/**
+ * resolves a scope expression to component → envs:
+ * - (empty): all components, all envs
+ * - `env:component`: one env of one component
+ * - `env` / `env:`: one env, every component that has it
+ * - `:component`: every env of one component
+ */
+export const resolveSecretsScope = async (
+  scope?: string,
+): Promise<SecretsScope> => {
+  const all = await getAllComponentsWithAllEnvsHierarchical();
+  if (Object.keys(all).length === 0) {
+    throw new Error("no catladder config found");
+  }
+  if (!scope) {
+    return all;
+  }
+
+  const [env, componentName] = scope.split(":").map((x) => x || null);
+
+  if (componentName && !(componentName in all)) {
+    throw new Error(
+      `unknown component "${componentName}" — components: ${Object.keys(all).join(", ")}`,
+    );
+  }
+
+  const scoped = Object.fromEntries(
+    Object.entries(all)
+      .filter(([name]) => !componentName || name === componentName)
+      .map(([name, envs]): [string, string[]] => [
+        name,
+        env ? envs.filter((e) => e === env) : envs,
+      ])
+      .filter(([, envs]) => envs.length > 0),
+  );
+
+  if (Object.keys(scoped).length === 0) {
+    const allEnvs = [...new Set(Object.values(all).flat())];
+    throw new Error(
+      `no component has the environment "${env}" — environments: ${allEnvs.join(", ")}`,
+    );
+  }
+  return scoped;
+};
+
+/**
+ * all valid scope expressions, for autocompletion: `env:component`
+ * pairs plus the `env:` (one env, all components) and `:component`
+ * (one component, all envs) forms
+ */
+export const secretsScopeChoices = async (): Promise<string[]> => {
+  const all = await getAllComponentsWithAllEnvsHierarchical();
+  const pairs = Object.entries(all).flatMap(([componentName, envs]) =>
+    envs.map((env) => `${env}:${componentName}`),
+  );
+  const envs = [...new Set(Object.values(all).flat())];
+  return [
+    ...pairs,
+    ...envs.map((env) => `${env}:`),
+    ...Object.keys(all).map((componentName) => `:${componentName}`),
+  ];
+};
+
+/** one env of one component together with its (filtered) secret keys */
+export type ScopedSecretKeys = Array<{
+  env: string;
+  componentName: string;
+  keys: string[];
+}>;
+
+/** the secret keys declared in the config for one env of one component */
+export const getDeclaredSecretKeys = async (
+  env: string,
+  componentName: string,
+): Promise<string[]> => {
+  const { secretEnvVarKeys, jobOnlyVars } = await getEnvironment(
+    env,
+    componentName,
+  );
+  return [
+    ...jobOnlyVars.build.secretEnvVarKeys,
+    ...jobOnlyVars.deploy.secretEnvVarKeys,
+    ...secretEnvVarKeys,
+  ]
+    .filter((k) => !k.hidden)
+    .map((k) => k.key);
+};
+
+/**
+ * expands a scope to its declared secret keys, optionally filtered to
+ * `onlyKeys`. Throws when a requested key is not declared anywhere in
+ * the scope (typo protection); env/component pairs without matching
+ * keys are dropped.
+ */
+export const getScopedSecretKeys = async (
+  scope: SecretsScope,
+  onlyKeys?: string[],
+): Promise<ScopedSecretKeys> => {
+  const scoped: ScopedSecretKeys = [];
+  const declaredAnywhere = new Set<string>();
+  for (const [componentName, envs] of Object.entries(scope)) {
+    for (const env of envs) {
+      const declared = await getDeclaredSecretKeys(env, componentName);
+      declared.forEach((key) => declaredAnywhere.add(key));
+      const keys = onlyKeys
+        ? declared.filter((key) => onlyKeys.includes(key))
+        : declared;
+      if (keys.length > 0) {
+        scoped.push({ env, componentName, keys });
+      }
+    }
+  }
+
+  const unknownKeys = (onlyKeys ?? []).filter(
+    (key) => !declaredAnywhere.has(key),
+  );
+  if (unknownKeys.length > 0) {
+    throw new Error(
+      `these secrets are not declared in this scope: ${unknownKeys.join(", ")}` +
+        `\ndeclared secrets: ${[...declaredAnywhere].join(", ")}`,
+    );
+  }
+  if (scoped.length === 0) {
+    throw new Error("there are no secrets to configure in this scope");
+  }
+  return scoped;
+};
+
+/** human-readable one-line-per-component summary of a scope */
+export const formatScope = (scoped: ScopedSecretKeys): string => {
+  const byComponent = new Map<string, string[]>();
+  for (const { env, componentName } of scoped) {
+    byComponent.set(componentName, [
+      ...(byComponent.get(componentName) ?? []),
+      env,
+    ]);
+  }
+  return [...byComponent.entries()]
+    .map(([componentName, envs]) => `- ${componentName}: ${envs.join(", ")}`)
+    .join("\n");
+};

package/src/commands/security/commandSecurityAuditCiJob.ts

@@ -8,8 +8,7 @@ import {
   createSecurityAuditMergeRequest,
   SECURITY_AUDIT_FILE_NAME,
 } from "../../security/createSecurityAuditMergeRequest";
-
-const GITLAB_HOST = "https://git.panter.ch";
+import { getGitlabHostUrl } from "../../utils/gitlabHost";
 
 export const commandSecurityAuditCiJob = defineCommand({
   name: "security audit ci-job",
@@ -52,7 +51,7 @@ export const commandSecurityAuditCiJob = defineCommand({
       const projectId = await ctx.get("projectId");
       const userId = await ctx.get("userId");
 
-      const api = new Gitlab({ host: GITLAB_HOST, token });
+      const api = new Gitlab({ host: await getGitlabHostUrl(), token });
 
       const mr = await createSecurityAuditMergeRequest({
         api,

package/src/commands/security/commandSecurityAuditCreate.ts

@@ -1,8 +1,7 @@
 import { defineCommand } from "../../core/defineCommand";
 import { Gitlab } from "@gitbeaker/rest";
 import { createSecurityAuditMergeRequest } from "../../security/createSecurityAuditMergeRequest";
-
-const GITLAB_HOST = "https://git.panter.ch";
+import { getGitlabHostUrl } from "../../utils/gitlabHost";
 
 export const commandSecurityAuditCreate = defineCommand({
   name: "security audit create",
@@ -36,7 +35,7 @@ export const commandSecurityAuditCreate = defineCommand({
     const projectId = await ctx.get("projectId");
     const userId = await ctx.get("userId");
 
-    const api = new Gitlab({ host: GITLAB_HOST, token });
+    const api = new Gitlab({ host: await getGitlabHostUrl(), token });
 
     const result = await createSecurityAuditMergeRequest({
       api,

package/src/completion.ts

@@ -157,8 +157,8 @@ _catladder_find_binary() {
       return
     fi
     # Also check for catladder-dev (monorepo development)
-    if [[ -x "$dir/cli/bin/catladder-dev" ]]; then
-      echo "$dir/cli/bin/catladder-dev"
+    if [[ -x "$dir/apps/cli/bin/catladder-dev" ]]; then
+      echo "$dir/apps/cli/bin/catladder-dev"
       return
     fi
     dir="\${dir:h}"