@catladder/cli 0.0.0-next-4e7c6848 → 0.0.0-next-8b9ad5ca

package/package.json

@@ -53,7 +53,7 @@
     }
   ],
   "license": "MIT",
-  "version": "0.0.0-next-4e7c6848",
+  "version": "0.0.0-next-8b9ad5ca",
   "scripts": {
     "lint": "eslint \"src/**/*.ts\"",
     "lint:fix": "eslint \"src/**/*.ts\" --fix",

package/src/adapters/baseContext.ts

@@ -6,6 +6,8 @@ import type {
   InputsSchema,
   OptionalInput,
 } from "../core/types";
+import type { VaultManager } from "../vault/VaultManager";
+import { createVaultManagerGetter } from "./vaultManagerAccess";
 
 /**
  * Base class for CommandContext implementations.
@@ -19,9 +21,17 @@ export abstract class BaseContext<TInputs extends InputsSchema>
 
   constructor(protected readonly command: CommandDef<TInputs>) {}
 
+  abstract readonly interactive: boolean;
+
   abstract log(message: string): void;
   abstract confirm(message: string): Promise<boolean>;
 
+  private readonly vaultManagerGetter = createVaultManagerGetter(() => this);
+
+  getVaultManager(): Promise<VaultManager> {
+    return this.vaultManagerGetter();
+  }
+
   /**
    * Resolve a single input value. Called once per input (result is cached).
    * Implementations check pre-supplied values, prompt interactively, use defaults, or throw.

package/src/adapters/nonInteractive.ts

@@ -0,0 +1,30 @@
+import type { IO } from "../core/types";
+import { NonInteractiveError } from "../core/types";
+import type { SecretsMode } from "../vault";
+import { createVaultManagerGetter } from "./vaultManagerAccess";
+
+/**
+ * the context of non-interactive runs (catenv via direnv): logging
+ * goes to stderr, any required interaction fails with a clear error.
+ */
+export function createNonInteractiveIO(
+  options: { vaultMode?: SecretsMode } = {},
+): IO {
+  const io: IO = {
+    interactive: false,
+    getVaultManager: createVaultManagerGetter(() => io, {
+      mode: options.vaultMode,
+    }),
+    log(message: string): void {
+      // stderr: stdout may be evaluated (direnv)
+      console.error(message);
+    },
+    async confirm(message: string): Promise<boolean> {
+      throw new NonInteractiveError(`confirmation ("${message}")`);
+    },
+    async promptDirect(spec): Promise<never> {
+      throw new NonInteractiveError(`input "${spec.name}"`);
+    },
+  };
+  return io;
+}

package/src/adapters/programmatic.ts

@@ -7,6 +7,7 @@ import type {
 } from "../core/types";
 import { MissingInputError } from "../core/types";
 import { BaseContext } from "./baseContext";
+import { createVaultManagerGetter } from "./vaultManagerAccess";
 
 export interface ProgrammaticOptions {
   /** Pre-supplied values keyed by input name */
@@ -21,6 +22,8 @@ class ProgrammaticContext<
   private inputs: Record<string, unknown>;
   private onLog?: (message: string) => void;
 
+  readonly interactive = false;
+
   constructor(command: CommandDef<TInputs>, options: ProgrammaticOptions) {
     super(command);
     this.inputs = options.inputs ?? {};
@@ -66,7 +69,9 @@ export function createProgrammaticContext<TInputs extends InputsSchema>(
 export function createProgrammaticIO(options: ProgrammaticOptions = {}): IO {
   const { inputs = {}, onLog } = options;
 
-  return {
+  const io: IO = {
+    interactive: false,
+    getVaultManager: createVaultManagerGetter(() => io),
     log(message: string): void {
       onLog?.(message);
     },
@@ -85,4 +90,5 @@ export function createProgrammaticIO(options: ProgrammaticOptions = {}): IO {
       throw new MissingInputError(spec.name, spec.message);
     },
   };
+  return io;
 }

package/src/adapters/terminal.ts

@@ -8,6 +8,7 @@ import type {
 } from "../core/types";
 import { MissingInputError } from "../core/types";
 import { BaseContext } from "./baseContext";
+import { createVaultManagerGetter } from "./vaultManagerAccess";
 
 /**
  * Resolve choices from a choices function.
@@ -93,6 +94,10 @@ class TerminalContext<
     super(command);
   }
 
+  get interactive(): boolean {
+    return this.options.interactive ?? process.stdin.isTTY === true;
+  }
+
   log(message: string): void {
     console.log(message);
   }
@@ -180,7 +185,9 @@ export function createTerminalContext<TInputs extends InputsSchema>(
  * Create a bare IO instance for terminal helper functions.
  */
 export function createTerminalIO(options: TerminalOptions = {}): IO {
-  return {
+  const io: IO = {
+    interactive: options.interactive ?? process.stdin.isTTY === true,
+    getVaultManager: createVaultManagerGetter(() => io),
     log(message: string): void {
       console.log(message);
     },
@@ -204,4 +211,5 @@ export function createTerminalIO(options: TerminalOptions = {}): IO {
       throw new MissingInputError(spec.name, spec.message);
     },
   };
+  return io;
 }

package/src/adapters/vaultManagerAccess.ts

@@ -0,0 +1,25 @@
+import { getProjectConfig } from "../config/getProjectConfig";
+import type { IO } from "../core/types";
+import type { VaultManagerOptions } from "../vault/VaultManager";
+import { VaultManager } from "../vault/VaultManager";
+
+/**
+ * lazily creates the vault manager of a context (once per context),
+ * loading the project config on first use
+ */
+export const createVaultManagerGetter = (
+  io: () => IO,
+  options: Omit<VaultManagerOptions, "io"> = {},
+): (() => Promise<VaultManager>) => {
+  let managerPromise: Promise<VaultManager> | undefined;
+  return () =>
+    (managerPromise ??= (async () => {
+      const config = await getProjectConfig();
+      if (!config) {
+        throw new Error(
+          "no catladder config found — the secrets vault needs a project config",
+        );
+      }
+      return new VaultManager(config, { ...options, io: io() });
+    })());
+};

package/src/apps/catenv/catenv.ts

@@ -8,10 +8,13 @@ import {
   createCatenvContext,
   generatePipelineFiles,
 } from "@catladder/pipeline";
+import type { SecretsMode } from "../../vault";
+import { createNonInteractiveIO } from "../../adapters/nonInteractive";
 
 type Options = {
   verbose?: boolean;
   printVariables?: boolean;
+  vaultMode?: SecretsMode;
 };
 export default async (choice?: Choice, options?: Options) => {
   const config = await getProjectConfig();
@@ -19,7 +22,8 @@ export default async (choice?: Choice, options?: Options) => {
     return;
   }
 
-  const context = createCatenvContext(config);
+  const io = createNonInteractiveIO({ vaultMode: options?.vaultMode });
+  const context = { ...createCatenvContext(config), io };
   if (options?.verbose) {
     printVerboseBanner();
   }

package/src/apps/catenv/printVariables.ts

@@ -1,4 +1,6 @@
-import type { CatenvContext, Config } from "@catladder/pipeline";
+import type { Config } from "@catladder/pipeline";
+import type { IO } from "../../core/types";
+import type { CatenvCliContext } from "./types";
 import { getEnvVarsResolved } from "../../config/getProjectConfig";
 import type { Choice } from "./types";
 import {
@@ -7,21 +9,25 @@ import {
   sanitizeEnvVarName,
 } from "./utils";
 
-const getAllVariablesToPrint = async (config: Config, choice?: Choice) => {
+const getAllVariablesToPrint = async (
+  io: IO,
+  config: Config,
+  choice?: Choice,
+) => {
   const { env, currentComponent } = await getCurrentComponentAndEnvFromChoice(
     config,
     choice,
   );
 
   if (currentComponent) {
-    return await getEnvVarsResolved(null, env, currentComponent);
+    return await getEnvVarsResolved(io, env, currentComponent);
   } else {
     // when in a monorep and not in a subapp, merge all env vars.
     // this is not 100% correct, but better than not exporting any vars at all
     // so we also add prefixed variants
     return await Object.keys(config.components).reduce(
       async (acc, componentName) => {
-        const subappvars = await getEnvVarsResolved(null, env, componentName);
+        const subappvars = await getEnvVarsResolved(io, env, componentName);
         delete subappvars["_ALL_ENV_VAR_KEYS"];
         return {
           ...(await acc),
@@ -41,10 +47,14 @@ const getAllVariablesToPrint = async (config: Config, choice?: Choice) => {
 };
 
 export const printVariables = async (
-  context: CatenvContext,
+  context: CatenvCliContext,
   choice?: Choice,
 ) => {
-  const variables = await getAllVariablesToPrint(context.config, choice);
+  const variables = await getAllVariablesToPrint(
+    context.io,
+    context.config,
+    choice,
+  );
 
   console.log(makeExportKeyValuestring(variables));
 };

package/src/apps/catenv/types.ts

@@ -1,3 +1,12 @@
+import type { CatenvContext } from "@catladder/pipeline";
+import type { IO } from "../../core/types";
+
+/**
+ * the pipeline's catenv context extended with the cli io context
+ * (non-interactive when run via direnv)
+ */
+export type CatenvCliContext = CatenvContext & { io: IO };
+
 import type { VariableValue } from "@catladder/pipeline";
 
 export type Choice = {

package/src/apps/catenv/writeDotEnvFiles.ts

@@ -7,10 +7,10 @@ import {
   getCurrentComponentAndEnvFromChoice,
   makeKeyValueString,
 } from "./utils";
-import type { CatenvContext } from "@catladder/pipeline";
+import type { CatenvCliContext } from "./types";
 
 export const writeDotEnvFiles = async (
-  context: CatenvContext,
+  context: CatenvCliContext,
   choice?: Choice,
 ) => {
   const { env, currentComponent } = await getCurrentComponentAndEnvFromChoice(
@@ -32,7 +32,7 @@ export const writeDotEnvFiles = async (
   const gitRoot = await getGitRoot();
 
   for (const componentName of componentsToActuallyWriteDotEnvNow) {
-    const variables = await getEnvVarsResolved(null, env, componentName);
+    const variables = await getEnvVarsResolved(context.io, env, componentName);
     delete variables["_ALL_ENV_VAR_KEYS"];
     const componentDir = getComponentFullPath(
       gitRoot,

package/src/apps/cli/commands/project/commandConfigSecrets.ts

@@ -1,4 +1,9 @@
-import { getSecretVarName } from "@catladder/pipeline";
+import {
+  getEnabledPipelineTypes,
+  getSecretVarName,
+  getVaultConfig,
+} from "@catladder/pipeline";
+import type { Config } from "@catladder/pipeline";
 import { stripIndents } from "common-tags";
 import { difference } from "lodash-es";
 import type { IO } from "../../../../core/types";
@@ -8,9 +13,15 @@ import {
   getEnvVarsResolved,
   getJobOnlyEnvVarsResolved,
   getProjectComponents,
+  getProjectConfig,
   parseChoice,
 } from "../../../../config/getProjectConfig";
+import {
+  getConfiguredGithubRepo,
+  pushSecretsToGithub,
+} from "../../../../commands/project/commandSecretsSyncGithub";
 import { editAsFile } from "../../../../utils/editAsFile";
+import { isGhAuthenticated } from "../../../../utils/github";
 import { upsertAllVariables } from "../../../../utils/gitlab";
 import { delay } from "../../../../utils/promise";
 
@@ -169,27 +180,101 @@ const doItFor = async (
       }
     }
   }
+  const config = await getProjectConfig();
+  if (!config) {
+    throw new Error("no catladder config found");
+  }
+
   instance.log("");
-  instance.log("upserting all variables, please wait...");
+  instance.log("writing all secrets to the vault, please wait...");
   instance.log("");
   for (const [componentName, envs] of Object.entries(envAndComponents)) {
     for (const env of envs) {
-      instance.log("upserting " + env + ":" + componentName + "...\n");
-      await upsertAllVariables(
-        instance,
-        valuesToEdit[componentName][env],
-        env,
-        componentName,
-      );
+      instance.log("writing " + env + ":" + componentName + "...\n");
+      await (
+        await instance.getVaultManager()
+      ).writeSecrets(env, componentName, valuesToEdit[componentName][env]);
       instance.log("");
       instance.log("✅ " + env + ":" + componentName);
       instance.log("--------------------------------\n");
     }
   }
+
+  await mirrorSecretsToCiBackends(
+    instance,
+    config,
+    envAndComponents,
+    valuesToEdit,
+  );
+
   instance.log("done! 😻");
   instance.log("");
 };
 
+/**
+ * the vault is the source of truth; every enabled CI backend receives
+ * a mirrored copy of the edited secrets (gitlab variables, github
+ * secrets), so all CI systems stay in sync
+ */
+const mirrorSecretsToCiBackends = async (
+  instance: IO,
+  config: Config,
+  envAndComponents: { [componentName: string]: string[] },
+  valuesToEdit: Vars,
+) => {
+  const enabled = getEnabledPipelineTypes(config);
+
+  // gitlab mirror (unless gitlab is the vault itself, then it is
+  // already written)
+  if (getVaultConfig(config).type !== "gitlab" && enabled.includes("gitlab")) {
+    for (const [componentName, envs] of Object.entries(envAndComponents)) {
+      for (const env of envs) {
+        await upsertAllVariables(
+          instance,
+          valuesToEdit[componentName][env],
+          env,
+          componentName,
+        );
+      }
+    }
+    instance.log("✅ mirrored to gitlab");
+  }
+
+  if (!enabled.includes("github")) {
+    return;
+  }
+  const secrets = Object.entries(envAndComponents).flatMap(
+    ([componentName, envs]) =>
+      envs.flatMap((env) =>
+        Object.entries(valuesToEdit[componentName][env] ?? {}).map(
+          ([key, value]) => ({
+            name: getSecretVarName(env, componentName, key),
+            value:
+              typeof value === "string" ? value : JSON.stringify(value ?? ""),
+          }),
+        ),
+      ),
+  );
+  if (secrets.length === 0) {
+    return;
+  }
+  if (!(await isGhAuthenticated())) {
+    instance.log(
+      "⚠️ github pipeline is enabled, but the github cli (gh) is not authenticated — run `project secrets-sync-github` later to mirror the secrets",
+    );
+    return;
+  }
+  const repo = await getConfiguredGithubRepo();
+  if (!repo) {
+    instance.log(
+      "⚠️ github pipeline is enabled, but no github repository was found — run `project secrets-sync-github` later to mirror the secrets",
+    );
+    return;
+  }
+  instance.log(`mirroring ${secrets.length} secrets to github ('${repo}')...`);
+  await pushSecretsToGithub(instance, repo, secrets);
+};
+
 export const projectConfigSecrets = async (io: IO, envComponent?: string) => {
   if (!envComponent) {
     const allEnvAndcomponents = await getAllComponentsWithAllEnvsHierarchical();

package/src/catenv.ts

@@ -1,7 +1,8 @@
-import { Command } from "commander";
+import { Command, Option } from "commander";
 import catenv from "./apps/catenv/catenv";
 import { parseChoice } from "./config/parseChoice";
 import packageInfos from "./packageInfos";
+import type { SecretsMode } from "./vault";
 
 const program = new Command();
 
@@ -12,14 +13,33 @@ program
   .argument("[envComponent]", "env or env:component")
   .option("-v, --verbose", "verbose output")
   .option("-p, --print-variables", "print variables")
+  .addOption(
+    new Option(
+      "--vault-mode <mode>",
+      [
+        "how to interact with the secrets vault:",
+        "auto: contact the vault when cached secrets are missing (may prompt to unlock)",
+        "no-prompt: like auto, but fail instead of prompting",
+        "offline: never contact the vault, use only locally cached secrets",
+        "refresh: ignore the local cache once and refresh it from the vault",
+      ].join("\n"),
+    )
+      .choices(["auto", "no-prompt", "offline", "refresh"])
+      .default("auto"),
+  )
   .action(
     (
       envComponent: string | undefined,
-      opts: { verbose?: boolean; printVariables?: boolean },
+      opts: {
+        verbose?: boolean;
+        printVariables?: boolean;
+        vaultMode: SecretsMode;
+      },
     ) => {
       catenv(envComponent ? parseChoice(envComponent) : null, {
         verbose: opts.verbose ?? false,
         printVariables: opts.printVariables ?? false,
+        vaultMode: opts.vaultMode,
       }).then(() => {
         // we have to exit manually, because we have some file watches
         process.exit();

package/src/commands/index.ts

@@ -15,6 +15,7 @@ export { commandOpenGit } from "./project/commandOpenGit";
 export { commandCloudSqlProxy } from "./project/commandCloudSqlProxy";
 export { commandConfigSecrets } from "./project/commandConfigSecrets";
 export { commandSecretsClearBackups } from "./project/commandSecretsClearBackups";
+export { commandSecretsSyncGithub } from "./project/commandSecretsSyncGithub";
 export { commandOpenLogs } from "./project/commandOpenLogs";
 export { commandTriggerCronjobGeneral } from "./project/k8s/commandTriggerCronjob";
 export { commandRenewToken } from "./project/commandRenewToken";

package/src/commands/project/commandSecretsSyncGithub.ts

@@ -0,0 +1,149 @@
+import {
+  getEnabledPipelineTypes,
+  getPipelineGitRemote,
+  getSecretVarName,
+} from "@catladder/pipeline";
+import {
+  getAllComponentsWithAllEnvsHierarchical,
+  getEnvironment,
+  getProjectConfig,
+} from "../../config/getProjectConfig";
+import { defineCommand } from "../../core/defineCommand";
+import type { IO } from "../../core/types";
+import {
+  getGithubRepoFromRemote,
+  isGhAuthenticated,
+  setGithubSecret,
+} from "../../utils/github";
+
+/**
+ * all secrets declared in the config, with their values read from the
+ * vault (github secrets are write-only and can never act as a source)
+ */
+export const collectSecretsFromVault = async (io: IO) => {
+  const envAndComponents = await getAllComponentsWithAllEnvsHierarchical();
+
+  const names: string[] = [];
+  for (const [componentName, envs] of Object.entries(envAndComponents)) {
+    for (const env of envs) {
+      const { secretEnvVarKeys, jobOnlyVars } = await getEnvironment(
+        env,
+        componentName,
+      );
+      const keys = [
+        ...secretEnvVarKeys,
+        ...jobOnlyVars.build.secretEnvVarKeys,
+        ...jobOnlyVars.deploy.secretEnvVarKeys,
+      ];
+      keys.forEach(({ key }) =>
+        names.push(getSecretVarName(env, componentName, key)),
+      );
+    }
+  }
+
+  const values = await (await io.getVaultManager()).readSecrets(names, io);
+  const secrets = names
+    .filter((name) => values[name] !== undefined)
+    .map((name) => ({ name, value: values[name] }));
+  const missing = names.filter((name) => values[name] === undefined);
+  return { secrets, missing };
+};
+
+/**
+ * pushes secrets to github repository secrets via the gh cli
+ */
+export const pushSecretsToGithub = async (
+  io: IO,
+  repo: string,
+  secrets: { name: string; value: string }[],
+) => {
+  for (const { name, value } of secrets) {
+    await setGithubSecret(repo, name, value);
+    io.log(`✅ ${name}`);
+  }
+};
+
+/**
+ * resolves the github repository from the configured git remote
+ * (`pipelines.github.gitRemote`, defaults to origin)
+ */
+export const getConfiguredGithubRepo = async (): Promise<
+  string | undefined
+> => {
+  const config = await getProjectConfig();
+  if (!config) {
+    return undefined;
+  }
+  return getGithubRepoFromRemote(getPipelineGitRemote(config, "github"));
+};
+
+/**
+ * copies the catladder secrets from the vault to github actions
+ * repository secrets, keeping the exact same names, so the github
+ * pipeline can reference them as ${{ secrets.<NAME> }}.
+ */
+export const commandSecretsSyncGithub = defineCommand({
+  name: "project secrets-sync-github",
+  description:
+    "copies all secrets from the vault to github repo secrets (same names), for the github pipeline",
+  group: "project",
+  inputs: {
+    repo: {
+      type: "string",
+      message: "github repository (owner/name)",
+      positional: true,
+      required: false,
+    },
+  },
+  execute: async (ctx) => {
+    const config = await getProjectConfig();
+    if (!config) {
+      throw new Error("no catladder config found");
+    }
+    const enabled = getEnabledPipelineTypes(config);
+    if (!enabled.includes("github")) {
+      throw new Error(
+        "the github pipeline is not enabled in `pipelines` — nothing to sync to",
+      );
+    }
+    if (!(await isGhAuthenticated())) {
+      throw new Error(
+        "the github cli (gh) is not installed or not authenticated — run `gh auth login` first",
+      );
+    }
+    const repo = (await ctx.get("repo")) ?? (await getConfiguredGithubRepo());
+    if (!repo) {
+      throw new Error(
+        `no github repository found on the '${getPipelineGitRemote(config, "github")}' remote — pass it as owner/name or configure pipelines.github.gitRemote`,
+      );
+    }
+
+    const { secrets, missing } = await collectSecretsFromVault(ctx);
+
+    if (missing.length > 0) {
+      ctx.log(
+        `⚠️ ${missing.length} secrets are not set in the vault and will be skipped:`,
+      );
+      missing.forEach((name) => ctx.log(`  - ${name}`));
+      ctx.log("");
+    }
+    if (secrets.length === 0) {
+      ctx.log("no secrets to sync 🤷");
+      return;
+    }
+
+    ctx.log(`about to set ${secrets.length} secrets on '${repo}':`);
+    secrets.forEach(({ name }) => ctx.log(`  - ${name}`));
+    ctx.log("");
+    const confirmed = await ctx.confirm("continue? 🤔");
+    if (!confirmed) {
+      throw new Error("abort");
+    }
+
+    await pushSecretsToGithub(ctx, repo, secrets);
+    ctx.log("");
+    ctx.log(
+      "done! 😻 the github pipeline can use them as ${{ secrets.<NAME> }}",
+    );
+  },
+});

package/src/config/getProjectConfig.ts

@@ -12,7 +12,7 @@ import {
 } from "@catladder/pipeline";
 
 import type { IO } from "../core/types";
-import { getAllVariables, getVariableValueByRawName } from "../utils/gitlab";
+import { getVariableValueByRawName } from "../utils/gitlab";
 
 import { getGitRoot } from "../utils/projects";
 
@@ -121,10 +121,24 @@ export const getGitlabVar = async (
 };
 
 const resolveSecrets = async (
-  io: IO | null,
+  io: IO,
   varSets: EnvironmentEnvVars[],
 ): Promise<Record<string, string>> => {
-  const allVariablesInGitlab = await getAllVariables(io);
+  // the secret variables actually referenced by the env var values —
+  // when they are all in the local cache, the vault is not contacted
+  const referencedNames = [
+    ...new Set(
+      varSets.flatMap(({ envVars }) =>
+        Object.values(envVars).flatMap(
+          (value) => `${value}`.match(/\$CL_\w+/g) ?? [],
+        ),
+      ),
+    ),
+  ].map((reference) => reference.slice(1));
+
+  const vaultManager = await io.getVaultManager();
+  const secrets = await vaultManager.readSecrets(referencedNames, io);
+  const secretEntries = Object.entries(secrets);
 
   return Object.fromEntries(
     varSets.flatMap(({ envVars, secretEnvVarKeys }) =>
@@ -137,9 +151,9 @@ const resolveSecrets = async (
         )
         .map(([key, value]) => [
           key,
-          allVariablesInGitlab.reduce(
-            (acc, curr) =>
-              acc.replace(new RegExp("\\$" + curr.key, "g"), curr.value),
+          secretEntries.reduce(
+            (acc, [name, secretValue]) =>
+              acc.replace(new RegExp("\\$" + name, "g"), secretValue),
             `${value}`,
           ),
         ]),
@@ -148,7 +162,7 @@ const resolveSecrets = async (
 };
 
 export const getEnvVarsResolved = async (
-  io: IO | null,
+  io: IO,
   env: string,
   componentName: string | null,
 ) => {