@catladder/cli 0.0.0-next-4e7c6848 → 0.0.0-next-6a4450e0

package/src/config/getProjectConfig.ts

@@ -12,7 +12,7 @@ import {
 } from "@catladder/pipeline";
 
 import type { IO } from "../core/types";
-import { getAllVariables, getVariableValueByRawName } from "../utils/gitlab";
+import { getVariableValueByRawName } from "../utils/gitlab";
 
 import { getGitRoot } from "../utils/projects";
 
@@ -121,10 +121,24 @@ export const getGitlabVar = async (
 };
 
 const resolveSecrets = async (
-  io: IO | null,
+  io: IO,
   varSets: EnvironmentEnvVars[],
 ): Promise<Record<string, string>> => {
-  const allVariablesInGitlab = await getAllVariables(io);
+  // the secret variables actually referenced by the env var values —
+  // when they are all in the local cache, the vault is not contacted
+  const referencedNames = [
+    ...new Set(
+      varSets.flatMap(({ envVars }) =>
+        Object.values(envVars).flatMap(
+          (value) => `${value}`.match(/\$CL_\w+/g) ?? [],
+        ),
+      ),
+    ),
+  ].map((reference) => reference.slice(1));
+
+  const vaultManager = await io.getVaultManager();
+  const secrets = await vaultManager.readSecrets(referencedNames, io);
+  const secretEntries = Object.entries(secrets);
 
   return Object.fromEntries(
     varSets.flatMap(({ envVars, secretEnvVarKeys }) =>
@@ -137,9 +151,9 @@ const resolveSecrets = async (
         )
         .map(([key, value]) => [
           key,
-          allVariablesInGitlab.reduce(
-            (acc, curr) =>
-              acc.replace(new RegExp("\\$" + curr.key, "g"), curr.value),
+          secretEntries.reduce(
+            (acc, [name, secretValue]) =>
+              acc.replace(new RegExp("\\$" + name, "g"), secretValue),
             `${value}`,
           ),
         ]),
@@ -148,7 +162,7 @@ const resolveSecrets = async (
 };
 
 export const getEnvVarsResolved = async (
-  io: IO | null,
+  io: IO,
   env: string,
   componentName: string | null,
 ) => {

package/src/core/types.ts

@@ -1,3 +1,4 @@
+import type { VaultManager } from "../vault/VaultManager";
 import type { ComponentContext } from "@catladder/pipeline";
 
 // ─── Input Definitions ───────────────────────────────────────────────
@@ -76,6 +77,19 @@ export type InputsSchema = Record<string, InputDef>;
  * that are NOT part of a command's declared input schema.
  */
 export interface IO {
+  /**
+   * whether this context can interact with the user (prompts, editors).
+   * Non-interactive contexts throw {@link NonInteractiveError} when an
+   * interaction would be required.
+   */
+  readonly interactive: boolean;
+
+  /**
+   * the secrets vault of this context, created lazily from the project
+   * config
+   */
+  getVaultManager(): Promise<VaultManager>;
+
   log(message: string): void;
   /**
    * Ask for confirmation. Returns true/false.
@@ -131,6 +145,15 @@ export interface CommandDef<TInputs extends InputsSchema = InputsSchema> {
 
 // ─── Errors ──────────────────────────────────────────────────────────
 
+export class NonInteractiveError extends Error {
+  constructor(interaction: string) {
+    super(
+      `${interaction} requires an interactive terminal, but this context is not interactive`,
+    );
+    this.name = "NonInteractiveError";
+  }
+}
+
 export class MissingInputError extends Error {
   constructor(
     public readonly inputName: string,

package/src/git/gitProjectInformation.ts

@@ -12,17 +12,19 @@ export const getProjectRootPath = async (): Promise<string> => {
 /**
  * Get the git remote hostname and project path from the git config.
  */
-export const getGitRemoteHostAndPath = async (): Promise<{
+export const getGitRemoteHostAndPath = async (
+  remoteName = "origin",
+): Promise<{
   gitRemoteHost: string;
   gitRemotePath: string;
 }> => {
-  const remoteUrl = await gitConfigGet("remote.origin.url");
+  const remoteUrl = await gitConfigGet(`remote.${remoteName}.url`);
   const remoteReg = /(https:\/\/|git@)([^:/]+)[:/]([^.]*)(\.git)?/;
   const match = remoteUrl.match(remoteReg) ?? [];
   const [, , gitRemoteHost, gitRemotePath] = match;
   if (!gitRemoteHost?.length || !gitRemotePath?.length) {
     throw new Error(
-      `Failed to parse git remote hostname and path from git configs remote.origin.url! ${remoteUrl}`,
+      `Failed to parse git remote hostname and path from git configs remote.${remoteName}.url! ${remoteUrl}`,
     );
   }
   return { gitRemoteHost, gitRemotePath };

package/src/utils/github.ts

@@ -0,0 +1,76 @@
+import { execFile as execFileCb, spawn } from "child_process";
+import { promisify } from "util";
+
+const execFile = promisify(execFileCb);
+
+/**
+ * whether the github cli (gh) is installed and authenticated
+ */
+export const isGhAuthenticated = async (): Promise<boolean> => {
+  try {
+    await execFile("gh", ["auth", "status"]);
+    return true;
+  } catch {
+    return false;
+  }
+};
+
+/**
+ * the github repository (owner/name) a git remote points to.
+ * Which remote belongs to github is configured per pipeline type
+ * (`pipelines.github.gitRemote`, defaults to origin).
+ */
+export const getGithubRepoFromRemote = async (
+  remoteName: string,
+): Promise<string | undefined> => {
+  try {
+    const { stdout } = await execFile("git", ["remote", "get-url", remoteName]);
+    const match = stdout.match(
+      /github\.com[:/]([\w.-]+\/[\w.-]+?)(\.git)?\s*$/,
+    );
+    return match?.[1];
+  } catch {
+    return undefined;
+  }
+};
+
+const ghSetViaStdin = (
+  kind: "secret" | "variable",
+  repo: string,
+  name: string,
+  value: string,
+): Promise<void> =>
+  new Promise<void>((resolve, reject) => {
+    const child = spawn("gh", [kind, "set", name, "--repo", repo], {
+      stdio: ["pipe", "ignore", "pipe"],
+    });
+    let stderr = "";
+    child.stderr?.on("data", (data) => (stderr += data));
+    child.on("error", reject);
+    child.on("close", (code) =>
+      code === 0
+        ? resolve()
+        : reject(new Error(`gh ${kind} set ${name} failed: ${stderr.trim()}`)),
+    );
+    child.stdin?.end(value);
+  });
+
+/**
+ * sets a github actions repository secret via the gh cli.
+ * The value is passed via stdin so it never shows up in a process list.
+ */
+export const setGithubSecret = async (
+  repo: string,
+  name: string,
+  value: string,
+): Promise<void> => ghSetViaStdin("secret", repo, name, value);
+
+/**
+ * sets a github actions repository variable (plain, unmasked) via the
+ * gh cli — for vault values classified as not sensitive
+ */
+export const setGithubVariable = async (
+  repo: string,
+  name: string,
+  value: string,
+): Promise<void> => ghSetViaStdin("variable", repo, name, value);

package/src/vault/BitwardenVault.ts

@@ -0,0 +1,161 @@
+import { getSecretVarName } from "@catladder/pipeline";
+import type { Config } from "@catladder/pipeline";
+import { parse, stringify } from "yaml";
+import type { IO } from "../core/types";
+import { getBwSession, runBw, runBwJson } from "./bw";
+import type { SecretsVault } from "./types";
+
+const DEFAULT_COLLECTION = "catladder";
+
+const SECURE_NOTE_TYPE = 2;
+
+type BwCollection = { id: string; name: string; organizationId?: string };
+type BwItem = {
+  id: string;
+  name: string;
+  notes?: string | null;
+  organizationId?: string | null;
+  collectionIds?: string[];
+};
+
+/**
+ * one yaml note per env and component:
+ * `<customerName>/<appName>/<env>/<componentName>/secrets.yml`
+ */
+const itemName = (config: Config, env: string, componentName: string): string =>
+  `${config.customerName}/${config.appName}/${env}/${componentName}/secrets.yml`;
+
+const itemPrefix = (config: Config) =>
+  `${config.customerName}/${config.appName}/`;
+
+/**
+ * parses env and componentName back out of an item name
+ */
+const parseItemName = (config: Config, name: string) => {
+  const match = name
+    .slice(itemPrefix(config).length)
+    .match(/^([^/]+)\/([^/]+)\/secrets\.yml$/);
+  return match ? { env: match[1], componentName: match[2] } : null;
+};
+
+const findCollection = async (
+  session: string,
+  collectionName: string,
+): Promise<BwCollection> => {
+  const collections = await runBwJson<BwCollection[]>(session, [
+    "list",
+    "collections",
+    "--search",
+    collectionName,
+  ]);
+  const collection = collections.find((c) => c.name === collectionName);
+  if (!collection) {
+    throw new Error(
+      `bitwarden collection '${collectionName}' not found — create it and make sure you have access`,
+    );
+  }
+  return collection;
+};
+
+const encodePayload = (payload: unknown) =>
+  Buffer.from(JSON.stringify(payload), "utf-8").toString("base64");
+
+export class BitwardenVault implements SecretsVault {
+  readonly id: string;
+
+  private readonly collectionName: string;
+
+  constructor(
+    private readonly config: Config,
+    options: { collection?: string },
+    /**
+     * whether a locked bitwarden may be unlocked interactively
+     * (false with catenv --vault-mode no-prompt)
+     */
+    private readonly allowPrompt: boolean,
+  ) {
+    this.collectionName = options.collection ?? DEFAULT_COLLECTION;
+    this.id = `bitwarden:${this.collectionName}`;
+  }
+
+  private session(io: IO | null) {
+    return getBwSession(io, { allowPrompt: this.allowPrompt });
+  }
+
+  async readAllSecrets(io: IO | null): Promise<Record<string, string>> {
+    const { config, collectionName } = this;
+    const bwSession = await this.session(io);
+    const collection = await findCollection(bwSession, collectionName);
+    const items = await runBwJson<BwItem[]>(bwSession, [
+      "list",
+      "items",
+      "--collectionid",
+      collection.id,
+    ]);
+
+    const secrets: Record<string, string> = {};
+    for (const item of items) {
+      const parsed = parseItemName(config, item.name);
+      if (!parsed || !item.notes) {
+        continue;
+      }
+      const values = parse(item.notes) as Record<string, unknown> | null;
+      Object.entries(values ?? {}).forEach(([key, value]) => {
+        secrets[getSecretVarName(parsed.env, parsed.componentName, key)] =
+          typeof value === "string" ? value : JSON.stringify(value);
+      });
+    }
+    return secrets;
+  }
+
+  async writeSecrets(
+    io: IO,
+    env: string,
+    componentName: string,
+    secrets: Record<string, unknown>,
+  ): Promise<void> {
+    const { config, collectionName } = this;
+    const bwSession = await this.session(io);
+    const collection = await findCollection(bwSession, collectionName);
+    const name = itemName(config, env, componentName);
+
+    const existing = (
+      await runBwJson<BwItem[]>(bwSession, [
+        "list",
+        "items",
+        "--collectionid",
+        collection.id,
+        "--search",
+        name,
+      ])
+    ).find((item) => item.name === name);
+
+    const notes = stringify(secrets);
+
+    if (existing) {
+      await runBw(bwSession, [
+        "edit",
+        "item",
+        existing.id,
+        encodePayload({ ...existing, notes }),
+      ]);
+    } else {
+      await runBw(bwSession, [
+        "create",
+        "item",
+        encodePayload({
+          type: SECURE_NOTE_TYPE,
+          secureNote: { type: 0 },
+          name,
+          notes,
+          ...(collection.organizationId
+            ? {
+                organizationId: collection.organizationId,
+                collectionIds: [collection.id],
+              }
+            : {}),
+        }),
+      ]);
+    }
+  }
+}

package/src/vault/GitlabVault.ts

@@ -0,0 +1,25 @@
+import type { IO } from "../core/types";
+import { getAllVariables, upsertAllVariables } from "../utils/gitlab";
+import type { SecretsVault } from "./types";
+
+/**
+ * the legacy (and default) vault: the gitlab project variables double
+ * as the secret store
+ */
+export class GitlabVault implements SecretsVault {
+  readonly id = "gitlab";
+
+  async readAllSecrets(io: IO | null): Promise<Record<string, string>> {
+    const variables = await getAllVariables(io);
+    return Object.fromEntries(variables.map((v) => [v.key, v.value]));
+  }
+
+  async writeSecrets(
+    io: IO,
+    env: string,
+    componentName: string,
+    secrets: Record<string, unknown>,
+  ): Promise<void> {
+    await upsertAllVariables(io, secrets, env, componentName);
+  }
+}

package/src/vault/VaultManager.ts

@@ -0,0 +1,211 @@
+import type { Config } from "@catladder/pipeline";
+import {
+  getEnvironment,
+  getSecretVarName,
+  getVaultConfig,
+} from "@catladder/pipeline";
+import type { IO } from "../core/types";
+import { BitwardenVault } from "./BitwardenVault";
+import type { CachedSecrets } from "./cache";
+import {
+  readSecretsCache,
+  updateSecretsCache,
+  writeSecretsCache,
+} from "./cache";
+import { GitlabVault } from "./GitlabVault";
+import type { SecretsMode, SecretsVault } from "./types";
+
+/**
+ * only secrets of this env are cached locally — the other envs'
+ * secrets have no business sitting on developer machines
+ */
+const CACHED_ENV = "local";
+
+export type VaultManagerOptions = {
+  mode?: SecretsMode;
+  /**
+   * default IO for vault interactions (logging, unlock prompts).
+   * Method-level io takes precedence.
+   */
+  io?: IO | null;
+};
+
+/**
+ * owns the configured vault, the secrets mode and the local cache.
+ * The vault is the readable source of truth for secrets; CI backends
+ * only receive mirrored copies.
+ */
+export class VaultManager {
+  readonly mode: SecretsMode;
+
+  readonly vault: SecretsVault;
+
+  private readonly io: IO | null;
+
+  /** refresh mode bypasses the cache only once per manager */
+  private refreshed = false;
+
+  constructor(
+    private readonly config: Config,
+    options: VaultManagerOptions = {},
+  ) {
+    this.mode = options.mode ?? "auto";
+    this.io = options.io ?? null;
+    this.vault = this.createVault();
+  }
+
+  private createVault(): SecretsVault {
+    const vaultConfig = getVaultConfig(this.config);
+    switch (vaultConfig.type) {
+      case "gitlab":
+        return new GitlabVault();
+      case "bitwarden":
+        return new BitwardenVault(
+          this.config,
+          vaultConfig,
+          this.mode === "auto" || this.mode === "refresh",
+        );
+    }
+  }
+
+  /**
+   * the vault secrets, served from the local cache when it contains
+   * all required names; otherwise (depending on the mode) refreshed
+   * from the vault. Only the "local" env is (re-)cached.
+   */
+  async readSecrets(
+    requiredNames: string[],
+    io: IO | null = this.io,
+  ): Promise<Record<string, string>> {
+    const bypassCache = this.mode === "refresh" && !this.refreshed;
+    const cache = bypassCache ? null : await readSecretsCache();
+    // known entries include recorded misses (null values), so declared
+    // but intentionally unset secrets don't force a vault refresh
+    const known =
+      cache?.vaultId === this.vault.id
+        ? flattenSecrets(cache.secrets)
+        : undefined;
+
+    const missing = requiredNames.filter((name) => !(name in (known ?? {})));
+    // nothing missing (or nothing required at all): don't touch the vault
+    if (missing.length === 0 && (known || requiredNames.length === 0)) {
+      return withoutMisses(known ?? {});
+    }
+
+    if (this.mode === "offline") {
+      if (missing.length > 0) {
+        io?.log(
+          `⚠️ vault is offline (--vault-mode offline) and ${missing.length} secrets are not in the local cache: ${missing.join(", ")}`,
+        );
+      }
+      return withoutMisses(known ?? {});
+    }
+
+    const secrets = await this.vault.readAllSecrets(io);
+    this.refreshed = true;
+    await writeSecretsCache(this.vault.id, await this.cacheableSubset(secrets));
+    return secrets;
+  }
+
+  /**
+   * writes secrets of one env/component to the vault and updates the
+   * local cache (write-through, "local" env only)
+   */
+  async writeSecrets(
+    env: string,
+    componentName: string,
+    secrets: Record<string, unknown>,
+    io: IO | null = this.io,
+  ): Promise<void> {
+    if (!io) {
+      throw new Error("writing to the vault requires an interactive context");
+    }
+    await this.vault.writeSecrets(io, env, componentName, secrets);
+    if (env === CACHED_ENV) {
+      await updateSecretsCache(
+        this.vault.id,
+        env,
+        componentName,
+        Object.fromEntries(
+          Object.entries(secrets).map(([key, value]) => [
+            key,
+            typeof value === "string" ? value : JSON.stringify(value),
+          ]),
+        ),
+      );
+    }
+  }
+
+  /**
+   * the subset of the vault secrets that may be cached: the declared
+   * secrets of the "local" env, per component, as a hierarchy
+   */
+  private async cacheableSubset(
+    flatSecrets: Record<string, string>,
+  ): Promise<CachedSecrets> {
+    const local: CachedSecrets[string] = {};
+    for (const componentName of Object.keys(this.config.components)) {
+      const keys = await this.declaredSecretKeys(componentName);
+      const values = Object.fromEntries(
+        keys.map((key): [string, string | null] => [
+          key,
+          // null records "absent in the vault" (a known miss)
+          flatSecrets[getSecretVarName(CACHED_ENV, componentName, key)] ?? null,
+        ]),
+      );
+      if (Object.keys(values).length > 0) {
+        local[componentName] = values;
+      }
+    }
+    return Object.keys(local).length > 0 ? { [CACHED_ENV]: local } : {};
+  }
+
+  private async declaredSecretKeys(componentName: string): Promise<string[]> {
+    try {
+      const { secretEnvVarKeys, jobOnlyVars } = await getEnvironment({
+        config: this.config,
+        componentName,
+        env: CACHED_ENV,
+      });
+      return [
+        ...secretEnvVarKeys,
+        ...jobOnlyVars.build.secretEnvVarKeys,
+        ...jobOnlyVars.deploy.secretEnvVarKeys,
+      ].map(({ key }) => key);
+    } catch {
+      // components without a local env simply have nothing to cache
+      return [];
+    }
+  }
+}
+
+/**
+ * the cache hierarchy as a flat map of raw secret variable names
+ * (null values = recorded misses)
+ */
+const flattenSecrets = (
+  secrets: CachedSecrets,
+): Record<string, string | null> =>
+  Object.fromEntries(
+    Object.entries(secrets).flatMap(([env, components]) =>
+      Object.entries(components).flatMap(([componentName, values]) =>
+        Object.entries(values).map(([key, value]) => [
+          getSecretVarName(env, componentName, key),
+          value,
+        ]),
+      ),
+    ),
+  );
+
+/**
+ * only actual values — recorded misses stay unresolved, like any other
+ * unset variable
+ */
+const withoutMisses = (
+  known: Record<string, string | null>,
+): Record<string, string> =>
+  Object.fromEntries(
+    Object.entries(known).filter(
+      (entry): entry is [string, string] => entry[1] !== null,
+    ),
+  );