@catladder/cli 0.0.0-main-8dbb6c76 → 0.0.0-main-a6b45abe

package/src/vault/BitwardenVault.ts

@@ -0,0 +1,161 @@
+import { getSecretVarName } from "@catladder/pipeline";
+import type { Config } from "@catladder/pipeline";
+import { parse, stringify } from "yaml";
+import type { IO } from "../core/types";
+import { getBwSession, runBw, runBwJson } from "./bw";
+import type { SecretsVault } from "./types";
+
+const DEFAULT_COLLECTION = "catladder";
+
+const SECURE_NOTE_TYPE = 2;
+
+type BwCollection = { id: string; name: string; organizationId?: string };
+type BwItem = {
+  id: string;
+  name: string;
+  notes?: string | null;
+  organizationId?: string | null;
+  collectionIds?: string[];
+};
+
+/**
+ * one yaml note per env and component:
+ * `<customerName>/<appName>/<env>/<componentName>/secrets.yml`
+ */
+const itemName = (config: Config, env: string, componentName: string): string =>
+  `${config.customerName}/${config.appName}/${env}/${componentName}/secrets.yml`;
+
+const itemPrefix = (config: Config) =>
+  `${config.customerName}/${config.appName}/`;
+
+/**
+ * parses env and componentName back out of an item name
+ */
+const parseItemName = (config: Config, name: string) => {
+  const match = name
+    .slice(itemPrefix(config).length)
+    .match(/^([^/]+)\/([^/]+)\/secrets\.yml$/);
+  return match ? { env: match[1], componentName: match[2] } : null;
+};
+
+const findCollection = async (
+  session: string,
+  collectionName: string,
+): Promise<BwCollection> => {
+  const collections = await runBwJson<BwCollection[]>(session, [
+    "list",
+    "collections",
+    "--search",
+    collectionName,
+  ]);
+  const collection = collections.find((c) => c.name === collectionName);
+  if (!collection) {
+    throw new Error(
+      `bitwarden collection '${collectionName}' not found — create it and make sure you have access`,
+    );
+  }
+  return collection;
+};
+
+const encodePayload = (payload: unknown) =>
+  Buffer.from(JSON.stringify(payload), "utf-8").toString("base64");
+
+export class BitwardenVault implements SecretsVault {
+  readonly id: string;
+
+  private readonly collectionName: string;
+
+  constructor(
+    private readonly config: Config,
+    options: { collection?: string },
+    /**
+     * whether a locked bitwarden may be unlocked interactively
+     * (false with catenv --vault-mode no-prompt)
+     */
+    private readonly allowPrompt: boolean,
+  ) {
+    this.collectionName = options.collection ?? DEFAULT_COLLECTION;
+    this.id = `bitwarden:${this.collectionName}`;
+  }
+
+  private session(io: IO | null) {
+    return getBwSession(io, { allowPrompt: this.allowPrompt });
+  }
+
+  async readAllSecrets(io: IO | null): Promise<Record<string, string>> {
+    const { config, collectionName } = this;
+    const bwSession = await this.session(io);
+    const collection = await findCollection(bwSession, collectionName);
+    const items = await runBwJson<BwItem[]>(bwSession, [
+      "list",
+      "items",
+      "--collectionid",
+      collection.id,
+    ]);
+
+    const secrets: Record<string, string> = {};
+    for (const item of items) {
+      const parsed = parseItemName(config, item.name);
+      if (!parsed || !item.notes) {
+        continue;
+      }
+      const values = parse(item.notes) as Record<string, unknown> | null;
+      Object.entries(values ?? {}).forEach(([key, value]) => {
+        secrets[getSecretVarName(parsed.env, parsed.componentName, key)] =
+          typeof value === "string" ? value : JSON.stringify(value);
+      });
+    }
+    return secrets;
+  }
+
+  async writeSecrets(
+    io: IO,
+    env: string,
+    componentName: string,
+    secrets: Record<string, unknown>,
+  ): Promise<void> {
+    const { config, collectionName } = this;
+    const bwSession = await this.session(io);
+    const collection = await findCollection(bwSession, collectionName);
+    const name = itemName(config, env, componentName);
+
+    const existing = (
+      await runBwJson<BwItem[]>(bwSession, [
+        "list",
+        "items",
+        "--collectionid",
+        collection.id,
+        "--search",
+        name,
+      ])
+    ).find((item) => item.name === name);
+
+    const notes = stringify(secrets);
+
+    if (existing) {
+      await runBw(bwSession, [
+        "edit",
+        "item",
+        existing.id,
+        encodePayload({ ...existing, notes }),
+      ]);
+    } else {
+      await runBw(bwSession, [
+        "create",
+        "item",
+        encodePayload({
+          type: SECURE_NOTE_TYPE,
+          secureNote: { type: 0 },
+          name,
+          notes,
+          ...(collection.organizationId
+            ? {
+                organizationId: collection.organizationId,
+                collectionIds: [collection.id],
+              }
+            : {}),
+        }),
+      ]);
+    }
+  }
+}

package/src/vault/GitlabVault.ts

@@ -0,0 +1,25 @@
+import type { IO } from "../core/types";
+import { getAllVariables, upsertAllVariables } from "../utils/gitlab";
+import type { SecretsVault } from "./types";
+
+/**
+ * the legacy (and default) vault: the gitlab project variables double
+ * as the secret store
+ */
+export class GitlabVault implements SecretsVault {
+  readonly id = "gitlab";
+
+  async readAllSecrets(io: IO | null): Promise<Record<string, string>> {
+    const variables = await getAllVariables(io);
+    return Object.fromEntries(variables.map((v) => [v.key, v.value]));
+  }
+
+  async writeSecrets(
+    io: IO,
+    env: string,
+    componentName: string,
+    secrets: Record<string, unknown>,
+  ): Promise<void> {
+    await upsertAllVariables(io, secrets, env, componentName);
+  }
+}

package/src/vault/VaultManager.ts

@@ -0,0 +1,214 @@
+import type { Config } from "@catladder/pipeline";
+import {
+  getEnvironment,
+  getSecretVarName,
+  getVaultConfig,
+} from "@catladder/pipeline";
+import type { IO } from "../core/types";
+import { BitwardenVault } from "./BitwardenVault";
+import type { CachedSecrets } from "./cache";
+import {
+  readSecretsCache,
+  updateSecretsCache,
+  writeSecretsCache,
+} from "./cache";
+import { GitlabVault } from "./GitlabVault";
+import type { SecretsMode, SecretsVault } from "./types";
+
+/**
+ * only secrets of this env are cached locally — the other envs'
+ * secrets have no business sitting on developer machines
+ */
+const CACHED_ENV = "local";
+
+export type VaultManagerOptions = {
+  mode?: SecretsMode;
+  /**
+   * default IO for vault interactions (logging, unlock prompts).
+   * Method-level io takes precedence.
+   */
+  io?: IO | null;
+};
+
+/**
+ * owns the configured vault, the secrets mode and the local cache.
+ * The vault is the readable source of truth for secrets; CI backends
+ * only receive mirrored copies.
+ */
+export class VaultManager {
+  readonly mode: SecretsMode;
+
+  readonly vault: SecretsVault;
+
+  private readonly io: IO | null;
+
+  /** refresh mode bypasses the cache only once per manager */
+  private refreshed = false;
+
+  constructor(
+    private readonly config: Config,
+    options: VaultManagerOptions = {},
+  ) {
+    this.mode = options.mode ?? "auto";
+    this.io = options.io ?? null;
+    this.vault = this.createVault();
+  }
+
+  private createVault(): SecretsVault {
+    const vaultConfig = getVaultConfig(this.config);
+    switch (vaultConfig.type) {
+      case "gitlab":
+        return new GitlabVault();
+      case "bitwarden":
+        return new BitwardenVault(
+          this.config,
+          vaultConfig,
+          // unlock prompts need a mode that allows them AND an IO that
+          // can actually prompt (catenv under direnv/turbo/CI cannot)
+          (this.mode === "auto" || this.mode === "refresh") &&
+            (this.io?.interactive ?? true),
+        );
+    }
+  }
+
+  /**
+   * the vault secrets, served from the local cache when it contains
+   * all required names; otherwise (depending on the mode) refreshed
+   * from the vault. Only the "local" env is (re-)cached.
+   */
+  async readSecrets(
+    requiredNames: string[],
+    io: IO | null = this.io,
+  ): Promise<Record<string, string>> {
+    const bypassCache = this.mode === "refresh" && !this.refreshed;
+    const cache = bypassCache ? null : await readSecretsCache();
+    // known entries include recorded misses (null values), so declared
+    // but intentionally unset secrets don't force a vault refresh
+    const known =
+      cache?.vaultId === this.vault.id
+        ? flattenSecrets(cache.secrets)
+        : undefined;
+
+    const missing = requiredNames.filter((name) => !(name in (known ?? {})));
+    // nothing missing (or nothing required at all): don't touch the vault
+    if (missing.length === 0 && (known || requiredNames.length === 0)) {
+      return withoutMisses(known ?? {});
+    }
+
+    if (this.mode === "offline") {
+      if (missing.length > 0) {
+        io?.log(
+          `⚠️ vault is offline (--vault-mode offline) and ${missing.length} secrets are not in the local cache: ${missing.join(", ")}`,
+        );
+      }
+      return withoutMisses(known ?? {});
+    }
+
+    const secrets = await this.vault.readAllSecrets(io);
+    this.refreshed = true;
+    await writeSecretsCache(this.vault.id, await this.cacheableSubset(secrets));
+    return secrets;
+  }
+
+  /**
+   * writes secrets of one env/component to the vault and updates the
+   * local cache (write-through, "local" env only)
+   */
+  async writeSecrets(
+    env: string,
+    componentName: string,
+    secrets: Record<string, unknown>,
+    io: IO | null = this.io,
+  ): Promise<void> {
+    if (!io) {
+      throw new Error("writing to the vault requires an interactive context");
+    }
+    await this.vault.writeSecrets(io, env, componentName, secrets);
+    if (env === CACHED_ENV) {
+      await updateSecretsCache(
+        this.vault.id,
+        env,
+        componentName,
+        Object.fromEntries(
+          Object.entries(secrets).map(([key, value]) => [
+            key,
+            typeof value === "string" ? value : JSON.stringify(value),
+          ]),
+        ),
+      );
+    }
+  }
+
+  /**
+   * the subset of the vault secrets that may be cached: the declared
+   * secrets of the "local" env, per component, as a hierarchy
+   */
+  private async cacheableSubset(
+    flatSecrets: Record<string, string>,
+  ): Promise<CachedSecrets> {
+    const local: CachedSecrets[string] = {};
+    for (const componentName of Object.keys(this.config.components)) {
+      const keys = await this.declaredSecretKeys(componentName);
+      const values = Object.fromEntries(
+        keys.map((key): [string, string | null] => [
+          key,
+          // null records "absent in the vault" (a known miss)
+          flatSecrets[getSecretVarName(CACHED_ENV, componentName, key)] ?? null,
+        ]),
+      );
+      if (Object.keys(values).length > 0) {
+        local[componentName] = values;
+      }
+    }
+    return Object.keys(local).length > 0 ? { [CACHED_ENV]: local } : {};
+  }
+
+  private async declaredSecretKeys(componentName: string): Promise<string[]> {
+    try {
+      const { secretEnvVarKeys, jobOnlyVars } = await getEnvironment({
+        config: this.config,
+        componentName,
+        env: CACHED_ENV,
+      });
+      return [
+        ...secretEnvVarKeys,
+        ...jobOnlyVars.build.secretEnvVarKeys,
+        ...jobOnlyVars.deploy.secretEnvVarKeys,
+      ].map(({ key }) => key);
+    } catch {
+      // components without a local env simply have nothing to cache
+      return [];
+    }
+  }
+}
+
+/**
+ * the cache hierarchy as a flat map of raw secret variable names
+ * (null values = recorded misses)
+ */
+const flattenSecrets = (
+  secrets: CachedSecrets,
+): Record<string, string | null> =>
+  Object.fromEntries(
+    Object.entries(secrets).flatMap(([env, components]) =>
+      Object.entries(components).flatMap(([componentName, values]) =>
+        Object.entries(values).map(([key, value]) => [
+          getSecretVarName(env, componentName, key),
+          value,
+        ]),
+      ),
+    ),
+  );
+
+/**
+ * only actual values — recorded misses stay unresolved, like any other
+ * unset variable
+ */
+const withoutMisses = (
+  known: Record<string, string | null>,
+): Record<string, string> =>
+  Object.fromEntries(
+    Object.entries(known).filter(
+      (entry): entry is [string, string] => entry[1] !== null,
+    ),
+  );

package/src/vault/bw.ts

@@ -0,0 +1,110 @@
+import { execFile as execFileCb, spawn } from "child_process";
+import { promisify } from "util";
+import type { IO } from "../core/types";
+import {
+  getPreference,
+  hasPreference,
+  setPreference,
+} from "../utils/preferences";
+
+const execFile = promisify(execFileCb);
+
+const SESSION_PREFERENCE = "bwsession";
+
+class BitwardenLockedError extends Error {
+  constructor() {
+    super(
+      "bitwarden is locked and prompting is disabled — run `bw unlock` or catenv without --vault-mode no-prompt",
+    );
+  }
+}
+
+/**
+ * runs an interactive bw command that prints a session key on stdout
+ * (login/unlock) while the user types the password on the terminal
+ */
+const interactiveSessionCommand = (args: string[]): Promise<string> =>
+  new Promise((resolve, reject) => {
+    const child = spawn("bw", [...args, "--raw"], {
+      stdio: ["inherit", "pipe", "inherit"],
+    });
+    let session = "";
+    child.stdout?.on("data", (data) => (session += data.toString("utf-8")));
+    child.on("error", reject);
+    child.on("close", (code) =>
+      code === 0 && session.trim()
+        ? resolve(session.trim())
+        : reject(new Error(`bw ${args[0]} failed`)),
+    );
+  });
+
+const getBwStatus = async (
+  session: string | null,
+): Promise<"unlocked" | "locked" | "unauthenticated"> => {
+  try {
+    const { stdout } = await execFile("bw", ["status"], {
+      env: { ...process.env, ...(session ? { BW_SESSION: session } : {}) },
+    });
+    return JSON.parse(stdout).status;
+  } catch {
+    return "unauthenticated";
+  }
+};
+
+/**
+ * a working bw session: reuses the cached one, otherwise unlocks (or
+ * logs in) interactively — unless prompting is disabled
+ */
+export const getBwSession = async (
+  io: IO | null,
+  { allowPrompt }: { allowPrompt: boolean },
+): Promise<string> => {
+  const cached = (await hasPreference(SESSION_PREFERENCE))
+    ? String(await getPreference(SESSION_PREFERENCE))
+    : null;
+
+  const status = await getBwStatus(cached);
+  if (status === "unlocked" && cached) {
+    return cached;
+  }
+  if (!allowPrompt) {
+    throw new BitwardenLockedError();
+  }
+
+  io?.log("");
+  io?.log(
+    status === "unauthenticated"
+      ? "# Please login to Bitwarden:"
+      : "# Bitwarden is locked, please unlock:",
+  );
+  io?.log("");
+  const session = await interactiveSessionCommand([
+    status === "unauthenticated" ? "login" : "unlock",
+  ]);
+  await setPreference(SESSION_PREFERENCE, session);
+  await runBw(session, ["sync"]);
+  return session;
+};
+
+/**
+ * runs a bw command with the given session and returns stdout
+ */
+export const runBw = async (
+  session: string,
+  args: string[],
+): Promise<string> => {
+  const { stdout } = await execFile(
+    "bw",
+    [...args, "--raw", "--nointeraction"],
+    {
+      env: { ...process.env, BW_SESSION: session },
+      maxBuffer: 50 * 1024 * 1024,
+    },
+  );
+  return stdout;
+};
+
+export const runBwJson = async <T>(
+  session: string,
+  args: string[],
+): Promise<T> => JSON.parse(await runBw(session, args));

package/src/vault/cache.ts

@@ -0,0 +1,80 @@
+import { mkdir, readFile, writeFile } from "fs/promises";
+import { join } from "path";
+import { getProjectRootPath } from "../git/gitProjectInformation";
+
+/**
+ * hierarchical: env -> componentName -> key -> value.
+ * Only the "local" env is ever cached — other envs' secrets have no
+ * business sitting on developer machines.
+ *
+ * `null` records a declared secret that was absent in the vault at the
+ * last refresh (e.g. deploy credentials that intentionally don't exist
+ * for local) — a known miss, so it doesn't force re-contacting the
+ * vault on every run.
+ */
+export type CachedSecrets = {
+  [env: string]: { [componentName: string]: { [key: string]: string | null } };
+};
+
+/**
+ * local, project-scoped cache of the vault secrets, so that frequent
+ * catenv runs don't have to contact the vault (e.g. unlock bitwarden)
+ * every time. Plaintext — like the .env files catenv writes anyway —
+ * but never committed (the folder ignores itself).
+ */
+type SecretsCache = {
+  /** the vault this cache was filled from */
+  vaultId: string;
+  updatedAt: string;
+  secrets: CachedSecrets;
+};
+
+const getCacheFolder = async () =>
+  join(await getProjectRootPath(), ".catladder");
+
+const getCachePath = async () => join(await getCacheFolder(), "secrets.json");
+
+export const readSecretsCache = async (): Promise<SecretsCache | null> => {
+  try {
+    return JSON.parse(await readFile(await getCachePath(), "utf-8"));
+  } catch {
+    return null;
+  }
+};
+
+export const writeSecretsCache = async (
+  vaultId: string,
+  secrets: CachedSecrets,
+): Promise<void> => {
+  const folder = await getCacheFolder();
+  await mkdir(folder, { recursive: true });
+  // the folder ignores itself, so the cache can never be committed
+  await writeFile(join(folder, ".gitignore"), "*\n");
+  const cache: SecretsCache = {
+    vaultId,
+    updatedAt: new Date().toISOString(),
+    secrets,
+  };
+  await writeFile(await getCachePath(), JSON.stringify(cache, null, 2));
+};
+
+/**
+ * merges freshly written secrets of one env/component into the cache
+ * (write-through from config-secrets), so the next catenv run doesn't
+ * need the vault
+ */
+export const updateSecretsCache = async (
+  vaultId: string,
+  env: string,
+  componentName: string,
+  values: Record<string, string>,
+): Promise<void> => {
+  const existing = await readSecretsCache();
+  const secrets: CachedSecrets =
+    existing?.vaultId === vaultId ? existing.secrets : {};
+  secrets[env] = {
+    ...secrets[env],
+    [componentName]: { ...secrets[env]?.[componentName], ...values },
+  };
+  await writeSecretsCache(vaultId, secrets);
+};

package/src/vault/index.ts

@@ -0,0 +1,5 @@
+export * from "./types";
+export * from "./VaultManager";
+export { GitlabVault } from "./GitlabVault";
+export { BitwardenVault } from "./BitwardenVault";
+export { updateSecretsCache } from "./cache";

package/src/vault/types.ts

@@ -0,0 +1,42 @@
+import type { IO } from "../core/types";
+
+/**
+ * a secrets vault is the readable source of truth for secret values.
+ * CI backends (gitlab variables, github secrets) only receive mirrored
+ * copies of what is stored here.
+ */
+export type SecretsVault = {
+  /**
+   * identity of the vault (incl. its config), used to invalidate the
+   * local cache when the vault changes
+   */
+  id: string;
+
+  /**
+   * all secrets as a flat map of raw secret variable names
+   * (CL_<ENV>_<COMPONENT>_<KEY>) to values
+   */
+  readAllSecrets(io: IO | null): Promise<Record<string, string>>;
+
+  /**
+   * upserts the secrets of one env and component
+   * (keys as declared in the config, not the raw variable names)
+   */
+  writeSecrets(
+    io: IO,
+    env: string,
+    componentName: string,
+    secrets: Record<string, unknown>,
+  ): Promise<void>;
+};
+
+/**
+ * how secret resolution may interact with the vault:
+ * - auto (default): contact the vault when the local cache is
+ *   incomplete, prompting to unlock if necessary
+ * - no-prompt: contact the vault only if that needs no interaction;
+ *   fail on missing secrets instead of prompting to unlock
+ * - offline: never contact the vault, use only the local cache
+ * - refresh: ignore the local cache once and refresh it from the vault
+ */
+export type SecretsMode = "auto" | "no-prompt" | "offline" | "refresh";

package/tsconfig.json

@@ -13,8 +13,10 @@
     "incremental": true,
     "resolveJsonModule": true,
     "paths": {
-      "@catladder/pipeline": ["../pipeline/src/index.ts"]
+      "@catladder/bash": ["../../packages/bash/src/index.ts"],
+      "@catladder/pipeline": ["../../packages/pipeline/src/index.ts"]
     }
   },
-  "include": ["./src/**/*"]
+  "include": ["./src/**/*"],
+  "exclude": ["**/__tests__", "**/*.test.ts", "**/*.test.tsx"]
 }

package/babel-inline.json

@@ -1,11 +0,0 @@
-{
-  "plugins": [
-    "@babel/plugin-syntax-typescript",
-    [
-      "transform-inline-environment-variables",
-      {
-        "include": ["INLINE_PIPELINE_IMAGE_TAG", "INLINE_DOCKER_REGISTRY"]
-      }
-    ]
-  ]
-}

package/dist/cli/src/adapters/baseContext.d.ts

@@ -1,22 +0,0 @@
-import type { CommandContext, CommandDef, InputDef, InputResultType, InputsSchema, OptionalInput } from "../core/types";
-/**
- * Base class for CommandContext implementations.
- * Handles caching, the `get()` method, and `promptDirect` delegation.
- * Subclasses only implement the adapter-specific resolution logic.
- */
-export declare abstract class BaseContext<TInputs extends InputsSchema> implements CommandContext<TInputs> {
-  protected readonly command: CommandDef<TInputs>;
-  private cache;
-  constructor(command: CommandDef<TInputs>);
-  abstract log(message: string): void;
-  abstract confirm(message: string): Promise<boolean>;
-  /**
-   * Resolve a single input value. Called once per input (result is cached).
-   * Implementations check pre-supplied values, prompt interactively, use defaults, or throw.
-   */
-  protected abstract resolveInput<P extends InputDef>(name: string, spec: P): Promise<InputResultType<P>>;
-  get<K extends keyof TInputs & string>(name: K): Promise<OptionalInput<TInputs[K]>>;
-  promptDirect<P extends InputDef>(spec: P & {
-    name: string;
-  }): Promise<InputResultType<P>>;
-}