@catladder/cli 0.0.0-main-8dbb6c76 → 0.0.0-main-a6b45abe

package/src/commands/project/commandConfigSecrets.ts

@@ -1,31 +1,92 @@
+import { stripIndents } from "common-tags";
 import { defineCommand } from "../../core/defineCommand";
-import { projectConfigSecrets } from "../../apps/cli/commands/project/commandConfigSecrets";
-import { envAndComponents } from "../../apps/cli/commands/project/utils/autocompletions";
+import { editAsFile } from "../../utils/editAsFile";
+import { delay } from "../../utils/promise";
+import {
+  buildSecretsDocument,
+  checkSecretsDocument,
+  collectSecretsWrites,
+  writeSecretsAndMirror,
+} from "./secrets/document";
+import {
+  formatScope,
+  getScopedSecretKeys,
+  resolveSecretsScope,
+  secretsScopeChoices,
+} from "./secrets/scope";
+import { parseKeysInput } from "./secrets/parseKeysInput";
 
-// This command delegates to the existing implementation which uses
-// editAsFile (opens editor) and has retry loops with confirm prompts.
-// Full migration to CommandIO would require reworking the editor flow,
-// so for now we bridge it.
 export const commandConfigSecrets = defineCommand({
   name: "project config-secrets",
-  description: "setup/update secrets stored in pass",
+  description: "setup/update secrets in the vault via your editor",
   group: "project",
   inputs: {
     envComponent: {
       type: "string",
-      message: "environment:component",
+      message: "scope: env:component, env:, :component or empty for everything",
       positional: true,
       required: false,
-      choices: async () => envAndComponents(),
+      choices: async () => secretsScopeChoices(),
+    },
+    key: {
+      type: "string",
+      message: "only edit these secrets (comma-separated keys)",
+      required: false,
     },
   },
   execute: async (ctx) => {
     const envComponent = await ctx.get("envComponent");
-    // Bridge: the existing implementation accepts CommandInstance-like
-    // objects for logging and prompting. Since it uses this.prompt()
-    // internally (not ctx.prompt), we pass ctx which satisfies the
-    // IO interface for logging. The interactive prompts inside
-    // projectConfigSecrets will need a deeper migration later.
-    await projectConfigSecrets(ctx, envComponent);
+    const keys = parseKeysInput(await ctx.get("key"));
+
+    if (!ctx.interactive) {
+      throw new Error(
+        "project config-secrets opens an editor and needs an interactive terminal — " +
+          "use `project secrets-pull`, `project secrets-push` or `project secrets-set` instead",
+      );
+    }
+
+    const scope = await resolveSecretsScope(envComponent);
+    const scoped = await getScopedSecretKeys(scope, keys);
+
+    let document = await buildSecretsDocument(ctx, scoped);
+    let hasErrors = true;
+    while (hasErrors) {
+      document = await editAsFile(
+        document,
+        stripIndents`
+        Please fill in all secrets for:
+
+        ${formatScope(scoped)}
+
+        `,
+      );
+      const problems = checkSecretsDocument(document, scoped, {
+        requireComplete: true,
+      });
+      hasErrors = problems.length > 0;
+      if (hasErrors) {
+        ctx.log("");
+        ctx.log("😿 Oh no! There is something wrong:");
+        ctx.log("");
+        problems.forEach((problem) => ctx.log(problem));
+        ctx.log("");
+
+        await delay(1000);
+        const shouldContinue = await ctx.confirm("Try again? 🤔");
+        if (!shouldContinue) {
+          throw new Error("abort");
+        }
+      }
+    }
+
+    const { writes, skipped } = collectSecretsWrites(document, scoped);
+    if (skipped.length > 0) {
+      ctx.log("");
+      ctx.log("⏭️ these secrets are still unset and will not be written:");
+      skipped.forEach((entry) => ctx.log(`  - ${entry}`));
+    }
+    await writeSecretsAndMirror(ctx, writes);
+    ctx.log("done! 😻");
+    ctx.log("");
   },
 });

package/src/commands/project/commandDoctor.ts

@@ -0,0 +1,21 @@
+import { doctorProject } from "../../apps/cli/commands/project/doctor";
+import { defineCommand } from "../../core/defineCommand";
+
+export const commandDoctor = defineCommand({
+  name: "project doctor",
+  description:
+    "Checks for drift between the config and the provisioned infrastructure (read-only): service accounts, IAM roles, gitlab tokens, github secrets, store entries.",
+  group: "project",
+  inputs: {
+    component: {
+      type: "string",
+      message: "component name",
+      positional: true,
+      required: false,
+    },
+  },
+  execute: async (ctx) => {
+    const component = await ctx.get("component");
+    await doctorProject(ctx, component);
+  },
+});

package/src/commands/project/commandGitlabCi.ts

@@ -2,6 +2,7 @@ import { exec } from "child-process-promise";
 import { last } from "lodash-es";
 import open from "open";
 import { defineCommand } from "../../core/defineCommand";
+import { getGitlabHostUrl } from "../../utils/gitlabHost";
 import {
   doGitlabRequest,
   getGitlabToken,
@@ -99,7 +100,7 @@ export const commandCiJobLog = defineCommand({
       const trace = await exec(
         `curl -s --header "PRIVATE-TOKEN: ${await getGitlabToken(
           ctx,
-        )}" "https://git.panter.ch/api/v4/projects/${projectId}/jobs/${id}/trace"`,
+        )}" "${await getGitlabHostUrl()}/api/v4/projects/${projectId}/jobs/${id}/trace"`,
       );
 
       const job = await doGitlabRequest(

package/src/commands/project/commandSecretsSyncGithub.ts

@@ -0,0 +1,244 @@
+import type { Config } from "@catladder/pipeline";
+import {
+  getEnabledPipelineTypes,
+  getPipelineGitRemote,
+  getSecretVarName,
+  GithubBackend,
+} from "@catladder/pipeline";
+import {
+  getAllComponentsWithAllEnvsHierarchical,
+  getEnvironment,
+  getProjectConfig,
+} from "../../config/getProjectConfig";
+import { defineCommand } from "../../core/defineCommand";
+import type { IO } from "../../core/types";
+import {
+  ensureGithubEnvironment,
+  getGithubRepoFromRemote,
+  isGhAuthenticated,
+  setGithubSecret,
+  setGithubVariable,
+} from "../../utils/github";
+
+/**
+ * all secrets declared in the config, with their values read from the
+ * vault (github secrets are write-only and can never act as a source)
+ */
+export const collectSecretsFromVault = async (io: IO, envFilter?: string[]) => {
+  const envAndComponents = await getAllComponentsWithAllEnvsHierarchical();
+
+  const kinds = new Map<string, "secret" | "variable">();
+  for (const [componentName, envs] of Object.entries(envAndComponents)) {
+    for (const env of envs) {
+      if (envFilter && !envFilter.includes(env)) {
+        continue;
+      }
+      const { secretEnvVarKeys, jobOnlyVars } = await getEnvironment(
+        env,
+        componentName,
+      );
+      const keys = [
+        ...secretEnvVarKeys,
+        ...jobOnlyVars.build.secretEnvVarKeys,
+        ...jobOnlyVars.deploy.secretEnvVarKeys,
+      ];
+      keys.forEach(({ key, kind }) =>
+        kinds.set(getSecretVarName(env, componentName, key), kind ?? "secret"),
+      );
+    }
+  }
+
+  const names = [...kinds.keys()];
+  const values = await (await io.getVaultManager()).readSecrets(names, io);
+  const secrets = names
+    .filter((name) => values[name] !== undefined)
+    .map((name) => ({
+      name,
+      value: values[name],
+      kind: kinds.get(name) ?? ("secret" as const),
+    }));
+  const missing = names.filter((name) => values[name] === undefined);
+  return { secrets, missing };
+};
+
+/** marker for repo-level (environment-less) targets */
+const REPO_LEVEL = "";
+
+/**
+ * derives which github environment(s) each secret/variable must live in
+ * by scanning the GENERATED workflows: a `${{ secrets.X }}` / `${{
+ * vars.X }}` reference in a job's env targets that job's environment
+ * (or the repo level when the job has none). Deriving from the real
+ * output means the sync can never drift from what the workflows expect.
+ *
+ * Environment secrets are the answer to github's cap of 100 secrets
+ * per REPO (a multi-env project easily exceeds it): the cap applies
+ * per environment instead.
+ */
+export const collectSecretTargets = async (
+  config: Config,
+): Promise<Map<string, Set<string>>> => {
+  const workflows = await new GithubBackend().createWorkflows(config);
+  const targets = new Map<string, Set<string>>();
+  for (const workflow of Object.values(workflows)) {
+    for (const job of Object.values(workflow.jobs)) {
+      const environment =
+        typeof job.environment === "object"
+          ? job.environment.name
+          : job.environment;
+      for (const value of Object.values(job.env ?? {})) {
+        if (typeof value !== "string") continue;
+        for (const match of value.matchAll(
+          /\$\{\{\s*(?:secrets|vars)\.([A-Za-z0-9_]+)\s*\}\}/g,
+        )) {
+          const name = match[1];
+          if (!targets.has(name)) {
+            targets.set(name, new Set());
+          }
+          targets.get(name)!.add(environment ?? REPO_LEVEL);
+        }
+      }
+    }
+  }
+  return targets;
+};
+
+/**
+ * pushes vault values to github via the gh cli: sensitive ones as
+ * secrets (masked), "variable"-kind ones as variables (plain) — each
+ * into the environment(s) whose jobs reference them
+ */
+export const pushSecretsToGithub = async (
+  io: IO,
+  repo: string,
+  secrets: { name: string; value: string; kind: "secret" | "variable" }[],
+  targets: Map<string, Set<string>>,
+) => {
+  const environments = [
+    ...new Set(
+      [...targets.values()].flatMap((envs) =>
+        [...envs].filter((env) => env !== REPO_LEVEL),
+      ),
+    ),
+  ];
+  for (const environment of environments) {
+    await ensureGithubEnvironment(repo, environment);
+    io.log(`🌍 environment ${environment}`);
+  }
+
+  for (const { name, value, kind } of secrets) {
+    const envs = targets.get(name);
+    if (!envs) {
+      io.log(`⏭️ ${name} (not referenced by any generated workflow)`);
+      continue;
+    }
+    for (const environment of envs) {
+      const suffix = environment === REPO_LEVEL ? "" : ` → ${environment}`;
+      if (kind === "variable") {
+        await setGithubVariable(repo, name, value, environment || undefined);
+        io.log(`📖 ${name} (variable)${suffix}`);
+      } else {
+        await setGithubSecret(repo, name, value, environment || undefined);
+        io.log(`✅ ${name}${suffix}`);
+      }
+    }
+  }
+};
+
+/**
+ * resolves the github repository from the configured git remote
+ * (`pipelines.github.gitRemote`, defaults to origin)
+ */
+export const getConfiguredGithubRepo = async (): Promise<
+  string | undefined
+> => {
+  const config = await getProjectConfig();
+  if (!config) {
+    return undefined;
+  }
+  return getGithubRepoFromRemote(getPipelineGitRemote(config, "github"));
+};
+
+/**
+ * copies the catladder secrets from the vault to github actions
+ * repository secrets, keeping the exact same names, so the github
+ * pipeline can reference them as ${{ secrets.<NAME> }}.
+ */
+export const commandSecretsSyncGithub = defineCommand({
+  name: "project secrets-sync-github",
+  description:
+    "copies all secrets from the vault to github repo secrets (same names), for the github pipeline",
+  group: "project",
+  inputs: {
+    repo: {
+      type: "string",
+      message: "github repository (owner/name)",
+      positional: true,
+      required: false,
+    },
+    env: {
+      type: "string",
+      message:
+        "only sync these environments (comma-separated, e.g. review,dev) — github allows at most 100 repo secrets",
+      positional: true,
+      required: false,
+    },
+  },
+  execute: async (ctx) => {
+    const config = await getProjectConfig();
+    if (!config) {
+      throw new Error("no catladder config found");
+    }
+    const enabled = getEnabledPipelineTypes(config);
+    if (!enabled.includes("github")) {
+      throw new Error(
+        "the github pipeline is not enabled in `pipelines` — nothing to sync to",
+      );
+    }
+    if (!(await isGhAuthenticated())) {
+      throw new Error(
+        "the github cli (gh) is not installed or not authenticated — run `gh auth login` first",
+      );
+    }
+    const repo = (await ctx.get("repo")) ?? (await getConfiguredGithubRepo());
+    if (!repo) {
+      throw new Error(
+        `no github repository found on the '${getPipelineGitRemote(config, "github")}' remote — pass it as owner/name or configure pipelines.github.gitRemote`,
+      );
+    }
+
+    const envFilter = (await ctx.get("env"))?.split(",").map((e) => e.trim());
+    const { secrets, missing } = await collectSecretsFromVault(ctx, envFilter);
+
+    if (missing.length > 0) {
+      ctx.log(
+        `⚠️ ${missing.length} secrets are not set in the vault and will be skipped:`,
+      );
+      missing.forEach((name) => ctx.log(`  - ${name}`));
+      ctx.log("");
+    }
+    if (secrets.length === 0) {
+      ctx.log("no secrets to sync 🤷");
+      return;
+    }
+
+    ctx.log(`about to set ${secrets.length} secrets on '${repo}':`);
+    secrets.forEach(({ name }) => ctx.log(`  - ${name}`));
+    ctx.log("");
+    const confirmed = await ctx.confirm("continue? 🤔");
+    if (!confirmed) {
+      throw new Error("abort");
+    }
+
+    await pushSecretsToGithub(
+      ctx,
+      repo,
+      secrets,
+      await collectSecretTargets(config),
+    );
+    ctx.log("");
+    ctx.log(
+      "done! 😻 the github pipeline can use them as ${{ secrets.<NAME> }}",
+    );
+  },
+});

package/src/commands/project/secrets/__tests__/document.test.ts

@@ -0,0 +1,122 @@
+import { describe, expect, it } from "vitest";
+import {
+  FILL_ME,
+  checkSecretsDocument,
+  collectSecretsWrites,
+} from "../document";
+import { parseKeysInput } from "../parseKeysInput";
+import type { ScopedSecretKeys } from "../scope";
+
+const scoped: ScopedSecretKeys = [
+  { env: "dev", componentName: "web", keys: ["API_KEY", "SMTP_PASSWORD"] },
+  { env: "prod", componentName: "web", keys: ["API_KEY", "SMTP_PASSWORD"] },
+  { env: "dev", componentName: "worker", keys: ["QUEUE_TOKEN"] },
+];
+
+describe("checkSecretsDocument", () => {
+  it("accepts a complete document", () => {
+    const problems = checkSecretsDocument(
+      {
+        web: {
+          dev: { API_KEY: "a", SMTP_PASSWORD: "b" },
+          prod: { API_KEY: "c", SMTP_PASSWORD: "d" },
+        },
+        worker: { dev: { QUEUE_TOKEN: "e" } },
+      },
+      scoped,
+      { requireComplete: true },
+    );
+    expect(problems).toEqual([]);
+  });
+
+  it("accepts a partial document when completeness is not required", () => {
+    const problems = checkSecretsDocument(
+      { web: { prod: { API_KEY: "rotated" } } },
+      scoped,
+      { requireComplete: false },
+    );
+    expect(problems).toEqual([]);
+  });
+
+  it("reports missing keys when completeness is required", () => {
+    const problems = checkSecretsDocument(
+      { web: { prod: { API_KEY: "rotated" } } },
+      scoped,
+      { requireComplete: true },
+    );
+    expect(problems).toEqual([
+      '"dev:web": "API_KEY" has not been provided',
+      '"dev:web": "SMTP_PASSWORD" has not been provided',
+      '"prod:web": "SMTP_PASSWORD" has not been provided',
+      '"dev:worker": "QUEUE_TOKEN" has not been provided',
+    ]);
+  });
+
+  it("rejects undeclared keys, components and envs", () => {
+    const problems = checkSecretsDocument(
+      {
+        web: {
+          dev: { TYPO_KEY: "a" },
+          stage: { API_KEY: "b" },
+        },
+        api: { dev: { API_KEY: "c" } },
+      },
+      scoped,
+      { requireComplete: false },
+    );
+    expect(problems).toEqual([
+      '"dev:web": "TYPO_KEY" is not declared in the config (or filtered out by --key)',
+      '"stage:web" is not in this scope — remove it or widen the scope',
+      '"api" is not a component in this scope — remove it or widen the scope',
+    ]);
+  });
+});
+
+describe("collectSecretsWrites", () => {
+  it("collects only provided values and skips placeholders", () => {
+    const { writes, skipped } = collectSecretsWrites(
+      {
+        web: {
+          dev: { API_KEY: "a", SMTP_PASSWORD: FILL_ME },
+          prod: { API_KEY: null as unknown as string },
+        },
+        worker: { dev: {} },
+      },
+      scoped,
+    );
+    expect(writes).toEqual([
+      { env: "dev", componentName: "web", secrets: { API_KEY: "a" } },
+    ]);
+    expect(skipped).toEqual(["dev:web SMTP_PASSWORD", "prod:web API_KEY"]);
+  });
+
+  it("keeps objects and empty strings as values", () => {
+    const { writes, skipped } = collectSecretsWrites(
+      { web: { dev: { API_KEY: "", SMTP_PASSWORD: { user: "a" } } } },
+      scoped,
+    );
+    expect(writes).toEqual([
+      {
+        env: "dev",
+        componentName: "web",
+        secrets: { API_KEY: "", SMTP_PASSWORD: { user: "a" } },
+      },
+    ]);
+    expect(skipped).toEqual([]);
+  });
+});
+
+describe("parseKeysInput", () => {
+  it("splits and trims comma-separated keys", () => {
+    expect(parseKeysInput("API_KEY, SMTP_PASSWORD ,")).toEqual([
+      "API_KEY",
+      "SMTP_PASSWORD",
+    ]);
+  });
+
+  it("returns undefined for empty input", () => {
+    expect(parseKeysInput(undefined)).toBeUndefined();
+    expect(parseKeysInput("")).toBeUndefined();
+    expect(parseKeysInput(" , ")).toBeUndefined();
+  });
+});

package/src/commands/project/secrets/commandSecretsList.ts

@@ -0,0 +1,75 @@
+import { getSecretVarName } from "@catladder/pipeline";
+import { defineCommand } from "../../../core/defineCommand";
+import {
+  getScopedSecretKeys,
+  resolveSecretsScope,
+  secretsScopeChoices,
+} from "./scope";
+import { parseKeysInput } from "./parseKeysInput";
+
+export const commandSecretsList = defineCommand({
+  name: "project secrets-list",
+  description:
+    "lists the declared secrets of a scope with their set/unset status (values only with --reveal)",
+  group: "project",
+  inputs: {
+    envComponent: {
+      type: "string",
+      message: "scope: env:component, env:, :component or empty for everything",
+      positional: true,
+      required: false,
+      choices: async () => secretsScopeChoices(),
+    },
+    key: {
+      type: "string",
+      message: "only these secrets (comma-separated keys)",
+      required: false,
+    },
+    reveal: {
+      type: "boolean",
+      message: "also print the secret values",
+      default: false,
+    },
+    check: {
+      type: "boolean",
+      message: "exit with an error when any secret is unset (for CI/agents)",
+      default: false,
+    },
+  },
+  execute: async (ctx) => {
+    const scope = await resolveSecretsScope(await ctx.get("envComponent"));
+    const scoped = await getScopedSecretKeys(
+      scope,
+      parseKeysInput(await ctx.get("key")),
+    );
+    const reveal = await ctx.get("reveal");
+
+    const names = scoped.flatMap(({ env, componentName, keys }) =>
+      keys.map((key) => getSecretVarName(env, componentName, key)),
+    );
+    const values = await (await ctx.getVaultManager()).readSecrets(names, ctx);
+
+    let unsetCount = 0;
+    let setCount = 0;
+    for (const { env, componentName, keys } of scoped) {
+      for (const key of keys) {
+        const value = values[getSecretVarName(env, componentName, key)];
+        if (value === undefined) {
+          unsetCount += 1;
+          ctx.log(`🚨 ${env}:${componentName} ${key} (unset)`);
+        } else {
+          setCount += 1;
+          ctx.log(
+            `✅ ${env}:${componentName} ${key}` + (reveal ? ` = ${value}` : ""),
+          );
+        }
+      }
+    }
+    ctx.log("");
+    ctx.log(`${setCount} set, ${unsetCount} unset`);
+
+    if ((await ctx.get("check")) && unsetCount > 0) {
+      throw new Error(`${unsetCount} secrets are unset`);
+    }
+  },
+});

package/src/commands/project/secrets/commandSecretsPull.ts

@@ -0,0 +1,55 @@
+import { writeFile } from "fs-extra";
+import { stringify } from "yaml";
+import { defineCommand } from "../../../core/defineCommand";
+import { buildSecretsDocument } from "./document";
+import {
+  getScopedSecretKeys,
+  resolveSecretsScope,
+  secretsScopeChoices,
+} from "./scope";
+import { parseKeysInput } from "./parseKeysInput";
+
+export const commandSecretsPull = defineCommand({
+  name: "project secrets-pull",
+  description:
+    "prints the current secrets of a scope as yaml (unset ones as a placeholder) — edit and feed to secrets-push",
+  group: "project",
+  inputs: {
+    envComponent: {
+      type: "string",
+      message: "scope: env:component, env:, :component or empty for everything",
+      positional: true,
+      required: false,
+      choices: async () => secretsScopeChoices(),
+    },
+    key: {
+      type: "string",
+      message: "only these secrets (comma-separated keys)",
+      required: false,
+    },
+    out: {
+      type: "string",
+      message: "write the yaml to this file instead of stdout",
+      required: false,
+    },
+  },
+  execute: async (ctx) => {
+    const scope = await resolveSecretsScope(await ctx.get("envComponent"));
+    const scoped = await getScopedSecretKeys(
+      scope,
+      parseKeysInput(await ctx.get("key")),
+    );
+    const document = await buildSecretsDocument(ctx, scoped);
+    const yaml = stringify(document, { aliasDuplicateObjects: false });
+
+    const out = await ctx.get("out");
+    if (out) {
+      await writeFile(out, yaml);
+      ctx.log(
+        `wrote secrets to ${out} — ⚠️ contains secret values, don't commit it`,
+      );
+    } else {
+      ctx.log(yaml);
+    }
+  },
+});