@catladder/cli 0.0.0-main-8dbb6c76 → 0.0.0-main-a6b45abe

package/src/apps/cli/commands/project/setup/index.ts

@@ -1,32 +1,62 @@
+import {
+  createCatenvContext,
+  generateAgentSkills,
+  getEnabledPipelineTypes,
+} from "@catladder/pipeline";
 import type { IO } from "../../../../../core/types";
-import { getAllPipelineContexts } from "../../../../../config/getProjectConfig";
+import {
+  getAllPipelineContexts,
+  getProjectConfig,
+} from "../../../../../config/getProjectConfig";
 import { setupAccessTokens } from "./setupAccessTokens";
 import { setupContext } from "./setupContext";
 import { setupTopic } from "./setupTopic";
 import { setupAgents } from "./setupAgents";
+import { setupGithub } from "./setupGithub";
 import { logSection } from "./logSection";
+import { ensureGcloudProjectNumbers } from "./ensureGcloudProjectNumbers";
 
 export const setupProject = async (
   instance: IO,
   onlyComponents?: string | string[],
 ) => {
+  // refresh the agent skills first — store-independent, so it survives a
+  // later failure (e.g. missing gcloud auth in ensureGcloudProjectNumbers).
+  // This makes `catladder project setup` the one command that also lands
+  // the skills, so agents helping a migrating project get them even when
+  // catenv itself can't run yet (e.g. before the store is populated).
+  const config = await getProjectConfig();
+  if (config) {
+    await generateAgentSkills(createCatenvContext(config));
+  }
+
+  // must come first: creating contexts already requires the store
+  await ensureGcloudProjectNumbers(instance);
   const allContext = await getAllPipelineContexts(onlyComponents);
   instance.log("will setup those contexts:");
   allContext.forEach((context) => {
     instance.log(` - ${context.name}:${context.env}`);
   });
 
+  const enabledPipelines = config ? getEnabledPipelineTypes(config) : [];
   await logSection(instance, "base setup", async () => {
-    await setupAccessTokens(instance);
+    // gitlab-specific provisioning (access token, agent webhooks) —
+    // meaningless (and failing) on a github-only project
+    if (enabledPipelines.includes("gitlab")) {
+      await setupAccessTokens(instance);
+      await setupAgents(instance);
+    }
     await setupTopic(instance);
-    await setupAgents(instance);
+    if (config) {
+      await setupGithub(instance, config);
+    }
   });
   for (const context of allContext) {
     await setupContext(instance, context);
   }
 
   instance.log("");
-  instance.log("gitlab is ready! 🥂");
+  instance.log(`${enabledPipelines.join(" + ") || "gitlab"} is ready! 🥂`);
   instance.log("\n");
   instance.log("do not forget to make sure that:");
   [

package/src/apps/cli/commands/project/setup/setupAccessTokens.ts

@@ -1,13 +1,20 @@
 import type { IO } from "../../../../../core/types";
 import { add, format } from "date-fns";
-import { doGitlabRequest, getProjectInfo } from "../../../../../utils/gitlab";
+import {
+  doGitlabRequest,
+  doGitlabRequestAllPages,
+  getProjectInfo,
+} from "../../../../../utils/gitlab";
 
 const TOKEN_NAME = "semantic-release";
 
 export const setupAccessTokens = async (instance: IO) => {
   const { id: projectId } = await getProjectInfo(instance);
 
-  const projectTokens = await doGitlabRequest(
+  // all pages: rotated tokens accumulate one (inactive) entry per
+  // rotation, so the active one quickly moves past the first page —
+  // missing it here created a brand-new duplicate token on every setup
+  const projectTokens = await doGitlabRequestAllPages(
     instance,
     `projects/${projectId}/access_tokens`,
   );

package/src/apps/cli/commands/project/setup/setupCloudRun.ts

@@ -1,16 +1,54 @@
 import type { ComponentContext } from "@catladder/pipeline";
 import {
   GCLOUD_DEPLOY_CREDENTIALS_KEY,
-  GCLOUD_RUN_CANONICAL_HOST_SUFFIX,
   isOfDeployType,
 } from "@catladder/pipeline";
 
 import type { IO } from "../../../../../core/types";
 import { upsertGcloudArtifactsRegistry } from "../../../../../gcloud/artifactsRegistry";
-import { getCloudRunDomainSuffix } from "../../../../../gcloud/cloudRun/getCloudRunDomainSuffix";
 import { enableGCloudServices } from "../../../../../gcloud/enableServices";
+import { fetchGcloudProjectNumber } from "../../../../../gcloud/getProjectNumber";
 import { upsertGcloudServiceAccountAndSaveSecret } from "../../../../../gcloud/serviceAccounts";
-import { upsertAllVariables } from "../../../../../utils/gitlab";
+import { updateCatladderStore } from "../../../../../store/updateCatladderStore";
+
+export const CLOUD_RUN_DEPLOY_SA_NAME = "cl-d";
+
+/**
+ * the gcloud services the given deploy config requires.
+ *
+ * Shared with `project doctor`: setup enables them, doctor compares
+ * them against what is actually enabled — config edits after setup
+ * (e.g. adding `cloudSql`) would otherwise drift silently.
+ */
+export const getCloudRunRequiredServices = (config: {
+  cloudSql?: unknown;
+}): string[] => [
+  "run.googleapis.com",
+  "artifactregistry.googleapis.com",
+  "cloudscheduler.googleapis.com", // for scheduling jobs
+  "cloudresourcemanager.googleapis.com", // only required to get google cloud project number
+  ...(config.cloudSql
+    ? ["sqladmin.googleapis.com", "sql-component.googleapis.com"]
+    : []),
+];
+
+/**
+ * the IAM roles the deploy service account needs for the given deploy
+ * config. Shared with `project doctor` (same rationale as
+ * {@link getCloudRunRequiredServices}) — the motivating case: a
+ * component gets `cloudSql` after its setup ran, deploys keep working,
+ * and only the review teardown hits the missing `cloudsql.admin`.
+ */
+export const getCloudRunDeployRoles = (config: {
+  cloudSql?: unknown;
+}): string[] => [
+  // TODO: run.admin is a bit much, would be better to follow https://stackoverflow.com/questions/55788714/deploying-to-cloud-run-with-a-custom-service-account-failed-with-iam-serviceacco
+  "roles/artifactregistry.repoAdmin",
+  "roles/run.admin",
+  "roles/iam.serviceAccountUser",
+  "roles/cloudscheduler.admin", // for scheduling
+  ...(config.cloudSql ? ["roles/cloudsql.admin"] : []),
+];
 
 export const setupCloudRun = async (
   instance: IO,
@@ -25,18 +63,7 @@ export const setupCloudRun = async (
   // enable services
 
   instance.log("enable required services...");
-  await enableGCloudServices(
-    [
-      "run.googleapis.com",
-      "artifactregistry.googleapis.com",
-      "cloudscheduler.googleapis.com", // for scheduling jobs
-      "cloudresourcemanager.googleapis.com", // only required to get google cloud project number
-      ...(config.cloudSql
-        ? ["sqladmin.googleapis.com", "sql-component.googleapis.com"]
-        : []),
-    ],
-    config,
-  );
+  await enableGCloudServices(getCloudRunRequiredServices(config), config);
   instance.log("upsert artifacts registry...");
   await upsertGcloudArtifactsRegistry(config);
 
@@ -48,37 +75,23 @@ export const setupCloudRun = async (
       projectId: config.projectId,
       displayName: "Catladder gcloud deploy",
       description: "This service account deploys to google cloud",
-      name: "cl-d",
-      // TODO: run.admin is a bit much, would be better to follow https://stackoverflow.com/questions/55788714/deploying-to-cloud-run-with-a-custom-service-account-failed-with-iam-serviceacco
-      roles: [
-        "roles/artifactregistry.repoAdmin",
-        "roles/run.admin",
-        "roles/iam.serviceAccountUser",
-        "roles/cloudscheduler.admin", // for scheduling
-        ...(config.cloudSql ? ["roles/cloudsql.admin"] : []),
-      ],
+      name: CLOUD_RUN_DEPLOY_SA_NAME,
+      roles: getCloudRunDeployRoles(config),
     },
     GCLOUD_DEPLOY_CREDENTIALS_KEY,
   );
 
-  // gcloud run automatically gives us a hostname
-  // but the pipeline need to know it before deployment
-  // so we get this through some magic and add this as a ci/cd variable
-  // the pipeline than can use this to construct the canonical host
-  instance.log(
-    "get service domain suffix... that might take a while initially",
-  );
-  const suffix = await getCloudRunDomainSuffix(config);
-  instance.log("domain suffix: " + suffix);
+  // the project number is needed at generation time to construct cloud
+  // run's deterministic urls (<service>-<projectNumber>.<region>.run.app)
+  instance.log("fetch gcloud project number...");
+  const projectNumber = await fetchGcloudProjectNumber(config.projectId);
+  instance.log("project number: " + projectNumber);
 
-  await upsertAllVariables(
-    instance,
-    {
-      [GCLOUD_RUN_CANONICAL_HOST_SUFFIX]: suffix,
+  await updateCatladderStore((store) => ({
+    ...store,
+    gcloudProjects: {
+      ...store.gcloudProjects,
+      [config.projectId]: { projectNumber },
     },
-    context.env,
-    context.name,
-    false, // backup
-    false, // masked
-  );
+  }));
 };

package/src/apps/cli/commands/project/setup/setupGithub.ts

@@ -0,0 +1,117 @@
+import type { Config } from "@catladder/pipeline";
+import {
+  AGGREGATE_CHECK_JOB_NAME,
+  getEnabledPipelineTypes,
+  getPipelineGitRemote,
+} from "@catladder/pipeline";
+import { getConfiguredGithubRepo } from "../../../../../commands/project/commandSecretsSyncGithub";
+import type { IO } from "../../../../../core/types";
+import { ghApiJson, isGhAuthenticated } from "../../../../../utils/github";
+
+/**
+ * configures github merge gating — the sane default gitlab ships
+ * built-in and a fresh github repository lacks entirely: without it a
+ * PR is mergeable the moment it opens, checks still running, and the
+ * auto-merge button never appears.
+ *
+ * - repo settings: allow auto-merge, delete merged head branches
+ * - branch protection on the default branch: require the generated
+ *   `catladder ✅` aggregate check (the one stable context — individual
+ *   job names change with every component add/rename)
+ *
+ * Deliberately non-destructive: existing protection settings
+ * (enforce-admins, required reviews, restrictions, strictness) are
+ * preserved; the aggregate context is merged into the required checks
+ * rather than replacing them. Admins can still bypass per PR — this is
+ * a default, not a cage.
+ */
+export const setupGithub = async (instance: IO, config: Config) => {
+  if (!getEnabledPipelineTypes(config).includes("github")) {
+    return;
+  }
+  if (!(await isGhAuthenticated())) {
+    instance.log(
+      "⚠️ github cli (gh) not authenticated — skipping github merge-gating setup (run: gh auth login)",
+    );
+    return;
+  }
+  const repo = await getConfiguredGithubRepo();
+  if (!repo) {
+    instance.log(
+      `⚠️ no github repository found on the '${getPipelineGitRemote(config, "github")}' remote — skipping github merge-gating setup`,
+    );
+    return;
+  }
+
+  const repoInfo = await ghApiJson<{
+    default_branch: string;
+    allow_auto_merge: boolean;
+    delete_branch_on_merge: boolean;
+  }>("GET", `repos/${repo}`);
+
+  if (!repoInfo.allow_auto_merge || !repoInfo.delete_branch_on_merge) {
+    await ghApiJson("PATCH", `repos/${repo}`, {
+      allow_auto_merge: true,
+      delete_branch_on_merge: true,
+    });
+    instance.log("✅ auto-merge + delete-merged-branches enabled");
+  } else {
+    instance.log("✅ auto-merge already enabled");
+  }
+
+  const branch = repoInfo.default_branch;
+  const protection = await ghApiJson<any>(
+    "GET",
+    `repos/${repo}/branches/${branch}/protection`,
+  ).catch(() => null); // 404: branch not protected yet
+
+  const existingChecks: Array<{ context: string; app_id?: number }> =
+    protection?.required_status_checks?.checks ?? [];
+  if (existingChecks.some((c) => c.context === AGGREGATE_CHECK_JOB_NAME)) {
+    instance.log(
+      `✅ '${AGGREGATE_CHECK_JOB_NAME}' already required on ${branch}`,
+    );
+    return;
+  }
+
+  // merge into whatever protection exists; PUT replaces the whole rule,
+  // so every preserved setting must be carried over explicitly
+  await ghApiJson("PUT", `repos/${repo}/branches/${branch}/protection`, {
+    required_status_checks: {
+      strict: protection?.required_status_checks?.strict ?? false,
+      checks: [
+        ...existingChecks.map(({ context, app_id }) => ({
+          context,
+          ...(app_id !== undefined ? { app_id } : {}),
+        })),
+        { context: AGGREGATE_CHECK_JOB_NAME },
+      ],
+    },
+    enforce_admins: protection?.enforce_admins?.enabled ?? false,
+    required_pull_request_reviews: protection?.required_pull_request_reviews
+      ? {
+          required_approving_review_count:
+            protection.required_pull_request_reviews
+              .required_approving_review_count ?? 0,
+          dismiss_stale_reviews:
+            protection.required_pull_request_reviews.dismiss_stale_reviews ??
+            false,
+          require_code_owner_reviews:
+            protection.required_pull_request_reviews
+              .require_code_owner_reviews ?? false,
+        }
+      : null,
+    restrictions: protection?.restrictions
+      ? {
+          users: protection.restrictions.users.map((u: any) => u.login),
+          teams: protection.restrictions.teams.map((t: any) => t.slug),
+          apps: protection.restrictions.apps.map((a: any) => a.slug),
+        }
+      : null,
+    allow_force_pushes: protection?.allow_force_pushes?.enabled ?? false,
+    allow_deletions: protection?.allow_deletions?.enabled ?? false,
+  });
+  instance.log(
+    `✅ branch protection on ${branch}: '${AGGREGATE_CHECK_JOB_NAME}' required (merges now wait for the pipeline)`,
+  );
+};

package/src/catci.ts

@@ -0,0 +1,203 @@
+/**
+ * catci — catladder's CI companion.
+ *
+ * A minimal, treeshaken bundle of the CLI commands that CI jobs need.
+ * It is materialized into `.catladder-generated/catci/` by pipeline
+ * generation, so jobs can run it with plain `node` — always the exact
+ * version that generated the pipeline, with no install step and no
+ * catladder baked into job images.
+ *
+ * Deliberately no CLI framework: the surface is tiny and the callers
+ * are generated scripts, not humans.
+ */
+import { Gitlab } from "@gitbeaker/rest";
+import {
+  evaluateSecurityAudit,
+  makeSecurityAuditOverview,
+} from "./security/evaluateSecurityAudit";
+import {
+  createSecurityAuditMergeRequest,
+  SECURITY_AUDIT_FILE_NAME,
+} from "./security/createSecurityAuditMergeRequest";
+import {
+  changesetsReleaseJob,
+  dispatchTaggedReleaseWorkflow,
+} from "./release/changesetsReleaseJob";
+import { changesetCheckJob } from "./release/changesetCheckJob";
+import {
+  githubQueueCheckJob,
+  githubQueuedGuardJob,
+} from "./release/githubQueuedRelease";
+import { npmPublishJob, parseNpmPublishArgs } from "./publish/npmPublishJob";
+import { getGitlabHostUrl } from "./utils/gitlabHost";
+
+const USAGE = `catci — catladder CI companion
+
+usage:
+  catci security-audit ci-job <path> <gitlab-token> <main-branch> <project-id> <user-id>
+      evaluates ${SECURITY_AUDIT_FILE_NAME}; creates a gitlab MR with the
+      audit template when the document is missing (and exits 1)
+
+  catci security-audit check <path>
+      evaluates ${SECURITY_AUDIT_FILE_NAME}; exits 1 with instructions when
+      the document is missing or unanswered (no remediation — works on
+      any CI, including github)
+
+  catci release changesets
+      consumes pending .changeset/*.md files: computes the next version
+      from the last v* tag, writes the changelog, commits, tags and
+      pushes (no-op when no changesets are pending)
+
+  catci release changeset-check
+      MR/PR pipelines: reports which changesets the merge request adds,
+      what is pending and what version merging would release (job log,
+      changeset-report.md, sticky MR/PR comment where a token allows);
+      exits 1 when the merge request adds no changeset
+
+  catci release dispatch-tagged-workflow <tag>
+      github only: dispatches the generated taggedRelease workflow for
+      the tag (tags pushed with the job token don't trigger it)
+
+  catci release github-queue-check
+      github only, first step of the manual create-release task:
+      releases right away when the main workflow run for HEAD is green
+      (queued=false step output), queues the release when it is still
+      running (queued=true, marker ref for the release-on-green
+      workflow), fails when it concluded red
+
+  catci release github-queued-guard <head-sha> <conclusion>
+      github only, guard of the release-on-green workflow: consumes the
+      queue marker when it matches the completed main run and decides
+      whether the release step runs (release=true step output)
+
+  catci publish npm --dir <dir> --env-type <envType> [--access <access>] [--registry <url>] [--dist-tag <tag>]
+      npmPackage deploy job: derives version + dist-tag from the
+      pipeline trigger (tagged release -> tag version @latest; branch/MR
+      -> 0.0.0-<slug>-<sha> canary, with next/beta branches publishing
+      under their own dist-tag), stamps package.json and runs npm
+      publish (authenticated via the NPM_TOKEN secret)
+`;
+
+const fail = (message: string): never => {
+  console.error(message);
+  process.exit(1);
+};
+
+/**
+ * shared gate: evaluates the audit document and prints the overview.
+ * Returns null when the document could not be evaluated (missing or
+ * unparseable) so the caller can decide on remediation.
+ */
+const evaluateGate = async (path: string) => {
+  const evaluation = await evaluateSecurityAudit({ path });
+  if (evaluation.isErr()) {
+    return null;
+  }
+  if (evaluation.value.score.answeredTopics === 0) {
+    fail(
+      `audit document has no answered topics\n` +
+        `please answer security topics in ${SECURITY_AUDIT_FILE_NAME} by adding responsible people and check/cross in the table`,
+    );
+  }
+  console.log(makeSecurityAuditOverview(evaluation.value));
+  return evaluation.value;
+};
+
+const securityAuditCheck = async (path: string) => {
+  const evaluation = await evaluateGate(path);
+  if (!evaluation) {
+    fail(
+      `could not evaluate ${SECURITY_AUDIT_FILE_NAME}\n` +
+        `please add a ${SECURITY_AUDIT_FILE_NAME} security audit document to the repository ` +
+        `(run \`catladder security audit create\` locally to generate the template)`,
+    );
+  }
+};
+
+const securityAuditCiJob = async (
+  path: string,
+  token: string,
+  mainBranch: string,
+  projectId: string,
+  userId: string,
+) => {
+  const evaluation = await evaluateGate(path);
+  if (evaluation) {
+    return;
+  }
+  console.log("could not evaluate security audit document");
+  console.log("creating new merge request with security audit template...");
+
+  const api = new Gitlab({ host: await getGitlabHostUrl(), token });
+  const mr = await createSecurityAuditMergeRequest({
+    api,
+    mainBranch,
+    projectId,
+    userId: parseInt(userId),
+  });
+
+  if (mr.isErr()) {
+    fail(
+      `could not create merge request with security audit template: ${mr.error}`,
+    );
+    return;
+  }
+  console.log("security audit merge request created successfully");
+  console.log(
+    `please finish the MR by updating ${SECURITY_AUDIT_FILE_NAME}: ${mr.value.web_url}`,
+  );
+  process.exit(1);
+};
+
+const main = async () => {
+  const [group, command, ...args] = process.argv.slice(2);
+  if (group === "security-audit" && command === "ci-job" && args.length === 5) {
+    const [path, token, mainBranch, projectId, userId] = args;
+    return securityAuditCiJob(path, token, mainBranch, projectId, userId);
+  }
+  if (group === "security-audit" && command === "check" && args.length === 1) {
+    return securityAuditCheck(args[0]);
+  }
+  if (group === "release" && command === "changesets" && args.length === 0) {
+    return changesetsReleaseJob();
+  }
+  if (
+    group === "release" &&
+    command === "changeset-check" &&
+    args.length === 0
+  ) {
+    return changesetCheckJob();
+  }
+  if (
+    group === "release" &&
+    command === "dispatch-tagged-workflow" &&
+    args.length === 1
+  ) {
+    return dispatchTaggedReleaseWorkflow(args[0]);
+  }
+  if (
+    group === "release" &&
+    command === "github-queue-check" &&
+    args.length === 0
+  ) {
+    return githubQueueCheckJob();
+  }
+  if (
+    group === "release" &&
+    command === "github-queued-guard" &&
+    args.length === 2
+  ) {
+    return githubQueuedGuardJob(args[0], args[1]);
+  }
+  if (group === "publish" && command === "npm") {
+    const options = parseNpmPublishArgs(args);
+    if (options) {
+      return npmPublishJob(options);
+    }
+  }
+  fail(USAGE);
+};
+
+main().catch((error) => {
+  fail(`catci failed: ${error instanceof Error ? error.message : error}`);
+});

package/src/catenv.ts

@@ -1,7 +1,8 @@
-import { Command } from "commander";
+import { Command, Option } from "commander";
 import catenv from "./apps/catenv/catenv";
 import { parseChoice } from "./config/parseChoice";
 import packageInfos from "./packageInfos";
+import type { SecretsMode } from "./vault";
 
 const program = new Command();
 
@@ -12,14 +13,45 @@ program
   .argument("[envComponent]", "env or env:component")
   .option("-v, --verbose", "verbose output")
   .option("-p, --print-variables", "print variables")
+  .addOption(
+    new Option(
+      "--vault-mode <mode>",
+      [
+        "how to interact with the secrets vault:",
+        "auto: contact the vault when cached secrets are missing (may prompt to unlock)",
+        "no-prompt: like auto, but fail instead of prompting",
+        "offline: never contact the vault, use only locally cached secrets",
+        "refresh: ignore the local cache once and refresh it from the vault",
+      ].join("\n"),
+    )
+      .choices(["auto", "no-prompt", "offline", "refresh"])
+      .default("auto"),
+  )
+  .option(
+    "--non-interactive",
+    "never prompt (token setup, vault unlock) — assumed automatically when not attached to a terminal (direnv, turbo, CI)",
+  )
   .action(
     (
       envComponent: string | undefined,
-      opts: { verbose?: boolean; printVariables?: boolean },
+      opts: {
+        verbose?: boolean;
+        printVariables?: boolean;
+        vaultMode: SecretsMode;
+        nonInteractive?: boolean;
+      },
     ) => {
+      // interactive iff a human is attached: direnv captures stdout,
+      // turbo/CI pipe it — all of those auto-degrade to non-interactive
+      const interactive =
+        !opts.nonInteractive &&
+        process.stdin.isTTY === true &&
+        process.stdout.isTTY === true;
       catenv(envComponent ? parseChoice(envComponent) : null, {
         verbose: opts.verbose ?? false,
         printVariables: opts.printVariables ?? false,
+        vaultMode: opts.vaultMode,
+        interactive,
       }).then(() => {
         // we have to exit manually, because we have some file watches
         process.exit();

package/src/commands/index.ts

@@ -14,12 +14,18 @@ export { commandOpenEnv } from "./project/commandOpenEnv";
 export { commandOpenGit } from "./project/commandOpenGit";
 export { commandCloudSqlProxy } from "./project/commandCloudSqlProxy";
 export { commandConfigSecrets } from "./project/commandConfigSecrets";
+export { commandSecretsPull } from "./project/secrets/commandSecretsPull";
+export { commandSecretsPush } from "./project/secrets/commandSecretsPush";
+export { commandSecretsSet } from "./project/secrets/commandSecretsSet";
+export { commandSecretsList } from "./project/secrets/commandSecretsList";
 export { commandSecretsClearBackups } from "./project/commandSecretsClearBackups";
+export { commandSecretsSyncGithub } from "./project/commandSecretsSyncGithub";
 export { commandOpenLogs } from "./project/commandOpenLogs";
 export { commandTriggerCronjobGeneral } from "./project/k8s/commandTriggerCronjob";
 export { commandRenewToken } from "./project/commandRenewToken";
 export { commandGetMyTotalWorktime } from "./project/commandGetMyTotalWorktime";
 export { commandSetup } from "./project/commandSetup";
+export { commandDoctor } from "./project/commandDoctor";
 export { commandCiJobOpen, commandCiJobLog } from "./project/commandGitlabCi";
 export { commandProjectRestoreDb } from "./project/commandProjectRestoreDb";
 export { commandSecurityEvaluate } from "./project/commandSecurityEvaluate";