@catladder/cli 0.0.0-next-8b9ad5ca → 0.0.0-next-1cd29c1f

package/package.json

@@ -53,7 +53,7 @@
     }
   ],
   "license": "MIT",
-  "version": "0.0.0-next-8b9ad5ca",
+  "version": "0.0.0-next-1cd29c1f",
   "scripts": {
     "lint": "eslint \"src/**/*.ts\"",
     "lint:fix": "eslint \"src/**/*.ts\" --fix",

package/src/apps/cli/commands/project/commandConfigSecrets.ts

@@ -243,15 +243,35 @@ const mirrorSecretsToCiBackends = async (
   if (!enabled.includes("github")) {
     return;
   }
+  const kinds = new Map<string, "secret" | "variable">();
+  for (const [componentName, envs] of Object.entries(envAndComponents)) {
+    for (const env of envs) {
+      const { secretEnvVarKeys, jobOnlyVars } = await getEnvironment(
+        env,
+        componentName,
+      );
+      [
+        ...secretEnvVarKeys,
+        ...jobOnlyVars.build.secretEnvVarKeys,
+        ...jobOnlyVars.deploy.secretEnvVarKeys,
+      ].forEach(({ key, kind }) =>
+        kinds.set(getSecretVarName(env, componentName, key), kind ?? "secret"),
+      );
+    }
+  }
   const secrets = Object.entries(envAndComponents).flatMap(
     ([componentName, envs]) =>
       envs.flatMap((env) =>
         Object.entries(valuesToEdit[componentName][env] ?? {}).map(
-          ([key, value]) => ({
-            name: getSecretVarName(env, componentName, key),
-            value:
-              typeof value === "string" ? value : JSON.stringify(value ?? ""),
-          }),
+          ([key, value]) => {
+            const name = getSecretVarName(env, componentName, key);
+            return {
+              name,
+              value:
+                typeof value === "string" ? value : JSON.stringify(value ?? ""),
+              kind: kinds.get(name) ?? ("secret" as const),
+            };
+          },
         ),
       ),
   );

package/src/apps/cli/commands/project/setup/setupCloudRun.ts

@@ -1,16 +1,15 @@
 import type { ComponentContext } from "@catladder/pipeline";
 import {
   GCLOUD_DEPLOY_CREDENTIALS_KEY,
-  GCLOUD_RUN_CANONICAL_HOST_SUFFIX,
   isOfDeployType,
 } from "@catladder/pipeline";
 
 import type { IO } from "../../../../../core/types";
 import { upsertGcloudArtifactsRegistry } from "../../../../../gcloud/artifactsRegistry";
-import { getCloudRunDomainSuffix } from "../../../../../gcloud/cloudRun/getCloudRunDomainSuffix";
 import { enableGCloudServices } from "../../../../../gcloud/enableServices";
+import { fetchGcloudProjectNumber } from "../../../../../gcloud/getProjectNumber";
 import { upsertGcloudServiceAccountAndSaveSecret } from "../../../../../gcloud/serviceAccounts";
-import { upsertAllVariables } from "../../../../../utils/gitlab";
+import { updateCatladderStore } from "../../../../../store/updateCatladderStore";
 
 export const setupCloudRun = async (
   instance: IO,
@@ -61,24 +60,17 @@ export const setupCloudRun = async (
     GCLOUD_DEPLOY_CREDENTIALS_KEY,
   );
 
-  // gcloud run automatically gives us a hostname
-  // but the pipeline need to know it before deployment
-  // so we get this through some magic and add this as a ci/cd variable
-  // the pipeline than can use this to construct the canonical host
-  instance.log(
-    "get service domain suffix... that might take a while initially",
-  );
-  const suffix = await getCloudRunDomainSuffix(config);
-  instance.log("domain suffix: " + suffix);
+  // the project number is needed at generation time to construct cloud
+  // run's deterministic urls (<service>-<projectNumber>.<region>.run.app)
+  instance.log("fetch gcloud project number...");
+  const projectNumber = await fetchGcloudProjectNumber(config.projectId);
+  instance.log("project number: " + projectNumber);
 
-  await upsertAllVariables(
-    instance,
-    {
-      [GCLOUD_RUN_CANONICAL_HOST_SUFFIX]: suffix,
+  await updateCatladderStore((store) => ({
+    ...store,
+    gcloudProjects: {
+      ...store.gcloudProjects,
+      [config.projectId]: { projectNumber },
     },
-    context.env,
-    context.name,
-    false, // backup
-    false, // masked
-  );
+  }));
 };

package/src/commands/project/commandSecretsSyncGithub.ts

@@ -14,18 +14,22 @@ import {
   getGithubRepoFromRemote,
   isGhAuthenticated,
   setGithubSecret,
+  setGithubVariable,
 } from "../../utils/github";
 
 /**
  * all secrets declared in the config, with their values read from the
  * vault (github secrets are write-only and can never act as a source)
  */
-export const collectSecretsFromVault = async (io: IO) => {
+export const collectSecretsFromVault = async (io: IO, envFilter?: string[]) => {
   const envAndComponents = await getAllComponentsWithAllEnvsHierarchical();
 
-  const names: string[] = [];
+  const kinds = new Map<string, "secret" | "variable">();
   for (const [componentName, envs] of Object.entries(envAndComponents)) {
     for (const env of envs) {
+      if (envFilter && !envFilter.includes(env)) {
+        continue;
+      }
       const { secretEnvVarKeys, jobOnlyVars } = await getEnvironment(
         env,
         componentName,
@@ -35,31 +39,42 @@ export const collectSecretsFromVault = async (io: IO) => {
         ...jobOnlyVars.build.secretEnvVarKeys,
         ...jobOnlyVars.deploy.secretEnvVarKeys,
       ];
-      keys.forEach(({ key }) =>
-        names.push(getSecretVarName(env, componentName, key)),
+      keys.forEach(({ key, kind }) =>
+        kinds.set(getSecretVarName(env, componentName, key), kind ?? "secret"),
       );
     }
   }
 
+  const names = [...kinds.keys()];
   const values = await (await io.getVaultManager()).readSecrets(names, io);
   const secrets = names
     .filter((name) => values[name] !== undefined)
-    .map((name) => ({ name, value: values[name] }));
+    .map((name) => ({
+      name,
+      value: values[name],
+      kind: kinds.get(name) ?? ("secret" as const),
+    }));
   const missing = names.filter((name) => values[name] === undefined);
   return { secrets, missing };
 };
 
 /**
- * pushes secrets to github repository secrets via the gh cli
+ * pushes vault values to github via the gh cli: sensitive ones as repo
+ * secrets (masked), "variable"-kind ones as repo variables (plain)
  */
 export const pushSecretsToGithub = async (
   io: IO,
   repo: string,
-  secrets: { name: string; value: string }[],
+  secrets: { name: string; value: string; kind: "secret" | "variable" }[],
 ) => {
-  for (const { name, value } of secrets) {
-    await setGithubSecret(repo, name, value);
-    io.log(`✅ ${name}`);
+  for (const { name, value, kind } of secrets) {
+    if (kind === "variable") {
+      await setGithubVariable(repo, name, value);
+      io.log(`📖 ${name} (variable)`);
+    } else {
+      await setGithubSecret(repo, name, value);
+      io.log(`✅ ${name}`);
+    }
   }
 };
 
@@ -94,6 +109,13 @@ export const commandSecretsSyncGithub = defineCommand({
       positional: true,
       required: false,
     },
+    env: {
+      type: "string",
+      message:
+        "only sync these environments (comma-separated, e.g. review,dev) — github allows at most 100 repo secrets",
+      positional: true,
+      required: false,
+    },
   },
   execute: async (ctx) => {
     const config = await getProjectConfig();
@@ -118,7 +140,8 @@ export const commandSecretsSyncGithub = defineCommand({
       );
     }
 
-    const { secrets, missing } = await collectSecretsFromVault(ctx);
+    const envFilter = (await ctx.get("env"))?.split(",").map((e) => e.trim());
+    const { secrets, missing } = await collectSecretsFromVault(ctx, envFilter);
 
     if (missing.length > 0) {
       ctx.log(

package/src/gcloud/getProjectNumber.ts

@@ -0,0 +1,20 @@
+import { exec } from "child-process-promise";
+
+/**
+ * resolves the numeric gcloud project number for a project id.
+ * Requires `cloudresourcemanager.googleapis.com` to be enabled.
+ */
+export const fetchGcloudProjectNumber = async (
+  projectId: string,
+): Promise<string> => {
+  const result = await exec(
+    `gcloud projects describe ${projectId} --format="value(projectNumber)"`,
+  );
+  const projectNumber = result.stdout.trim();
+  if (!/^\d+$/.test(projectNumber)) {
+    throw new Error(
+      `could not determine the gcloud project number for "${projectId}" (got: "${projectNumber}")`,
+    );
+  }
+  return projectNumber;
+};

package/src/store/updateCatladderStore.ts

@@ -0,0 +1,33 @@
+import { mkdir, writeFile } from "fs/promises";
+import { dirname } from "path";
+
+import type { CatladderStore } from "@catladder/pipeline";
+import {
+  catladderStoreSchema,
+  getCatladderStorePath,
+  readCatladderStore,
+} from "@catladder/pipeline";
+import { stringify } from "yaml";
+
+import { getGitRoot } from "../utils/projects";
+
+const HEADER = `# managed by catladder — written by \`catladder project setup\`.
+# Non-secret, machine-fetched values needed to generate the pipeline.
+# Check this file in. If it is missing, rerun \`catladder project setup\`.
+`;
+
+/**
+ * applies an update to `.catladder-store/store.yml` at the git root,
+ * creating the folder on first use
+ */
+export const updateCatladderStore = async (
+  update: (store: CatladderStore) => CatladderStore,
+) => {
+  const root = (await getGitRoot()) ?? process.cwd();
+  const store = readCatladderStore(root);
+  const updated = catladderStoreSchema.parse(update(store));
+  const path = getCatladderStorePath(root);
+  await mkdir(dirname(path), { recursive: true });
+  await writeFile(path, HEADER + stringify(updated));
+  return updated;
+};

package/src/utils/github.ts

@@ -34,17 +34,14 @@ export const getGithubRepoFromRemote = async (
   }
 };
 
-/**
- * sets a github actions repository secret via the gh cli.
- * The value is passed via stdin so it never shows up in a process list.
- */
-export const setGithubSecret = async (
+const ghSetViaStdin = (
+  kind: "secret" | "variable",
   repo: string,
   name: string,
   value: string,
 ): Promise<void> =>
   new Promise<void>((resolve, reject) => {
-    const child = spawn("gh", ["secret", "set", name, "--repo", repo], {
+    const child = spawn("gh", [kind, "set", name, "--repo", repo], {
       stdio: ["pipe", "ignore", "pipe"],
     });
     let stderr = "";
@@ -53,7 +50,27 @@ export const setGithubSecret = async (
     child.on("close", (code) =>
       code === 0
         ? resolve()
-        : reject(new Error(`gh secret set ${name} failed: ${stderr.trim()}`)),
+        : reject(new Error(`gh ${kind} set ${name} failed: ${stderr.trim()}`)),
     );
     child.stdin?.end(value);
   });
+
+/**
+ * sets a github actions repository secret via the gh cli.
+ * The value is passed via stdin so it never shows up in a process list.
+ */
+export const setGithubSecret = async (
+  repo: string,
+  name: string,
+  value: string,
+): Promise<void> => ghSetViaStdin("secret", repo, name, value);
+
+/**
+ * sets a github actions repository variable (plain, unmasked) via the
+ * gh cli — for vault values classified as not sensitive
+ */
+export const setGithubVariable = async (
+  repo: string,
+  name: string,
+  value: string,
+): Promise<void> => ghSetViaStdin("variable", repo, name, value);

package/dist/cli/src/gcloud/cloudRun/getCloudRunDomainSuffix.d.ts

@@ -1,2 +0,0 @@
-import type { DeployConfigCloudRun } from "../../../../pipeline/src/index.js";
-export declare const getCloudRunDomainSuffix: (config: DeployConfigCloudRun) => Promise<any>;

package/dist/cli/src/gcloud/cloudRun/getCloudRunDomainSuffix.js

@@ -1,29 +0,0 @@
-"use strict";
-
-Object.defineProperty(exports, "__esModule", {
-  value: true
-});
-exports.getCloudRunDomainSuffix = void 0;
-const child_process_promise_1 = require("child-process-promise");
-const getSuffixFromService = service => {
-  const fullUrl = service.status.url;
-  const name = service.metadata.name;
-  return fullUrl.replace("https://" + name + "-", "");
-};
-const getCloudRunDomainSuffix = async config => {
-  /* google cloud run domains are partially predictable, they have a fixed suffix that depends on the region and project, but we don't know it beforehand
-     So what we do is either:
-   - get one service and extract its suffix
-   - create a dummy project, extract the suffix and save that as a gitlab variable
-  
-  
-  */
-  const dummyServiceName = "cl-dummy-service-delete-me";
-  const existingServices = await (0, child_process_promise_1.exec)(`gcloud run services list --format=json  --project="${config.projectId}"  --region=${config.region} --limit=1`).then(r => JSON.parse(r.stdout));
-  if (existingServices.length > 0) {
-    return getSuffixFromService(existingServices[0]);
-  }
-  const result = await (0, child_process_promise_1.exec)(`gcloud run deploy ${dummyServiceName} --region=${config.region} --allow-unauthenticated --project ${config.projectId} --image=us-docker.pkg.dev/cloudrun/container/hello --format=json`).then(r => JSON.parse(r.stdout));
-  return await getSuffixFromService(result);
-};
-exports.getCloudRunDomainSuffix = getCloudRunDomainSuffix;

package/dist/cli/src/gcloud/cloudRun/getCloudRunDomainSuffix.js.map

@@ -1 +0,0 @@
-{"version":3,"file":"getCloudRunDomainSuffix.js","sourceRoot":"","sources":["../../../../../src/gcloud/cloudRun/getCloudRunDomainSuffix.ts"],"names":[],"mappings":";;;AACA,iEAA6C;AAE7C,MAAM,oBAAoB,GAAG,CAAC,OAAY,EAAE,EAAE;IAC5C,MAAM,OAAO,GAAG,OAAO,CAAC,MAAM,CAAC,GAAG,CAAC;IACnC,MAAM,IAAI,GAAG,OAAO,CAAC,QAAQ,CAAC,IAAI,CAAC;IAEnC,OAAO,OAAO,CAAC,OAAO,CAAC,UAAU,GAAG,IAAI,GAAG,GAAG,EAAE,EAAE,CAAC,CAAC;AACtD,CAAC,CAAC;AACK,MAAM,uBAAuB,GAAG,KAAK,EAAE,MAA4B,EAAE,EAAE;IAC5E;;;;;;;MAOE;IAEF,MAAM,gBAAgB,GAAG,4BAA4B,CAAC;IAEtD,MAAM,gBAAgB,GAAG,MAAM,IAAA,4BAAI,EACjC,sDAAsD,MAAM,CAAC,SAAS,eAAe,MAAM,CAAC,MAAM,YAAY,CAC/G,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC;IAEpC,IAAI,gBAAgB,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QAChC,OAAO,oBAAoB,CAAC,gBAAgB,CAAC,CAAC,CAAC,CAAC,CAAC;IACnD,CAAC;IAED,MAAM,MAAM,GAAG,MAAM,IAAA,4BAAI,EACvB,qBAAqB,gBAAgB,aAAa,MAAM,CAAC,MAAM,sCAAsC,MAAM,CAAC,SAAS,mEAAmE,CACzL,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC;IAEpC,OAAO,MAAM,oBAAoB,CAAC,MAAM,CAAC,CAAC;AAC5C,CAAC,CAAC;AAzBW,QAAA,uBAAuB,2BAyBlC"}

package/src/gcloud/cloudRun/getCloudRunDomainSuffix.ts

@@ -1,35 +0,0 @@
-import type { DeployConfigCloudRun } from "@catladder/pipeline";
-import { exec } from "child-process-promise";
-
-const getSuffixFromService = (service: any) => {
-  const fullUrl = service.status.url;
-  const name = service.metadata.name;
-
-  return fullUrl.replace("https://" + name + "-", "");
-};
-export const getCloudRunDomainSuffix = async (config: DeployConfigCloudRun) => {
-  /* google cloud run domains are partially predictable, they have a fixed suffix that depends on the region and project, but we don't know it beforehand
-
-  So what we do is either:
-   - get one service and extract its suffix
-   - create a dummy project, extract the suffix and save that as a gitlab variable
-  
-  
-  */
-
-  const dummyServiceName = "cl-dummy-service-delete-me";
-
-  const existingServices = await exec(
-    `gcloud run services list --format=json  --project="${config.projectId}"  --region=${config.region} --limit=1`,
-  ).then((r) => JSON.parse(r.stdout));
-
-  if (existingServices.length > 0) {
-    return getSuffixFromService(existingServices[0]);
-  }
-
-  const result = await exec(
-    `gcloud run deploy ${dummyServiceName} --region=${config.region} --allow-unauthenticated --project ${config.projectId} --image=us-docker.pkg.dev/cloudrun/container/hello --format=json`,
-  ).then((r) => JSON.parse(r.stdout));
-
-  return await getSuffixFromService(result);
-};