@vaharoni/devops 1.0.47

package/src/libs/discovery/process-package-json.ts

@@ -0,0 +1,47 @@
+import chalk from "chalk";
+import fs from "fs";
+import { globSync } from "glob";
+import path from "path";
+import { packageFileNodeSchema, type PackageData } from "../../types";
+import { PackageDataProcessor } from "./process-common";
+
+const rootPkgJsonPath = path.join(process.cwd(), "package.json");
+const rootPkgJson = JSON.parse(fs.readFileSync(rootPkgJsonPath, "utf8")) as {
+  workspaces: string[];
+};
+
+const _workspaces: Record<string, PackageData> = {};
+let _workspacesLoaded = false;
+
+export function nodeWorkspaces() {
+  if (!_workspacesLoaded) {
+    const allPackageJsons = (rootPkgJson.workspaces ?? []).flatMap((workspaceGlob) =>
+      globSync(path.join(workspaceGlob, "package.json"))
+    );
+
+    const processor = new PackageDataProcessor({
+      language: "node",
+      pathList: allPackageJsons,
+      zodSchema: packageFileNodeSchema,
+      fileParser: JSON.parse,
+      nameExtractor: (data) => data.name,
+    });
+
+    processor.convert((data) => {
+      const { dependencies, ...rest } = data;
+      const dependencyNames = Object.keys(dependencies ?? []);
+      return { ...rest, dependencyNames: processor.filterDependencies(dependencyNames) };
+    }).forEach(pkgData => {
+      _workspaces[pkgData.name] = pkgData
+    })
+
+    console.warn(
+      chalk.yellow(
+        // prettier-ignore
+        `Node workspace discovery initialized. Workspaces found: ${Object.keys(_workspaces).join(", ")}`
+      )
+    );
+    _workspacesLoaded = true;
+  }
+  return _workspaces;
+}

package/src/libs/discovery/process-pyproject-toml.ts

@@ -0,0 +1,43 @@
+import chalk from "chalk";
+import { globSync } from "glob";
+import { packageFilePythonSchema, type PackageData } from "../../types";
+import { PackageDataProcessor } from "./process-common";
+import TOML from '@iarna/toml';
+
+const _workspaces: Record<string, PackageData> = {};
+let _workspacesLoaded = false;
+
+export function pythonWorkspaces() {
+  if (!_workspacesLoaded) {
+    const allPyprojectTomls = globSync("**/*/pyproject.toml");
+    const processor = new PackageDataProcessor({
+      language: "python",
+      pathList: allPyprojectTomls,
+      zodSchema: packageFilePythonSchema,
+      fileParser: TOML.parse,
+      nameExtractor: (data) => data.project.name,
+    });
+
+    processor.convert((data) => {
+      const deployment = data.tool?.devops?.deployment;
+      const scripts = data.tool?.devops?.scripts;
+      const dependencyNames = data.project.dependencies ?? [];
+      return {
+        scripts,
+        deployment,
+        dependencyNames: processor.filterDependencies(dependencyNames)
+      }
+    }).forEach(pkgData => {
+      _workspaces[pkgData.name] = pkgData
+    })
+
+    console.warn(
+      chalk.yellow(
+        // prettier-ignore
+        `Python workspace discovery initialized. Workspaces found: ${Object.keys(_workspaces).join(", ")}`
+      )
+    );
+    _workspacesLoaded = true;
+  }
+  return _workspaces;
+}

package/src/libs/git-helpers.ts

@@ -0,0 +1,32 @@
+import { CommandExecutor } from "../cli/common";
+
+export function commitExists(sha: string) {
+  const statusCode = new CommandExecutor(
+    `git merge-base --is-ancestor "${sha}" HEAD`
+  ).exec({ onlyStatusCode: true });
+  return statusCode === 0;
+}
+
+export function firstCommit() {
+  const res = new CommandExecutor(`git rev-list --max-parents=0 HEAD`, {
+    quiet: true,
+  }).exec();
+  return res?.trim();
+}
+
+export function isAffected(
+  path: string,
+  opts: { baseSha?: string; headSha?: string; skipCheck?: boolean } = {}
+) {
+  const baseSha = opts.baseSha || "HEAD^";
+  const headSha = opts.headSha || "HEAD";
+  // When in doubt, assume it's affected
+  if (!opts.skipCheck && (!commitExists(baseSha) || !commitExists(headSha)))
+    return true;
+
+  const statusCode = new CommandExecutor(
+    `git diff --quiet ${baseSha} ${headSha} -- ${path}`,
+    { quiet: true }
+  ).exec({ onlyStatusCode: true });
+  return statusCode !== 0;
+}

package/src/libs/hetzner/reg-secret.ts

@@ -0,0 +1,54 @@
+import { CommandExecutor } from "../../cli/common";
+import { getConst } from "../config";
+import { envToNamespace } from "../k8s-constants";
+import { kubectlCommand } from "../k8s-helpers";
+
+function isApplicable() {
+  const infra = getConst("infra");
+  if (infra !== "hetzner") {
+    console.warn(
+      "Setting up registry permissions is only needed for Harbor in a Hetzner setup"
+    );
+    return false;
+  }
+  return true;
+}
+
+export function copySecretHarborToNamespace(monorepoEnv: string) {
+  if (!isApplicable()) return;
+
+  const cmd = kubectlCommand("get secret harbor-registry-secret -o json", {
+    monorepoEnv,
+    namespace: "harbor",
+  });
+  const secretStr = new CommandExecutor(cmd, { quiet: true }).exec();
+  const secretJson = JSON.parse(secretStr);
+  const {
+    apiVersion,
+    data,
+    kind,
+    metadata: { name },
+    type,
+  } = secretJson;
+
+  const relevantParts = {
+    apiVersion,
+    data,
+    kind,
+    metadata: { name, namespace: envToNamespace(monorepoEnv) },
+    type,
+  };
+  // prettier-ignore
+  const copyCmd = `echo '${JSON.stringify(relevantParts)}' | kubectl apply -f -`;
+  new CommandExecutor(copyCmd, { quiet: true }).exec();
+}
+
+export function patchServiceAccountImagePullSecret(monorepoEnv: string) {
+  if (!isApplicable()) return;
+
+  const cmd = kubectlCommand(
+    `patch serviceaccount default -p '{"imagePullSecrets": [{"name": "harbor-registry-secret"}]}'`,
+    { monorepoEnv }
+  );
+  new CommandExecutor(cmd, { quiet: true }).exec();
+}

package/src/libs/k8s-constants.ts

@@ -0,0 +1,83 @@
+import { getConst, getImageData } from "./config";
+
+export const MISSING_DOMAIN_KEY_ERROR = "$$$MISSING_DOMAIN_KEY$$$"
+const DEFAULT_REMOTE_ENVS = ["staging", "production"];
+const DEFAULT_LOCAL_ENVS = ["development", "test"];
+
+let _remoteSupportedEnvs: string[];
+function remoteSupportedEnvs() {
+  if (_remoteSupportedEnvs) return _remoteSupportedEnvs;
+  const extra = getConst("extra-remote-environments", { ignoreIfInvalid: true }) ?? [];
+  _remoteSupportedEnvs = [...DEFAULT_REMOTE_ENVS, ...extra];
+  return _remoteSupportedEnvs;
+}
+
+let _localSupportedEnvs: string[];
+function localSupportedEnvs() {
+  if (_localSupportedEnvs) return _localSupportedEnvs;
+  const extra = getConst("extra-local-environments", { ignoreIfInvalid: true }) ?? [];
+  _localSupportedEnvs = [...DEFAULT_LOCAL_ENVS, ...extra];
+  return _localSupportedEnvs
+}
+
+export function allSupportedEnvs() {
+  return [...remoteSupportedEnvs(), ...localSupportedEnvs()];
+}
+
+function validateEnv(monorepoEnv?: string) {
+  if (!monorepoEnv) throw new Error("MONOREPO_ENV cannot be empty");
+  if (!remoteSupportedEnvs().includes(monorepoEnv)) {
+    console.error(
+      `MONOREPO_ENV must be one of: ${remoteSupportedEnvs().join(", ")}. Can be set using --env flag.`
+    );
+    process.exit(1);
+  }
+}
+
+export function envToNamespace(monorepoEnv?: string) {
+  validateEnv(monorepoEnv);
+  return `${getConst("project-name")}-${monorepoEnv}`;
+}
+
+export function secretName() {
+  return `${getConst("project-name")}-secret`;
+}
+
+export function imageDebugName(image: string) {
+  return `${image}-debug`;
+}
+
+export function imageConfigMap(image: string) {
+  return `image-config-${image}`;
+}
+
+export function containerRegistryRepoName(image: string, monorepoEnv: string) {
+  validateEnv(monorepoEnv);
+  return `${getConst("project-name")}-${monorepoEnv}-${image}`;
+}
+
+export function containerRegistryPath() {
+  return [getConst("registry-base-url"), getConst("registry-name")].join("/");
+}
+
+export function containerRegistryRepoPath(
+  image: string,
+  monorepoEnv: string,
+  gitSha: string
+) {
+  return [
+    getConst("registry-base-url"),
+    getConst("registry-name"),
+    [containerRegistryRepoName(image, monorepoEnv), gitSha].join(":"),
+  ].join("/");
+}
+
+export function domainNameForEnv(image: string, monorepoEnv: string) {
+  const imageData = getImageData(image); 
+  const value = imageData["domains"]?.[monorepoEnv] ?? MISSING_DOMAIN_KEY_ERROR;
+  return value;
+}
+
+export function dbMigrateJobName(gitSha: string) {
+  return `db-migrate-job-${gitSha.slice(0, 8)}`;
+}

package/src/libs/k8s-db.ts

@@ -0,0 +1,83 @@
+import { CommandExecutor } from "../cli/common";
+import { kubectlCommand } from "./k8s-helpers";
+
+export function getDbList() {
+  const cmd = kubectlCommand(`get sgcluster -A`);
+  const res = new CommandExecutor(cmd, { quiet: true }).exec();
+  if (!res) return null;
+  return res;
+}
+
+export function getDbBackups() {
+  const cmd = kubectlCommand(`get sgbkp -A`);
+  const res = new CommandExecutor(cmd, { quiet: true }).exec();
+  if (!res) return null;
+  return res;
+}
+
+export function getDbAdminPassword() {
+  const cmd = kubectlCommand(
+    `get secrets/stackgres-restapi-admin -o jsonpath="{.data}"`,
+    { namespace: "stackgres" }
+  );
+  const res = new CommandExecutor(cmd, { quiet: true }).exec();
+  if (!res) return null;
+  try {
+    const resJson = JSON.parse(res);
+    const password = atob(resJson["clearPassword"]);
+    const user = atob(resJson["k8sUsername"]);
+    return { user, password };
+  } catch {
+    return null;
+  }
+}
+
+export function getDbPasswords(namespace: string) {
+  const cmd = kubectlCommand(`get secrets/${namespace} -o jsonpath="{.data}"`, {
+    namespace,
+  });
+  const res = new CommandExecutor(cmd, { quiet: true }).exec();
+  if (!res) return null;
+  try {
+    const resJson = JSON.parse(res);
+    const superUser = atob(resJson["superuser-username"]);
+    const superPassword = atob(resJson["superuser-password"]);
+    const authenticatorUser = atob(resJson["authenticator-username"]);
+    const authenticatorPassword = atob(resJson["authenticator-password"]);
+    const replicationUser = atob(resJson["replication-username"]);
+    const replicationPassword = atob(resJson["replication-password"]);
+    return {
+      superUser,
+      superPassword,
+      authenticatorUser,
+      authenticatorPassword,
+      replicationUser,
+      replicationPassword,
+    };
+  } catch {
+    return null;
+  }
+}
+
+export function connectToPatroni(namespace: string) {
+  const cmd = kubectlCommand(
+    `exec -it ${namespace}-0 -c patroni -- /bin/bash`,
+    { namespace }
+  );
+  new CommandExecutor(cmd).spawn();
+}
+
+export function connectToPsql(namespace: string) {
+  const cmd = kubectlCommand(
+    `exec -it ${namespace}-0 -c postgres-util -- psql`,
+    { namespace }
+  );
+  new CommandExecutor(cmd).spawn();
+}
+
+export function establishTunnel(namespace: string, port: string) {
+  const cmd = kubectlCommand(`port-forward pod/${namespace}-0 ${port}:5432`, {
+    namespace,
+  });
+  new CommandExecutor(cmd).spawn();
+}

package/src/libs/k8s-generate.ts

@@ -0,0 +1,211 @@
+import type { PackageData, TemplateDbMigrateObject, TemplateDebugObject, TemplateDeploymentObject, TemplateSharedContext } from "../types";
+import { BASE_SECRET_KEY } from "./k8s-namespace";
+import { containerRegistryRepoPath, dbMigrateJobName, domainNameForEnv, envToNamespace, imageDebugName, MISSING_DOMAIN_KEY_ERROR, secretName } from "./k8s-constants";
+import { getWorkspaceScale } from "./k8s-image-config";
+import path from "path";
+import yaml from "yaml";
+import fs from 'fs';
+import { globSync } from "glob";
+import _, { template } from 'lodash';
+import Handlebars from "handlebars";
+import { getImageData } from "./config";
+import { getImageDescendentData } from "./discovery/images";
+
+const MANIFEST_FOLDER_PATH = path.join(process.cwd(), '.devops/manifests');
+const MANIFEST_INDEX_FILE_PATH = path.join(MANIFEST_FOLDER_PATH, '_index.yaml');
+const DB_MIGRATE_TEMPLATE_NAME = 'db-migrate';
+
+type RenderFn = (template: string) => string;
+
+// = Interface
+
+export function generateImageDeployments(
+  monorepoEnv: string,
+  image: string,
+  gitSha: string
+) {
+  const generator = new ImageContextGenerator(monorepoEnv, image, gitSha);
+  const apps = getImageDescendentData(image)
+    .filter((packageData) => packageData.deployment)
+    .flatMap((projectData) => {
+      const context = generator.getDeployment(projectData);
+      const renderFn = (template: string) => Handlebars.compile(template)(context);
+      return generateManifestForDeployment(projectData.rootPath, projectData.deployment!.template, renderFn);
+    });
+  const debug = generateDebugDeployment(monorepoEnv, image, gitSha);
+  const manifest = [debug, ...apps].join("\n---\n");
+  return ensureProperDomainsPresent(manifest, monorepoEnv, image);
+}
+
+export function generateWorkspaceDeployment(packageData: PackageData, monorepoEnv: string, image: string, gitSha: string) {
+  const generator = new ImageContextGenerator(monorepoEnv, image, gitSha);
+  const context = generator.getDeployment(packageData);
+  const renderFn = (template: string) => Handlebars.compile(template)(context);
+  const manifest = generateManifestForDeployment(packageData.rootPath, packageData.deployment!.template, renderFn).join("\n---\n");
+  return ensureProperDomainsPresent(manifest, monorepoEnv, image);
+}
+
+export function generateDebugDeployment(
+  monorepoEnv: string,
+  image: string,
+  gitSha: string
+) {
+  const generator = new ImageContextGenerator(monorepoEnv, image, gitSha);
+  const context = generator.getDebug();
+  const renderFn = (template: string) => Handlebars.compile(template)(context);
+  const debugTemplate = getImageData(image)["debug-template"];
+  return generateManifestsFromTemplateName(debugTemplate, renderFn).map(x => yaml.stringify(x)).join("\n---\n");
+}
+
+export function generateDbMigrateJob(
+  monorepoEnv: string,
+  image: string,
+  gitSha: string
+) {
+  const generator = new ImageContextGenerator(monorepoEnv, image, gitSha);
+  const context = generator.getDbMigrate();
+  const renderFn = (template: string) => Handlebars.compile(template)(context);
+  return generateManifestsFromTemplateName(DB_MIGRATE_TEMPLATE_NAME, renderFn).map(x => yaml.stringify(x)).join("\n---\n");
+}
+
+// = Verification
+
+function ensureProperDomainsPresent(manifest: string, monorepoEnv: string, image: string) {
+  if (manifest.includes(MISSING_DOMAIN_KEY_ERROR)) {
+    console.error(`The image ${image} does not have a domain defined for the environment ${monorepoEnv}. Please add it to the .devops/config/images.yaml.`);
+    process.exit(1);
+  }
+  return manifest;
+}
+
+// = Generation logic
+
+// L1 generation: composite
+
+function generateManifestForDeployment(rootPath: string, templateName: string, renderFn: RenderFn): string[] {
+  const defaults = generateManifestsFromTemplateName(templateName, renderFn);
+  const overrides = generateManifestFromFilesInFolder(rootPath, renderFn);
+  const keyExtractor = (manifest: { kind: string, metadata: { name: string } }) => `${manifest.kind}-${manifest.metadata.name}`;
+  const defaultTemplateLookup = _.keyBy(defaults, keyExtractor);
+  const overrideTemplateLookup = _.keyBy(overrides, keyExtractor);
+  const mergedTemplates = _.merge(defaultTemplateLookup, overrideTemplateLookup);
+  return Object.values(mergedTemplates).map(x => yaml.stringify(x));
+}
+
+// L2 generation
+
+function generateManifestsFromTemplateName(templateName: string, renderFn: RenderFn): object[] {
+  const entries = manifestFilesForTemplate(templateName);
+  if (!entries) {
+    console.error(`No entries found for ${templateName} in ${MANIFEST_INDEX_FILE_PATH}`);
+    process.exit(1);
+  }
+  return generateManifestsFromFileList(entries.map(entry => path.join(MANIFEST_FOLDER_PATH, entry)), renderFn);  
+}
+
+// L2 generation
+
+function generateManifestFromFilesInFolder(folderPath: string, renderFn: RenderFn): object[] {
+  const manifestOverridePath = path.join(folderPath, 'manifests');
+  if (!fs.existsSync(manifestOverridePath)) {
+    return []
+  }
+  const files = globSync(path.join(manifestOverridePath, '**/*')).filter(x => fs.lstatSync(x).isFile());
+  return generateManifestsFromFileList(files, renderFn);
+}
+
+// L3 generation
+
+function generateManifestsFromFileList(filesList: string[], renderFn: RenderFn): object[] {
+  return filesList.flatMap((filePath) => {
+    try {
+      const manifestFileStr = fs.readFileSync(filePath, 'utf8');
+      const renderedStr = renderFn(manifestFileStr);
+      const res = yaml.parseAllDocuments(renderedStr);
+      res.forEach((doc) => {
+        if (!doc.get("kind") || !doc.getIn(["metadata", "name"])) {
+          console.error(`Invalid manifest file ${filePath}: kind and metadata.name must be present`);
+          console.error(doc.toString())
+          process.exit(1);
+        }          
+      })
+      return res.map(x => x.toJSON());
+    } catch (e) {
+      if (e instanceof Error) {
+        console.error(`Could not parse ${filePath}: ${e.message}`);
+      } else {
+        console.error(`Could not parse ${filePath}`);
+      }
+      process.exit(1);
+    }
+  })
+}
+
+// Infra: the template index file
+
+let _manifestIndex: Record<string, string[]>;
+function manifestFilesForTemplate(template: string) {
+  if (!_manifestIndex) {
+    try {
+      const indexFileStr = fs.readFileSync(MANIFEST_INDEX_FILE_PATH, 'utf8');
+      _manifestIndex = yaml.parse(indexFileStr)
+    } catch {
+      console.error(`Unable to process ${MANIFEST_INDEX_FILE_PATH}`);
+      process.exit(1)
+    }
+  }
+  return _manifestIndex[template];
+}
+
+// Infra: the image context generator
+
+export class ImageContextGenerator {
+  replicaMap: Record<string, number>;
+  imageContext: Omit<TemplateSharedContext, 'project_name'>;
+
+  constructor(public monorepoEnv: string, public image: string, public gitSha: string) {
+    this.replicaMap = getWorkspaceScale(monorepoEnv, image);
+    this.imageContext = {
+      monorepo_env: monorepoEnv,
+      namespace: envToNamespace(monorepoEnv),
+      env_secret_name: secretName(),
+      env_base_secret_key: BASE_SECRET_KEY,
+      domain_name: domainNameForEnv(image, monorepoEnv),
+      image_path: containerRegistryRepoPath(image, monorepoEnv, gitSha),
+    }
+  }
+
+  getDeployment(pkgData: PackageData): TemplateDeploymentObject {
+    if (!pkgData.deployment) {
+      console.error(`The deployment key is missing for workspace ${pkgData.name}`);
+      process.exit(1);
+    }
+    return {
+      // Basic context
+      project_name: pkgData.name,
+      ...this.imageContext,
+      // Defaults that can be overriden by pkgData.deployment
+      app_name: pkgData.name,
+      subdomain: pkgData.deployment.service_name,
+      // This may override the defaults above
+      ...pkgData.deployment,
+      // Override from config map
+      replicas: this.replicaMap[pkgData.name] ?? 1,
+    }
+  }
+
+  getDbMigrate(): TemplateDbMigrateObject {
+    return {
+      ...this.imageContext,
+      db_migrate_job_name: dbMigrateJobName(this.gitSha),
+    }
+  }
+
+  getDebug(): TemplateDebugObject {
+    return {
+      ...this.imageContext,
+      debug_pod_name: imageDebugName(this.image)
+    }
+  }
+}
+

package/src/libs/k8s-helpers.ts

@@ -0,0 +1,59 @@
+import { CommandExecutor } from "../cli/common";
+import { envToNamespace } from "./k8s-constants";
+import fs from "fs";
+
+const TMP_MANIFEST_FOLDER = "tmp/k8s-manifests";
+
+export function kubectlCommand(
+  cmd: string,
+  opts: { monorepoEnv?: string; namespace?: string } = {}
+) {
+  if (opts.monorepoEnv || opts.namespace) {
+    const namespace = opts.namespace ?? envToNamespace(opts.monorepoEnv);
+    return `kubectl -n ${namespace} ${cmd}`;
+  } else {
+    return `kubectl ${cmd}`;
+  }
+}
+
+export function upsertConfigMapCommand(monorepoEnv: string, configMapName: string, data: Record<string, string>) {
+  const manifest = { 
+    apiVersion: 'v1',
+    kind: 'ConfigMap',
+    metadata: {
+      name: configMapName,
+      namespace: envToNamespace(monorepoEnv),
+      labels: {
+        env: monorepoEnv
+      }
+    },
+    data
+  }
+  const applyCmd = kubectlCommand(`apply -f -`, { monorepoEnv });
+  return `echo '${JSON.stringify(manifest)}' | ${applyCmd}`;
+}
+
+export function patchSecretKeyCommand(monorepoEnv: string, secretName: string, secretKey: string, secretValue: string) {
+  const redactedCommand = kubectlCommand(
+    `patch secret ${secretName} -p='{"stringData": {"${secretKey}": **REDACTED**}}'`,
+    { monorepoEnv }
+  );
+  const fullCommand = redactedCommand.replace(
+    "**REDACTED**",
+    JSON.stringify(secretValue)
+  );
+  return { fullCommand, redactedCommand };
+}
+
+export function applyHandler(
+  filePrefix: string,
+  command: "apply" | "delete",
+  manifest: string
+) {
+  if (!fs.existsSync(TMP_MANIFEST_FOLDER)) {
+    fs.mkdirSync(TMP_MANIFEST_FOLDER, { recursive: true });
+  }
+  const path = `${TMP_MANIFEST_FOLDER}/${filePrefix}-${Date.now()}.yaml`;
+  fs.writeFileSync(path, manifest);
+  new CommandExecutor(kubectlCommand(`${command} -f ${path}`)).exec();
+}