@vaharoni/devops 1.0.47

package/src/devops.ts

@@ -0,0 +1,119 @@
+#!/usr/bin/env bun
+// This file behaves as a façade for various devops scripts
+import { CLICommandParser, printUsageAndExit } from "./cli/common";
+import affected from "./cli/affected";
+import constant from "./cli/constant";
+import console from "./cli/console";
+import db from "./cli/db";
+import dml from "./cli/dml";
+import registry from "./cli/registry";
+import env from "./cli/env";
+import exec from "./cli/exec";
+import prepBuild from "./cli/prep-build";
+import prisma from "./cli/prisma";
+import run from "./cli/run";
+import runMany from "./cli/run-many";
+import test from "./cli/test";
+import init from "./cli/init";
+import redis from "./cli/redis";
+import internalCurl from "./cli/internal-curl";
+import jwt from "./cli/jwt";
+import namespace from "./cli/namespace";
+import image from "./cli/image";
+import template from "./cli/template";
+import job from "./cli/job";
+
+const [_node, _scriptPath, ...commandArgs] = process.argv;
+
+const allImports = [
+  // day-to-day
+  init,
+  run,
+  runMany,
+  exec,
+  env,
+  prisma,
+  dml,
+  db,
+  redis,
+  console,
+  test,
+  //= Infra
+  namespace,
+  image,
+  template,
+  job,
+  //= Deployment
+  prepBuild,
+  affected,
+  constant,
+  registry,
+  internalCurl,
+  jwt
+];
+
+const commands: {
+  [key: string]: {
+    oneLiner: string;
+    keyExamples: string;
+    run: CallableFunction;
+    key: string;
+  };
+} = {};
+allImports.forEach((imported) => {
+  Object.entries(imported).forEach(([key, object]) => {
+    const { oneLiner, keyExamples, run } = object;
+    commands[key] = { oneLiner, keyExamples, run, key };
+  });
+});
+
+const keyLength = Math.max(...Object.keys(commands).map((x) => x.length)) + 10;
+const newLine = "\n    ";
+
+const GENERAL_USAGE = `
+Devops utilities for the monorepo.
+
+USAGE
+    devops <command> <args> <env-options>
+
+CHOOSING ENV with <env-options>
+    By default, all commands run under the env specified in MONOREPO_ENV env variable, or development if it does not exist.
+    The test command is an exception: its environment is forced to test. 
+    When running in development, the env files config/.env.development and config/.env.global are injected, where the 
+    former overrides the latter.
+
+    To override the environment in which a command is executed in, use --env <some-env>. This overrides the MONOREPO_ENV variable.
+    You can use it anywhere in the command, i.e. the following are equivalent:
+    $ devops --env staging run project:task
+    $ devops run project:task --env staging
+
+    Supported environments: development, staging, test, and production.
+
+    Certain commands like run, exec, and prisma ensure that the env variables exist and correspond to env.yaml files in the repo. 
+    To skip this check, use --skip-env-check.
+
+
+COMMANDS
+    ${Object.values(commands)
+      .map((cmd) =>
+        [cmd.key, " ".repeat(keyLength - cmd.key.length), cmd.oneLiner].join("")
+      )
+      .join(newLine)}
+`;
+
+// EXAMPLES
+//     ${Object.values(commands)
+//       .map((cmd) =>
+//         cmd.keyExamples
+//           .split("\n")
+//           .map((x) => x.trim())
+//           .filter(Boolean)
+//           .join(newLine)
+//       )
+//       .join(newLine)}
+
+const commandObj = new CLICommandParser(commandArgs,);
+const chosenCommand = commands[commandObj.command];
+if (!chosenCommand) printUsageAndExit(GENERAL_USAGE);
+
+chosenCommand.run(commandObj);

package/src/index.ts

@@ -0,0 +1,3 @@
+export * from "./types/index";
+export * from "./app-support/discovery/service-endpoint";
+export * from "./app-support/crypto";

package/src/libs/affected-entities.ts

@@ -0,0 +1,71 @@
+import { getImageNames } from "./config";
+import { getWorkspace } from "./discovery";
+import { getImageDescendentData, getWorkspaceImages } from "./discovery/images";
+import { commitExists, isAffected } from "./git-helpers";
+import { getImageVersion } from "./k8s-image-config";
+
+type AffectedOpts = {
+  baseSha?: string;
+  headSha?: string;
+  fromLiveVersion?: boolean;
+  monorepoEnv?: string;
+};
+
+export function isImageAffected(image: string, opts: AffectedOpts = {}) {
+  const headSha = opts.headSha ?? "HEAD";
+  let baseSha;
+  if (opts.fromLiveVersion) {
+    if (!opts.monorepoEnv) {
+      throw new Error("monorepoEnv is required when fromLiveVersion is true");
+    }
+    baseSha = getImageVersion(opts.monorepoEnv, image);
+    if (!baseSha) return true;
+  }
+  baseSha ??= opts.baseSha ?? "HEAD^";
+  const descendentData = getImageDescendentData(image);
+  if (!commitExists(baseSha) || !commitExists(headSha)) return true;
+
+  for (const { rootPath } of descendentData) {
+    if (isAffected(rootPath, { baseSha, headSha, skipCheck: true })) {
+      return true;
+    }
+  }
+  return false;
+}
+
+export function findImagesAffected(opts: AffectedOpts = {}) {
+  return getImageNames().filter((imageName) =>
+    isImageAffected(imageName, opts)
+  );
+}
+
+export function isWorkspaceAffected(
+  workspaceName: string,
+  opts: { baseSha?: string; headSha?: string } = {}
+) {
+  const data = getWorkspace(workspaceName);
+  return isAffected(data.rootPath, opts);
+}
+
+export function findImagesWithAffectedWorkspace(
+  workspaceName: string,
+  opts: AffectedOpts = {}
+) {
+  const res = []
+  const headSha = opts.headSha ?? "HEAD";
+  const defaultBaseSha = opts.baseSha ?? "HEAD^";
+  const rootPath = getWorkspace(workspaceName).rootPath;
+  if (opts.fromLiveVersion && !opts.monorepoEnv) {
+    throw new Error("monorepoEnv is required when fromLiveVersion is true");
+  }
+  for (const imageName of getWorkspaceImages(workspaceName)) {
+    const baseSha = opts.fromLiveVersion
+      ? getImageVersion(opts.monorepoEnv!, imageName)
+      : defaultBaseSha;
+
+    if (isAffected(rootPath, { baseSha, headSha })) {
+      res.push(imageName);
+    }
+  }
+  return res;
+}

package/src/libs/config.ts

@@ -0,0 +1,117 @@
+import { readFileSync } from "fs";
+import yaml from "yaml";
+import path from "path";
+import { getWorkspace } from "./discovery";
+import { constFileSchema, imageFileSchema, type ConstFileSchema, type ImageFileSchema, type SingleImageSchema, type SingleTemplateSchema } from "../types";
+
+const constantsFilePath = path.join(process.cwd(), ".devops/config/constants.yaml");
+const imagesFilePath = path.join(process.cwd(), ".devops/config/images.yaml");
+
+// We want these to be lazy loaded so that calling devops in a context that does not need the config files won't fail
+export const { getConst } = processConstFile();
+export const { getImageData, getImageNames, getTemplateData } = processImagesFile();
+
+// Process config/constants.yaml
+
+function processConstFile() {
+  let constants: { valid: boolean, data?: ConstFileSchema };
+  function constFileData() {
+    if (constants) return constants;
+    try {
+      const constantsYaml = readFileSync(constantsFilePath, "utf8");
+      constants = yaml.parse(constantsYaml);
+    } catch (e) {
+      // This is only a warning - the file may not exist, which is fine if getConst is called with ignoreIfInvalid
+      console.warn("Warning: cannot read .devops/config/constants.yaml");
+      return { valid: false };
+    }
+    const parseRes = constFileSchema.safeParse(constants);
+    if (parseRes.error) {
+      // This is an error - if the file exists, it must be valid
+      console.error(`Error parsing config/constants.yaml: ${parseRes.error.toString()}`);
+      process.exit(1);
+    }
+    constants = { valid: true, data: parseRes.data };
+    return constants;
+  }
+
+  function getConst<T extends keyof ConstFileSchema>(key: T, opts: { ignoreIfInvalid?: boolean } = {}): ConstFileSchema[T] | undefined {
+    const { valid, data } = constFileData();
+    if (!valid && !opts.ignoreIfInvalid) { 
+      console.error(".devops/config/constants.yaml is invalid");
+      process.exit(1);
+    }
+    const value = data?.[key];
+    if (!value && !opts.ignoreIfInvalid) {
+      console.error(`Missing constant in .devops/config/constants.yaml: ${key}`);
+      process.exit(1);
+    }
+    return value;
+  }
+
+  return { getConst };
+}
+
+// Process config/images.yaml
+
+function processImagesFile() {
+  let images: ImageFileSchema;
+  function imagesFileData() {
+    if (images) return images;
+    try {
+      const imagesYaml = readFileSync(imagesFilePath, "utf8");
+      images = yaml.parse(imagesYaml);
+    } catch (e) {
+      console.error("Error reading .devops/config/images.yaml");
+      process.exit(1);
+    }
+    const parseRes = imageFileSchema.safeParse(images);
+    if (parseRes.error) {
+      console.error(
+        `Error parsing config/images.yaml: ${parseRes.error.toString()}`
+      );
+      process.exit(1);
+    }
+    return images;
+  }
+
+  function getImageData(imageName: string): SingleImageSchema {
+    const imageData = imagesFileData()['images'][imageName];
+    if (!imageData) {
+      console.error(
+        `Image ${imageName} not found in .devops/config/images.yaml`
+      );
+      process.exit(1);
+    }
+
+    imageData.applications.forEach((project: string) => {
+      const data = getWorkspace(project);
+      if (!data) {
+        console.error(
+          `Project ${project} not found for image ${imageName} in .devops/config/images.yaml`
+        );
+        process.exit(1);
+      }
+    });
+
+    return imageData;
+  }
+
+  function getTemplateData(templateName: string): SingleTemplateSchema {
+    const templateData = imagesFileData()['templates'][templateName];
+    if (!templateData) {
+      console.error(
+        `Template ${templateName} not found in .devops/config/images.yaml`
+      );
+      process.exit(1);
+    }
+
+    return templateData;
+  }
+
+  function getImageNames() {
+    return Object.keys(imagesFileData()["images"]);
+  }
+
+  return { getImageData, getImageNames, getTemplateData };
+}

package/src/libs/digital-ocean/container-reg.ts

@@ -0,0 +1,81 @@
+import { CommandExecutor } from "../../cli/common";
+import { z } from "zod";
+import { getConst } from "../config";
+
+const repoTagMetadataSchema = z.object({
+  // What we rely on
+  tag: z.string().optional(),
+  updated_at: z.string(),
+  manifest_digest: z.string(),
+  // Other fields that existed in the output
+  // registry_name: z.string().optional(),
+  // repository: z.string().optional(),
+  // compressed_size_bytes: z.number().optional(),
+  // size_bytes: z.number().optional(),
+});
+const repoTagMetadataSchemaOutput = z.array(repoTagMetadataSchema);
+
+type RepoTagMetadata = z.infer<typeof repoTagMetadataSchema>;
+
+/** The metadata is returned in descending order (most recent first) */
+function getRepoTagMetadata(repoName: string): RepoTagMetadata[] {
+  // Get the metadata for the tags in the repository
+  const cmd = `doctl registry repository list-tags ${repoName} -o json`;
+  const res = new CommandExecutor(cmd, { quiet: true }).exec();
+  if (!res) return [];
+  try {
+    const parsed = JSON.parse(res);
+    const parseRes = repoTagMetadataSchemaOutput.safeParse(parsed);
+    if (parseRes.error) {
+      console.error(
+        `Error schema-parsing output from "${cmd}": ${parseRes.error.toString()}`
+      );
+      console.error(">>> Command output");
+      console.error(res);
+      process.exit(1);
+    }
+    return parseRes.data
+      .filter((data) => data.tag)
+      .sort(
+        (a, b) =>
+          new Date(b.updated_at).getTime() - new Date(a.updated_at).getTime()
+      );
+  } catch (e) {
+    console.error(`Error JSON-parsing output from "${cmd}": ${res}`);
+    process.exit(1);
+  }
+}
+
+function deleteRepoTag(repoName: string, tag: string) {
+  const cmd = `doctl registry repository delete-tag ${repoName} ${tag} --force`;
+  new CommandExecutor(cmd).exec();
+}
+
+function stargGarbageCollection(registryName: string) {
+  const cmd = `doctl registry garbage-collection start --include-untagged-manifests ${registryName} --force`;
+  new CommandExecutor(cmd).exec();
+}
+
+export function prune(
+  /** To keep the image-related constants simple, this accepts the full URL including the prefix registry.digitalocean.com */
+  registryFullName: string,
+  /** The name of the repository inside the registry */
+  repoName: string
+) {
+  const infra = getConst("infra");
+  if (infra !== "digitalocean") {
+    console.warn(
+      "Pruning is only supported for the DigitalOcean container registry"
+    );
+    return;
+  }
+  const tags = getRepoTagMetadata(repoName);
+  const versionsToKeep = Number(getConst("image-versions-to-keep"));
+  if (!tags.length || tags.length <= versionsToKeep) return;
+  const tagsToDelete = tags.slice(versionsToKeep);
+  tagsToDelete.forEach((tag) => {
+    deleteRepoTag(repoName, tag.tag!);
+  });
+  const registryName = registryFullName.split("/").slice(-1)[0];
+  stargGarbageCollection(registryName);
+}

package/src/libs/discovery/dependencies.spec.ts

@@ -0,0 +1,25 @@
+
+import { describe, it, expect } from 'vitest';
+import { WorkspaceDependencies } from './dependencies';
+
+describe('createDependencyResolver', () => {
+  const projects = {
+    '@local/a': { language: "node" as const, rootPath: 'applications/a', name: '@local/a', dependencyNames: ['@local/b', '@local/d', 'external1'] },
+    '@local/b': { language: "node" as const, rootPath: 'applications/b', name: '@local/b', dependencyNames: ['@local/c', 'external2'] },
+    '@local/c': { language: "node" as const, rootPath: 'applications/c', name: '@local/c', dependencyNames: [ 'external3' ] },
+    '@local/d': { language: "node" as const, rootPath: 'applications/d', name: '@local/d', dependencyNames: [] },
+    '@local/e': { language: "node" as const, rootPath: 'applications/e', name: '@local/e', dependencyNames: ['@local/f'] },
+    '@local/f': { language: "node" as const, rootPath: 'applications/f', name: '@local/f', dependencyNames: ['@local/e'] } 
+  };
+
+  it('should return dependents of a given project', () => {
+    const resolver = new WorkspaceDependencies(() => projects);
+
+    expect(resolver.getDependents('@local/a')).toEqual(['@local/a', '@local/b', '@local/c', '@local/d']);
+    expect(resolver.getDependents('@local/b')).toEqual(['@local/b', '@local/c']);
+    expect(resolver.getDependents('@local/c')).toEqual(['@local/c']);
+    expect(resolver.getDependents('@local/d')).toEqual(['@local/d']);
+    expect(resolver.getDependents('@local/e')).toEqual(['@local/e', '@local/f']);
+    expect(resolver.getDependents('@local/f')).toEqual(['@local/f', '@local/e']);
+  });
+});

package/src/libs/discovery/dependencies.ts

@@ -0,0 +1,73 @@
+import chalk from "chalk";
+import type { PackageData } from "../../types";
+
+export class WorkspaceDependencies {
+  dependencies: Record<string, DependencyNode> = {};
+  loaded = false;
+  workspaces: Record<string, PackageData> = {};
+
+  constructor(public getAllProjects: () => Record<string, PackageData>) {}
+
+  _getOrCreate(name: string) {
+    let node = this.dependencies[name];
+    if (!node) {
+      node = new DependencyNode(name);
+      this.dependencies[name] = node;
+    }
+    return node;
+  }
+
+  _buildTree() {
+    this.workspaces = this.getAllProjects();
+    this.loaded = true;
+    for (const workspace of Object.keys(this.workspaces)) {
+      const node = this._getOrCreate(workspace);
+      const data = this.workspaces[workspace];
+      for (const dep of data.dependencyNames ?? []) {
+        if (this.workspaces[dep]) {
+          node.dependsOn.add(dep);
+        }
+      }
+    }
+  }
+
+  getDependents(workspaceName: string) {
+    if (!this.loaded) {
+      this._buildTree();
+    }
+    const node = this.dependencies[workspaceName];
+    if (!node) {
+      console.error(chalk.red(`\nWorkspace ${workspaceName} not found\n`));
+      process.exit(1);
+    }
+    return node.flattenDependents(this.dependencies);
+  }
+}
+
+class DependencyNode {
+  name: string;
+  dependsOn: Set<string>;
+
+  constructor(name: string) {
+    this.name = name;
+    this.dependsOn = new Set();
+  }
+
+  flattenDependents(
+    allDependencies: Record<string, DependencyNode>,
+    visited?: Set<string>
+  ) {
+    visited ??= new Set();
+    visited.add(this.name);
+    const notVisitedDependents: string[] = [];
+    for (const dep of this.dependsOn) {
+      if (!visited.has(dep)) {
+        const node = allDependencies[dep];
+        notVisitedDependents.push(
+          ...node.flattenDependents(allDependencies, visited)
+        );
+      }
+    }
+    return [this.name, ...notVisitedDependents];
+  }
+}

package/src/libs/discovery/images.ts

@@ -0,0 +1,57 @@
+import { getWorkspace, workspaceDirectoryForLanguage } from ".";
+import type { PackageData } from "../../types";
+import { getImageData, getImageNames } from "../config";
+import { WorkspaceDependencies } from "./dependencies";
+
+// = From images to workspaces
+
+const _imageDescendents: Record<string, PackageData[]> = {};
+let _imageDescendentsLoaded = false;
+
+function imageDescendents() {
+  if (!_imageDescendentsLoaded) {
+    for (const imageName of getImageNames()) {
+      const descendents = new Set<string>();
+      const imageData = getImageData(imageName);
+      const workspaces = workspaceDirectoryForLanguage(imageData.language);
+      const dependencyResolver = new WorkspaceDependencies(() => workspaces)
+      imageData.applications.forEach((workspace) => {
+        dependencyResolver.getDependents(workspace).forEach((name) => descendents.add(name));
+      });
+      _imageDescendents[imageName] = Array.from(descendents).map(name => workspaces[name]);
+    }
+    _imageDescendentsLoaded = true;
+  }
+  return _imageDescendents;
+}
+
+/** The dependent workspaces are specified in config/images.yaml */
+export function getImageDescendentData(imageName: string) {
+  return (
+    imageDescendents()[imageName] ?? []
+  );
+}
+
+// = From workspace to images
+
+const _workspaceImages: Record<string, string[]> = {};
+let _workspaceImagesLoaded = false;
+
+function workspaceImages() {
+  if (!_workspaceImagesLoaded) {
+    for (const [imageName, descendents] of Object.entries(imageDescendents())) {
+      for (const packageData of descendents) {
+        _workspaceImages[packageData.name] ??= [];
+        _workspaceImages[packageData.name].push(imageName);
+      }
+    }
+    _workspaceImagesLoaded = true;
+  }
+  return _workspaceImages;
+}
+
+
+export function getWorkspaceImages(workspaceName: string) {
+  const _verifyPresence = getWorkspace(workspaceName);
+  return workspaceImages()[workspaceName] ?? [];
+}

package/src/libs/discovery/index.ts

@@ -0,0 +1,60 @@
+import chalk from "chalk";
+import { nodeWorkspaces } from "./process-package-json";
+import { pythonWorkspaces } from "./process-pyproject-toml";
+import type { SupportedLanguages, WorkspaceIndex } from "../../types";
+
+const _workspaces: WorkspaceIndex = {};
+let _workspacesLoaded = false;
+
+export function workspaces() {
+  if (_workspacesLoaded) return _workspaces;
+  const nodeWorkspacesData = nodeWorkspaces();
+  const pythonWorkspacesData = pythonWorkspaces();
+
+  Object.values(nodeWorkspacesData).forEach(data => {
+    _workspaces[data.name] = {
+      rootPath: data.rootPath,
+      packageDataEntries: [data]
+    };
+  })
+
+  Object.values(pythonWorkspacesData).forEach(data => {
+    const existing = _workspaces[data.name];
+    if (existing) {
+      if (existing.rootPath !== data.rootPath) {
+        // prettier-ignore
+        console.error(chalk.red(`\nWorkspace ${data.name} has conflicting root paths:\n\t${existing.rootPath}\n\t${data.rootPath}\n`));
+        process.exit(1);
+      }
+    } else {
+      _workspaces[data.name] = {
+        rootPath: data.rootPath,
+        packageDataEntries: []
+      };
+    }
+    _workspaces[data.name].packageDataEntries.push(data);
+  })
+  _workspacesLoaded = true;
+  return _workspaces;
+}
+
+export function workspaceDirectoryForLanguage(language: SupportedLanguages) {
+  switch (language) {
+    case "node":
+      return nodeWorkspaces();
+    case "python":
+      return pythonWorkspaces();
+    default:
+      throw new Error(`Unsupported language: ${language}`);
+  }
+}
+
+export function getWorkspace(workspaceName: string) {
+  const workspace = workspaces()[workspaceName];
+  if (!workspace) {
+    console.error(chalk.red(`\nWorkspace ${workspaceName} not found\n`));
+    // The gha relies on the 13 exit code for "not found"
+    process.exit(13);
+  }
+  return workspace;
+}

package/src/libs/discovery/process-common.ts

@@ -0,0 +1,55 @@
+import type { ZodSchema } from "zod";
+import type { PackageData, SupportedLanguages } from "../../types";
+import fs from "fs";
+import path from "path"
+
+export const IGNORED_PATHS = ["node_modules/", "venv/"];
+
+type PackageDataProcessorConfig<T> = {
+  language: SupportedLanguages;
+  pathList: string[];
+  zodSchema: ZodSchema<T>;
+  fileParser: (fileStringData: string) => unknown;
+  nameExtractor: (data: T) => string;
+}
+
+export class PackageDataProcessor<T> {
+  workspaceNames: Set<string> = new Set();
+  loadedFiles: Record<string, { name: string, data: T }> = {};
+  language: SupportedLanguages;
+  nameExtractor: (data: T) => string;
+
+  constructor(config: PackageDataProcessorConfig<T>) {
+    this.language = config.language;
+    this.nameExtractor = config.nameExtractor;
+    this._runFirstPass(config.pathList, config.zodSchema, config.fileParser);
+  }
+
+  _runFirstPass(pathList: string[], zodSchema: ZodSchema<T>, fileParser: (fileStringData: string) => unknown) {
+    pathList.forEach((packageFilePath) => {
+      if (IGNORED_PATHS.find(path => packageFilePath.includes(path))) return;
+      const unsafeData = fileParser(fs.readFileSync(packageFilePath, "utf8"));
+      const parsedRes = zodSchema.safeParse(unsafeData);
+      if (parsedRes.error) {
+        console.error(`Error parsing ${packageFilePath}: ${parsedRes.error}`);
+        process.exit(1);
+      }
+      const rootDir = path.dirname(packageFilePath);
+      const name = this.nameExtractor(parsedRes.data)
+      this.workspaceNames.add(name);
+      this.loadedFiles[rootDir] = { name, data: parsedRes.data };
+    });
+  }
+
+  // Should be used by the iterator sent to convert in order to filter out dependencies that are not workspaces
+  filterDependencies(dependencyNames: string[]) {
+    return dependencyNames.filter((name) => this.workspaceNames.has(name));
+  }
+
+  convert(iterator: (data: T) => Omit<PackageData, 'rootPath' | 'language' | 'name'>): PackageData[] {
+    return Object.entries(this.loadedFiles).map(([rootPath, { name, data }]) => {
+      const res = iterator(data);
+      return { rootPath, language: this.language, name, ...res };
+    })
+  }
+}