@vaharoni/devops 1.0.47

package/src/target-templates/db/db/db_client_test.py

@@ -0,0 +1,46 @@
+import pytest_asyncio
+from prisma import Prisma
+from contextlib import asynccontextmanager
+
+"""Usage:
+    Add the file tests/conftest.py with the following content:
+
+    from db.db_client_test import *
+"""
+
+# To open ipython at a particular point, do this:
+#
+# Add imports:
+#   from IPython import embed
+#   import nest_asyncio
+#
+# Insert this to the right position in the code:
+#   nest_asyncio.apply() 
+#   embed()
+
+class RollbackTransaction(Exception): pass
+
+class TransactionProxy:
+    def __init__(self, transaction):
+        self._transaction = transaction
+    
+    def __getattr__(self, name):
+        return getattr(self._transaction, name)
+
+    @asynccontextmanager
+    async def tx(self):
+        yield self._transaction
+
+@pytest_asyncio.fixture(scope='function')
+async def db_connection():
+    db = Prisma()
+    await db.connect()
+    try:
+        async with db.tx() as transaction:
+            tx = TransactionProxy(transaction)
+            yield tx
+            raise RollbackTransaction('rollback')
+    except RollbackTransaction:
+        pass
+    finally:
+        await db.disconnect()

package/src/target-templates/db/db-client-test.ts

@@ -0,0 +1,140 @@
+// This code is used in db/prisma-setup-vitest.ts to run test code in the context of a rollbacked transaction.
+// It is an improvement of:
+//    https://github.com/prisma/prisma-client-extensions/tree/main/callback-free-itx
+//
+// The main idea is to return a single client object that knows whether a transaction is active.
+// If a transaction is active, it routes all calls to that transaction client.
+// If no transaction is active, it routes all calls to the original client.
+// To activate a transaction, use $begin().
+//
+// See:
+//    https://github.com/prisma/prisma/issues/12458
+//    https://github.com/prisma/prisma-client-extensions/pull/52
+//    https://www.prisma.io/docs/orm/more/help-and-troubleshooting/help-articles/nextjs-prisma-client-dev-practices
+//
+import { Prisma, PrismaClient } from '@prisma/client';
+
+const ROLLBACK = { [Symbol.for('prisma.client.extension.rollback')]: true };
+
+export const prismaClientSingleton = () => {
+  let _txClient: FlatTransactionClient | null = null;
+  const prismaClientOptions = process.env['PRISMA_DEBUG']
+    ? { log: ['query' as const] }
+    : undefined;
+
+  const extendedClient = new PrismaClient(prismaClientOptions).$extends({
+    client: {
+      async $begin() {
+        const prisma = Prisma.getExtensionContext(this);
+        let setTxClient: (txClient: Prisma.TransactionClient) => void;
+        let commit: () => void;
+        let rollback: () => void;
+
+        // a promise for getting the tx inner client
+        const txClient = new Promise<Prisma.TransactionClient>((res) => {
+          setTxClient = (txClient) => res(txClient);
+        });
+
+        // a promise for controlling the transaction
+        const txPromise = new Promise((_res, _rej) => {
+          commit = () => _res(undefined);
+          rollback = () => _rej(ROLLBACK);
+        });
+
+        // opening a transaction to control externally
+        if (
+          '$transaction' in prisma &&
+          typeof prisma.$transaction === 'function'
+        ) {
+          // [AA] Note the change from the github repo to actually make rollbacks work
+          const tx = prisma
+            .$transaction((txClient: unknown) => {
+              setTxClient(txClient as unknown as Prisma.TransactionClient);
+              return txPromise;
+            })
+            .catch((e: unknown) => {
+              if (e === ROLLBACK) return;
+              throw e;
+            });
+
+          // return a proxy TransactionClient with `$commit` and `$rollback` methods
+          _txClient = new Proxy(await txClient, {
+            get(target, prop) {
+              if (prop === '$commit') {
+                return () => {
+                  _txClient = null;
+                  commit();
+                  return tx;
+                };
+              }
+              if (prop === '$rollback') {
+                return () => {
+                  _txClient = null;
+                  rollback();
+                  return tx;
+                };
+              }
+              if (prop === '$transaction') {
+                return async (fn: CallableFunction) => {
+                  return fn(target);
+                };
+              }
+              return target[prop as keyof typeof target];
+            },
+          }) as FlatTransactionClient;
+
+          return _txClient;
+        }
+
+        throw new Error('Transactions are not supported by this client');
+      },
+    },
+  });
+
+  /* eslint-disable */
+  // @ts-ignore
+  const client = new Proxy(extendedClient, {
+    get(_target, prop) {
+      if (_txClient) {
+        return _txClient[prop as keyof typeof _txClient];
+      } else {
+        return extendedClient[prop as keyof typeof extendedClient];
+      }
+    },
+  });
+  /* eslint-enable */
+
+  return client as unknown as ExtendedTransactionClient | FlatTransactionClient;
+};
+
+declare global {
+  // eslint-disable-next-line no-var
+  var prismaTestGlobal:
+    | ExtendedTransactionClient
+    | FlatTransactionClient
+    | undefined;
+}
+
+export type ExtendedTransactionClient = Prisma.TransactionClient & {
+  $begin: () => Promise<FlatTransactionClient>;
+};
+
+export type FlatTransactionClient = Prisma.TransactionClient & {
+  $commit: () => Promise<void>;
+  $rollback: () => Promise<void>;
+};
+
+const prisma: ExtendedTransactionClient | FlatTransactionClient =
+  globalThis.prismaTestGlobal ?? prismaClientSingleton();
+
+globalThis.prismaTestGlobal = prisma;
+
+export { prisma };
+
+export async function startTx() {
+  await (prisma as ExtendedTransactionClient).$begin();
+}
+
+export async function rollbackTx() {
+  await (prisma as FlatTransactionClient).$rollback();
+}

package/src/target-templates/db/db-client.ts

@@ -0,0 +1,19 @@
+// See https://www.prisma.io/docs/orm/more/help-and-troubleshooting/help-articles/nextjs-prisma-client-dev-practices
+import { PrismaClient } from '@prisma/client';
+
+declare global {
+  // eslint-disable-next-line no-var
+  var prismaGlobal: PrismaClient | undefined;
+}
+
+const prismaClientSingleton = () => {
+  return new PrismaClient();
+};
+
+const prisma = (globalThis.prismaGlobal ??
+  prismaClientSingleton()) as PrismaClient;
+
+globalThis.prismaGlobal = prisma;
+
+export { prisma };
+export * from '@prisma/client';

package/src/target-templates/db/env.yaml

@@ -0,0 +1,4 @@
+# Prisma supports the native connection string format for PostgreSQL, MySQL, SQLite, SQL Server, MongoDB and CockroachDB.
+# See the documentation for all the connection string options: https://pris.ly/d/connection-strings
+
+- DATABASE_URL

package/src/target-templates/db/package.json

@@ -0,0 +1,17 @@
+{
+  "name": "db",
+  "main": "db-client.ts",
+  "type": "module",
+  "private": true,
+  "scripts": {
+    "generate": "prisma generate --generator js-client",
+    "migrate-deploy": "prisma migrate deploy"
+  },
+  "dependencies": {
+    "@prisma/client": "^6.5.0",
+    "prisma": "^6.5.0"
+  },
+  "devDependencies": {
+    "vitest": "^3.0.9"
+  }
+}

package/src/target-templates/db/prisma/schema.prisma

@@ -0,0 +1,24 @@
+// This is your Prisma schema file,
+// learn more about it in the docs: https://pris.ly/d/prisma-schema
+
+// Looking for ways to speed up your queries, or scale easily with your serverless or edge functions?
+// Try Prisma Accelerate: https://pris.ly/cli/accelerate-init
+
+generator client {
+  provider = "prisma-client-js"
+}
+
+generator js-client {
+  provider        = "prisma-client-js"
+  previewFeatures = ["postgresqlExtensions"]
+}
+
+generator python-client {
+  provider        = "prisma-client-py"
+  previewFeatures = ["postgresqlExtensions"]
+}
+
+datasource db {
+  provider = "postgresql"
+  url      = env("DATABASE_URL")
+}

package/src/target-templates/db/prisma-setup-vitest.ts

@@ -0,0 +1,27 @@
+import { afterEach, beforeEach, vi } from 'vitest';
+import { prisma, rollbackTx, startTx } from './db-client-test';
+
+/**
+ * Usage:
+ *  Create a `vitest.config.ts` in the project's folder with the following content.
+ *  Adjust the path to the setup file based on the nesting of the project's folder.
+ 
+  import { defineConfig } from 'vitest/config'
+
+  export default defineConfig({
+    test: {
+      setupFiles: ['../../db/prisma-setup-vitest.ts']
+    },
+  })
+  
+ */
+
+vi.mock('db', () => ({ prisma }));
+
+beforeEach(async () => {
+  await startTx();
+});
+
+afterEach(async () => {
+  await rollbackTx();
+});

package/src/target-templates/db/pyproject.toml

@@ -0,0 +1,14 @@
+[project]
+name = "db"
+version = "0.1.0"
+description = ""
+authors = []
+requires-python = ">=3.12"
+dependencies = ["prisma (>=0.15.0,<0.16.0)"]
+
+[tool.devops.scripts]
+generate = "prisma generate --generator python-client"
+
+[build-system]
+requires = ["hatchling"]
+build-backend = "hatchling.build"

package/src/target-templates/db/tsconfig.json

@@ -0,0 +1,3 @@
+{
+  "extends": "../tsconfig.json"
+}

package/src/target-templates/devops

@@ -0,0 +1,3 @@
+#! /usr/bin/env bash
+
+bunx devops "$@"

package/src/target-templates/devopspy

@@ -0,0 +1,3 @@
+#! /usr/bin/env bash
+
+uv run devopspy "$@"

package/src/target-templates/dml/package.json

@@ -0,0 +1,7 @@
+{
+  "name": "dml",
+  "type": "module",
+  "dependencies": {
+    "db": "workspace:*"
+  }
+}

package/src/target-templates/dml/tsconfig.json

@@ -0,0 +1,3 @@
+{
+  "extends": "../tsconfig.json"
+}

package/src/target-templates/libs/example-node-lib/bun.lock

@@ -0,0 +1,27 @@
+{
+  "lockfileVersion": 1,
+  "workspaces": {
+    "": {
+      "name": "example-node-lib",
+      "devDependencies": {
+        "@types/bun": "latest",
+      },
+      "peerDependencies": {
+        "typescript": "^5",
+      },
+    },
+  },
+  "packages": {
+    "@types/bun": ["@types/bun@1.2.5", "", { "dependencies": { "bun-types": "1.2.5" } }, "sha512-w2OZTzrZTVtbnJew1pdFmgV99H0/L+Pvw+z1P67HaR18MHOzYnTYOi6qzErhK8HyT+DB782ADVPPE92Xu2/Opg=="],
+
+    "@types/node": ["@types/node@22.13.11", "", { "dependencies": { "undici-types": "~6.20.0" } }, "sha512-iEUCUJoU0i3VnrCmgoWCXttklWcvoCIx4jzcP22fioIVSdTmjgoEvmAO/QPw6TcS9k5FrNgn4w7q5lGOd1CT5g=="],
+
+    "@types/ws": ["@types/ws@8.5.14", "", { "dependencies": { "@types/node": "*" } }, "sha512-bd/YFLW+URhBzMXurx7lWByOu+xzU9+kb3RboOteXYDfW+tr+JZa99OyNmPINEGB/ahzKrEuc8rcv4gnpJmxTw=="],
+
+    "bun-types": ["bun-types@1.2.5", "", { "dependencies": { "@types/node": "*", "@types/ws": "~8.5.10" } }, "sha512-3oO6LVGGRRKI4kHINx5PIdIgnLRb7l/SprhzqXapmoYkFl5m4j6EvALvbDVuuBFaamB46Ap6HCUxIXNLCGy+tg=="],
+
+    "typescript": ["typescript@5.8.2", "", { "bin": { "tsc": "bin/tsc", "tsserver": "bin/tsserver" } }, "sha512-aJn6wq13/afZp/jT9QZmwEjDqqvSGp1VT5GVg+f/t6/oVyrgXM6BY1h9BRh/O5p3PlUPAe+WuiEZOmb/49RqoQ=="],
+
+    "undici-types": ["undici-types@6.20.0", "", {}, "sha512-Ny6QZ2Nju20vw1SRHe3d9jVu6gJ+4e3+MMpqu7pqE5HT6WsTSlce++GQmK5UXS8mzV8DSYHrQH+Xrf2jVcuKNg=="],
+  }
+}

package/src/target-templates/libs/example-node-lib/index.ts

@@ -0,0 +1,3 @@
+export function test() {
+  return "test";
+}

package/src/target-templates/libs/example-node-lib/package.json

@@ -0,0 +1,12 @@
+{
+  "name": "example-node-lib",
+  "module": "index.ts",
+  "type": "module",
+  "private": true,
+  "devDependencies": {
+    "@types/bun": "latest"
+  },
+  "peerDependencies": {
+    "typescript": "^5"
+  }
+}

package/src/target-templates/libs/example-node-lib/tsconfig.json

@@ -0,0 +1,3 @@
+{
+  "extends": "../../tsconfig.json"
+}

package/src/target-templates/libs/example-python-lib/pyproject.toml

@@ -0,0 +1,11 @@
+[project]
+name = "example-python-lib"
+version = "0.1.0"
+description = ""
+authors = []
+requires-python = ">=3.12"
+dependencies = []
+
+[build-system]
+requires = ["hatchling"]
+build-backend = "hatchling.build"

package/src/target-templates/libs/example-python-lib/src/example_python_lib/__init__.py

@@ -0,0 +1,2 @@
+def test():
+    return "test"

package/src/target-templates/pyproject.toml

@@ -0,0 +1,19 @@
+[project]
+name = "devops-test"
+version = "0.1.0"
+description = ""
+authors = []
+readme = "README.md"
+requires-python = ">=3.12"
+dependencies = ["devops-python"]
+
+[tool.uv.workspace]
+members = [
+  "db",
+  "applications/example-data-pipeline",
+  "applications/example-python",
+  "libs/example-python-lib",
+]
+
+[tool.uv.sources]
+devops-python = { git = "https://github.com/vaharoni/devops-python.git" }

package/src/target-templates/tsconfig.json

@@ -0,0 +1,27 @@
+{
+  "compilerOptions": {
+    // Enable latest features
+    "lib": ["ESNext", "DOM"],
+    "target": "ESNext",
+    "module": "ESNext",
+    "moduleDetection": "force",
+    "jsx": "react-jsx",
+    "allowJs": true,
+
+    // Bundler mode
+    "moduleResolution": "bundler",
+    "allowImportingTsExtensions": true,
+    "verbatimModuleSyntax": true,
+    "noEmit": true,
+
+    // Best practices
+    "strict": true,
+    "skipLibCheck": true,
+    "noFallthroughCasesInSwitch": true,
+
+    // Some stricter flags (disabled by default)
+    "noUnusedLocals": false,
+    "noUnusedParameters": false,
+    "noPropertyAccessFromIndexSignature": false
+  }
+}

package/src/types/index.ts

@@ -0,0 +1,173 @@
+import { z } from "zod";
+
+export const SUPPORTED_LANGUAGES = ["python", "node"] as const;
+export type SupportedLanguages = typeof SUPPORTED_LANGUAGES[number];
+
+// Config files
+
+export const constFileSchema = z.object({
+  "project-name": z.string(),
+  "infra": z.enum(["hetzner", "digitalocean"]),
+  "image-versions-to-keep": z.number(),
+  "registry-base-url": z.string(),
+  "registry-name": z.string(),
+  "extra-remote-environments": z.array(z.string()),
+  "extra-local-environments": z.array(z.string()),
+})
+export type ConstFileSchema = z.infer<typeof constFileSchema>;
+
+const singleTemplateSchema = z.object({
+  "copy-common": z.boolean().optional(),
+  "extra-content": z.array(z.string()).optional(),
+})
+export type SingleTemplateSchema = z.infer<typeof singleTemplateSchema>;
+
+const singleImageSchema = z.object({
+  "image-template": z.string(),
+  "language": z.enum(SUPPORTED_LANGUAGES),
+  "domains": z.record(z.string()).optional(),
+  "debug-template": z.string(),
+  "can-db-migrate": z.boolean().optional(),
+  applications: z.array(z.string()),
+});
+export type SingleImageSchema = z.infer<typeof singleImageSchema>;
+
+export const imageFileSchema = z.object({
+  templates: z.record(singleTemplateSchema),
+  images: z.record(singleImageSchema),
+})
+export type ImageFileSchema = z.infer<typeof imageFileSchema>;
+
+// # Workspace Discovery
+
+// We need a term to describe the files that contain the project data (e.g. package.json, pyproject.toml).
+// While they are typically called "manifests" in the context of package managers, we cannot use that term here due to 
+// ambiguity with kubernetes manifests. We therefore refer to them as "package files".
+//
+// Terminology:
+// - Workspace             - a project that is part of the monorepo. Can be either a library or an application. We borrow this term from npm workspaces.
+// - Package File          - language-specific file that resides inside the workspace's root folder and contains project data (e.g. package.json, pyproject.toml)
+// - Package Data          - a language-agnostic abstraction of the relevant data in a package file. Package Files are converted into Package Data.
+// - Workspace Index Entry - a record that maps workspace names to where they reside and their package data. A workspace may contain multiple 
+//                           package files. In such case, their content must use the same "name" key so that we can always translate paths to names and vice versa.
+//                    
+
+// ## Workspace Discovery: Language-agnostic constructs
+
+// The keys are the workspace names
+export type WorkspaceIndex = Record<string, WorkspaceIndexEntry>;
+
+export type WorkspaceIndexEntry = {
+  rootPath: string;
+  packageDataEntries: PackageData[];
+};
+
+export type PackageData = Omit<PackageFileNode, 'dependencies'> & {
+  // Duplication with WorkspaceIndexEntry intentional to simplify types
+  rootPath: string;
+  language: SupportedLanguages;
+  /** The list of monorepo-internal packages this project depends on */
+  dependencyNames: string[];
+}
+
+const deploymentSchema = z.object({
+  /** The name of the composite template to build in order to support this deployment */
+  template: z.string(),
+  /** The name used by kubernetes objects. If ommitted, the name field from the enclosing context (e.g. package.json) is used. */
+  app_name: z.string().optional(),
+  /** This will be the name of the internal service used for intra-cluster communication */
+  service_name: z.string().optional(),
+  /** For services, the port to expose */
+  port: z.number().optional(),
+  /** This will be the subdomain of the exposed ingress. If not provided, taken from service_name */
+  subdomain: z.string().optional(),
+  /** For cron jobs */
+  cronJobs: z.array(
+    z.object({
+      /** The name of the cron job. Used to identify in k8s. Should be unique among all cron jobs. */
+      name: z.string().optional(),
+      /** The cron schedule */
+      cron: z.string().optional(),
+      /** The curl command to invoke. Must be an array. */
+      curl: z.array(z.string()).optional(),
+    })
+  ).optional(),
+  prefectFlows: z.array(
+    z.object({
+      /** The name of the container inside the prefect pod */
+      flow_name: z.string(),
+      /** The path to the file that runs the flow, relative to the project folder. E.g. src/data_pipeline/main.py */
+      script_path: z.string(),
+    })
+  ).optional()
+  /** The catchall below allows any other fields that might be in the package.json file */
+}).catchall(z.any());
+export type Deployment = z.infer<typeof deploymentSchema>;
+
+// ## Workspace Discovery: Languae-specific constructs
+
+export const packageFileNodeSchema = z.object({
+  /** This is considered the project name */
+  name: z.string(),
+  /** Scripts exposed by this project. */
+  scripts: z.record(z.string()).optional(),
+  /** The dependencies of the project */
+  dependencies: z.record(z.string()).optional(),
+  /** When a deployment data is available, a k8s deployment is created */
+  deployment: deploymentSchema.optional(),
+});
+export type PackageFileNode = z.infer<typeof packageFileNodeSchema>;
+
+export const packageFilePythonSchema = z.object({
+  project: z.object({
+    name: z.string(),
+    dependencies: z.string().array().optional(),
+  }),
+  tool: z.object({
+    devops: z.object({
+      deployment: deploymentSchema.optional(),
+      scripts: z.record(z.string()).optional(),
+    }).optional(),
+  }).optional(),
+})
+export type PackageFilePython = z.infer<typeof packageFilePythonSchema>;
+
+// # k8s template generation
+
+// When we generate k8s templates, we need to provide a context object that contains all the necessary information to render the template.
+// There are 3 contexts in which we generate templates, and thus 3 types of context objects:
+// - Deployment     - used to render the deployable k8s entities a workspace declared in its package file (as part of image deployment)
+// - Debug          - used to render a "debug instance" for an image the user can console into (as part of an image deployment)
+// - DbMigrateJob   - used to render a job that runs database migrations using one of the images that support it (as part of the orchestrating git-push process)
+
+export type TemplateSharedContext = {
+  /** The monorepo environment */
+  monorepo_env: string;
+  /** The k8s namespace that the monorepo environment is associated with */
+  namespace: string;
+  /** The domain name associated with the monorepo environment (from constnts.yaml) */
+  domain_name: string;
+  /** The path to the image in the container registry */
+  image_path: string;
+  /** The secret name of the monorepo environment */
+  env_secret_name: string;
+  /** The name of the key inside the environment's secret that contains the base secret (constant) */
+  env_base_secret_key: string;
+}
+
+export type TemplateDeploymentObject = TemplateSharedContext & Deployment & {
+  /** Same as pkgData.name */
+  project_name: string;
+  /** The number of replicas as defined by running devops scale */
+  replicas: number;
+}
+
+export type TemplateDbMigrateObject = TemplateSharedContext & {
+  /** The name of the migration job to create */
+  db_migrate_job_name: string;
+}
+
+export type TemplateDebugObject = TemplateSharedContext & {
+  /** The name of the debug pod */
+  debug_pod_name: string;
+}