@postxl/generators 0.2.2

package/dist/backend-core/template/README.md

@@ -0,0 +1,97 @@
+# Backend
+
+## Folder structure
+
+We use [NestJS](https://github.com/nestjs/nest) with the [Fastify](https://fastify.dev/) adapter as the framework for our backends. Each backend consists of two apps - api and cli - that you find under `apps` directory:
+
+- `apps/api` the server-providing app that talks to the web application,
+
+Additional micro-services can be added as separate apps in this directory.
+
+Furthermore, you can find everything else (i.e. all building blocks of apps) inside the `lib` directory. Each folder inside `lib` contains a separate nest module that can be used by any app. For detailed description of each module, see the [Modules](#modules) section below.
+
+## Data Flow
+
+The backend implements a kind-of Model-View-Update (MVU) architecture. The data flow is as follows:
+
+```
+          +-------------+
+          |  Front-end  |
+          +-------------+
+                ^
+                | tRPC
+                v
+          +-------------+
+          | Router/API  |
+          +-------------+
+            ^          \
+           /            \ Dispatch
+          /              \ actions
+         / Business logic v
++-----------+      +----------+
+|   View    |      |  Update  |
++-----------+      +----------+
+        ^               /
+         \             / Mutations
+          \           v
+         +-------------+              +------+
+         |    Model    | <----------  | Seed |
+         +-------------+  Mutations   +------+
+                ^
+                | Prisma
+                v
+         +-------------+
+         |   Database  |
+         +-------------+
+```
+
+#### Useful Links
+
+- NestJS: https://github.com/nestjs/nest
+- tRPC: https://trpc.io/
+- Fastify: https://fastify.dev/
+- PassportJS: https://duckduckgo.com/?q=passport+js&ia=web
+
+---
+
+## Cheat Sheet
+
+### Changes to schema
+
+The normal Prisma workflow applies here, i.e.:
+
+1. Modify the schema in `schema.prisma`
+2. Run `pnpm prisma migrate dev` to generate a migration
+3. This will update the generated project files. Ensure that the changes are correct, esp. regarding any ejected files.
+4. Run `pnpm prisma migrate deploy` to apply the migration to the production database
+
+### Seeding
+
+Seeding is done via seed migrations defined in `seed-data/src`.
+At the application start, the seed service takes these migrations and checks if they have already been applied to the database.
+In case they have not, it will apply them by triggering a corresponding seed action for each migration (enabling detailed logging).
+
+Migrations are defined as `Actions`. We currently support three types of migrations:
+
+1. Static data (`Action_Seed_Data`) - any constant data can be provided here in form of an array of `SeedData` objects.
+   We use an array here to enable multiple subsequent steps, e.g. to first create items and then link them (via an update).
+   See `migrations/001-based-data` for an example.
+2. Excel import (`Action_Seed_Excel`) - any Excel file can be provided here. The file will be parsed into `SeedData` with a to be provided decoder.
+   See `migrations/002-excel-example` for an example.
+3. Custom logic (`Action_Seed_Custom`) - any custom logic can be provided here. This is useful for more complex migrations that cannot be handled by the other two types.
+   See `migrations/003-custom-logic` for an example.
+
+### Add new actions/mutations
+
+Actions are typically scoped to a specific update service, i.e. should be defined in the corresponding update service file.
+
+To add a new action, you typically need to create/extend the following in the service file:
+
+- Define the action type, e.g. `export type Action_Post_Create = { scope: 'post'; type: 'create'; payload: PostCreateDTO }`
+- For convenience for the dispatching routes, define a corresponding `createAction` function, e.g. `export const createActionPostCreate = (payload: PostCreateDTO): Action_Post_Create => ({ scope: 'post', type: 'create', payload })`
+- Add the action to the service's `Action` type, e.g. `export type Action_Post = Action_Post_Create | ...`
+- Add the result type to the service's `ActionResult` type, e.g. `export type ActionResult_Post = {  create: Post, ... }`
+- Add the action handler as a method to the service, e.g. `public async create({ item, execution }: { item: PostCreateDTO; execution: IActionExecution }): Promise<Post> { ... }`
+  Ensure that the method returns the same type as defined in the `ActionResult` type!
+  Note that the `execution` parameter is required and will be provided by the dispatcher (see below). This execution context is required for any CRUD changes of any model's repository.
+- Extend the `dispatch` method to handle the new action, e.g. `case 'post': return this.dispatchPost(action, execution)`

package/dist/backend-core/template/apps/api/README.md

@@ -0,0 +1,3 @@
+# API
+
+REST API module written with NestJS and TRPC.

package/dist/backend-core/template/apps/api/src/api.controller.ts

@@ -0,0 +1,14 @@
+import { Controller, Get, Logger } from '@nestjs/common'
+
+@Controller('/')
+export class ApiController {
+  private readonly logger = new Logger(ApiController.name)
+
+  // eslint-disable-next-line @typescript-eslint/no-useless-constructor, @typescript-eslint/no-empty-function
+  constructor() {}
+
+  @Get('health')
+  health(): string {
+    return 'OK'
+  }
+}

package/dist/backend-core/template/apps/api/src/console.logger.ts

@@ -0,0 +1,55 @@
+import { ConsoleLogger, Injectable, LogLevel } from '@nestjs/common'
+
+import { yellow } from '@postxl/utils'
+
+const DATETIME_FORMATTER = new Intl.DateTimeFormat(undefined, {
+  hour: 'numeric',
+  minute: 'numeric',
+  second: 'numeric',
+})
+
+const maxContextLength = 15
+
+@Injectable()
+export class CompactConsoleLogger extends ConsoleLogger {
+  constructor(private readonly port: number) {
+    super()
+  }
+  protected override formatMessage(
+    logLevel: LogLevel,
+    message: unknown,
+    _pidMessage: string,
+    formattedLogLevel: string,
+    contextMessage: string,
+    timestampDiff: string,
+  ) {
+    // Base logger will take care of stringification
+    const output = this.stringifyMessage(message, logLevel) as string
+
+    // pidMessage = this.colorize(pidMessage, logLevel)
+    formattedLogLevel = logLevel === 'error' || logLevel === 'warn' ? this.colorize(formattedLogLevel, logLevel) : ''
+
+    return `${this.getTimestamp()} ${contextMessage} ${formattedLogLevel}${output}${timestampDiff} ${contextMessage}\n`
+  }
+
+  protected override formatContext(context: string): string {
+    if (context) {
+      return yellow(`[${context.slice(0, maxContextLength)}]`.padEnd(maxContextLength + 2, ' '))
+    }
+    return ''.padEnd(maxContextLength, ' ')
+  }
+
+  protected override getTimestamp(): string {
+    return DATETIME_FORMATTER.format(Date.now())
+  }
+}
+
+function toMb(bytes: number): string {
+  return `${(bytes / 1024 / 1024).toFixed(0)}m`
+}
+
+export function getMemoryUsage(): string {
+  const { rss, heapUsed, heapTotal } = process.memoryUsage()
+  const percent = Math.round((heapUsed / heapTotal) * 100)
+  return `RSS:${toMb(rss)}/U:${toMb(heapUsed)}/T:${toMb(heapTotal)}/${percent}%`
+}

package/dist/backend-core/template/apps/api/src/e2e.ts

@@ -0,0 +1,458 @@
+/**
+ * E2E Backend Isolation Helper
+ * ============================
+ *
+ * This server is designed to facilitate reliable and isolated end-to-end (E2E) testing of the backend.
+ * It achieves this by managing multiple backend instances, each running on a separate port, and routing
+ * test requests to the appropriate backend based on custom headers.
+ *
+ * ## Concept
+ *
+ * - **Default Backend:** A single backend instance is always running and serves as the default for requests
+ *   that do not specify a test context.
+ * - **Spec File Backends:** For each test spec file (e.g., e2e/specs/test.spec.ts), a dedicated backend instance
+ *   can be started. This allows tests in different files to run in complete isolation, with their own state.
+ * - **Request Routing:** Incoming requests are routed to the correct backend instance based on custom HTTP headers
+ *   (`x-test-spec-file` and `x-test-test-title`). If no headers are present, the request is routed to the default backend.
+ * - **Lifecycle Management:** Backends are started and stopped via explicit API calls (`/startBackend`, `/stopBackend`).
+ *   This is controlled using the helper functions `startBackend` and `stopBackend` in the tests (mostly in `test.beforeAll`).
+ *   Backends are automatically shut down if they are idle for a configurable period.
+ *
+ * ## How It Works
+ *
+ * 1. **Starting a Backend for a Spec File**
+ *    - The test runner (e.g., Playwright) sends a `PUT /startBackend` request with headers:
+ *      - `x-test-spec-file`: The path or identifier of the test spec file.
+ *      - `x-test-test-title`: The title of the test (for logging/tracking).
+ *    - The helper spins up a new backend instance on a free port, or reuses an existing one for that spec file.
+ *
+ * 2. **Routing Requests**
+ *    - All test requests (API, UI, etc.) are proxied through this helper server.
+ *    - The helper inspects the headers to determine which backend instance should handle the request.
+ *    - If the headers are missing or invalid, the default backend is used.
+ *
+ * 3. **Stopping a Backend**
+ *    - The test runner sends a `POST /stopBackend` request with the same headers to indicate that the backend
+ *      for a spec file can be shut down (after a debounce period).
+ *    - Backends are also automatically shut down if idle for a configured timeout (e.g., 180 seconds).
+ *
+ * ## Example Usage (Playwright)
+ *
+ * ```typescript
+ * // In your Playwright test setup:
+ * import { startBackend, stopBackend } from 'fixtures/handle-backend'
+ *
+ * test.beforeAll(async ({}, testInfo) => {
+ *   await startBackend('file', testInfo)
+ * })
+ *
+ * test.afterAll(async ({}, testInfo) => {
+ *   await stopBackend('file', testInfo)
+ * })
+ * ```
+ *
+ * The `startBackend` and `stopBackend` helpers send the appropriate requests to this server,
+ * ensuring that each spec file gets its own isolated backend.
+ *
+ * ## Benefits
+ *
+ * - **Isolation:** Each spec file can run in parallel with its own backend, avoiding cross-test interference.
+ * - **Performance:** Only the required backends are started, and idle ones are shut down automatically.
+ * - **Debuggability:** Backend lifecycle and routing are logged for easier debugging.
+ *
+ * ## API Endpoints
+ *
+ * - `PUT /startBackend` - Start or reuse a backend for a spec file.
+ * - `POST /stopBackend` - Schedule shutdown of a backend for a spec file.
+ * - `POST /shutdown` - Shut down all backends and the helper server.
+ * - `/*` - Proxy all other requests to the appropriate backend.
+ *
+ */
+
+import { INestApplication } from '@nestjs/common'
+
+import { getMemoryUsage } from 'apps/api/src/console.logger'
+import express, { Express, Request, Response } from 'express'
+import http, { Server } from 'node:http'
+import { format } from 'node:util'
+import z from 'zod'
+
+import { checkPortAvailability } from '@postxl/utils'
+
+import { bootstrap } from './main'
+
+const fixedDate = new Date(Date.UTC(2025, 8, 1, 0, 0, 0))
+
+// @ts-expect-error This is for testing
+globalThis._date = class extends Date {}
+
+// @ts-expect-error This is for testing
+globalThis.Date = class extends Date {
+  constructor(...args: any[]) {
+    if (args.length === 0) {
+      super(fixedDate.getTime())
+    } else if (args.length === 1) {
+      super(args[0])
+    } else {
+      super(args[0], args[1], args[2], args[3], args[4], args[5], args[6])
+    }
+  }
+  static override now() {
+    return fixedDate.getTime()
+  }
+}
+
+function now(): Date {
+  // @ts-expect-error This is for testing
+  return new globalThis._date()
+}
+
+const zSpecId = z.string().brand('SpecId')
+type SpecId = z.infer<typeof zSpecId>
+const zTestId = z.string().brand('TestId')
+
+// eslint-disable-next-line @typescript-eslint/no-unused-vars
+const zPort = z.number().min(1024).max(65535).brand('Port')
+type Port = z.infer<typeof zPort>
+
+const zSpecFromHeader = z
+  .object({
+    'x-test-spec-file': zSpecId,
+    'x-test-test-title': zTestId,
+  })
+  .transform((val) => ({
+    specFile: val['x-test-spec-file'],
+    testTitle: val['x-test-test-title'],
+  }))
+
+function parseTestParamsFromHeader(req: Request): z.infer<typeof zSpecFromHeader> | undefined {
+  const specParsed = zSpecFromHeader.safeParse(req.headers)
+  if (!specParsed.success) {
+    return undefined
+  }
+  return specParsed.data
+}
+
+type Backend = Backend_Starting | Backend_Running | Backend_Stopping
+type Backend_Shared = {
+  port: Port
+  spec: SpecId
+  isDefault: boolean
+  requestsServed: number
+}
+type Backend_Starting = {
+  status: 'starting'
+  promise: Promise<Backend_Running>
+} & Backend_Shared
+type Backend_Running = {
+  status: 'running'
+  nestApp: INestApplication
+  lastRequestRouted: Date
+} & Backend_Shared
+type Backend_Stopping = {
+  status: 'stopping'
+  promise: Promise<void>
+} & Backend_Shared
+
+const defaultBackendId = zSpecId.parse('default-backend')
+
+const backends = new Map<SpecId, Backend>()
+const usedPorts = new Set<Port>()
+
+async function getNextAvailablePort(): Promise<Port> {
+  let port: Port = 4000 as Port
+  while (usedPorts.has(port) || !(await checkPortAvailability(port))) {
+    port = (port + 1) as Port
+  }
+  usedPorts.add(port)
+  return port
+}
+
+async function startBackend(spec: SpecId, isDefault: boolean): Promise<Backend_Running> {
+  const existing = backends.get(spec)
+  if (existing) {
+    switch (existing.status) {
+      case 'starting':
+        return existing.promise
+      case 'running':
+        return existing
+      case 'stopping':
+        await existing.promise
+        break
+      default:
+        throw new Error(`Unknown backend status: ${(existing as any).status}`)
+    }
+  }
+  const port = await getNextAvailablePort()
+  console.debug(`Starting backend for spec ${spec} on port ${port}`)
+  const promise = bootstrap({ port, isTest: true, enableLog: true }).then((nestApp) => {
+    const runningBackend: Backend_Running = {
+      status: 'running',
+      nestApp,
+      lastRequestRouted: now(),
+      port,
+      spec,
+      requestsServed: 0,
+      isDefault,
+    }
+    backends.set(spec, runningBackend)
+    console.debug(`Backend for spec ${spec} is running on port ${port}`)
+    return runningBackend
+  })
+  const startingBackend: Backend_Starting = {
+    status: 'starting',
+    port,
+    promise,
+    spec,
+    requestsServed: 0,
+    isDefault,
+  }
+  backends.set(spec, startingBackend)
+  return promise
+}
+
+async function shutdownBackend(spec: SpecId, force: boolean): Promise<void> {
+  const backend = backends.get(spec)
+  if (!backend) {
+    console.error(`No backend found for spec ${spec}`)
+    return
+  }
+
+  if (backend.isDefault && !force) {
+    console.debug(`Not shutting down default backend on port ${backend.port}`)
+    return
+  }
+
+  let closingPromise: Promise<void>
+  switch (backend.status) {
+    case 'starting':
+      closingPromise = backend.promise.then((backend) => backend.nestApp.close())
+      break
+    case 'stopping':
+      return backend.promise
+    case 'running':
+      closingPromise = backend.nestApp.close()
+      break
+    default:
+      throw new Error(`Unknown backend status: ${(backend as any).status}`)
+  }
+
+  const promise = closingPromise.then(() => {
+    console.warn(`${backend.port} backend shut down!`)
+    usedPorts.delete(backend.port)
+    backends.delete(spec)
+  })
+
+  console.log(`Shutting down backend for spec ${spec} on port ${backend.port}`)
+  backends.set(spec, {
+    status: 'stopping',
+    promise,
+    isDefault: backend.isDefault,
+    port: backend.port,
+    requestsServed: backend.requestsServed,
+    spec: backend.spec,
+  })
+
+  return promise
+}
+
+function checkStaleBackends() {
+  const status = []
+  const backendsToShutdown: Backend[] = []
+
+  const nowTime = now().getTime()
+  for (const [spec, backend] of backends) {
+    status.push({ spec, backend })
+
+    if (backend.status !== 'running' || backend.isDefault) {
+      continue
+    }
+    const lastTestStarted = nowTime - backend.lastRequestRouted.getTime()
+    if (lastTestStarted > 180_000) {
+      backendsToShutdown.push(backend)
+    }
+  }
+
+  if (status.length > 0) {
+    console.debug(
+      `\n\n---- Current backend status (${now().toLocaleTimeString()}):  ----
+${getMemoryUsage()}
+${status
+  .toSorted((a, b) => a.backend.port - b.backend.port)
+  .map(
+    ({ spec, backend }) =>
+      `${backend.port} (${backend.status}) (${spec}) ${
+        backend.status === 'running' ? format((nowTime - backend.lastRequestRouted.getTime()) / 1000) : 0
+      }s, (spec: ${spec}) / ${backend.requestsServed} requests served`,
+  )
+  .join('\n')}\n-----------------------------------\n\n`,
+    )
+  } else {
+    console.debug(
+      `\n\n---- Current backend status (${now().toLocaleTimeString()}):  ----\n${getMemoryUsage()}\nNo backends running\n-----------------------------------\n\n`,
+    )
+  }
+
+  for (const backendRef of backendsToShutdown) {
+    console.debug(`No tests for over 180s. Shutting down backend on port ${backendRef.port}`)
+    void shutdownBackend(backendRef.spec, false)
+  }
+}
+
+setInterval(() => {
+  checkStaleBackends()
+}, 10_000)
+
+const app: Express = express()
+app.disable('x-powered-by') // else Sonar complains about security concern
+
+let expressServer: Server | null = null
+
+app.put('/startBackend', async (req: Request, res: Response) => {
+  console.log('Received request to start backend')
+  const params = parseTestParamsFromHeader(req)
+  if (!params) {
+    res.status(400).json({ error: 'Invalid headers' })
+    return
+  }
+  const { specFile } = params
+  console.log(`Starting backend for spec ${specFile}`)
+  let backend = backends.get(specFile)
+  if (!backend) {
+    backend = await startBackend(specFile, false)
+    console.debug(`Started backend for spec ${specFile} on port ${backend.port}`)
+  } else if (backend.status === 'starting') {
+    backend = await backend.promise
+  } else if (backend.status === 'stopping') {
+    await backend.promise
+    backend = await startBackend(specFile, false)
+    console.debug(`Restarted backend for spec ${specFile} on port ${backend.port}`)
+  }
+
+  if (backend.status !== 'running') {
+    res.status(500).json({ error: `Backend for spec ${specFile} is not running` })
+    return
+  }
+
+  backend.lastRequestRouted = now()
+  res.status(200).json({ port: backend.port })
+})
+
+app.put('/stopBackend', async (req: Request, res: Response) => {
+  const params = parseTestParamsFromHeader(req)
+  if (!params) {
+    console.error('Invalid headers for stopBackend')
+    res.status(400).json({ error: 'Invalid headers' })
+    return
+  }
+  const { specFile } = params
+  await shutdownBackend(specFile, false)
+  res.status(200).json({})
+})
+
+app.post('/shutdown', async (_req: Request, res: Response) => {
+  const promises = []
+  for (const spec of backends.keys()) {
+    promises.push(shutdownBackend(spec, true))
+  }
+
+  await Promise.all(promises)
+  console.debug('All backends shut down')
+  res.status(204).end()
+
+  await new Promise<void>((resolve) => {
+    if (expressServer) {
+      expressServer.close(() => {
+        console.debug('E2E helper server shut down')
+        resolve()
+      })
+    } else {
+      console.log('E2E helper server already shut down')
+      resolve()
+    }
+  })
+  process.kill(process.pid, 'SIGTERM')
+})
+
+let defaultBackend: Backend | null = null
+// prettier-ignore
+void startBackend(defaultBackendId, true).then((app) => { // NOSONAR - we will switch to top-level await later
+  defaultBackend = app
+  console.debug(`Default backend started on port ${app.port}`)
+})
+
+function stripSearchParams(originalUrl: string): string {
+  if (originalUrl.includes('?')) {
+    return originalUrl.split('?')[0]
+  }
+  return originalUrl
+}
+
+/**
+ * The server accepts all requests and forwards them to either the defaultNestApp or the backend that
+ * was created for the spec file
+ */
+app.use('/', (req, res) => {
+  const params = parseTestParamsFromHeader(req)
+
+  const backend = params ? (backends.get(params.specFile) ?? defaultBackend) : defaultBackend
+  if (!backend) {
+    res.status(503).send('Server not ready')
+    return
+  }
+  if (backend.status !== 'running') {
+    res.status(503).send('Server not ready')
+    return
+  }
+
+  backend.lastRequestRouted = now()
+  backend.requestsServed++
+  const requestId = backend.requestsServed
+
+  const target = { protocol: 'http:', host: 'localhost', port: backend.port }
+
+  const options = {
+    protocol: target.protocol,
+    hostname: target.host,
+    port: target.port,
+    path: req.originalUrl,
+    method: req.method,
+    headers: req.headers,
+  }
+
+  const startTime = process.hrtime.bigint()
+  // prettier-ignore
+  const proxyReq = http.request(options, (proxyRes) => { // NOSONAR - this server only runs in the CI environment for testing
+    const statusCode = proxyRes.statusCode ?? 502
+    res.writeHead(statusCode, proxyRes.headers as http.OutgoingHttpHeaders)
+    proxyRes.pipe(res, { end: true })
+
+    proxyRes.on('end', () => {
+      const endTime = process.hrtime.bigint()
+      const durationMs = Number(endTime - startTime) / 1_000_000
+      console.debug(
+        `[Proxy ${requestId}] ${req.method} ${stripSearchParams(req.originalUrl)} -> port ${backend.port} (${params?.testTitle}) took ${durationMs.toFixed(2)}ms`
+      )
+    })
+  })
+
+  req.pipe(proxyReq, { end: true })
+
+  proxyReq.on('error', (err) => {
+    res.status(502).send('Proxy error: ' + err.message)
+    const endTime = process.hrtime.bigint()
+    const durationMs = Number(endTime - startTime) / 1_000_000
+    console.debug(
+      `[Proxy ${requestId}] ${req.method} ${stripSearchParams(req.originalUrl)} -> port ${backend.port} (${
+        backend.spec
+      }) errored after ${durationMs.toFixed(2)}ms: ${err.message}`,
+    )
+  })
+})
+
+app.use(express.json())
+
+// Start server on port 3001
+expressServer = app.listen(Number(process.env.API_PORT) || 3001, '0.0.0.0', () => {
+  console.debug(`E2E helper server listening on http://localhost:${process.env.API_PORT || 3001}`)
+})

package/dist/backend-core/template/apps/api/test/base.test.ts

@@ -0,0 +1,7 @@
+import { describe, expect, test } from '@jest/globals'
+
+describe('backend', () => {
+  test('should work', () => {
+    expect(1).toBe(1)
+  })
+})

package/dist/backend-core/template/apps/api/tsconfig.app.json

@@ -0,0 +1,9 @@
+{
+  "extends": "../../tsconfig.json",
+  "compilerOptions": {
+    "declaration": false,
+    "outDir": "../../dist/apps/api"
+  },
+  "include": ["src/**/*"],
+  "exclude": ["node_modules", "dist", "test", "**/*spec.ts"]
+}

package/dist/backend-core/template/jest.config.ts

@@ -0,0 +1,39 @@
+import type { Config } from 'jest'
+
+import { pathsToModuleNameMapper } from 'ts-jest'
+
+import { compilerOptions } from './tsconfig.json'
+
+const config: Config = {
+  displayName: 'backend',
+  moduleFileExtensions: ['js', 'json', 'ts'],
+  rootDir: '.',
+  testRegex: '.*\\.test\\.ts$',
+  transform: {
+    '^.+\\.(t|j)s$': 'ts-jest',
+  },
+  testEnvironment: 'node',
+  roots: [
+    //'<rootDir>/libs/',
+    '<rootDir>/apps/',
+  ],
+  moduleNameMapper: pathsToModuleNameMapper(compilerOptions.paths, {
+    prefix: '<rootDir>/',
+  }),
+  prettierPath: null,
+  coverageReporters: [
+    [
+      'lcov',
+      {
+        // We need this setting so the lcov.info file has the paths relative to
+        // the project root. Without this, Sonar not be able to map the coverage to
+        // the right files in case different packages have the same relative path.
+        // This setting does not work on Windows - however, we only need it for the CI
+        projectRoot: '../../../../',
+      },
+    ],
+    'html',
+  ],
+}
+
+export default config