iec-builder 0.1.0

package/src/services/builder.ts

@@ -0,0 +1,1800 @@
+import { exec as execCb } from 'child_process'
+import { createHash } from 'crypto'
+import { promisify } from 'util'
+
+const execAsync = promisify(execCb)
+import { simpleGit, SimpleGit } from 'simple-git'
+import { mkdir, rm, access, readFile, readdir } from 'fs/promises'
+import { join, resolve, dirname } from 'path'
+import { fileURLToPath } from 'url'
+import { parse as parseYaml } from 'yaml'
+import { env } from '../config/env.js'
+import { getBuildsCollection, getServicesCollection, publishBuildEvent } from './database.js'
+import { configureDnsForService, buildDnsHostname } from './cloudflare.js'
+import { registerOrUpdateDomainInKoko, registerOrUpdateServiceInKoko, registerDatabaseInKoko, registerAvailableScopes, checkScopeGrant, autoSeedScopeGrant, syncOnboardingToKoko } from './koko.js'
+import { extractTarball } from './storage.js'
+import { parseCatalogInfo, detectFramework, type ParsedCatalog } from './catalog.js'
+import { WalletClient } from '@insureco/wallet'
+import { checkDeployReserve, deriveWalletId, isDeployGateSkipped } from './deploy-gate.js'
+import { resolveDependencies, type ResolveDependenciesResult } from './dependency-resolver.js'
+import { generateDockerfile, extractNextPublicEnvVars, validateNonRootDockerfile, patchCustomDockerfile } from './dockerfile-generator.js'
+import { provisionDatabases, provisionConsumedDatabases, buildConnectionString, type ProvisionResult } from './db-provisioner.js'
+import { provisionOAuthClient } from './oauth-provisioner.js'
+import { getBioClient } from './bio-client.js'
+import { provisionStorage, calculateStorageGasPerMonth } from './storage-provisioner.js'
+import { validateConfigCompleteness, computeConfigDefaults } from './config-validator.js'
+import type { Build, BuildStatus, CredentialRotation, RotationHistoryEntry, DatabaseCredential, DatabaseSharingConfig, StorageCredential, Service, CustomDomainRecord, DeployDiagnostics } from '../models/types.js'
+import { captureDeployDiagnostics } from './pod-diagnostics.js'
+import {
+  isVaultEnabled,
+  isVaultHealthy,
+  ensureMongoDbConnection,
+  ensureVaultMongoRole,
+  ensureVaultRedisRole,
+  ensureVaultNeo4jRole,
+  ensureVaultPolicy,
+  ensureKubeAuthRole,
+  ensureServiceAccount,
+  getVaultAgentAnnotations,
+  addMinioPolicyRules,
+  type VaultAnnotations,
+  type VaultDatabaseRoleConfig,
+} from './vault-client.js'
+import { logger } from '../utils/logger.js'
+
+/**
+ * Provision databases via Vault dynamic credentials.
+ * Creates Vault roles, policies, K8s auth bindings, and ServiceAccount.
+ * Returns Vault Agent annotations for pod injection.
+ * Falls back gracefully — caller should catch errors and fall back to Phase 1.
+ */
+// Database types with working Vault connections.
+// Redis and Neo4j connections are TODO (Phase 2c/2d) — skip them to
+// avoid Vault Agent init loops on non-existent connections.
+const VAULT_SUPPORTED_DB_TYPES = new Set<string>(['mongodb'])
+
+async function provisionDatabasesVault(
+  serviceName: string,
+  environment: string,
+  namespace: string,
+  databases: ReadonlyArray<{ type: 'mongodb' | 'redis' | 'neo4j'; name?: string }>
+): Promise<VaultAnnotations> {
+  // Only provision databases that have working Vault connections
+  const vaultDatabases = databases.filter((db) => VAULT_SUPPORTED_DB_TYPES.has(db.type))
+
+  if (vaultDatabases.length === 0) {
+    return {}
+  }
+
+  // 1. Ensure K8s ServiceAccount (needed for Vault K8s auth)
+  ensureServiceAccount(serviceName, namespace)
+
+  // 1b. Ensure Vault database connections exist (idempotent, one-time setup per DB type)
+  const hasMongo = vaultDatabases.some((db) => db.type === 'mongodb')
+  if (hasMongo && env.DB_MONGODB_ADMIN_URI) {
+    const connectionUri = env.DB_MONGODB_ADMIN_URI.replace('localhost', env.DB_MONGODB_HOST)
+    await ensureMongoDbConnection('mongodb-tawa', connectionUri)
+  }
+
+  // 2. Ensure Vault roles for each database
+  const dbTypes: string[] = []
+  for (const db of vaultDatabases) {
+    const dbName = db.name || `${serviceName}-${environment}`
+    const config: VaultDatabaseRoleConfig = {
+      serviceName,
+      environment,
+      databaseName: dbName,
+      role: 'readWrite',
+      dbType: db.type,
+    }
+
+    if (db.type === 'mongodb') {
+      await ensureVaultMongoRole(config)
+    }
+    dbTypes.push(db.type)
+  }
+
+  // 3. Ensure Vault policy + K8s auth binding
+  await ensureVaultPolicy(serviceName, environment, dbTypes)
+  await ensureKubeAuthRole(serviceName, environment, namespace)
+
+  // 4. Build Vault Agent annotations (only for supported types)
+  return getVaultAgentAnnotations(serviceName, environment, vaultDatabases)
+}
+
+/**
+ * Merge new database credentials with existing ones.
+ * Replaces entries matching (type, environment), keeps others.
+ * Returns a new array (immutable).
+ */
+function mergeCredentials(
+  existing: ReadonlyArray<DatabaseCredential>,
+  incoming: ReadonlyArray<DatabaseCredential>
+): DatabaseCredential[] {
+  const incomingKeys = new Set(
+    incoming.map((c) => `${c.type}:${c.environment}`)
+  )
+
+  const kept = existing.filter(
+    (c) => !incomingKeys.has(`${c.type}:${c.environment}`)
+  )
+
+  return [...kept, ...incoming]
+}
+
+/**
+ * Inject git credentials into clone URLs for known hosts.
+ *
+ * Handles three cases:
+ * 1. Forgejo HTTPS:  https://git.insureco.io/org/repo.git
+ *    → https://forgejo-token:TOKEN@git.insureco.io/org/repo.git
+ *
+ * 2. GitHub HTTPS:   https://github.com/org/repo.git
+ *    → https://x-access-token:TOKEN@github.com/org/repo.git
+ *
+ * 3. GitHub SSH:     git@github.com:org/repo.git
+ *    → https://x-access-token:TOKEN@github.com/org/repo.git
+ */
+function injectGitCredentials(repoUrl: string): string {
+  // Forgejo — convert SSH to HTTPS and inject token
+  if (env.FORGEJO_TOKEN) {
+    try {
+      const forgejoHost = new URL(env.FORGEJO_URL).host
+
+      // SSH format: git@git.insureco.io:org/repo.git
+      const sshMatch = repoUrl.match(new RegExp(`^git@${forgejoHost.replace('.', '\\.')}:(.+)$`))
+      if (sshMatch) {
+        return `https://forgejo-token:${env.FORGEJO_TOKEN}@${forgejoHost}/${sshMatch[1]}`
+      }
+
+      // HTTPS format: https://git.insureco.io/org/repo.git
+      const repoUrlObj = new URL(repoUrl)
+      if (repoUrlObj.host === forgejoHost) {
+        return `${repoUrlObj.protocol}//forgejo-token:${env.FORGEJO_TOKEN}@${repoUrlObj.host}${repoUrlObj.pathname}`
+      }
+    } catch {
+      // Not a valid URL and not SSH format, continue
+    }
+  }
+
+  // GitHub — convert SSH to HTTPS and inject token
+  if (env.GITHUB_TOKEN) {
+    // SSH format: git@github.com:org/repo.git
+    const sshMatch = repoUrl.match(/^git@github\.com:(.+)$/)
+    if (sshMatch) {
+      return `https://x-access-token:${env.GITHUB_TOKEN}@github.com/${sshMatch[1]}`
+    }
+
+    // HTTPS format: https://github.com/org/repo.git
+    try {
+      const repoUrlObj = new URL(repoUrl)
+      if (repoUrlObj.host === 'github.com') {
+        return `https://x-access-token:${env.GITHUB_TOKEN}@github.com${repoUrlObj.pathname}`
+      }
+    } catch {
+      // Not a valid URL, return as-is
+    }
+  }
+
+  return repoUrl
+}
+
+/**
+ * Extract the organization/owner from a git repo URL.
+ * Handles HTTPS (https://github.com/org/repo.git) and SSH (git@github.com:org/repo.git)
+ */
+function extractRepoOrg(repoUrl: string): string | undefined {
+  // Try HTTPS format first: https://host[:port]/org/repo.git
+  try {
+    const url = new URL(repoUrl)
+    const parts = url.pathname.split('/').filter(Boolean)
+    if (parts.length >= 2) return parts[0]
+  } catch {
+    // Not a valid URL — try SSH format below
+  }
+
+  // SSH format: git@host:org/repo.git
+  const sshMatch = repoUrl.match(/^[^@]+@[^:]+:([^/]+)\//)
+  if (sshMatch) return sshMatch[1]
+
+  return undefined
+}
+
+/** Escape a value for use in helm --set (commas and braces need escaping) */
+function escapeHelmValue(value: string): string {
+  const str = value ?? ''
+  return str.replace(/,/g, '\\,').replace(/\{/g, '\\{').replace(/\}/g, '\\}')
+}
+
+/** Escape a value for safe shell interpolation */
+function escapeShellArg(value: string): string {
+  const str = value ?? ''
+  const sanitized = str.replace(/\0/g, '')
+  return `'${sanitized.replace(/'/g, "'\\''")}'`
+}
+
+/** Validate that a string is a valid env var key name */
+const ENV_KEY_REGEX = /^[A-Za-z_][A-Za-z0-9_]*$/
+function validateEnvKey(key: string): void {
+  if (!ENV_KEY_REGEX.test(key)) {
+    throw new Error(`Invalid config key name: ${key}`)
+  }
+}
+
+interface IecYamlConfig {
+  dockerfile?: string
+  buildContext?: string
+  helmChart?: string
+}
+
+/**
+ * Read .iec.yaml from the app directory to get monorepo build configuration
+ */
+async function readIecYaml(workDir: string, appPath?: string): Promise<IecYamlConfig | null> {
+  const configPath = appPath
+    ? join(workDir, appPath, '.iec.yaml')
+    : join(workDir, '.iec.yaml')
+
+  try {
+    await access(configPath)
+    const content = await readFile(configPath, 'utf-8')
+    const config = parseYaml(content) as IecYamlConfig
+    logger.info({ configPath, config }, 'Loaded .iec.yaml config')
+    return config
+  } catch {
+    return null
+  }
+}
+
+/**
+ * Auto-discover the helm chart path within a work directory.
+ * Convention: check helm/, then helm/serviceName/, then scan helm subdirectories.
+ */
+async function resolveHelmChartPath(
+  workDir: string,
+  serviceName: string,
+  appPath?: string
+): Promise<string> {
+  const baseDir = appPath ? join(workDir, appPath) : workDir
+
+  // 1. Check helm/ directly (Chart.yaml at root of helm/)
+  const helmRoot = join(baseDir, 'helm')
+  try {
+    await access(join(helmRoot, 'Chart.yaml'))
+    return 'helm'
+  } catch { /* not here */ }
+
+  // 2. Convention: helm/<serviceName>/
+  const conventionPath = join(helmRoot, serviceName)
+  try {
+    await access(join(conventionPath, 'Chart.yaml'))
+    return `helm/${serviceName}`
+  } catch { /* not here */ }
+
+  // 3. Scan helm/*/ for any Chart.yaml (single chart in helm/)
+  try {
+    const entries = await readdir(helmRoot, { withFileTypes: true })
+    for (const entry of entries) {
+      if (entry.isDirectory()) {
+        try {
+          await access(join(helmRoot, entry.name, 'Chart.yaml'))
+          logger.info({ discovered: `helm/${entry.name}` }, 'Auto-discovered helm chart')
+          return `helm/${entry.name}`
+        } catch { /* no Chart.yaml here */ }
+      }
+    }
+  } catch { /* helm/ doesn't exist or isn't readable */ }
+
+  // 4. Check chart/ directory (alternative convention)
+  try {
+    await access(join(baseDir, 'chart', 'Chart.yaml'))
+    return 'chart'
+  } catch { /* not here */ }
+
+  // 5. Fallback — use built-in default chart
+  const builderDir = dirname(fileURLToPath(import.meta.url))
+  const defaultChart = resolve(builderDir, '../../helm/default-service')
+  logger.info({ defaultChart }, 'No helm chart found in project, using built-in default')
+  return defaultChart
+}
+
+export async function executeBuild(buildId: string): Promise<void> {
+  const builds = getBuildsCollection()
+  const services = getServicesCollection()
+
+  const build = await builds.findOne({ id: buildId })
+  if (!build) {
+    logger.error({ buildId }, 'Build not found')
+    return
+  }
+
+  const service = await services.findOne({ id: build.serviceId })
+  if (!service) {
+    await updateBuildStatus(buildId, 'failed', 'Service not found')
+    return
+  }
+
+  // Backfill build org from service if missing (handles webhook builds pre-Phase 4.1)
+  if (!build.org && service.org) {
+    await builds.updateOne(
+      { id: buildId },
+      { $set: { org: service.org, updatedAt: new Date().toISOString() } }
+    )
+    // Intentional mutation: keep local build in sync with DB update
+    build.org = service.org
+  }
+
+  const workDir = join(env.WORKSPACE_DIR, buildId)
+  let deployPhaseReached = false
+
+  try {
+    // Clone repository or extract tarball
+    await updateBuildStatus(buildId, 'cloning')
+    const actualCommitSha = await cloneOrExtract(build.repoUrl, build.branch, build.commitSha, workDir)
+
+    // Update build with actual commit sha
+    await builds.updateOne(
+      { id: buildId },
+      { $set: { commitSha: actualCommitSha, updatedAt: new Date().toISOString() } }
+    )
+    await appendBuildLog(buildId, `Source ready @ ${actualCommitSha}`)
+
+    // Read .iec.yaml for monorepo configuration
+    const iecConfig = await readIecYaml(workDir, service.appPath)
+    if (iecConfig) {
+      await appendBuildLog(buildId, `Found .iec.yaml config in ${service.appPath || 'root'}`)
+    }
+
+    // Determine dockerfile path and build context for monorepos
+    // Priority: .iec.yaml > service config > defaults
+    let dockerfilePath = iecConfig?.dockerfile || service.dockerfilePath || 'Dockerfile'
+    // Only treat dockerfilePath as explicit if it's a non-default value.
+    // service.dockerfilePath defaults to 'Dockerfile' in the schema, which is not an explicit opt-in.
+    const hasExplicitDockerfile = !!(iecConfig?.dockerfile || (service.dockerfilePath && service.dockerfilePath !== 'Dockerfile'))
+    let buildContext = workDir
+
+    // Resolve buildContext from .iec.yaml or service config
+    const contextOverride = iecConfig?.buildContext || service.buildContext
+    if (contextOverride) {
+      const baseDir = service.appPath ? join(workDir, service.appPath) : workDir
+      buildContext = resolve(baseDir, contextOverride)
+      await appendBuildLog(buildId, `Using build context: ${buildContext} (from ${contextOverride})`)
+    }
+
+    // For monorepos with appPath, resolve dockerfile relative to appPath
+    if (service.appPath) {
+      if (!dockerfilePath.startsWith('/')) {
+        dockerfilePath = join(service.appPath, dockerfilePath)
+      }
+    }
+
+    // Always parse catalog-info.yaml — provisioning (databases, OAuth, dependencies)
+    // needs it regardless of whether the project ships its own Dockerfile.
+    // For monorepos with appPath, look in the app subdirectory first.
+    const catalogDir = service.appPath ? join(workDir, service.appPath) : workDir
+    let catalog: ParsedCatalog | null = await parseCatalogInfo(catalogDir)
+
+    if (catalog) {
+      await appendBuildLog(buildId, `Detected framework: ${catalog.framework}`)
+    }
+
+    // app-root annotation: tells the builder where package.json lives (e.g., "backend")
+    // Sets the Docker build context so COPY commands are relative to the app root.
+    // Only applies when no explicit buildContext override exists (.iec.yaml or service config).
+    if (catalog?.appRoot && buildContext === workDir) {
+      const appRootDir = join(workDir, catalog.appRoot)
+      try {
+        await access(appRootDir)
+        buildContext = appRootDir
+        await appendBuildLog(buildId, `Using app root: ${catalog.appRoot}`)
+      } catch {
+        throw new Error(
+          `app-root directory "${catalog.appRoot}" not found in repository. ` +
+          `Verify the insureco.io/app-root annotation matches an existing directory.`
+        )
+      }
+    }
+
+    // Dockerfile strategy:
+    // - Default: builder generates an optimized Dockerfile (devs never need one)
+    // - Opt-in: power users set insureco.io/custom-dockerfile: 'true' to bring their own
+    // - Explicit: .iec.yaml or service config specifies a custom dockerfile path
+    let useCustomDockerfile = hasExplicitDockerfile || catalog?.customDockerfile === true
+
+    // If custom Dockerfile is requested but the file doesn't exist, fall back to auto-generation.
+    // This handles services with stale dockerfilePath in the DB or removed Dockerfiles.
+    if (useCustomDockerfile) {
+      const fullCustomPath = dockerfilePath.startsWith('/')
+        ? dockerfilePath
+        : join(workDir, dockerfilePath)
+      try {
+        await access(fullCustomPath)
+      } catch {
+        await appendBuildLog(
+          buildId,
+          `[warn] Custom Dockerfile not found at ${dockerfilePath} — falling back to auto-generation`
+        )
+        useCustomDockerfile = false
+      }
+    }
+
+    if (env.ENABLE_AUTO_DOCKERFILE === 'true' && !useCustomDockerfile) {
+      // Warn if a Dockerfile exists but is being ignored (vibe coder probably generated it by accident)
+      try {
+        const projectDockerfile = join(buildContext, 'Dockerfile')
+        await access(projectDockerfile)
+        await appendBuildLog(
+          buildId,
+          `[warn] Found Dockerfile in project — ignoring in favor of builder-generated Dockerfile. ` +
+          `To use your own, add 'insureco.io/custom-dockerfile: "true"' to your catalog-info.yaml annotations.`
+        )
+      } catch {
+        // No Dockerfile in project — expected, nothing to warn about
+      }
+
+      if (catalog) {
+        const generatedPath = await generateDockerfile(buildContext, catalog)
+        dockerfilePath = generatedPath.replace(buildContext + '/', '')
+        await appendBuildLog(buildId, `Generated Dockerfile for ${catalog.framework}`)
+      } else {
+        // Try to detect framework without catalog
+        const detectedFramework = await detectFramework(buildContext)
+        if (detectedFramework !== 'unknown') {
+          await appendBuildLog(buildId, `Auto-detected framework: ${detectedFramework}`)
+
+          // Extract NEXT_PUBLIC_* env vars for Next.js
+          const envVars = detectedFramework === 'nextjs'
+            ? await extractNextPublicEnvVars(buildContext)
+            : []
+
+          catalog = {
+            name: service.name,
+            description: '',
+            catalogVersion: '0.1.0',
+            framework: detectedFramework,
+            nodeVersion: '20',
+            buildCommand: 'npm run build',
+            startCommand: 'npm start',
+            outputDir: detectedFramework === 'nextjs' ? '.next' : 'dist',
+            port: 3000,
+            healthEndpoint: '/health',
+            envVars,
+            routes: [],
+            cronjobs: [],
+            databases: [],
+            consumesDatabase: [],
+            internalDependencies: [],
+            dependencies: [],
+            storage: [],
+            lifecycle: 'production',
+            owner: 'unknown',
+            type: 'service',
+            podTier: 'nano',
+            configDeclarations: [],
+            modules: [],
+            externalDependencies: [],
+            copyPaths: [],
+          }
+
+          const generatedPath = await generateDockerfile(buildContext, catalog)
+          dockerfilePath = generatedPath.replace(buildContext + '/', '')
+          await appendBuildLog(buildId, `Generated Dockerfile for ${detectedFramework}`)
+        }
+      }
+    } else if (useCustomDockerfile) {
+      await appendBuildLog(buildId, `Using custom Dockerfile: ${dockerfilePath}`)
+
+      // Auto-patch custom Dockerfiles for platform compliance:
+      // - Replace non-numeric USER directives with USER 1001 (K8s runAsNonRoot)
+      // - Inject Vault entrypoint wrapper if missing (when Vault is enabled)
+      const fullCustomPath = dockerfilePath.startsWith('/')
+        ? dockerfilePath
+        : join(buildContext, dockerfilePath)
+      const patches = await patchCustomDockerfile(fullCustomPath, env.VAULT_ENABLED === 'true')
+      for (const patch of patches) {
+        await appendBuildLog(buildId, `[auto-patch] ${patch.message}`)
+      }
+    }
+
+    // Deploy gate: check wallet balance before deploying
+    if (env.ENABLE_K8S_DEPLOY === 'true' && env.WALLET_URL && catalog) {
+      if (isDeployGateSkipped(service.name)) {
+        await appendBuildLog(buildId, `Deploy gate skipped: ${service.name} is a platform service`)
+      } else {
+        const ownerId = catalog.owner
+        const walletId = deriveWalletId(ownerId)
+        const podTier = catalog.podTier || 'nano'
+
+        // Auto-create wallet if it doesn't exist (safety net for missed Bio-id registration)
+        try {
+          const walletClient = new WalletClient({
+            baseUrl: env.WALLET_URL,
+            serviceKey: env.INTERNAL_SERVICE_KEY,
+            retries: 1,
+            timeoutMs: 8_000,
+          })
+          await walletClient.ensureWallet({
+            ownerId,
+            ownerType: 'organization',
+            ownerName: ownerId,
+          })
+        } catch {
+          // fail-open: wallet service may be unavailable
+        }
+
+        const reserveResult = await checkDeployReserve(env.WALLET_URL, walletId, podTier, env.INTERNAL_SERVICE_KEY)
+
+        // Storage gas adds to the reserve requirement
+        const storageGas = calculateStorageGasPerMonth(catalog.storage || [])
+        const totalRequired = reserveResult.requiredTokens + (storageGas * 3)
+        const totalShortfall = Math.max(0, totalRequired - reserveResult.currentBalance)
+
+        if (totalShortfall > 0) {
+          throw new Error(
+            `Deploy gate: insufficient gas reserve. Required: ${totalRequired} tokens ` +
+            `(hosting: ${reserveResult.requiredTokens}, storage: ${storageGas * 3}), ` +
+            `available: ${reserveResult.currentBalance}. ` +
+            `Shortfall: ${totalShortfall} tokens. Top up your wallet to deploy.`
+          )
+        }
+
+        const storageNote = storageGas > 0 ? ` (includes ${storageGas} gas/mo storage)` : ''
+        await appendBuildLog(buildId, `Deploy gate passed: ${reserveResult.currentBalance}/${reserveResult.requiredTokens} tokens${storageNote}`)
+      }
+    }
+
+    // Config preflight: validate required config declarations before building
+    if (catalog?.configDeclarations && catalog.configDeclarations.length > 0) {
+      const storedConfig = service.config || {}
+      const storedSecretKeys = Object.keys(service.secrets || {})
+      const validation = validateConfigCompleteness(
+        catalog.configDeclarations,
+        storedConfig,
+        storedSecretKeys,
+      )
+
+      if (!validation.valid) {
+        throw new Error(
+          `Config preflight failed: missing required config vars: ${validation.missing.join(', ')}. ` +
+          `Set them with \`tawa config set\` or \`tawa config push\` before deploying.`
+        )
+      }
+
+      await appendBuildLog(buildId, `Config preflight passed (${catalog.configDeclarations.length} declarations checked)`)
+
+      // Cache declarations on service record for push endpoint
+      await services.updateOne(
+        { id: service.id },
+        { $set: { configDeclarations: catalog.configDeclarations, updatedAt: new Date().toISOString() } }
+      )
+    }
+
+    // Build output preflight: detect no-op build script with default output-dir
+    if (catalog && catalog.outputDir === 'dist') {
+      try {
+        const pkgPath = join(buildContext, 'package.json')
+        const pkg = JSON.parse(await readFile(pkgPath, 'utf-8'))
+        const buildScript = pkg?.scripts?.build || ''
+        const isNoOp = /^\s*(echo\s|exit\s*0|true|:)\s*/i.test(buildScript)
+
+        if (isNoOp) {
+          throw new Error(
+            `Build output preflight failed: build script is a no-op ("${buildScript.trim()}") ` +
+            `but output-dir is "dist" (default). The generated Dockerfile will fail copying /app/dist. ` +
+            `Set \`insureco.io/output-dir\` to your source directory (e.g. "src") and ` +
+            `\`insureco.io/start-command\` accordingly in catalog-info.yaml.`
+          )
+        }
+      } catch (err: any) {
+        if (err.message.startsWith('Build output preflight failed')) throw err
+      }
+      await appendBuildLog(buildId, 'Build output preflight passed')
+    }
+
+    // Validate Dockerfile runs as non-root (skip for static/nginx which needs root)
+    if (catalog?.framework !== 'static') {
+      const fullDockerfilePath = dockerfilePath.startsWith('/')
+        ? dockerfilePath
+        : join(buildContext, dockerfilePath)
+      await validateNonRootDockerfile(fullDockerfilePath)
+      await appendBuildLog(buildId, 'Dockerfile non-root validation passed')
+    }
+
+    // Build Docker image
+    await updateBuildStatus(buildId, 'building')
+    const imageTag = `${env.DOCKER_REGISTRY}/${service.name}:${actualCommitSha.substring(0, 7)}`
+    await buildDockerImage(buildContext, imageTag, buildId, dockerfilePath)
+    await appendBuildLog(buildId, `Built image: ${imageTag}`)
+
+    // Push to registry
+    await updateBuildStatus(buildId, 'pushing')
+    await pushDockerImage(imageTag, buildId)
+    await appendBuildLog(buildId, `Pushed image to registry`)
+
+    // Update build with image tag
+    await builds.updateOne(
+      { id: buildId },
+      { $set: { imageTag, updatedAt: new Date().toISOString() } }
+    )
+
+    // Deploy to Kubernetes (optional - skip if not configured)
+    if (env.ENABLE_K8S_DEPLOY === 'true') {
+      await updateBuildStatus(buildId, 'deploying')
+      deployPhaseReached = true
+
+      // Use environment from build (set by CLI/API) - no more branch inference
+      const environment = build.environment
+      const namespace = service.namespace || `${service.name}-${environment}`
+
+      // Ensure namespace exists before any K8s operations (DB secrets, etc.)
+      await ensureNamespaceExists(namespace)
+
+      // Check Vault availability (feature-flagged + health-checked)
+      const vaultActive = isVaultEnabled() && await isVaultHealthy()
+      let vaultAnnotations: VaultAnnotations = {}
+
+      // Provision databases if catalog specifies them
+      let provisionedEnvVars: Record<string, string> = {}
+      let sharingConfigs: DatabaseSharingConfig[] = []
+      let dbResult: ProvisionResult = { envVars: {}, credentials: [] }
+      if (catalog?.databases && catalog.databases.length > 0) {
+        if (vaultActive) {
+          // Phase 2: Vault dynamic credentials — Agent sidecar injects DB env vars
+          try {
+            vaultAnnotations = await provisionDatabasesVault(
+              service.name, environment, namespace, catalog.databases
+            )
+            const vaultCount = catalog.databases.filter((db) => VAULT_SUPPORTED_DB_TYPES.has(db.type)).length
+            if (vaultCount > 0) {
+              await appendBuildLog(buildId, `Vault: provisioned ${vaultCount} dynamic database role(s)`)
+            }
+          } catch (vaultError) {
+            // Vault provisioning failed — fall back to Phase 1 static credentials for ALL databases
+            const msg = vaultError instanceof Error ? vaultError.message : 'Unknown error'
+            logger.warn({ error: msg }, 'Vault provisioning failed, falling back to static credentials')
+            await appendBuildLog(buildId, `[warn] Vault provisioning failed (${msg}), using static credentials`)
+            vaultAnnotations = {}
+          }
+
+          // Provision non-Vault databases (e.g. Redis, Neo4j) via Phase 1 static creds,
+          // or ALL databases if Vault provisioning failed above
+          const staticDatabases = Object.keys(vaultAnnotations).length > 0
+            ? catalog.databases.filter((db) => !VAULT_SUPPORTED_DB_TYPES.has(db.type))
+            : catalog.databases
+
+          if (staticDatabases.length > 0) {
+            dbResult = await provisionDatabases(
+              service.name, environment, namespace,
+              staticDatabases, service.databaseCredentials
+            )
+            provisionedEnvVars = { ...provisionedEnvVars, ...dbResult.envVars }
+          }
+        } else {
+          // Phase 1: Static credentials (fallback when Vault disabled or unhealthy)
+          dbResult = await provisionDatabases(
+            service.name, environment, namespace,
+            catalog.databases, service.databaseCredentials
+          )
+          provisionedEnvVars = { ...provisionedEnvVars, ...dbResult.envVars }
+        }
+
+        if (dbResult.credentials.length > 0) {
+          const mergedCreds = mergeCredentials(service.databaseCredentials || [], dbResult.credentials)
+          await services.updateOne(
+            { id: service.id },
+            { $set: { databaseCredentials: mergedCreds, updatedAt: new Date().toISOString() } }
+          )
+          await appendBuildLog(buildId, `Database credentials provisioned for ${dbResult.credentials.length} database(s)`)
+        }
+
+        if (!vaultActive || Object.keys(vaultAnnotations).length === 0) {
+          await appendBuildLog(buildId, `Provisioned ${catalog.databases.length} database(s)`)
+        }
+
+        // Register databases in Koko so koko-db CLI can discover them
+        // In Vault mode, dbResult is empty so this falls through to buildConnectionString()
+        let registeredCount = 0
+        await Promise.all(
+          catalog.databases.map(async (db) => {
+            const dbName = db.name || `${service.name}-${environment}`
+            const credential = dbResult.credentials.find(c => c.databaseName === dbName)
+            const connString = credential
+              ? dbResult.envVars[db.type === 'mongodb' ? 'MONGODB_URI' : db.type === 'redis' ? 'REDIS_URL' : 'NEO4J_URI']
+              : buildConnectionString(db.type, service.name, environment, db.name)
+            if (connString) {
+              const ok = await registerDatabaseInKoko(service.name, dbName, environment, connString, credential?.username)
+              if (ok) registeredCount++
+            }
+          })
+        )
+        await appendBuildLog(buildId, `Registered ${registeredCount}/${catalog.databases.length} database(s) in Koko`)
+
+        // Store database sharing config for owner services (used by consumers on their deploy)
+        sharingConfigs = catalog.databases
+          .filter(db => db.sharedWith && db.sharedWith.length > 0)
+          .map(db => ({
+            type: db.type,
+            databaseName: db.name || `${service.name}-${environment}`,
+            sharedWith: db.sharedWith!.map(s => ({ service: s.service, access: s.access })),
+          }))
+
+        if (sharingConfigs.length > 0) {
+          await services.updateOne(
+            { id: service.id },
+            { $set: { databaseSharing: sharingConfigs, updatedAt: new Date().toISOString() } }
+          )
+          await appendBuildLog(buildId, `Database sharing config stored (${sharingConfigs.length} shared database(s))`)
+        }
+
+        // Cache catalog spec on service (used by tawa-web databases page)
+        const catalogSpecUpdate = {
+          databases: catalog.databases.map(db => ({ type: db.type, name: db.name })),
+          storage: catalog.storage?.map(s => ({ tier: s.tier })) ?? [],
+        }
+        await services.updateOne(
+          { id: service.id },
+          { $set: { catalogSpec: catalogSpecUpdate, updatedAt: new Date().toISOString() } }
+        )
+      }
+
+      // Check scope grants before provisioning consumed databases
+      if (catalog?.consumesDatabase && catalog.consumesDatabase.length > 0) {
+        for (const consumed of catalog.consumesDatabase) {
+          const grantResult = await checkScopeGrant(consumed.service, service.name, consumed.type)
+
+          if (grantResult === null) {
+            // Koko unreachable — fail-open during rollout
+            await appendBuildLog(buildId, `[warn] Koko unreachable, skipping scope-grant check for ${consumed.service}:${consumed.type}`)
+          } else if (grantResult.granted) {
+            await appendBuildLog(buildId, `Scope grant approved for ${consumed.service}:${consumed.type}`)
+          } else if (grantResult.status === 'none') {
+            // No grant exists yet — fail-open during rollout (owner hasn't auto-seeded yet)
+            await appendBuildLog(buildId, `[warn] No scope grant found for ${consumed.service}:${consumed.type}, proceeding (rollout period)`)
+          } else if (grantResult.status === 'denied' || grantResult.status === 'revoked') {
+            throw new Error(
+              `Deploy blocked: access to ${consumed.service}'s ${consumed.type} database has been ${grantResult.status}. ` +
+              `Request access at https://tawa.insureco.io/console/scopes`
+            )
+          } else {
+            // pending — fail-open during rollout, will be fail-closed later
+            await appendBuildLog(buildId, `[warn] Scope grant pending for ${consumed.service}:${consumed.type}, proceeding (rollout period)`)
+          }
+        }
+      }
+
+      // Provision consumed databases (from other services)
+      if (catalog?.consumesDatabase && catalog.consumesDatabase.length > 0) {
+        const consumedResult = await provisionConsumedDatabases(
+          service.name,
+          environment,
+          namespace,
+          catalog.consumesDatabase,
+          service.databaseCredentials || [],
+          async (ownerName) => {
+            const ownerService = await services.findOne({ name: ownerName })
+            if (!ownerService) return null
+            return { name: ownerService.name, databaseSharing: ownerService.databaseSharing }
+          }
+        )
+
+        provisionedEnvVars = { ...provisionedEnvVars, ...consumedResult.envVars }
+
+        if (consumedResult.credentials.length > 0) {
+          const mergedCreds = mergeCredentials(service.databaseCredentials || [], consumedResult.credentials)
+          await services.updateOne(
+            { id: service.id },
+            { $set: { databaseCredentials: mergedCreds, updatedAt: new Date().toISOString() } }
+          )
+        }
+
+        await appendBuildLog(buildId, `Provisioned ${catalog.consumesDatabase.length} consumed database(s)`)
+      }
+
+      // Provision object storage (Vault-required, 0.3.0+ catalogs)
+      if (catalog?.storage && catalog.storage.length > 0) {
+        const storageResult = await provisionStorage(
+          service.name,
+          environment,
+          namespace,
+          catalog.storage,
+        )
+
+        // Add MinIO creds path to Vault policy (creates policy if it doesn't exist yet)
+        await addMinioPolicyRules(service.name, environment)
+
+        // If no databases were declared, Vault plumbing (ServiceAccount, K8s auth)
+        // hasn't been set up yet — create it now so the pod can authenticate to Vault
+        const noDatabases = !catalog?.databases || catalog.databases.length === 0
+        if (noDatabases) {
+          ensureServiceAccount(service.name, namespace)
+          await ensureKubeAuthRole(service.name, environment, namespace)
+
+          // Set base Vault Agent annotations (normally set by database provisioning)
+          const kubeRoleName = `svc-${service.name}-${environment}`
+          vaultAnnotations = {
+            'vault.hashicorp.com/agent-inject': 'true',
+            'vault.hashicorp.com/role': kubeRoleName,
+            'vault.hashicorp.com/agent-pre-populate-only': 'false',
+          }
+        }
+
+        // Merge Vault annotations for sidecar injection
+        vaultAnnotations = { ...vaultAnnotations, ...storageResult.vaultAnnotations }
+
+        if (storageResult.credential) {
+          const existingStorage = service.storageCredentials || []
+          const updatedStorage: StorageCredential[] = [
+            ...existingStorage.filter((c) => c.environment !== environment),
+            storageResult.credential,
+          ]
+
+          await services.updateOne(
+            { id: service.id },
+            { $set: { storageCredentials: updatedStorage, updatedAt: new Date().toISOString() } }
+          )
+        }
+
+        await appendBuildLog(
+          buildId,
+          `Provisioned ${catalog.storage.length} storage bucket(s) (${storageResult.gasPerMonth} gas/month)`
+        )
+      }
+
+      // Resolve unified dependencies (scope grant check + URL injection)
+      // This handles both new spec.dependencies (0.4.0+) and legacy internalDependencies/externalDependencies
+      const unifiedDeps = catalog?.dependencies || []
+      const hasScopedDeps = unifiedDeps.some(d => d.scopes.length > 0)
+      let depResult: ResolveDependenciesResult = { envVars: {}, warnings: [], blocked: [], grantResults: [] }
+
+      if (unifiedDeps.length > 0 && env.KOKO_URL) {
+        depResult = await resolveDependencies({
+          serviceName: service.name,
+          environment,
+          dependencies: unifiedDeps,
+          kokoUrl: env.KOKO_URL,
+          bioUrl: env.BIO_ID_URL,
+          bioInternalKey: env.BIO_INTERNAL_KEY || '',
+        })
+
+        provisionedEnvVars = { ...provisionedEnvVars, ...depResult.envVars }
+
+        for (const warning of depResult.warnings) {
+          await appendBuildLog(buildId, `[warn] ${warning}`)
+        }
+
+        // Create scope grant requests in Bio-ID for deps that don't have an active grant.
+        // Runs BEFORE the blocked check so that 'none' deps get a grant request created
+        // (triggering email notification) before the deploy is blocked.
+        if (hasScopedDeps && env.BIO_INTERNAL_KEY) {
+          const bioClient = getBioClient()
+          const clientId = `${service.name}-${environment}`
+          const grantStatusMap = new Map(depResult.grantResults.map(g => [g.service, g.status]))
+
+          for (const dep of unifiedDeps.filter(d => d.scopes.length > 0)) {
+            const status = grantStatusMap.get(dep.service)
+
+            // Already approved or pending — no need to create a new request
+            if (status === 'approved' || status === 'pending') {
+              await appendBuildLog(buildId, `Scope grant for ${dep.service} [${dep.scopes.join(', ')}]: ${status}`)
+              continue
+            }
+
+            // denied/revoked — already blocked, skip grant creation
+            if (status === 'denied' || status === 'revoked') {
+              continue
+            }
+
+            // none/unreachable — create a new scope grant request
+            try {
+              const result = await bioClient.createScopeGrant({
+                requestingServiceId: service.name,
+                targetServiceId: dep.service,
+                requestedScopes: [...dep.scopes],
+                credentialType: 'oauth_client',
+                credentialId: clientId,
+              })
+
+              if (result.success) {
+                await appendBuildLog(buildId, `Scope request created for ${dep.service} [${dep.scopes.join(', ')}] — pending approval`)
+              } else {
+                await appendBuildLog(buildId, `Scope request for ${dep.service} failed: ${result.error?.message || 'unknown'} (non-blocking)`)
+              }
+            } catch (error) {
+              const message = error instanceof Error ? error.message : 'unknown error'
+              await appendBuildLog(buildId, `Scope request for ${dep.service} failed: ${message} (non-blocking)`)
+            }
+          }
+        }
+
+        if (depResult.blocked.length > 0) {
+          const blockMessages = depResult.blocked.join('; ')
+          throw new Error(`Deploy blocked: ${blockMessages}`)
+        }
+
+        const directCount = unifiedDeps.filter(d => d.transport === 'direct').length
+        const gatewayCount = unifiedDeps.filter(d => d.transport === 'gateway').length
+        await appendBuildLog(
+          buildId,
+          `Resolved ${Object.keys(depResult.envVars).length} dependency URLs ` +
+          `(${directCount} direct, ${gatewayCount} gateway, ${depResult.warnings.length} warnings)`
+        )
+      }
+
+      // OAuth provisioning (opt-in via spec.auth or scoped dependencies)
+      if (catalog?.authMode === 'sso' || catalog?.authMode === 'service-only' || hasScopedDeps) {
+        // Collect dependency scopes for OAuth client (approved + unreachable pass; pending/none are now blocked)
+        const depScopes = unifiedDeps
+          .filter(d => d.scopes.length > 0)
+          .flatMap(d => d.scopes)
+        const oauthCreds = await provisionOAuthClient(service.name, environment, namespace, service.customDomains, build.requestedById, catalog?.authMode, service.id, service.oauthCredentials, hasScopedDeps, depScopes, build.org || service.org)
+        if (oauthCreds) {
+          // Resolve Bio-ID's public URL: prefer verified custom domain, fall back to platform hostname
+          const bioService = await getServicesCollection().findOne({ name: 'bio' })
+          const bioCustomDomain = bioService?.customDomains?.find(
+            (d: CustomDomainRecord) => d.dnsVerified && d.environment === environment
+          )
+          const bioIdUrl = bioCustomDomain
+            ? `https://${bioCustomDomain.domain}`
+            : `https://${buildDnsHostname('bio', environment)}`
+
+          // Resolve this service's public URL: prefer verified custom domain, fall back to platform hostname
+          const svcCustomDomain = (service.customDomains ?? []).find(
+            (d: CustomDomainRecord) => d.dnsVerified && d.environment === environment
+          )
+          const svcPublicUrl = svcCustomDomain
+            ? `https://${svcCustomDomain.domain}`
+            : `https://${buildDnsHostname(service.name, environment)}`
+
+          provisionedEnvVars = {
+            ...provisionedEnvVars,
+            BIO_CLIENT_ID: oauthCreds.clientId,
+            BIO_CLIENT_SECRET: oauthCreds.clientSecret,
+            BIO_ID_URL: bioIdUrl,
+            BIO_ID_CALLBACK_URL: `${svcPublicUrl}/api/auth/callback`,
+          }
+          const grantInfo = hasScopedDeps ? `, grants include client_credentials` : ''
+          await appendBuildLog(buildId, `OAuth client provisioned (${catalog?.authMode || 'dependencies'}): ${oauthCreds.clientId}, bio: ${bioIdUrl}, callback: ${svcPublicUrl}/api/auth/callback${grantInfo}`)
+        }
+      } else if (catalog?.authMode === 'none') {
+        await appendBuildLog(buildId, 'OAuth skipped (auth.mode: none)')
+      } else if (catalog) {
+        await appendBuildLog(buildId, 'OAuth skipped (spec.auth not configured)')
+      }
+
+      // Module registration in Bio-ID (opt-in via spec.modules)
+      if (catalog?.modules && catalog.modules.length > 0 && env.BIO_INTERNAL_KEY) {
+        const bioClient = getBioClient()
+        let serviceApiKey: string | undefined
+
+        for (const mod of catalog.modules) {
+          try {
+            const result = await bioClient.registerModule({
+              moduleId: mod.moduleId,
+              name: mod.name,
+              description: mod.description,
+              serviceId: service.name,
+              owner: catalog.owner,
+              homepage: mod.homepage,
+              scopes: [...mod.scopes],
+              defaultScopes: mod.defaultScopes ? [...mod.defaultScopes] : undefined,
+              onboarding: mod.onboarding ? { ...mod.onboarding } : undefined,
+            })
+            if (result.success) {
+              await appendBuildLog(buildId, `Module registered in Bio-ID: ${mod.moduleId}`)
+              // Capture the per-service API key returned by Bio-ID
+              const responseKey = (result as unknown as Record<string, unknown>).serviceApiKey as string | undefined
+              if (responseKey) {
+                serviceApiKey = responseKey
+              }
+            } else {
+              await appendBuildLog(buildId, `Module registration warning for ${mod.moduleId}: ${result.error?.message || 'unknown error'}`)
+            }
+          } catch (error) {
+            const message = error instanceof Error ? error.message : 'unknown error'
+            await appendBuildLog(buildId, `Module registration failed for ${mod.moduleId}: ${message} (non-blocking)`)
+          }
+        }
+
+        // Inject the per-service API key (scoped to this service, not the builder's master key)
+        if (serviceApiKey) {
+          provisionedEnvVars = {
+            ...provisionedEnvVars,
+            BIO_INTERNAL_KEY: serviceApiKey,
+          }
+          await appendBuildLog(buildId, `Service API key provisioned for ${catalog.modules.length} module(s)`)
+        }
+      }
+
+      // Apply config defaults from catalog declarations (lowest priority)
+      if (catalog?.configDeclarations && catalog.configDeclarations.length > 0) {
+        const configDefaults = computeConfigDefaults(
+          catalog.configDeclarations,
+          service.config || {},
+          Object.keys(service.secrets || {}),
+        )
+        if (Object.keys(configDefaults).length > 0) {
+          provisionedEnvVars = { ...configDefaults, ...provisionedEnvVars }
+          await appendBuildLog(buildId, `Applied ${Object.keys(configDefaults).length} config default(s): ${Object.keys(configDefaults).join(', ')}`)
+        }
+      }
+
+      await deployToKubernetes(service, imageTag, environment, workDir, buildId, iecConfig, provisionedEnvVars, catalog?.port, catalog?.healthEndpoint, vaultAnnotations, catalog?.framework)
+      await appendBuildLog(buildId, `Deployed to Kubernetes namespace: ${namespace}`)
+
+      // Register service in Koko after successful deployment
+      // Extract repo name and org from URL (e.g., git@github.com:insurecosys/iec-koko.git -> iec-koko)
+      const repoName = service.repoUrl.split('/').pop()?.replace('.git', '') || service.name
+      const repoOrg = extractRepoOrg(service.repoUrl)
+
+      // Owner priority: build.org (CLI auth) > service.org (stored) > catalog.owner > fallback
+      const serviceOwner = build.org || service.org || catalog?.owner || repoOrg
+
+      const kokoService = await registerOrUpdateServiceInKoko(
+        service.name,  // Use service name as ID, not UUID
+        service.name,
+        service.description || `${service.name} service`,
+        repoName,
+        namespace,
+        service.port || 3000,
+        service.healthEndpoint || '/health',
+        service.tags || [],
+        catalog?.routes || [],
+        serviceOwner,
+        repoOrg,
+        {
+          podTier: catalog?.podTier || 'nano',
+          storageTiers: catalog?.storage?.map((s) => s.tier) ?? [],
+          homepage: catalog?.homepage,
+        },
+      )
+      if (kokoService) {
+        await appendBuildLog(buildId, `Service registered in Koko: ${kokoService.id}`)
+      }
+
+      // Register available scopes + auto-seed grants AFTER Koko knows about the service
+      if (sharingConfigs.length > 0) {
+        const scopesRegistered = await registerAvailableScopes(
+          service.name,
+          serviceOwner || service.name,
+          environment,
+          sharingConfigs
+        )
+        if (scopesRegistered > 0) {
+          await appendBuildLog(buildId, `Registered ${scopesRegistered}/${sharingConfigs.length} available scope(s) in Koko`)
+        }
+
+        // Auto-seed approved grants for all listed consumers (migration + backwards compat).
+        // The owner explicitly listed them in sharedWith, so auto-approve is safe.
+        for (const config of sharingConfigs) {
+          for (const consumer of config.sharedWith) {
+            await autoSeedScopeGrant(service.name, consumer.service, config.type, consumer.access)
+          }
+        }
+      }
+
+      // Sync onboarding spec to Koko (idempotent — cleans up if removed from catalog)
+      if (catalog?.onboarding) {
+        const synced = await syncOnboardingToKoko(
+          service.name,
+          catalog.onboarding.routes.map(r => ({
+            pattern: r.pattern,
+            steps: r.steps.map(s => ({ stepId: s.stepId, label: s.label, description: s.description })),
+          }))
+        )
+        if (synced) {
+          await appendBuildLog(buildId, `Onboarding spec synced to Koko (${catalog.onboarding.routes.length} route(s))`)
+        }
+      } else {
+        // No onboarding declared — clean up any previous spec
+        await syncOnboardingToKoko(service.name, undefined)
+      }
+
+      // Backfill service ownership if not yet set (handles services created before ownership tracking)
+      const ownershipUpdates: Record<string, string> = {}
+      if (!service.org && build.org) {
+        ownershipUpdates.org = build.org
+      }
+      if (!service.createdBy && build.requestedBy) {
+        ownershipUpdates.createdBy = build.requestedBy
+      }
+      if (Object.keys(ownershipUpdates).length > 0) {
+        await services.updateOne(
+          { id: service.id },
+          { $set: { ...ownershipUpdates, updatedAt: new Date().toISOString() } }
+        )
+        await appendBuildLog(buildId, `Service ownership backfilled: ${Object.keys(ownershipUpdates).join(', ')}`)
+      }
+
+      // Configure DNS after successful deployment
+      if (env.ENABLE_DNS_MANAGEMENT === 'true') {
+        const dnsRecord = await configureDnsForService(service.name, environment)
+        if (dnsRecord) {
+          const hostname = buildDnsHostname(service.name, environment)
+          await appendBuildLog(buildId, `DNS configured: ${hostname} -> ${dnsRecord.content}`)
+
+          // Register domain in Koko service registry
+          const domainBinding = await registerOrUpdateDomainInKoko(
+            hostname,
+            service.id,
+            environment as 'dev' | 'sandbox' | 'uat' | 'prod',
+            'platform',
+            env.CLOUDFLARE_ZONE_ID,
+            dnsRecord.id
+          )
+          if (domainBinding) {
+            await appendBuildLog(buildId, `Domain registered in Koko: ${domainBinding.id}`)
+          }
+        } else {
+          await appendBuildLog(buildId, `DNS configuration skipped (not configured)`)
+        }
+      }
+      // Post-deploy pod health check
+      try {
+        const diagnostics = await captureDeployDiagnostics(service.name, namespace)
+        const hasIssues = diagnostics.pods.some(
+          (p) => (p.phase !== 'Running' && p.phase !== 'Succeeded') || p.restartCount > 0
+        )
+        if (hasIssues) {
+          await appendBuildLog(buildId, `[post-deploy] ${diagnostics.summary}`)
+          for (const pod of diagnostics.pods) {
+            if (pod.phase !== 'Running' && pod.phase !== 'Succeeded') {
+              for (const event of pod.events.slice(-5)) {
+                await appendBuildLog(buildId, `[post-deploy]   ${event.type}: ${event.reason} - ${event.message}`)
+              }
+            }
+          }
+          await builds.updateOne({ id: buildId }, { $set: { diagnostics } })
+        } else {
+          await appendBuildLog(buildId, `[post-deploy] ${diagnostics.summary}`)
+        }
+      } catch (diagError) {
+        logger.warn({ buildId, error: diagError }, 'Failed to capture post-deploy diagnostics')
+      }
+    } else {
+      logger.warn({ buildId }, 'ENABLE_K8S_DEPLOY is not true — Kubernetes deployment SKIPPED')
+      await appendBuildLog(buildId, `WARNING: Kubernetes deployment was SKIPPED (ENABLE_K8S_DEPLOY is not 'true'). Image was built and pushed but NOT deployed to the cluster.`)
+    }
+
+    // Mark as completed
+    await updateBuildStatus(buildId, 'completed')
+    await appendBuildLog(buildId, `Build completed successfully`)
+
+    // Update service with last build info + credential rotation clock
+    const now = new Date()
+    const rotationTtlMs = env.ROTATION_TTL_DAYS * 24 * 60 * 60 * 1000
+    const isAutoRotation = build.requestedBy === 'rotation-scheduler@internal.tawa'
+    const isManualRefresh = build.requestedBy?.startsWith('refresh:')
+    const trigger: RotationHistoryEntry['trigger'] = isAutoRotation
+      ? 'auto-rotation'
+      : isManualRefresh
+        ? 'manual-refresh'
+        : 'deploy'
+
+    const newHistoryEntry: RotationHistoryEntry = {
+      rotatedAt: now.toISOString(),
+      trigger,
+      environment: build.environment,
+      buildId,
+    }
+
+    const existingHistory = service.credentialRotation?.rotationHistory ?? []
+    const credentialRotation: CredentialRotation = {
+      lastRotatedAt: now.toISOString(),
+      nextRotationAt: new Date(now.getTime() + rotationTtlMs).toISOString(),
+      rotationPolicy: service.credentialRotation?.rotationPolicy ?? 'auto',
+      rotationHistory: [newHistoryEntry, ...existingHistory].slice(0, 50),
+    }
+
+    await services.updateOne(
+      { id: service.id },
+      {
+        $set: {
+          lastBuildId: buildId,
+          lastBuildStatus: 'completed',
+          credentialRotation,
+          updatedAt: now.toISOString(),
+        }
+      }
+    )
+
+  } catch (error) {
+    const errorMessage = error instanceof Error ? error.message : 'Unknown error'
+    logger.error({ buildId, error: errorMessage }, 'Build failed')
+    await updateBuildStatus(buildId, 'failed', errorMessage)
+    await appendBuildLog(buildId, `Build failed: ${errorMessage}`)
+
+    // Capture pod diagnostics if failure occurred during deploy phase
+    if (deployPhaseReached) {
+      try {
+        const namespace = service.namespace || `${service.name}-${build.environment}`
+        const diagnostics = await captureDeployDiagnostics(service.name, namespace)
+        await appendBuildLog(buildId, `[diagnostics] ${diagnostics.summary}`)
+        for (const pod of diagnostics.pods) {
+          await appendBuildLog(buildId, `[diagnostics] Pod ${pod.name}: ${pod.phase} (restarts: ${pod.restartCount})`)
+          for (const event of pod.events.slice(-5)) {
+            await appendBuildLog(buildId, `[diagnostics]   ${event.type}: ${event.reason} - ${event.message}`)
+          }
+          if (pod.logs) {
+            await appendBuildLog(buildId, `[diagnostics] Container logs:\n${pod.logs}`)
+          }
+        }
+        await builds.updateOne({ id: buildId }, { $set: { diagnostics } })
+      } catch (diagError) {
+        logger.warn({ buildId, error: diagError }, 'Failed to capture deploy failure diagnostics')
+      }
+    }
+
+    await services.updateOne(
+      { id: service.id },
+      {
+        $set: {
+          lastBuildId: buildId,
+          lastBuildStatus: 'failed',
+          updatedAt: new Date().toISOString()
+        }
+      }
+    )
+  } finally {
+    // Cleanup workspace
+    try {
+      await rm(workDir, { recursive: true, force: true })
+    } catch {
+      // Ignore cleanup errors
+    }
+  }
+}
+
+/**
+ * Clone repository from git or extract tarball from GridFS
+ */
+async function cloneOrExtract(repoUrl: string, branch: string, commitSha: string | undefined, workDir: string): Promise<string> {
+  await mkdir(workDir, { recursive: true })
+
+  // Handle GridFS tarballs (from tawa push)
+  if (repoUrl.startsWith('gridfs://')) {
+    const tarballId = repoUrl.replace('gridfs://', '')
+    logger.info({ tarballId, workDir }, 'Extracting tarball from GridFS')
+    await extractTarball(tarballId, workDir)
+    return commitSha || 'push'
+  }
+
+  // Handle self-hosted git (push:// protocol) - tarball already uploaded
+  if (repoUrl.startsWith('push://')) {
+    logger.info({ repoUrl, workDir }, 'Source from push (tarball mode)')
+    return commitSha || 'push'
+  }
+
+  // Standard git clone — inject Forgejo token for private repos
+  const cloneUrl = injectGitCredentials(repoUrl)
+  const git: SimpleGit = simpleGit()
+  await git.clone(cloneUrl, workDir, ['--branch', branch, '--single-branch'])
+
+  const repoGit = simpleGit(workDir)
+
+  // If a specific commit SHA is provided (not "latest" or undefined), checkout that commit
+  if (commitSha && commitSha !== 'latest' && /^[a-f0-9]{7,40}$/i.test(commitSha)) {
+    await repoGit.checkout(commitSha)
+    return commitSha
+  }
+
+  // Otherwise, get the current HEAD commit
+  const log = await repoGit.log({ maxCount: 1 })
+  return log.latest?.hash || 'HEAD'
+}
+
+async function buildDockerImage(workDir: string, imageTag: string, buildId: string, dockerfilePath: string = 'Dockerfile'): Promise<void> {
+  const fullDockerfilePath = join(workDir, dockerfilePath)
+
+  logger.info({ imageTag, workDir, dockerfilePath: fullDockerfilePath }, 'Building Docker image')
+  await execAndLog(
+    `docker build -t ${imageTag} -f ${fullDockerfilePath} ${workDir}`,
+    buildId,
+    { label: 'docker build' }
+  )
+}
+
+/**
+ * Authenticate Docker CLI with the container registry.
+ * Credentials are passed via stdin to avoid shell escaping issues.
+ */
+async function ensureDockerAuth(buildId: string): Promise<void> {
+  if (!env.DOCKER_REGISTRY_TOKEN || !env.DOCKER_REGISTRY_USER) {
+    logger.warn('DOCKER_REGISTRY_USER or DOCKER_REGISTRY_TOKEN not set — skipping docker login (assuming local registry or pre-authenticated)')
+    return
+  }
+
+  const registryHost = env.DOCKER_REGISTRY.split('/')[0]
+  logger.info({ registryHost, user: env.DOCKER_REGISTRY_USER }, 'Authenticating with Docker registry')
+
+  await execAndLog(
+    `echo "${env.DOCKER_REGISTRY_TOKEN}" | docker login ${registryHost} -u ${env.DOCKER_REGISTRY_USER} --password-stdin`,
+    buildId,
+    { label: 'docker login' }
+  )
+}
+
+async function pushDockerImage(imageTag: string, buildId: string): Promise<void> {
+  // Ensure we're authenticated before pushing
+  await ensureDockerAuth(buildId)
+
+  logger.info({ imageTag }, 'Pushing image to registry')
+  await execAndLog(
+    `docker push ${imageTag}`,
+    buildId,
+    { label: 'docker push' }
+  )
+}
+
+async function ensureNamespaceExists(namespace: string): Promise<void> {
+  try {
+    await execAsync(`kubectl create namespace ${namespace} --dry-run=client -o yaml | kubectl apply -f -`, {
+      encoding: 'utf8',
+      shell: '/bin/bash',
+    })
+    logger.info({ namespace }, 'Namespace ensured')
+  } catch (error) {
+    logger.warn({ namespace, error }, 'Failed to ensure namespace (may already exist)')
+  }
+}
+
+async function deployToKubernetes(
+  service: Service,
+  imageTag: string,
+  environment: string,
+  workDir: string,
+  buildId: string,
+  iecConfig?: IecYamlConfig | null,
+  extraEnvVars?: Record<string, string>,
+  catalogPort?: number,
+  catalogHealthEndpoint?: string,
+  vaultAnnotations?: VaultAnnotations,
+  framework?: string
+): Promise<void> {
+
+  // Determine namespace: use service.namespace or generate from name-environment
+  const namespace = service.namespace || `${service.name}-${environment}`
+
+  // Resolve helm chart path — explicit config takes precedence, otherwise auto-discover
+  const explicitChart = iecConfig?.helmChart || service.helmChart
+  const helmChartPath = explicitChart
+    ? (service.appPath && !explicitChart.startsWith('/') ? join(service.appPath, explicitChart) : explicitChart)
+    : await resolveHelmChartPath(workDir, service.name, service.appPath)
+
+  const chart = helmChartPath.startsWith('/') ? helmChartPath : join(workDir, helmChartPath)
+  logger.info({ helmChartPath, chart }, 'Resolved helm chart path')
+
+  logger.info({ serviceName: service.name, namespace, imageTag, environment }, 'Deploying to Kubernetes')
+
+  // Ensure namespace exists (also called earlier for DB provisioning)
+  await ensureNamespaceExists(namespace)
+
+  // Enable Goldilocks resource recommendations for this namespace
+  try {
+    await execAsync(`kubectl label namespace ${namespace} goldilocks.fairwinds.com/enabled=true --overwrite`, {
+      encoding: 'utf8',
+      shell: '/bin/bash',
+    })
+  } catch (error) {
+    logger.warn({ namespace, error }, 'Failed to label namespace for Goldilocks')
+  }
+
+  // Apply default resource limits to prevent unbounded resource usage
+  try {
+    const limitRangeYaml = `
+apiVersion: v1
+kind: LimitRange
+metadata:
+  name: default-limits
+  namespace: ${namespace}
+spec:
+  limits:
+  - default:
+      cpu: "500m"
+      memory: "512Mi"
+    defaultRequest:
+      cpu: "50m"
+      memory: "64Mi"
+    max:
+      cpu: "2"
+      memory: "2Gi"
+    type: Container`
+    await execAsync(`echo '${limitRangeYaml}' | kubectl apply -f -`, {
+      encoding: 'utf8',
+      shell: '/bin/bash',
+    })
+    logger.info({ namespace }, 'LimitRange applied')
+  } catch (error) {
+    logger.warn({ namespace, error }, 'Failed to apply LimitRange')
+  }
+
+  // Ensure imagePullSecret exists in namespace (copy from default namespace)
+  try {
+    await execAsync(`kubectl get secret insureco -n ${namespace} 2>/dev/null || kubectl get secret insureco -n default -o yaml | sed 's/namespace: default/namespace: ${namespace}/' | kubectl apply -f -`, {
+      encoding: 'utf8',
+      shell: '/bin/bash'
+    })
+    logger.info({ namespace }, 'ImagePullSecret ensured')
+  } catch (error) {
+    logger.warn({ namespace, error }, 'Failed to ensure imagePullSecret')
+  }
+
+  // Build hostname based on environment
+  const hostname = buildDnsHostname(service.name, environment)
+
+  // Build helm command
+  const imageRepo = imageTag.split(':')[0]
+  const imageTagOnly = imageTag.split(':')[1]
+  let command = `helm upgrade --install ${service.name} ${chart} --namespace ${namespace} --set image.repository=${imageRepo} --set image.tag=${imageTagOnly}`
+
+  // Override service port if catalog specifies one (e.g., 80 for static/nginx, 3000 for Node.js)
+  if (catalogPort) {
+    command += ` --set service.port=${catalogPort}`
+  }
+
+  // Ensure imagePullSecrets matches the registry secret managed by the builder
+  command += ` --set imagePullSecrets[0].name=insureco`
+
+  // Set dynamic hostname for ingress (must include paths to avoid overwriting the whole object)
+  // Note: TLS is handled by Cloudflare proxy, not cert-manager
+  command += ` --set ingress.enabled=true`
+  command += ` --set ingress.className=nginx`
+  command += ` --set ingress.hosts[0].host=${hostname}`
+  command += ` --set ingress.hosts[0].paths[0].path=/`
+  command += ` --set ingress.hosts[0].paths[0].pathType=Prefix`
+
+  // Add verified custom domains as additional ingress hosts
+  const DOMAIN_RE = /^[a-z0-9]([a-z0-9-]*[a-z0-9])?(\.[a-z0-9]([a-z0-9-]*[a-z0-9])?)+$/
+  const verifiedCustomDomains = (service.customDomains || []).filter(
+    d => d.dnsVerified && d.environment === environment
+  )
+
+  let domainIdx = 0
+  for (const customDomain of verifiedCustomDomains) {
+    if (!DOMAIN_RE.test(customDomain.domain)) {
+      logger.warn({ domain: customDomain.domain }, 'Skipping invalid custom domain')
+      continue
+    }
+    const idx = domainIdx + 1  // [0] is the platform hostname
+    command += ` --set ingress.hosts[${idx}].host=${escapeShellArg(customDomain.domain)}`
+    command += ` --set ingress.hosts[${idx}].paths[0].path=/`
+    command += ` --set ingress.hosts[${idx}].paths[0].pathType=Prefix`
+    domainIdx++
+  }
+
+  if (verifiedCustomDomains.length > 0) {
+    await appendBuildLog(buildId, `Custom domains: ${verifiedCustomDomains.map(d => d.domain).join(', ')}`)
+  }
+
+  // Raise the nginx ingress body-size limit for services that declare storage
+  // (default 1m is too small for file uploads — Word docs, PDFs, etc.)
+  const hasStorage = vaultAnnotations?.['vault.hashicorp.com/agent-inject-secret-storage']
+  if (hasStorage) {
+    command += ` --set-json 'ingress.annotations={"nginx.ingress.kubernetes.io/proxy-body-size":"10m"}'`
+  }
+
+  // Canonical public URL for this service (prefers verified custom domain)
+  const primaryDomain = verifiedCustomDomains[0]?.domain
+  const serviceUrl = `https://${primaryDomain || hostname}`
+
+  // Set health probe paths from catalog (framework-specific, e.g. /api/health for nextjs)
+  // When Vault is active, liveness probe is overridden to exec-based (file check),
+  // so only set the readiness httpGet probe. The liveness override happens below.
+  const vaultActive = vaultAnnotations && Object.keys(vaultAnnotations).length > 0
+  if (catalogHealthEndpoint) {
+    if (!vaultActive) {
+      command += ` --set livenessProbe.httpGet.path=${catalogHealthEndpoint}`
+    }
+    command += ` --set readinessProbe.httpGet.path=${catalogHealthEndpoint}`
+  }
+
+  // Static uses nginx-unprivileged (non-root, UID 101, port 8080)
+  // Override default securityContext (UID 1001) to match nginx image user
+  if (framework === 'static') {
+    command += ` --set securityContext.runAsUser=101`
+    command += ` --set securityContext.fsGroup=101`
+    command += ` --set containerSecurityContext.allowPrivilegeEscalation=false`
+  }
+
+  // Set environment-specific URLs for portal service
+  // BIO_ID_URL is now resolved generically by the OAuth provisioner above
+  if (service.name === 'portal' && extraEnvVars) {
+    extraEnvVars.PORTAL_URL = serviceUrl
+    extraEnvVars.NEXT_PUBLIC_BIO_ID_URL = extraEnvVars.BIO_ID_URL || `https://${buildDnsHostname('bio', environment)}`
+    logger.info({ serviceName: service.name, portalUrl: serviceUrl }, 'Set portal-specific URLs')
+  }
+
+  // Set environment-specific URLs for bio service (OAuth provider)
+  // Bio needs to know its own public URL for JWT issuer and email links
+  if (service.name === 'bio' && extraEnvVars) {
+    extraEnvVars.BIO_ID_URL = serviceUrl
+    extraEnvVars.NEXTAUTH_URL = serviceUrl
+    extraEnvVars.JWT_ISSUER = serviceUrl
+    logger.info({ serviceName: service.name, bioUrl: serviceUrl }, 'Set URLs for bio')
+  }
+
+  // Add helm values from service config
+  if (service.helmValues) {
+    for (const [key, value] of Object.entries(service.helmValues)) {
+      command += ` --set ${key}=${value}`
+    }
+  }
+
+  // Inject platform default env vars (lowest precedence — user config can override)
+  // All deployed environments use 'production' — 'development' is for local dev only
+  const platformNodeEnv = 'production'
+  const platformDefaults: Record<string, string> = {
+    NODE_ENV: platformNodeEnv,
+    SERVICE_URL: serviceUrl,
+  }
+
+  for (const [key, value] of Object.entries(platformDefaults)) {
+    command += ` --set env.${key}=${escapeShellArg(value)}`
+  }
+  logger.info(
+    { serviceName: service.name, environment, NODE_ENV: platformNodeEnv },
+    'Injecting platform default env vars'
+  )
+
+  // Inject managed config vars (from `tawa config set`)
+  if (service.config && Object.keys(service.config).length > 0) {
+    // Warn if user has set NODE_ENV to a value that conflicts with the deploy target
+    const userNodeEnv = service.config.NODE_ENV
+    if (userNodeEnv && userNodeEnv !== platformNodeEnv) {
+      const warning = `Warning: NODE_ENV is set to "${userNodeEnv}" but deploying to "${environment}" (expected "${platformNodeEnv}"). User override will be used.`
+      logger.warn({ serviceName: service.name, userNodeEnv, expected: platformNodeEnv, environment }, warning)
+      await appendBuildLog(buildId, `⚠️  ${warning}`)
+    }
+
+    for (const [key, value] of Object.entries(service.config)) {
+      validateEnvKey(key)
+      command += ` --set env.${key}=${escapeShellArg(escapeHelmValue(value))}`
+    }
+    logger.info({ serviceName: service.name, keys: Object.keys(service.config) }, 'Injecting managed config vars')
+  }
+
+  // Inject provisioned env vars (from DB and OAuth provisioners)
+  if (extraEnvVars && Object.keys(extraEnvVars).length > 0) {
+    for (const [key, value] of Object.entries(extraEnvVars)) {
+      validateEnvKey(key)
+      command += ` --set env.${key}=${escapeShellArg(escapeHelmValue(value))}`
+    }
+    logger.info({ serviceName: service.name, keys: Object.keys(extraEnvVars) }, 'Injecting provisioned env vars')
+  }
+
+  // Create K8s Secret for managed secrets (from `tawa config set --secret`)
+  const podAnnotations: Record<string, string> = {}
+
+  if (service.secrets && Object.keys(service.secrets).length > 0) {
+    if (!process.env.CONFIG_ENCRYPTION_KEY) {
+      throw new Error('CONFIG_ENCRYPTION_KEY is required to deploy services with managed secrets')
+    }
+    const { decryptRecord } = await import('./crypto.js')
+    const decrypted = decryptRecord(service.secrets)
+    const secretName = `${service.name}-managed-secrets`
+
+    // Validate all key names before building shell command
+    for (const k of Object.keys(decrypted)) {
+      validateEnvKey(k)
+    }
+
+    const literals = Object.entries(decrypted)
+      .map(([k, v]) => `--from-literal=${k}=${escapeShellArg(v)}`)
+      .join(' ')
+
+    await execAndLog(
+      `kubectl create secret generic ${secretName} ${literals} --namespace ${namespace} --dry-run=client -o yaml | kubectl apply -f -`,
+      buildId,
+      { label: 'kubectl secret', shell: '/bin/bash' }
+    )
+
+    command += ` --set secretRef=${secretName}`
+
+    // Checksum of secret content forces pod restart when secrets change
+    // (otherwise same image tag = no pod rollout = stale secrets)
+    const secretHash = createHash('sha256')
+      .update(Object.keys(decrypted).sort().map(k => `${k}=${decrypted[k]}`).join('\n'))
+      .digest('hex')
+      .slice(0, 16)
+    podAnnotations['checksum/managed-secrets'] = secretHash
+
+    logger.info({ serviceName: service.name, keys: Object.keys(service.secrets) }, 'Injecting managed secrets')
+  }
+
+  // Zero-downtime rolling update strategy (credential rotation compliance)
+  command += ` --set strategy.type=RollingUpdate`
+  command += ` --set strategy.rollingUpdate.maxSurge=1`
+  command += ` --set strategy.rollingUpdate.maxUnavailable=0`
+
+  // Vault Agent annotations for dynamic database credentials
+  if (vaultActive) {
+    Object.assign(podAnnotations, vaultAnnotations)
+    command += ` --set serviceAccountName=${service.name}`
+
+    // Shared volume for Vault credential rotation liveness signaling
+    command += ` --set-json 'volumes=[{"name":"vault-signals","emptyDir":{"medium":"Memory"}}]'`
+    command += ` --set-json 'volumeMounts=[{"name":"vault-signals","mountPath":"/vault/signals"}]'`
+
+    // Override liveness probe: file-based check instead of httpGet
+    // Vault Agent removes /vault/signals/alive when credentials rotate at max_ttl,
+    // causing K8s to restart the container with fresh credentials.
+    // Explicitly null out httpGet to avoid conflict with chart defaults (custom charts
+    // may define livenessProbe.httpGet in values.yaml — Helm merges rather than replaces).
+    command += ` --set-json 'livenessProbe={"exec":{"command":["cat","/vault/signals/alive"]},"httpGet":null,"initialDelaySeconds":30,"periodSeconds":10}'`
+  }
+
+  // Inject pod annotations (secret checksum + Vault annotations merged)
+  if (Object.keys(podAnnotations).length > 0) {
+    const annotationsJson = JSON.stringify(podAnnotations)
+    command += ` --set-json podAnnotations=${escapeShellArg(annotationsJson)}`
+  }
+
+  command += ' --wait --timeout 5m --history-max 5'
+
+  // If the current release is in a failed state, uninstall it first so
+  // helm upgrade --install starts clean instead of fighting stacked failures.
+  try {
+    const { stdout: statusOut } = await execAsync(
+      `helm status ${service.name} -n ${namespace} -o json 2>/dev/null`,
+      { encoding: 'utf8', shell: '/bin/bash' }
+    )
+    const releaseStatus = JSON.parse(statusOut)
+    if (releaseStatus?.info?.status === 'failed') {
+      await execAndLog(
+        `helm uninstall ${service.name} -n ${namespace}`,
+        buildId,
+        { label: 'helm cleanup (failed release)' }
+      )
+    }
+  } catch {
+    // No existing release or helm status failed — proceed normally
+  }
+
+  await execAndLog(command, buildId, { label: 'helm deploy' })
+  logger.info({ serviceName: service.name, namespace, imageTag }, 'Deployed to Kubernetes')
+}
+
+async function updateBuildStatus(buildId: string, status: BuildStatus, error?: string): Promise<void> {
+  const builds = getBuildsCollection()
+  const now = new Date().toISOString()
+
+  const update: Partial<Build> = { status, updatedAt: now }
+
+  if (status === 'cloning') {
+    update.startedAt = now
+  }
+
+  if (status === 'completed' || status === 'failed') {
+    update.completedAt = now
+  }
+
+  if (error) {
+    update.error = error
+  }
+
+  await builds.updateOne({ id: buildId }, { $set: update })
+  await publishBuildEvent(buildId, status, { error })
+}
+
+async function appendBuildLog(buildId: string, log: string): Promise<void> {
+  const builds = getBuildsCollection()
+  const timestamp = new Date().toISOString()
+  await builds.updateOne(
+    { id: buildId },
+    { $push: { logs: `[${timestamp}] ${log}` } }
+  )
+}
+
+/**
+ * Mark builds stuck in active states (cloning/building/pushing/deploying) as failed.
+ * Called on startup to recover from crashes, and periodically to catch orphaned builds.
+ */
+export async function recoverStaleBuilds(maxAgeMinutes = 30): Promise<number> {
+  const builds = getBuildsCollection()
+  const cutoff = new Date(Date.now() - maxAgeMinutes * 60 * 1000).toISOString()
+
+  const result = await builds.updateMany(
+    {
+      status: { $in: ['cloning', 'building', 'pushing', 'deploying'] as const },
+      updatedAt: { $lt: cutoff },
+    },
+    {
+      $set: {
+        status: 'failed' as const,
+        error: 'Build timed out (server restart or stale process)',
+        completedAt: new Date().toISOString(),
+        updatedAt: new Date().toISOString(),
+      },
+    }
+  )
+
+  if (result.modifiedCount > 0) {
+    logger.warn({ recovered: result.modifiedCount, maxAgeMinutes }, 'Recovered stale builds')
+  }
+
+  return result.modifiedCount
+}
+
+/**
+ * Execute a shell command, capture stdout/stderr, and append output to build logs.
+ * Also pipes output to the process logger for PM2 visibility.
+ * Throws on non-zero exit code with the captured stderr.
+ */
+async function execAndLog(
+  command: string,
+  buildId: string,
+  options: { label?: string; shell?: string } = {}
+): Promise<string> {
+  const { label, shell } = options
+
+  try {
+    const { stdout } = await execAsync(command, {
+      encoding: 'utf8',
+      maxBuffer: 10 * 1024 * 1024,
+      shell: shell || '/bin/bash',
+    })
+
+    // Log significant output lines to build logs (skip blank lines, limit noise)
+    const lines = stdout.split('\n').filter((line: string) => line.trim())
+    if (lines.length > 0) {
+      // For large output (docker build), log last 30 lines to avoid flooding
+      const tail = lines.length > 30 ? lines.slice(-30) : lines
+      const prefix = label ? `[${label}] ` : ''
+      for (const line of tail) {
+        logger.info(line)
+      }
+      await appendBuildLog(buildId, `${prefix}${tail.join('\n')}`)
+    }
+
+    return stdout
+  } catch (error: unknown) {
+    const execError = error as { stdout?: string; stderr?: string; message?: string }
+    const stderr = execError.stderr || execError.message || 'Unknown error'
+    const stdout = execError.stdout || ''
+
+    // Log what we can from the failed command
+    const errorLines = stderr.split('\n').filter((line: string) => line.trim())
+    const outputLines = stdout.split('\n').filter((line: string) => line.trim())
+    const allLines = [...outputLines.slice(-20), ...errorLines.slice(-20)]
+    const prefix = label ? `[${label}] ` : ''
+
+    if (allLines.length > 0) {
+      for (const line of allLines) {
+        logger.error(line)
+      }
+      await appendBuildLog(buildId, `${prefix}${allLines.join('\n')}`)
+    }
+
+    throw error
+  }
+}
+
+/**
+ * Determine the environment from the branch name
+ * Branch to environment mapping:
+ * - main, master -> prod
+ * - develop, development -> sandbox
+ * - staging -> uat
+ * - feature/* -> sandbox (default for feature branches)
+ */
+function determineEnvironmentFromBranch(branch: string): string {
+  const lowerBranch = branch.toLowerCase()
+
+  if (lowerBranch === 'main' || lowerBranch === 'master') {
+    return 'prod'
+  }
+  if (lowerBranch === 'develop' || lowerBranch === 'development') {
+    return 'sandbox'
+  }
+  if (lowerBranch === 'staging' || lowerBranch === 'uat') {
+    return 'uat'
+  }
+  if (lowerBranch.startsWith('feature/') || lowerBranch.startsWith('feat/')) {
+    return 'sandbox'
+  }
+  if (lowerBranch.startsWith('hotfix/') || lowerBranch.startsWith('release/')) {
+    return 'prod'
+  }
+
+  // Default to sandbox for unknown branches
+  return 'sandbox'
+}