@o/scripts 0.16.19-canary.1790165522484

package/src/helpers/deploy-health.ts

@@ -0,0 +1,227 @@
+/**
+ * deployment health check helpers for uncloud deployments
+ * shared across repos via @o/scripts/helpers/deploy-health
+ */
+
+export interface SSHContext {
+  sshOpts: string
+  deployUser: string
+  deployHost: string
+  $: (
+    strings: TemplateStringsArray,
+    ...values: any[]
+  ) => { quiet: () => Promise<{ stdout: Buffer }> }
+}
+
+export interface ContainerInfo {
+  id: string
+  status: string
+  state: string
+  names: string
+  isRestarting: boolean
+}
+
+/**
+ * check for containers stuck in restart loop
+ * fails fast if any container is crash-looping
+ * excludes init containers (minio-init, migrate) that are expected to exit
+ */
+export async function checkRestartLoops(
+  ctx: SSHContext,
+  options?: { excludePatterns?: string[] }
+): Promise<{
+  hasLoop: boolean
+  services: string[]
+}> {
+  const { sshOpts, deployUser, deployHost, $ } = ctx
+  // default exclude patterns for init/oneshot containers
+  const excludePatterns = options?.excludePatterns ?? ['init', 'migrate']
+
+  try {
+    const sshCmd = `ssh ${sshOpts} ${deployUser}@${deployHost}`
+    const result =
+      await $`${sshCmd.split(' ')} "docker ps -a --format '{{.Names}} {{.Status}}'"`.quiet()
+    const lines = result.stdout.toString().trim().split('\n').filter(Boolean)
+
+    const loopingServices: string[] = []
+    for (const line of lines) {
+      if (line.includes('Restarting')) {
+        const name = line.split(' ')[0] || ''
+        // skip init/oneshot containers
+        const isExcluded = excludePatterns.some((pattern) => name.includes(pattern))
+        if (!isExcluded) {
+          loopingServices.push(name)
+        }
+      }
+    }
+
+    return { hasLoop: loopingServices.length > 0, services: loopingServices }
+  } catch {
+    return { hasLoop: false, services: [] }
+  }
+}
+
+/**
+ * get container status for all running containers
+ * extracts service name from container names (handles various naming patterns)
+ */
+export async function getContainerStatus(
+  ctx: SSHContext,
+  options?: { verbose?: boolean }
+): Promise<Map<string, ContainerInfo>> {
+  const { sshOpts, deployUser, deployHost, $ } = ctx
+  try {
+    const sshCmd = `ssh ${sshOpts} ${deployUser}@${deployHost}`
+    // use -a to see all containers including stopped/restarting
+    const result = await $`${sshCmd.split(' ')} "docker ps -a --format json"`.quiet()
+    const lines = result.stdout.toString().trim().split('\n').filter(Boolean)
+
+    const containers = new Map<string, ContainerInfo>()
+    for (const line of lines) {
+      try {
+        const container = JSON.parse(line)
+        const name = container.Names || ''
+        let serviceName: string | null = null
+
+        // try various naming patterns
+        // pattern 1: "projectname-service-1" or "projectname_service_1"
+        let match = name.match(/[^_-]+[_-]([^_-]+)[_-]\w+/)
+        if (match) {
+          serviceName = match[1]
+        } else {
+          // pattern 2: "service-xxxx" where xxxx is alphanumeric
+          match = name.match(/^([^_-]+)[_-]\w+$/)
+          if (match) {
+            serviceName = match[1]
+          } else {
+            // pattern 3: just the name before any dash or underscore
+            match = name.match(/^([^_-]+)/)
+            if (match) {
+              serviceName = match[1]
+            }
+          }
+        }
+
+        if (serviceName) {
+          const status = container.Status || ''
+          const state = container.State || ''
+          const isRestarting = state === 'restarting' || status.includes('Restarting')
+
+          containers.set(serviceName, {
+            id: container.ID,
+            status,
+            state,
+            names: container.Names,
+            isRestarting,
+          })
+        }
+      } catch {
+        // skip invalid json lines
+      }
+    }
+
+    return containers
+  } catch (error) {
+    if (options?.verbose) {
+      console.error('failed to get container status:', error)
+    }
+    return new Map()
+  }
+}
+
+/**
+ * check health status of a specific container
+ * returns 'healthy', 'checking' (mid-probe), or 'unhealthy'
+ */
+export async function checkContainerHealth(
+  ctx: SSHContext,
+  containerId: string
+): Promise<'healthy' | 'checking' | 'unhealthy'> {
+  const { sshOpts, deployUser, deployHost, $ } = ctx
+  try {
+    const sshCmd = `ssh ${sshOpts} ${deployUser}@${deployHost}`
+    const healthResult =
+      await $`${sshCmd.split(' ')} "docker inspect ${containerId} --format '{{if .State.Health}}{{.State.Health.Status}}{{else}}no-healthcheck{{end}}'"`.quiet()
+    const healthStatus = healthResult.stdout.toString().trim()
+
+    if (healthStatus === 'healthy') {
+      return 'healthy'
+    } else if (healthStatus === 'no-healthcheck') {
+      // fall back to checking if running for containers without healthcheck
+      const statusResult =
+        await $`${sshCmd.split(' ')} "docker inspect ${containerId} --format '{{.State.Status}}'"`.quiet()
+      const status = statusResult.stdout.toString().trim()
+      return status === 'running' ? 'healthy' : 'unhealthy'
+    } else if (healthStatus === 'starting') {
+      // mid-healthcheck probe, don't treat as failure yet
+      return 'checking'
+    }
+
+    return 'unhealthy'
+  } catch {
+    return 'unhealthy'
+  }
+}
+
+/**
+ * check http health endpoint from inside web container
+ */
+export async function checkHttpHealth(
+  ctx: SSHContext,
+  options?: { endpoint?: string; containerFilter?: string }
+): Promise<{
+  healthy: boolean
+  status?: number
+  message?: string
+}> {
+  const { sshOpts, deployUser, deployHost, $ } = ctx
+  const endpoint = options?.endpoint || 'http://localhost:8081/'
+  const containerFilter = options?.containerFilter || 'web'
+
+  try {
+    const sshCmd = `ssh ${sshOpts} ${deployUser}@${deployHost}`
+    const containersResult =
+      await $`${sshCmd.split(' ')} "docker ps --filter 'name=${containerFilter}' --format '{{.Names}}'"`.quiet()
+    const webContainer = containersResult.stdout.toString().trim()
+
+    if (!webContainer) {
+      return { healthy: false, message: 'web container not found' }
+    }
+
+    const result =
+      await $`${sshCmd.split(' ')} "docker exec ${webContainer} curl -so /dev/null -w '%{http_code}' -m 5 ${endpoint}"`.quiet()
+    const output = result.stdout.toString().trim()
+    const statusCode = parseInt(output, 10)
+
+    if (statusCode >= 200 && statusCode < 400) {
+      return { healthy: true, status: statusCode }
+    } else if (statusCode >= 400 && statusCode < 500) {
+      return { healthy: true, status: statusCode, message: 'server running' }
+    } else if (statusCode >= 500) {
+      return { healthy: false, status: statusCode, message: 'server error' }
+    }
+
+    return { healthy: false, message: 'app starting...' }
+  } catch {
+    return { healthy: false, message: 'app starting...' }
+  }
+}
+
+/**
+ * get recent logs from a container
+ */
+export async function getContainerLogs(
+  ctx: SSHContext,
+  containerId: string,
+  lines = 20
+): Promise<string[]> {
+  const { sshOpts, deployUser, deployHost, $ } = ctx
+  try {
+    const sshCmd = `ssh ${sshOpts} ${deployUser}@${deployHost}`
+    const result =
+      await $`${sshCmd.split(' ')} "docker logs ${containerId} --tail ${lines} 2>&1"`.quiet()
+    return result.stdout.toString().trim().split('\n')
+  } catch {
+    return []
+  }
+}

package/src/helpers/deploy-lock.ts

@@ -0,0 +1,231 @@
+/**
+ * deploy lock helper — prevents concurrent deploys with auto-expiry
+ * shared across repos via @o/scripts/helpers/deploy-lock
+ *
+ * uses mkdir for atomic lock acquisition (no race window).
+ * writes a meta.json inside the lock dir so stale locks can self-diagnose:
+ * when a deploy dies without releasing, the next deploy prints who left the
+ * lock (pid, host, git sha, CI run URL) instead of just a timestamp.
+ */
+
+import os from 'node:os'
+
+import { run } from './run'
+
+const DEFAULT_LOCK_PATH = '/tmp/deploy.lock'
+const DEFAULT_MAX_AGE_MIN = 15
+
+interface DeployLockOptions {
+  path?: string
+  maxAgeMin?: number
+  /**
+   * called before lock release on all exit paths: normal return, thrown
+   * error, and signal (SIGTERM/SIGINT/SIGHUP). use this to roll back
+   * partial state — e.g. restart a service that was stopped mid-deploy.
+   * cleanup errors are logged but do not prevent lock release.
+   */
+  cleanup?: () => Promise<void>
+}
+
+interface LockMeta {
+  pid: number
+  host: string
+  startedAt: string
+  gitSha: string
+  ciRunId: string
+  ciRunUrl: string
+  ciWorkflow: string
+  ciActor: string
+}
+
+function buildMeta(): LockMeta {
+  const repo = process.env.GITHUB_REPOSITORY || ''
+  const runId = process.env.GITHUB_RUN_ID || ''
+  const server = process.env.GITHUB_SERVER_URL || 'https://github.com'
+  const ciRunUrl = repo && runId ? `${server}/${repo}/actions/runs/${runId}` : ''
+  return {
+    pid: process.pid,
+    host: os.hostname(),
+    startedAt: new Date().toISOString(),
+    gitSha:
+      process.env.GIT_SHA || process.env.GITHUB_SHA || process.env.CI_COMMIT_SHA || '',
+    ciRunId: runId,
+    ciRunUrl,
+    ciWorkflow: process.env.GITHUB_WORKFLOW || '',
+    ciActor: process.env.GITHUB_ACTOR || '',
+  }
+}
+
+function toBase64(s: string): string {
+  return Buffer.from(s, 'utf-8').toString('base64')
+}
+
+async function sshBash(ssh: string, script: string, captureOutput = true) {
+  // base64-encode the script so we never have to escape quotes, $, backticks,
+  // newlines etc. at multiple layers (local bash → ssh → remote bash).
+  const b64 = toBase64(script)
+  return run(`${ssh} "echo ${b64} | base64 -d | bash"`, {
+    captureOutput,
+    silent: true,
+  })
+}
+
+export async function acquireDeployLock(
+  ssh: string,
+  opts?: DeployLockOptions
+): Promise<void> {
+  const lockPath = opts?.path || DEFAULT_LOCK_PATH
+  const maxAge = opts?.maxAgeMin || DEFAULT_MAX_AGE_MIN
+
+  // atomic cleanup-and-acquire in one round-trip:
+  // 1. if a stale lock exists (older than maxAge), print its metadata so the
+  //    caller can log who left it behind, then rm it.
+  // 2. try mkdir — atomic; succeeds iff no concurrent deploy holds the lock.
+  const acquireScript = `
+set -u
+LOCK=${lockPath}
+MAXAGE=${maxAge}
+if [ -d "$LOCK" ]; then
+  if [ -n "$(find "$LOCK" -maxdepth 0 -mmin +$MAXAGE 2>/dev/null)" ]; then
+    echo "STALE_META_START"
+    cat "$LOCK/meta.json" 2>/dev/null || echo "(no metadata file)"
+    echo "STALE_META_END"
+    rm -rf "$LOCK"
+  fi
+fi
+if mkdir "$LOCK" 2>/dev/null; then
+  echo ACQUIRED
+else
+  echo LOCKED
+fi
+`
+
+  const { stdout: acquireOut } = await sshBash(ssh, acquireScript)
+
+  // log any stale cleanup so the next deploy's logs show who left it
+  const staleMatch = acquireOut.match(/STALE_META_START\n([\s\S]*?)\nSTALE_META_END/)
+  if (staleMatch?.[1]) {
+    console.warn(`deploy-lock: cleared stale lock (older than ${maxAge}m) left by:`)
+    for (const line of staleMatch[1].split('\n')) {
+      console.warn(`  ${line}`)
+    }
+  }
+
+  if (!acquireOut.trim().endsWith('ACQUIRED')) {
+    // read the live holder's metadata so the error tells the user exactly
+    // which deploy is blocking — not just "a lock file exists".
+    const diagScript = `
+LOCK=${lockPath}
+echo "meta.json:"
+cat "$LOCK/meta.json" 2>/dev/null || echo "  (no metadata)"
+echo "mtime:"
+stat -c "  %y" "$LOCK" 2>/dev/null || echo "  (unknown)"
+`
+    let heldBy = '(diagnostics unavailable)'
+    try {
+      const { stdout } = await sshBash(ssh, diagScript)
+      heldBy = stdout.trim() || heldBy
+    } catch {
+      // best-effort
+    }
+
+    throw new Error(
+      `another deploy is in progress (${lockPath} exists on server)\n` +
+        `  held by:\n${heldBy
+          .split('\n')
+          .map((l) => `    ${l}`)
+          .join('\n')}\n` +
+        `  will auto-expire after ${maxAge} minutes, or remove manually: ${ssh} "rm -rf ${lockPath}"`
+    )
+  }
+
+  // write metadata inside the lock dir (best-effort; lock is already held
+  // so metadata failures are non-fatal — they only hurt future diagnostics).
+  const meta = buildMeta()
+  const metaJson = JSON.stringify(meta, null, 2)
+  const writeScript = `
+LOCK=${lockPath}
+cat > "$LOCK/meta.json" << 'DEPLOY_LOCK_META_EOF'
+${metaJson}
+DEPLOY_LOCK_META_EOF
+`
+  try {
+    await sshBash(ssh, writeScript, false)
+  } catch {
+    // non-fatal
+  }
+}
+
+export async function releaseDeployLock(
+  ssh: string,
+  opts?: DeployLockOptions
+): Promise<void> {
+  const lockPath = opts?.path || DEFAULT_LOCK_PATH
+  try {
+    await run(`${ssh} "rm -rf ${lockPath}"`, { silent: true })
+  } catch {
+    // best-effort, lock will auto-expire
+  }
+}
+
+/**
+ * run `fn` with the deploy lock held, releasing on ANY exit path —
+ * normal return, thrown error, or process signal (SIGTERM from CI cancel,
+ * SIGINT from ctrl-c, SIGHUP from terminal close).
+ *
+ * prefer this over raw acquire/release: a bare try/finally does NOT catch
+ * signals, so a CI-cancelled deploy would leak the lock for 15 minutes and
+ * block the next run. this helper installs signal handlers that release
+ * before exit, then removes them once `fn` completes.
+ *
+ * if `opts.cleanup` is provided, it runs before lock release on every exit
+ * path. use it to roll back partial deploy state (e.g. restart a stopped
+ * service) so a mid-flight crash doesn't leave prod in a broken state.
+ */
+export async function withDeployLock<T>(
+  ssh: string,
+  fn: () => Promise<T>,
+  opts?: DeployLockOptions
+): Promise<T> {
+  await acquireDeployLock(ssh, opts)
+
+  let released = false
+  const runCleanupAndRelease = async () => {
+    if (released) return
+    released = true
+    if (opts?.cleanup) {
+      try {
+        await opts.cleanup()
+      } catch (err) {
+        console.warn('deploy-lock: cleanup threw (non-fatal):', err)
+      }
+    }
+    await releaseDeployLock(ssh, opts)
+  }
+
+  // signal handlers — fire-and-forget cleanup+release with a safety timeout
+  // so the process always exits even if ssh hangs. the 15s budget covers
+  // user cleanup (e.g. restarting a service) plus the release rm call.
+  const signals: NodeJS.Signals[] = ['SIGTERM', 'SIGINT', 'SIGHUP']
+  const handlers = new Map<NodeJS.Signals, () => void>()
+  for (const sig of signals) {
+    const handler = () => {
+      const done = runCleanupAndRelease().catch(() => {})
+      const timeout = new Promise<void>((r) => setTimeout(r, 15_000))
+      Promise.race([done, timeout]).finally(() => {
+        process.exit(sig === 'SIGINT' ? 130 : 143)
+      })
+    }
+    handlers.set(sig, handler)
+    process.once(sig, handler)
+  }
+
+  try {
+    return await fn()
+  } finally {
+    for (const [sig, handler] of handlers) {
+      process.off(sig, handler)
+    }
+    await runCleanupAndRelease()
+  }
+}

package/src/helpers/ensure-s3-bucket.ts

@@ -0,0 +1,88 @@
+import {
+  CreateBucketCommand,
+  HeadBucketCommand,
+  PutBucketLifecycleConfigurationCommand,
+  PutBucketVersioningCommand,
+  S3Client,
+} from '@aws-sdk/client-s3'
+
+export async function ensureS3Bucket(
+  bucketName: string,
+  options?: {
+    region?: string
+    retentionDays?: number
+  }
+): Promise<void> {
+  const region = options?.region ?? 'us-west-1'
+  const retentionDays = options?.retentionDays ?? 30
+
+  const s3Client = new S3Client({
+    region,
+    credentials: {
+      accessKeyId: process.env.AWS_ACCESS_KEY_ID!,
+      secretAccessKey: process.env.AWS_SECRET_ACCESS_KEY!,
+    },
+  })
+
+  try {
+    // check if bucket exists
+    await s3Client.send(new HeadBucketCommand({ Bucket: bucketName }))
+    console.info(`✅ S3 bucket exists: ${bucketName}`)
+  } catch (error: any) {
+    if (error.name === 'NotFound' || error.$metadata?.httpStatusCode === 404) {
+      console.info(`📦 Creating S3 bucket: ${bucketName}`)
+
+      // create bucket
+      await s3Client.send(
+        new CreateBucketCommand({
+          Bucket: bucketName,
+          ...(region !== 'us-east-1' && {
+            CreateBucketConfiguration: {
+              LocationConstraint: region as any,
+            },
+          }),
+        })
+      )
+
+      // enable versioning for safety
+      await s3Client.send(
+        new PutBucketVersioningCommand({
+          Bucket: bucketName,
+          VersioningConfiguration: {
+            Status: 'Enabled',
+          },
+        })
+      )
+
+      // set lifecycle policy to automatically delete old backups
+      await s3Client.send(
+        new PutBucketLifecycleConfigurationCommand({
+          Bucket: bucketName,
+          LifecycleConfiguration: {
+            Rules: [
+              {
+                ID: 'delete-old-backups',
+                Status: 'Enabled',
+                Filter: {
+                  Prefix: 'db-backups/',
+                },
+                Expiration: {
+                  Days: retentionDays,
+                },
+                NoncurrentVersionExpiration: {
+                  NoncurrentDays: 7,
+                },
+              },
+            ],
+          },
+        })
+      )
+
+      console.info(
+        `✅ S3 bucket created with ${retentionDays} day retention: ${bucketName}`
+      )
+    } else {
+      throw error
+    }
+  }
+}

package/src/helpers/env-load.ts

@@ -0,0 +1,43 @@
+import { join } from 'node:path'
+
+export async function loadEnv(
+  environment: 'development' | 'production' | 'dev-prod',
+  options?: { optional?: string[] }
+) {
+  const { loadEnv: vxrnLoadEnv } = await import('vxrn/loadEnv')
+
+  // loads env into process.env
+  await vxrnLoadEnv(environment)
+
+  const previousMode = process.env.TAKEOUT_ENV_MODE
+  if (environment === 'development' || environment === 'production') {
+    process.env.TAKEOUT_ENV_MODE = environment
+  } else {
+    delete process.env.TAKEOUT_ENV_MODE
+  }
+
+  let Environment: Record<string, string>
+  try {
+    // import src/env.ts to get the env config applied (side effect: populates process.env)
+    const envModule = await import(join(process.cwd(), 'src/env.ts'))
+    Environment = envModule.server || {}
+  } finally {
+    if (previousMode === undefined) {
+      delete process.env.TAKEOUT_ENV_MODE
+    } else {
+      process.env.TAKEOUT_ENV_MODE = previousMode
+    }
+  }
+
+  // validate
+  for (const key in Environment) {
+    if (options?.optional?.includes(key)) {
+      continue
+    }
+    if (typeof Environment[key as keyof typeof Environment] === 'undefined') {
+      console.warn(`Missing key: ${key}`)
+    }
+  }
+
+  return Environment
+}

package/src/helpers/get-docker-host.ts

@@ -0,0 +1,37 @@
+#!/usr/bin/env bun
+
+import { execSync } from 'node:child_process'
+import { platform } from 'node:os'
+
+/**
+ * Gets the appropriate host address for Docker containers to connect back to the host machine
+ * - On macOS/Windows: host.docker.internal works out of the box
+ * - On Linux: We need to get the actual host IP from the docker bridge network
+ */
+export function getDockerHost(): string {
+  // In CI or on Linux, we need the actual host IP
+  if (platform() === 'linux') {
+    try {
+      // Get the host IP from docker network
+      const result = execSync(
+        "docker network inspect bridge --format '{{range .IPAM.Config}}{{.Gateway}}{{end}}'",
+        { encoding: 'utf-8' }
+      ).trim()
+
+      if (result) {
+        console.info(`Using Docker bridge gateway IP: ${result}`)
+        return result
+      }
+    } catch (error) {
+      console.warn('Failed to get Docker bridge IP, falling back to host.docker.internal')
+    }
+  }
+
+  // Default for macOS/Windows or fallback
+  return 'host.docker.internal'
+}
+
+// If run directly, output the host
+if (import.meta.main) {
+  console.info(getDockerHost())
+}

package/src/helpers/get-test-env.ts

@@ -0,0 +1,46 @@
+import { join } from 'node:path'
+
+import { loadEnv } from './env-load'
+import { getDockerHost } from './get-docker-host'
+
+export async function getTestEnv() {
+  const dockerHost = getDockerHost()
+  const devEnv = await loadEnv('development')
+
+  // import src/env.ts for server env fallback values
+  const envModule = await import(join(process.cwd(), 'src/env.ts'))
+  const serverEnvFallback = envModule.server || {}
+
+  // ports come from VITE_PORT_* in .env.development (loaded by loadEnv)
+  const dbPort = process.env.VITE_PORT_POSTGRES || '5433'
+  const appPort = process.env.VITE_PORT_WEB || '8081'
+  const minioPort = process.env.VITE_PORT_MINIO || '9200'
+
+  const dockerDbBase = `postgresql://user:password@127.0.0.1:${dbPort}`
+
+  return {
+    ...serverEnvFallback,
+    ...devEnv,
+    CI: 'true',
+    // Child test processes may import src/env.ts directly. Keep them pinned to
+    // development mode even when the parent release flow started from
+    // production-flavored env.
+    TAKEOUT_ENV_MODE: 'development',
+    ...(!process.env.DEBUG_BACKEND && {
+      ZERO_LOG_LEVEL: 'warn',
+    }),
+    DO_NOT_TRACK: '1',
+    // force localhost urls so cross-subdomain cookies don't activate in test
+    BETTER_AUTH_URL: `http://localhost:${appPort}`,
+    ONE_SERVER_URL: `http://localhost:${appPort}`,
+    ZERO_MUTATE_URL: `http://${dockerHost}:${appPort}/api/zero/push`,
+    ZERO_QUERY_URL: `http://${dockerHost}:${appPort}/api/zero/pull`,
+    ZERO_UPSTREAM_DB: `${dockerDbBase}/postgres`,
+    ZERO_CVR_DB: `${dockerDbBase}/zero_cvr`,
+    ZERO_CHANGE_DB: `${dockerDbBase}/zero_cdb`,
+    CLOUDFLARE_R2_ENDPOINT: `http://127.0.0.1:${minioPort}`,
+    CLOUDFLARE_R2_PUBLIC_URL: `http://127.0.0.1:${minioPort}`,
+    CLOUDFLARE_R2_ACCESS_KEY: 'minio',
+    CLOUDFLARE_R2_SECRET_KEY: 'minio_password',
+  }
+}