iec-builder 0.1.0

package/src/services/__tests__/troubleshooter.test.ts

@@ -0,0 +1,329 @@
+import { describe, it, expect, vi, beforeEach } from 'vitest'
+
+const { mockGetPods } = vi.hoisted(() => ({
+  mockGetPods: vi.fn(),
+}))
+
+vi.mock('../pod-diagnostics.js', () => ({
+  getPods: (...args: unknown[]) => mockGetPods(...args),
+}))
+
+vi.mock('../../utils/logger.js', () => ({
+  logger: { info: vi.fn(), warn: vi.fn(), error: vi.fn() },
+}))
+
+import { troubleshoot, type TroubleshootResult } from '../troubleshooter.js'
+import type { PodDiagnostic } from '../../models/types.js'
+
+function makePod(overrides: Partial<PodDiagnostic> = {}): PodDiagnostic {
+  return {
+    name: 'my-api-abc123',
+    phase: 'Running',
+    restartCount: 0,
+    containerStatuses: [{ name: 'my-api', ready: true, state: 'running' }],
+    events: [],
+    ...overrides,
+  }
+}
+
+describe('troubleshooter', () => {
+  beforeEach(() => {
+    vi.clearAllMocks()
+  })
+
+  describe('healthy service', () => {
+    it('returns healthy=true with no issues for running pods', async () => {
+      mockGetPods.mockResolvedValue([makePod()])
+
+      const result = await troubleshoot('my-api', 'my-api-prod', 'prod')
+      expect(result.healthy).toBe(true)
+      expect(result.issues).toHaveLength(0)
+      expect(result.service).toBe('my-api')
+      expect(result.namespace).toBe('my-api-prod')
+      expect(result.environment).toBe('prod')
+      expect(result.pods).toHaveLength(1)
+    })
+  })
+
+  describe('no pods', () => {
+    it('returns critical issue when namespace has no pods', async () => {
+      mockGetPods.mockResolvedValue([])
+
+      const result = await troubleshoot('my-api', 'my-api-prod', 'prod')
+      expect(result.healthy).toBe(false)
+      expect(result.issues).toHaveLength(1)
+      expect(result.issues[0].severity).toBe('critical')
+      expect(result.issues[0].category).toBe('deployment')
+      expect(result.issues[0].title).toBe('No Pods Found')
+    })
+  })
+
+  describe('crash-loop detection', () => {
+    it('detects CrashLoopBackOff in container', async () => {
+      mockGetPods.mockResolvedValue([
+        makePod({
+          restartCount: 5,
+          containerStatuses: [{
+            name: 'my-api',
+            ready: false,
+            state: 'waiting',
+            reason: 'CrashLoopBackOff',
+            exitCode: 1,
+          }],
+        }),
+      ])
+
+      const result = await troubleshoot('my-api', 'my-api-prod', 'prod')
+      expect(result.healthy).toBe(false)
+      const issue = result.issues.find((i) => i.title === 'CrashLoopBackOff')
+      expect(issue).toBeDefined()
+      expect(issue!.severity).toBe('critical')
+      expect(issue!.category).toBe('crash')
+      expect(issue!.detail).toContain('5 restarts')
+      expect(issue!.containerName).toBe('my-api')
+      expect(issue!.suggestion).toContain('tawa logs')
+    })
+
+    it('detects CrashLoopBackOff in init container', async () => {
+      mockGetPods.mockResolvedValue([
+        makePod({
+          phase: 'Init:Error',
+          containerStatuses: [{ name: 'my-api', ready: false, state: 'waiting' }],
+          initContainerStatuses: [{
+            name: 'vault-agent-init',
+            ready: false,
+            state: 'waiting',
+            reason: 'CrashLoopBackOff',
+          }],
+        }),
+      ])
+
+      const result = await troubleshoot('my-api', 'my-api-prod', 'prod')
+      expect(result.healthy).toBe(false)
+      const crashIssue = result.issues.find((i) => i.title === 'CrashLoopBackOff')
+      expect(crashIssue).toBeDefined()
+      expect(crashIssue!.containerName).toBe('vault-agent-init')
+    })
+  })
+
+  describe('image-pull detection', () => {
+    it('detects ImagePullBackOff', async () => {
+      mockGetPods.mockResolvedValue([
+        makePod({
+          phase: 'Pending',
+          containerStatuses: [{
+            name: 'my-api',
+            ready: false,
+            state: 'waiting',
+            reason: 'ImagePullBackOff',
+            message: 'Back-off pulling image',
+          }],
+        }),
+      ])
+
+      const result = await troubleshoot('my-api', 'my-api-prod', 'prod')
+      expect(result.healthy).toBe(false)
+      const issue = result.issues.find((i) => i.title === 'ImagePullBackOff')
+      expect(issue).toBeDefined()
+      expect(issue!.severity).toBe('critical')
+      expect(issue!.suggestion).toContain('registry')
+    })
+
+    it('detects ErrImagePull', async () => {
+      mockGetPods.mockResolvedValue([
+        makePod({
+          phase: 'Pending',
+          containerStatuses: [{
+            name: 'my-api',
+            ready: false,
+            state: 'waiting',
+            reason: 'ErrImagePull',
+          }],
+        }),
+      ])
+
+      const result = await troubleshoot('my-api', 'my-api-prod', 'prod')
+      const issue = result.issues.find((i) => i.title === 'ImagePullBackOff')
+      expect(issue).toBeDefined()
+    })
+  })
+
+  describe('OOMKilled detection', () => {
+    it('detects OOMKilled container', async () => {
+      mockGetPods.mockResolvedValue([
+        makePod({
+          restartCount: 1,
+          containerStatuses: [{
+            name: 'my-api',
+            ready: false,
+            state: 'terminated',
+            reason: 'OOMKilled',
+          }],
+        }),
+      ])
+
+      const result = await troubleshoot('my-api', 'my-api-prod', 'prod')
+      expect(result.healthy).toBe(false)
+      const issue = result.issues.find((i) => i.title === 'OOMKilled')
+      expect(issue).toBeDefined()
+      expect(issue!.severity).toBe('critical')
+      expect(issue!.suggestion).toContain('pod-tier')
+    })
+  })
+
+  describe('vault-init-failure detection', () => {
+    it('detects failed Vault init container', async () => {
+      mockGetPods.mockResolvedValue([
+        makePod({
+          phase: 'Init:Error',
+          containerStatuses: [{ name: 'my-api', ready: false, state: 'waiting' }],
+          initContainerStatuses: [{
+            name: 'vault-agent-init',
+            ready: false,
+            state: 'waiting',
+            message: 'vault auth failed',
+          }],
+        }),
+      ])
+
+      const result = await troubleshoot('my-api', 'my-api-prod', 'prod')
+      expect(result.healthy).toBe(false)
+      const issue = result.issues.find((i) => i.title === 'Vault Init Container Failed')
+      expect(issue).toBeDefined()
+      expect(issue!.severity).toBe('critical')
+      expect(issue!.category).toBe('vault')
+      expect(issue!.suggestion).toContain('Vault')
+    })
+  })
+
+  describe('config-error detection', () => {
+    it('detects CreateContainerConfigError', async () => {
+      mockGetPods.mockResolvedValue([
+        makePod({
+          containerStatuses: [{
+            name: 'my-api',
+            ready: false,
+            state: 'waiting',
+            reason: 'CreateContainerConfigError',
+            message: 'secret "my-api-managed-secrets" not found',
+          }],
+        }),
+      ])
+
+      const result = await troubleshoot('my-api', 'my-api-prod', 'prod')
+      expect(result.healthy).toBe(false)
+      const issue = result.issues.find((i) => i.title === 'Missing ConfigMap or Secret')
+      expect(issue).toBeDefined()
+      expect(issue!.detail).toContain('my-api-managed-secrets')
+      expect(issue!.suggestion).toContain('tawa config set')
+    })
+  })
+
+  describe('pending-scheduling detection', () => {
+    it('detects FailedScheduling event on Pending pod', async () => {
+      mockGetPods.mockResolvedValue([
+        makePod({
+          phase: 'Pending',
+          containerStatuses: [],
+          events: [{
+            type: 'Warning',
+            reason: 'FailedScheduling',
+            message: '0/3 nodes are available: insufficient cpu.',
+            age: '2m',
+          }],
+        }),
+      ])
+
+      const result = await troubleshoot('my-api', 'my-api-prod', 'prod')
+      const issue = result.issues.find((i) => i.title === 'Pod Pending (FailedScheduling)')
+      expect(issue).toBeDefined()
+      expect(issue!.severity).toBe('warning')
+      expect(issue!.detail).toContain('insufficient cpu')
+    })
+  })
+
+  describe('high-restarts detection', () => {
+    it('flags pods with 3+ restarts not in CrashLoopBackOff', async () => {
+      mockGetPods.mockResolvedValue([
+        makePod({
+          restartCount: 4,
+          containerStatuses: [{
+            name: 'my-api',
+            ready: true,
+            state: 'running',
+          }],
+        }),
+      ])
+
+      const result = await troubleshoot('my-api', 'my-api-prod', 'prod')
+      const issue = result.issues.find((i) => i.title === 'High Restart Count')
+      expect(issue).toBeDefined()
+      expect(issue!.severity).toBe('warning')
+      expect(issue!.detail).toContain('4 times')
+    })
+
+    it('does not flag high restarts when CrashLoopBackOff is already detected', async () => {
+      mockGetPods.mockResolvedValue([
+        makePod({
+          restartCount: 10,
+          containerStatuses: [{
+            name: 'my-api',
+            ready: false,
+            state: 'waiting',
+            reason: 'CrashLoopBackOff',
+          }],
+        }),
+      ])
+
+      const result = await troubleshoot('my-api', 'my-api-prod', 'prod')
+      const highRestartsIssue = result.issues.find((i) => i.title === 'High Restart Count')
+      expect(highRestartsIssue).toBeUndefined()
+      const crashIssue = result.issues.find((i) => i.title === 'CrashLoopBackOff')
+      expect(crashIssue).toBeDefined()
+    })
+  })
+
+  describe('mixed status', () => {
+    it('reports multiple issues from different pods sorted by severity', async () => {
+      mockGetPods.mockResolvedValue([
+        makePod({ name: 'healthy-pod' }),
+        makePod({
+          name: 'crash-pod',
+          restartCount: 5,
+          containerStatuses: [{
+            name: 'app',
+            ready: false,
+            state: 'waiting',
+            reason: 'CrashLoopBackOff',
+          }],
+        }),
+        makePod({
+          name: 'restart-pod',
+          restartCount: 4,
+          containerStatuses: [{
+            name: 'app',
+            ready: true,
+            state: 'running',
+          }],
+        }),
+      ])
+
+      const result = await troubleshoot('my-api', 'my-api-prod', 'prod')
+      expect(result.healthy).toBe(false)
+      expect(result.issues.length).toBeGreaterThanOrEqual(2)
+      // Critical issues should come first
+      expect(result.issues[0].severity).toBe('critical')
+      expect(result.pods).toHaveLength(3)
+    })
+  })
+
+  describe('result structure', () => {
+    it('includes analyzedAt timestamp', async () => {
+      mockGetPods.mockResolvedValue([makePod()])
+
+      const result = await troubleshoot('my-api', 'my-api-prod', 'prod')
+      expect(result.analyzedAt).toBeTruthy()
+      expect(new Date(result.analyzedAt).getTime()).not.toBeNaN()
+    })
+  })
+})

package/src/services/bio-client.ts

@@ -0,0 +1,189 @@
+import { env } from '../config/env.js'
+import { logger } from '../utils/logger.js'
+
+interface ApiResponse<T> {
+  success: boolean
+  data?: T
+  error?: {
+    code: string
+    message: string
+    details?: unknown
+  }
+}
+
+interface RequestOptions {
+  forwardedFor?: string
+  userAgent?: string
+}
+
+class BioClient {
+  private baseUrl: string
+  private internalKey: string
+
+  constructor(baseUrl: string, internalKey: string) {
+    this.baseUrl = baseUrl.replace(/\/$/, '')
+    this.internalKey = internalKey
+  }
+
+  private async request<T>(
+    method: string,
+    path: string,
+    body?: unknown,
+    options?: RequestOptions
+  ): Promise<ApiResponse<T>> {
+    const url = `${this.baseUrl}${path}`
+    const headers: Record<string, string> = {
+      'Content-Type': 'application/json',
+      'X-Internal-Key': this.internalKey,
+    }
+
+    if (options?.forwardedFor) {
+      headers['X-Forwarded-For'] = options.forwardedFor
+    }
+    if (options?.userAgent) {
+      headers['User-Agent'] = options.userAgent
+    }
+
+    try {
+      const response = await fetch(url, {
+        method,
+        headers,
+        body: body ? JSON.stringify(body) : undefined,
+      })
+
+      const json = await response.json() as ApiResponse<T>
+
+      if (!response.ok && !json.error) {
+        return {
+          success: false,
+          error: {
+            code: 'UPSTREAM_ERROR',
+            message: `Bio-ID returned ${response.status}`,
+          },
+        }
+      }
+
+      return json
+    } catch (error) {
+      logger.error({ error, url, method }, 'Bio-ID request failed')
+      return {
+        success: false,
+        error: {
+          code: 'UPSTREAM_ERROR',
+          message: 'Bio-ID service unavailable',
+        },
+      }
+    }
+  }
+
+  async listClients(options?: RequestOptions) {
+    return this.request<unknown[]>('GET', '/api/admin/oauth-clients', undefined, options)
+  }
+
+  async getClient(clientId: string, options?: RequestOptions) {
+    return this.request<unknown>('GET', `/api/admin/oauth-clients/${encodeURIComponent(clientId)}`, undefined, options)
+  }
+
+  async createClient(data: unknown, options?: RequestOptions) {
+    return this.request<unknown>('POST', '/api/admin/oauth-clients', data, options)
+  }
+
+  async updateClient(clientId: string, data: unknown, options?: RequestOptions) {
+    return this.request<unknown>('PATCH', `/api/admin/oauth-clients/${encodeURIComponent(clientId)}`, data, options)
+  }
+
+  async deleteClient(clientId: string, options?: RequestOptions) {
+    return this.request<unknown>('DELETE', `/api/admin/oauth-clients/${encodeURIComponent(clientId)}`, undefined, options)
+  }
+
+  async addRedirectUri(clientId: string, uri: string, options?: RequestOptions) {
+    return this.request<unknown>(
+      'POST',
+      `/api/admin/oauth-clients/${encodeURIComponent(clientId)}/redirect-uris`,
+      { uri },
+      options
+    )
+  }
+
+  async removeRedirectUri(clientId: string, uri: string, options?: RequestOptions) {
+    return this.request<unknown>(
+      'DELETE',
+      `/api/admin/oauth-clients/${encodeURIComponent(clientId)}/redirect-uris`,
+      { uri },
+      options
+    )
+  }
+
+  async regenerateSecret(clientId: string, options?: RequestOptions) {
+    return this.request<unknown>(
+      'POST',
+      `/api/admin/oauth-clients/${encodeURIComponent(clientId)}/regenerate-secret`,
+      undefined,
+      options
+    )
+  }
+
+  async listScopeGrants(filters: {
+    requestingServiceId?: string
+    targetServiceId?: string
+    credentialId?: string
+    status?: string
+  }, options?: RequestOptions) {
+    const params = new URLSearchParams()
+    if (filters.requestingServiceId) params.set('requestingServiceId', filters.requestingServiceId)
+    if (filters.targetServiceId) params.set('targetServiceId', filters.targetServiceId)
+    if (filters.credentialId) params.set('credentialId', filters.credentialId)
+    if (filters.status) params.set('status', filters.status)
+    const qs = params.toString()
+    return this.request<unknown[]>('GET', `/api/admin/scope-grants${qs ? `?${qs}` : ''}`, undefined, options)
+  }
+
+  async createScopeGrant(data: {
+    requestingServiceId: string
+    targetServiceId: string
+    requestedScopes: string[]
+    requestNote?: string
+    credentialType: 'oauth_client' | 'api_key'
+    credentialId: string
+  }, options?: RequestOptions) {
+    return this.request<unknown>('POST', '/api/admin/scope-grants', data, options)
+  }
+
+  async registerModule(data: {
+    moduleId: string
+    name: string
+    description: string
+    serviceId: string
+    owner?: string
+    homepage?: string
+    scopes: ReadonlyArray<{ code: string; name: string; description: string }>
+    defaultScopes?: readonly string[]
+    onboarding?: {
+      requirePasswordSetup?: boolean
+      requireProfileCompletion?: boolean
+      podOnboardingUrl?: string
+      welcomeMessage?: string
+      skipForExistingUsers?: boolean
+    }
+  }, options?: RequestOptions) {
+    return this.request<unknown>('POST', '/api/admin/modules', data, options)
+  }
+}
+
+let bioClientInstance: BioClient | null = null
+
+export function getBioClient(): BioClient {
+  if (!bioClientInstance) {
+    if (!env.BIO_INTERNAL_KEY) {
+      throw new Error('BIO_INTERNAL_KEY is not configured')
+    }
+    bioClientInstance = new BioClient(env.BIO_ID_URL, env.BIO_INTERNAL_KEY)
+  }
+  return bioClientInstance
+}
+
+export function extractRequestMeta(req: { headers: Record<string, string | string[] | undefined> }): RequestOptions {
+  const forwardedFor = (req.headers['x-forwarded-for'] as string) || (req.headers['x-real-ip'] as string)
+  const userAgent = req.headers['user-agent'] as string
+  return { forwardedFor, userAgent }
+}

package/src/services/build-queue.ts

@@ -0,0 +1,137 @@
+import { Queue, Worker, type Job } from 'bullmq'
+import { env } from '../config/env.js'
+import { logger } from '../utils/logger.js'
+import { executeBuild } from './builder.js'
+import { getBuildsCollection, getRedis } from './database.js'
+
+const QUEUE_NAME = 'iec-builds'
+const LEGACY_QUEUE_KEY = 'iec:build:queue'
+
+let buildQueue: Queue | null = null
+let buildWorker: Worker | null = null
+
+function parseRedisConnection() {
+  const url = new URL(env.REDIS_URL)
+  return {
+    host: url.hostname,
+    port: Number(url.port) || 6379,
+    ...(url.username ? { username: url.username } : {}),
+    ...(url.password ? { password: url.password } : {}),
+    maxRetriesPerRequest: null as null,  // Required for BullMQ blocking commands
+  }
+}
+
+export async function initBuildQueue(): Promise<void> {
+  const connection = parseRedisConnection()
+
+  buildQueue = new Queue(QUEUE_NAME, {
+    connection,
+    defaultJobOptions: {
+      attempts: 1,
+      removeOnComplete: 100,
+      removeOnFail: 200,
+    },
+  })
+
+  await drainLegacyQueue()
+
+  logger.info({ queue: QUEUE_NAME }, 'BullMQ build queue initialized')
+}
+
+/**
+ * One-time migration: drain any buildIds remaining in the legacy
+ * Redis list into BullMQ. After first deploy this is always a no-op.
+ */
+async function drainLegacyQueue(): Promise<void> {
+  try {
+    const redis = getRedis()
+    let migrated = 0
+
+    let buildId = await redis.rpop(LEGACY_QUEUE_KEY)
+    while (buildId) {
+      await enqueueBuild(buildId)
+      migrated++
+      buildId = await redis.rpop(LEGACY_QUEUE_KEY)
+    }
+
+    if (migrated > 0) {
+      logger.info({ migrated }, 'Drained legacy Redis list queue into BullMQ')
+    }
+  } catch (err) {
+    // Redis is connected at this point (connectRedis ran first); a failure here is unexpected
+    logger.error({ err }, 'Failed to drain legacy Redis queue — some builds may have been lost')
+  }
+}
+
+export async function enqueueBuild(buildId: string): Promise<void> {
+  if (!buildQueue) throw new Error('Build queue not initialized')
+  await buildQueue.add('build', { buildId }, { jobId: buildId })
+}
+
+export function initBuildWorker(): void {
+  if (!buildQueue) throw new Error('Build queue must be initialized before worker')
+
+  const connection = parseRedisConnection()
+  const concurrency = env.BUILD_CONCURRENCY
+
+  buildWorker = new Worker(
+    QUEUE_NAME,
+    async (job: Job<{ buildId: string }>) => {
+      const { buildId } = job.data
+
+      // Guard: if the build was already marked failed (e.g. by recoverStaleBuilds
+      // after a crash) or cancelled, skip re-execution. BullMQ marks this job as
+      // completed — intentional, since the build state is managed in MongoDB.
+      const builds = getBuildsCollection()
+      const build = await builds.findOne({ id: buildId })
+      if (!build || build.status === 'failed' || build.status === 'cancelled' || build.status === 'completed') {
+        logger.info({ buildId, status: build?.status }, 'Skipping build — already in terminal state')
+        return
+      }
+
+      logger.info({ buildId, jobId: job.id, concurrency }, 'Processing build')
+      await executeBuild(buildId)
+    },
+    {
+      connection,
+      concurrency,
+      lockDuration: 20 * 60 * 1000,   // 20 min — generous for slow Docker builds
+      lockRenewTime: 5 * 60 * 1000,   // renew every 5 min (4x before expiry)
+      stalledInterval: 5 * 60 * 1000,
+    },
+  )
+
+  buildWorker.on('completed', (job) => {
+    logger.info({ jobId: job.id, buildId: job.data.buildId }, 'Build job completed')
+  })
+
+  buildWorker.on('failed', (job, err) => {
+    logger.error({ jobId: job?.id, buildId: job?.data.buildId, err }, 'Build job failed')
+  })
+
+  buildWorker.on('stalled', (jobId) => {
+    logger.warn({ jobId }, 'Build job stalled — BullMQ will attempt recovery')
+  })
+
+  buildWorker.on('error', (err) => {
+    logger.error({ err }, 'BullMQ worker error')
+  })
+
+  logger.info({ concurrency, queue: QUEUE_NAME }, 'BullMQ build worker started')
+}
+
+export function getBuildQueue(): Queue | null {
+  return buildQueue
+}
+
+export async function closeBuildQueue(): Promise<void> {
+  if (buildWorker) {
+    logger.info('Closing BullMQ worker — waiting for active builds to finish...')
+    await buildWorker.close()
+    logger.info('BullMQ worker closed')
+  }
+  if (buildQueue) {
+    await buildQueue.close()
+    logger.info('BullMQ queue closed')
+  }
+}