iec-builder 0.1.0

package/src/services/storage-provisioner.ts

@@ -0,0 +1,216 @@
+import { env } from '../config/env.js'
+import { logger } from '../utils/logger.js'
+import {
+  buildBucketName,
+  buildBucketPolicy,
+  createMinioAdminClient,
+  ensureBucket,
+  setBucketQuota,
+  getStorageTierConfig,
+  DEFAULT_STORAGE_TIER,
+} from './storage-credential-manager.js'
+import {
+  isVaultEnabled,
+  getMinioAdminCreds,
+  configureMinioVaultRole,
+  buildMinioCredsPath,
+  type VaultAnnotations,
+} from './vault-client.js'
+import type { StorageTier, StorageCredential } from '../models/types.js'
+
+// ─────────────────────────────────────────────────────────────────────────────
+// Types
+// ─────────────────────────────────────────────────────────────────────────────
+
+export interface StorageSpec {
+  readonly name?: string
+  readonly tier?: StorageTier
+}
+
+export interface StorageProvisionResult {
+  readonly envVars: Record<string, string>
+  readonly vaultAnnotations: VaultAnnotations
+  readonly credential: StorageCredential | null
+  readonly gasPerMonth: number
+}
+
+// ─────────────────────────────────────────────────────────────────────────────
+// Provisioning
+// ─────────────────────────────────────────────────────────────────────────────
+
+/**
+ * Calculate total storage gas cost per month for deploy gate.
+ */
+export function calculateStorageGasPerMonth(
+  storageSpecs: ReadonlyArray<StorageSpec>,
+): number {
+  return storageSpecs.reduce((sum, spec) => {
+    const tier = spec.tier || DEFAULT_STORAGE_TIER
+    const tierConfig = getStorageTierConfig(tier)
+    return sum + tierConfig.gasPerMonth
+  }, 0)
+}
+
+/**
+ * Provision object storage (MinIO) for a service.
+ *
+ * Requires Vault — throws if Vault is not enabled (no static credential fallback).
+ *
+ * Flow:
+ * 1. Read MinIO admin creds from Vault KV
+ * 2. Create buckets via MinIO SDK (idempotent)
+ * 3. Set hard quotas per bucket via mc CLI
+ * 4. Build scoped IAM policy for all declared buckets
+ * 5. Create MinIO user + policy via mc CLI (admin operations)
+ * 6. Store per-service creds in Vault KV
+ * 7. Return Vault Agent annotations for pod credential injection
+ */
+export async function provisionStorage(
+  serviceName: string,
+  environment: string,
+  namespace: string,
+  storageSpecs: ReadonlyArray<StorageSpec>,
+): Promise<StorageProvisionResult> {
+  if (storageSpecs.length === 0) {
+    return { envVars: {}, vaultAnnotations: {}, credential: null, gasPerMonth: 0 }
+  }
+
+  // Storage requires Vault — no static credential fallback
+  if (!isVaultEnabled()) {
+    throw new Error(
+      'Storage provisioning requires Vault. Set VAULT_ENABLED=true and configure ' +
+      'the MinIO secrets engine in Vault before declaring spec.storage in catalog-info.yaml.'
+    )
+  }
+
+  // Read MinIO admin credentials from Vault KV (never stored on disk)
+  const adminCreds = await getMinioAdminCreds()
+
+  // Create MinIO S3 client for bucket management
+  const client = createMinioAdminClient(adminCreds.accessKey, adminCreds.secretKey)
+
+  // Build bucket names and tiers
+  const bucketEntries = storageSpecs.map((spec) => ({
+    name: buildBucketName(serviceName, environment, spec.name),
+    tier: spec.tier || DEFAULT_STORAGE_TIER,
+    suffix: spec.name,
+  }))
+
+  // Create buckets (via MinIO SDK) and set quotas (via mc CLI)
+  for (const entry of bucketEntries) {
+    await ensureBucket(client, entry.name)
+    await setBucketQuota(entry.name, entry.tier, adminCreds.accessKey, adminCreds.secretKey)
+  }
+
+  const bucketNames = bucketEntries.map((e) => e.name)
+
+  // Build scoped IAM policy for all declared buckets
+  const policyDocument = buildBucketPolicy(bucketNames)
+
+  // Configure Vault MinIO role with inline policy
+  // When pods read from minio/creds/{role}, the plugin creates an ephemeral
+  // MinIO user with this policy attached (dynamic credentials, lease-based)
+  await configureMinioVaultRole(serviceName, environment, JSON.stringify(policyDocument))
+
+  // Build Vault Agent annotations for pod injection
+  const vaultAnnotations = buildStorageVaultAnnotations(
+    serviceName,
+    environment,
+    bucketEntries,
+  )
+
+  // Build credential record
+  const credsPath = buildMinioCredsPath(serviceName, environment)
+  const credential: StorageCredential = {
+    type: 'minio',
+    vaultLeasePath: credsPath,
+    buckets: bucketNames,
+    tiers: bucketEntries.map((e) => e.tier),
+    environment: environment as StorageCredential['environment'],
+    provisionedAt: new Date().toISOString(),
+  }
+
+  const gasPerMonth = calculateStorageGasPerMonth(storageSpecs)
+
+  logger.info(
+    { serviceName, environment, buckets: bucketNames, gasPerMonth },
+    'Storage provisioned via Vault KV',
+  )
+
+  return {
+    envVars: {},  // Credentials injected via Vault Agent, not env vars
+    vaultAnnotations,
+    credential,
+    gasPerMonth,
+  }
+}
+
+// ─────────────────────────────────────────────────────────────────────────────
+// Vault Agent Annotations for Storage
+// ─────────────────────────────────────────────────────────────────────────────
+
+interface BucketEntry {
+  readonly name: string
+  readonly tier: StorageTier
+  readonly suffix?: string
+}
+
+/**
+ * Build Vault Agent pod annotations that inject S3 credentials into the pod.
+ *
+ * The Vault Agent Injector reads these annotations and:
+ * 1. Injects an init container that fetches credentials from the MinIO secrets engine
+ * 2. Writes credentials to /vault/secrets/storage as env-file format
+ * 3. Sidecar renews the lease and rotates creds automatically
+ *
+ * Credentials are dynamic — Vault creates ephemeral MinIO users on each read,
+ * and revokes them when the lease expires.
+ */
+function buildStorageVaultAnnotations(
+  serviceName: string,
+  environment: string,
+  bucketEntries: readonly BucketEntry[],
+): VaultAnnotations {
+  const endpoint = `http://${env.MINIO_HOST}:${env.MINIO_PORT}`
+  const credsPath = buildMinioCredsPath(serviceName, environment)
+
+  // Build the template that renders S3 credentials as env vars
+  // Always emit S3_{SUFFIX}_BUCKET (for createStorage('name')),
+  // plus S3_BUCKET alias for single-bucket services (for createStorage())
+  const namedLines = bucketEntries
+    .map((e) => {
+      const envSuffix = (e.suffix || 'default').toUpperCase().replace(/-/g, '_')
+      return `S3_${envSuffix}_BUCKET=${e.name}`
+    })
+    .join('\n')
+  const bucketLines = bucketEntries.length === 1
+    ? `S3_BUCKET=${bucketEntries[0].name}\n${namedLines}`
+    : namedLines
+
+  // Secrets engine data is at .Data (not .Data.data like KV v2)
+  // Inject both split (S3_HOST/S3_PORT) and combined (S3_ENDPOINT) for SDK compatibility:
+  //   MinIO SDK: new Client({ endPoint: S3_HOST, port: +S3_PORT, ... })
+  //   AWS SDK:   new S3Client({ endpoint: S3_ENDPOINT, ... })
+  const template = [
+    `{{- with secret "${credsPath}" -}}`,
+    `S3_HOST=${env.MINIO_HOST}`,
+    `S3_PORT=${env.MINIO_PORT}`,
+    `S3_ENDPOINT=${endpoint}`,
+    `S3_ACCESS_KEY_ID={{ .Data.access_key_id }}`,
+    `S3_SECRET_ACCESS_KEY={{ .Data.secret_access_key }}`,
+    `S3_REGION=us-east-1`,
+    bucketLines,
+    '{{- end -}}',
+  ].join('\n')
+
+  // Storage annotations are keyed with "storage" suffix to avoid collisions
+  // with database annotations (which use "mongodb", "redis", "neo4j" suffixes).
+  // The command annotation triggers on credential rotation — it removes the
+  // liveness signal file so K8s restarts the pod with fresh creds (same
+  // pattern as database annotations in vault-client.ts).
+  return {
+    'vault.hashicorp.com/agent-inject-secret-storage': credsPath,
+    'vault.hashicorp.com/agent-inject-template-storage': template,
+    'vault.hashicorp.com/agent-inject-command-storage': 'rm -f /vault/signals/alive',
+  }
+}

package/src/services/storage.ts

@@ -0,0 +1,274 @@
+import { MongoClient, GridFSBucket, ObjectId, Db, GridFSFile } from 'mongodb'
+import { Readable } from 'stream'
+import { createHash } from 'crypto'
+import { logger } from '../utils/logger.js'
+
+let gridfs: GridFSBucket | null = null
+let gridfsDb: Db | null = null
+
+/**
+ * Initialize GridFS bucket for tarball storage
+ */
+export function initGridFS(client: MongoClient, dbName?: string): void {
+  gridfsDb = client.db(dbName)
+  gridfs = new GridFSBucket(gridfsDb, {
+    bucketName: 'tarballs',
+    chunkSizeBytes: 1024 * 1024, // 1MB chunks for large files
+  })
+  logger.info('GridFS initialized for tarball storage')
+}
+
+/**
+ * Get the GridFS bucket instance
+ */
+export function getGridFS(): GridFSBucket {
+  if (!gridfs) {
+    throw new Error('GridFS not initialized. Call initGridFS first.')
+  }
+  return gridfs
+}
+
+function getGridFSDb(): Db {
+  if (!gridfsDb) {
+    throw new Error('GridFS not initialized. Call initGridFS first.')
+  }
+  return gridfsDb
+}
+
+export interface TarballMetadata {
+  serviceId: string
+  serviceName: string
+  version: string
+  commitSha: string
+  environment: string
+  uploadedBy: string
+  uploadedAt: string
+  size: number
+  checksum: string
+}
+
+export interface StoredTarball {
+  id: string
+  filename: string
+  metadata: TarballMetadata
+  length: number
+  uploadDate: Date
+}
+
+/**
+ * Upload a tarball to GridFS
+ */
+export async function uploadTarball(
+  stream: Readable,
+  filename: string,
+  metadata: Omit<TarballMetadata, 'uploadedAt' | 'size' | 'checksum'>
+): Promise<StoredTarball> {
+  const bucket = getGridFS()
+  const db = getGridFSDb()
+
+  // Calculate checksum while uploading
+  const hash = createHash('sha256')
+  let size = 0
+
+  const checksumStream = new Readable({
+    read() {}
+  })
+
+  stream.on('data', (chunk: Buffer) => {
+    hash.update(chunk)
+    size += chunk.length
+    checksumStream.push(chunk)
+  })
+
+  stream.on('end', () => {
+    checksumStream.push(null)
+  })
+
+  stream.on('error', (err) => {
+    checksumStream.destroy(err)
+  })
+
+  const fullMetadata: TarballMetadata = {
+    ...metadata,
+    uploadedAt: new Date().toISOString(),
+    size: 0,
+    checksum: '',
+  }
+
+  const uploadStream = bucket.openUploadStream(filename, {
+    metadata: fullMetadata,
+  })
+
+  return new Promise((resolve, reject) => {
+    checksumStream.pipe(uploadStream)
+
+    uploadStream.on('finish', async () => {
+      // Update metadata with final size and checksum
+      const checksum = hash.digest('hex')
+      fullMetadata.size = size
+      fullMetadata.checksum = checksum
+
+      // Update the file's metadata
+      await db.collection('tarballs.files').updateOne(
+        { _id: uploadStream.id },
+        { $set: { 'metadata.size': size, 'metadata.checksum': checksum } }
+      )
+
+      logger.info({ filename, size, checksum }, 'Tarball uploaded to GridFS')
+
+      resolve({
+        id: uploadStream.id.toString(),
+        filename,
+        metadata: fullMetadata,
+        length: size,
+        uploadDate: new Date(),
+      })
+    })
+
+    uploadStream.on('error', (error) => {
+      logger.error({ filename, error }, 'Failed to upload tarball')
+      reject(error)
+    })
+  })
+}
+
+/**
+ * Download a tarball from GridFS
+ */
+export function downloadTarball(fileId: string): Readable {
+  const bucket = getGridFS()
+  return bucket.openDownloadStream(new ObjectId(fileId))
+}
+
+/**
+ * Download a tarball by filename
+ */
+export function downloadTarballByFilename(filename: string): Readable {
+  const bucket = getGridFS()
+  return bucket.openDownloadStreamByName(filename)
+}
+
+/**
+ * List all tarballs for a service
+ */
+export async function listServiceTarballs(serviceId: string): Promise<StoredTarball[]> {
+  const db = getGridFSDb()
+
+  const files = await db
+    .collection<GridFSFile>('tarballs.files')
+    .find({ 'metadata.serviceId': serviceId })
+    .sort({ uploadDate: -1 })
+    .toArray()
+
+  return files.map((file) => ({
+    id: file._id.toString(),
+    filename: file.filename,
+    metadata: file.metadata as unknown as TarballMetadata,
+    length: file.length,
+    uploadDate: file.uploadDate,
+  }))
+}
+
+/**
+ * Get a specific tarball by ID
+ */
+export async function getTarball(fileId: string): Promise<StoredTarball | null> {
+  const db = getGridFSDb()
+
+  const file = await db.collection<GridFSFile>('tarballs.files').findOne({ _id: new ObjectId(fileId) })
+
+  if (!file) return null
+
+  return {
+    id: file._id.toString(),
+    filename: file.filename,
+    metadata: file.metadata as unknown as TarballMetadata,
+    length: file.length,
+    uploadDate: file.uploadDate,
+  }
+}
+
+/**
+ * Get the latest tarball for a service
+ */
+export async function getLatestTarball(serviceId: string): Promise<StoredTarball | null> {
+  const db = getGridFSDb()
+
+  const file = await db
+    .collection<GridFSFile>('tarballs.files')
+    .findOne(
+      { 'metadata.serviceId': serviceId },
+      { sort: { uploadDate: -1 } }
+    )
+
+  if (!file) return null
+
+  return {
+    id: file._id.toString(),
+    filename: file.filename,
+    metadata: file.metadata as unknown as TarballMetadata,
+    length: file.length,
+    uploadDate: file.uploadDate,
+  }
+}
+
+/**
+ * Delete a tarball
+ */
+export async function deleteTarball(fileId: string): Promise<void> {
+  const bucket = getGridFS()
+  await bucket.delete(new ObjectId(fileId))
+  logger.info({ fileId }, 'Tarball deleted from GridFS')
+}
+
+/**
+ * Delete all tarballs for a service (cleanup)
+ */
+export async function deleteServiceTarballs(serviceId: string): Promise<number> {
+  const bucket = getGridFS()
+  const db = getGridFSDb()
+
+  const files = await db
+    .collection<GridFSFile>('tarballs.files')
+    .find({ 'metadata.serviceId': serviceId })
+    .toArray()
+
+  for (const file of files) {
+    await bucket.delete(file._id)
+  }
+
+  logger.info({ serviceId, count: files.length }, 'Service tarballs deleted')
+  return files.length
+}
+
+/**
+ * Extract tarball to a directory
+ */
+export async function extractTarball(fileId: string, destDir: string): Promise<void> {
+  const { pipeline } = await import('stream/promises')
+  const { createWriteStream } = await import('fs')
+  const { mkdir } = await import('fs/promises')
+  const { exec } = await import('child_process')
+  const { promisify } = await import('util')
+  const { join } = await import('path')
+
+  const execAsync = promisify(exec)
+  const tarPath = join(destDir, 'source.tar.gz')
+
+  // Ensure destination exists
+  await mkdir(destDir, { recursive: true })
+
+  // Download tarball to temp file
+  const downloadStream = downloadTarball(fileId)
+  const writeStream = createWriteStream(tarPath)
+  await pipeline(downloadStream, writeStream)
+
+  // Extract tarball
+  await execAsync(`tar -xzf "${tarPath}" -C "${destDir}"`)
+
+  // Remove tarball after extraction
+  const { unlink } = await import('fs/promises')
+  await unlink(tarPath)
+
+  logger.info({ fileId, destDir }, 'Tarball extracted')
+}

package/src/services/troubleshooter.ts

@@ -0,0 +1,208 @@
+import { getPods } from './pod-diagnostics.js'
+import type { PodDiagnostic } from '../models/types.js'
+
+export type IssueSeverity = 'critical' | 'warning' | 'info'
+
+export interface TroubleshootIssue {
+  readonly severity: IssueSeverity
+  readonly category: string
+  readonly title: string
+  readonly detail: string
+  readonly suggestion: string
+  readonly podName?: string
+  readonly containerName?: string
+}
+
+export interface TroubleshootResult {
+  readonly service: string
+  readonly namespace: string
+  readonly environment: string
+  readonly analyzedAt: string
+  readonly healthy: boolean
+  readonly issues: ReadonlyArray<TroubleshootIssue>
+  readonly pods: ReadonlyArray<PodDiagnostic>
+}
+
+interface FailurePattern {
+  readonly name: string
+  readonly detect: (pod: PodDiagnostic) => TroubleshootIssue | null
+}
+
+const FAILURE_PATTERNS: ReadonlyArray<FailurePattern> = [
+  {
+    name: 'crash-loop',
+    detect: (pod) => {
+      const crashing = [...pod.containerStatuses, ...(pod.initContainerStatuses ?? [])]
+        .find((c) => c.reason === 'CrashLoopBackOff')
+      if (!crashing) return null
+      return {
+        severity: 'critical',
+        category: 'crash',
+        title: 'CrashLoopBackOff',
+        detail: `Container "${crashing.name}" is crash-looping (${pod.restartCount} restarts). Last exit code: ${crashing.exitCode ?? 'unknown'}.`,
+        suggestion: 'Check container logs for startup errors. Common causes: missing environment variable, failed database connection on startup, uncaught exception in entry point. Run `tawa logs <service>` to see the crash output.',
+        podName: pod.name,
+        containerName: crashing.name,
+      }
+    },
+  },
+  {
+    name: 'image-pull',
+    detect: (pod) => {
+      const failing = pod.containerStatuses.find(
+        (c) => c.reason === 'ImagePullBackOff' || c.reason === 'ErrImagePull'
+      )
+      if (!failing) return null
+      return {
+        severity: 'critical',
+        category: 'image',
+        title: 'ImagePullBackOff',
+        detail: `Container "${failing.name}" cannot pull its image. ${failing.message ?? ''}`.trim(),
+        suggestion: 'Verify the image exists in the registry. Check that imagePullSecrets are configured in the namespace. The builder pushes to registry.insureco.io/insureco.',
+        podName: pod.name,
+        containerName: failing.name,
+      }
+    },
+  },
+  {
+    name: 'oom-killed',
+    detect: (pod) => {
+      const oom = pod.containerStatuses.find((c) => c.reason === 'OOMKilled')
+      if (!oom) return null
+      return {
+        severity: 'critical',
+        category: 'resources',
+        title: 'OOMKilled',
+        detail: `Container "${oom.name}" was killed due to exceeding memory limits.`,
+        suggestion: 'Increase pod-tier in catalog-info.yaml (e.g., from "nano" to "small"). Each tier doubles the available memory.',
+        podName: pod.name,
+        containerName: oom.name,
+      }
+    },
+  },
+  {
+    name: 'vault-init-failure',
+    detect: (pod) => {
+      const vaultInit = (pod.initContainerStatuses ?? []).find(
+        (c) => c.name.includes('vault') && !c.ready
+      )
+      if (!vaultInit) return null
+      return {
+        severity: 'critical',
+        category: 'vault',
+        title: 'Vault Init Container Failed',
+        detail: `Init container "${vaultInit.name}" is not ready. State: ${vaultInit.state}. ${vaultInit.message ?? ''}`.trim(),
+        suggestion: 'Vault Agent could not authenticate or fetch secrets. Check: (1) ServiceAccount exists in namespace, (2) Vault K8s auth role is configured, (3) Vault is reachable from the cluster. Re-deploying usually fixes transient Vault issues.',
+        podName: pod.name,
+        containerName: vaultInit.name,
+      }
+    },
+  },
+  {
+    name: 'config-error',
+    detect: (pod) => {
+      const configErr = pod.containerStatuses.find(
+        (c) => c.reason === 'CreateContainerConfigError'
+      )
+      if (!configErr) return null
+      return {
+        severity: 'critical',
+        category: 'config',
+        title: 'Missing ConfigMap or Secret',
+        detail: `Container "${configErr.name}" cannot start because a required ConfigMap or Secret is missing. ${configErr.message ?? ''}`.trim(),
+        suggestion: 'Check that all referenced secrets exist in the namespace. If using managed secrets, ensure `tawa config set --secret` was run and a deploy has occurred since.',
+        podName: pod.name,
+        containerName: configErr.name,
+      }
+    },
+  },
+  {
+    name: 'pending-scheduling',
+    detect: (pod) => {
+      if (pod.phase !== 'Pending') return null
+      const schedEvent = pod.events.find((e) => e.reason === 'FailedScheduling')
+      if (!schedEvent) return null
+      return {
+        severity: 'warning',
+        category: 'scheduling',
+        title: 'Pod Pending (FailedScheduling)',
+        detail: `Pod is pending scheduling. ${schedEvent?.message ?? 'Cluster may lack resources.'}`,
+        suggestion: 'The cluster may not have enough CPU/memory to schedule this pod. Try a smaller pod-tier or wait for other pods to finish.',
+        podName: pod.name,
+      }
+    },
+  },
+  {
+    name: 'high-restarts',
+    detect: (pod) => {
+      if (pod.restartCount < 3) return null
+      const hasCrashLoop = pod.containerStatuses.some((c) => c.reason === 'CrashLoopBackOff')
+      if (hasCrashLoop) return null
+      return {
+        severity: 'warning',
+        category: 'stability',
+        title: 'High Restart Count',
+        detail: `Pod has restarted ${pod.restartCount} times. The container may be intermittently crashing.`,
+        suggestion: 'Check container logs (including --previous) for intermittent failures. Common causes: connection timeouts to databases, health check failures, OOM spikes.',
+        podName: pod.name,
+      }
+    },
+  },
+]
+
+function detectIssuesForPod(pod: PodDiagnostic): ReadonlyArray<TroubleshootIssue> {
+  const issues: TroubleshootIssue[] = []
+  for (const pattern of FAILURE_PATTERNS) {
+    const issue = pattern.detect(pod)
+    if (issue) {
+      issues.push(issue)
+    }
+  }
+  return issues
+}
+
+const SEVERITY_ORDER: Record<IssueSeverity, number> = {
+  critical: 0,
+  warning: 1,
+  info: 2,
+}
+
+export async function troubleshoot(
+  serviceName: string,
+  namespace: string,
+  environment: string
+): Promise<TroubleshootResult> {
+  const pods = await getPods(namespace, serviceName)
+
+  const issues: TroubleshootIssue[] = []
+
+  if (pods.length === 0) {
+    issues.push({
+      severity: 'critical',
+      category: 'deployment',
+      title: 'No Pods Found',
+      detail: `No pods found in namespace "${namespace}" for service "${serviceName}".`,
+      suggestion: 'The deployment may not have created any pods. Check that `tawa deploy` completed successfully and that the Helm release exists.',
+    })
+  }
+
+  for (const pod of pods) {
+    issues.push(...detectIssuesForPod(pod))
+  }
+
+  const sorted = [...issues].sort(
+    (a, b) => SEVERITY_ORDER[a.severity] - SEVERITY_ORDER[b.severity]
+  )
+
+  const hasCritical = sorted.some((i) => i.severity === 'critical')
+
+  return {
+    service: serviceName,
+    namespace,
+    environment,
+    analyzedAt: new Date().toISOString(),
+    healthy: !hasCritical,
+    issues: sorted,
+    pods,
+  }
+}