belt 0.2.4 → 0.2.6

data/lib/belt/cli/backup_runner.rb

@@ -0,0 +1,510 @@
+# frozen_string_literal: true
+
+require 'json'
+require 'open3'
+require 'tmpdir'
+require 'fileutils'
+
+module Belt
+  module CLI
+    class BackupRunner
+      def initialize(env, config, infra_dir:, app_name:)
+        @env = env
+        @config = config
+        @infra_dir = infra_dir
+        @app_name = app_name
+        @timestamp = Time.now.strftime('%Y%m%d-%H%M%S')
+        @backup_bucket = "#{@app_name}-backups-#{@env}"
+        @errors = []
+        @summary = []
+      end
+
+      def run
+        ensure_backup_bucket!
+        backup_dynamodb if @config.dynamodb?
+        backup_cognito if @config.cognito?
+        backup_s3_buckets if @config.s3_buckets.any?
+        cleanup_old_backups!
+        print_summary
+
+        abort "\n✗ Backup completed with errors (see above)" if @errors.any?
+      end
+
+      private
+
+      # ─── Backup Bucket ───────────────────────────────────────────────
+
+      def ensure_backup_bucket!
+        # Check if bucket exists
+        _, status = Open3.capture2('aws', 's3', 'ls', "s3://#{@backup_bucket}", err: File::NULL)
+        return if status.success?
+
+        puts "  📦 Creating backup bucket: #{@backup_bucket}"
+
+        run_aws('s3', 'mb', "s3://#{@backup_bucket}", '--region', detect_region)
+
+        # Enable versioning
+        run_aws('s3api', 'put-bucket-versioning',
+                '--bucket', @backup_bucket,
+                '--versioning-configuration', 'Status=Enabled')
+
+        # Block public access
+        run_aws('s3api', 'put-public-access-block',
+                '--bucket', @backup_bucket,
+                '--public-access-block-configuration',
+                'BlockPublicAcls=true,IgnorePublicAcls=true,BlockPublicPolicy=true,RestrictPublicBuckets=true')
+
+        puts '  ✅ Backup bucket created and secured'
+      end
+
+      # ─── DynamoDB ────────────────────────────────────────────────────
+
+      def backup_dynamodb
+        puts '  💾 DynamoDB backups...'
+        table_names = resolve_table_names
+
+        if table_names.empty?
+          @errors << 'Could not resolve DynamoDB table names'
+          puts '  ⚠️  No table names found'
+          return
+        end
+
+        # Filter tables if specific ones configured
+        if @config.dynamodb_tables.is_a?(Array)
+          table_names = table_names.select do |name|
+            @config.dynamodb_tables.intersect?(name)
+          end
+        end
+
+        # Ensure PITR is enabled on all tables
+        table_names.each do |table_name|
+          ensure_pitr(table_name)
+
+          # Create on-demand snapshots
+          create_snapshot(table_name)
+        end
+
+        @summary << "DynamoDB: #{table_names.size} table(s) — PITR verified + snapshot created"
+      end
+
+      def ensure_pitr(table_name)
+        output, status = Open3.capture2(
+          'aws', 'dynamodb', 'describe-continuous-backups',
+          '--table-name', table_name
+        )
+
+        if status.success?
+          data = JSON.parse(output)
+          pitr_status = data.dig('ContinuousBackupsDescription', 'PointInTimeRecoveryDescription',
+                                 'PointInTimeRecoveryStatus')
+
+          return if pitr_status == 'ENABLED'
+        end
+
+        puts "    Enabling PITR for #{short_table_name(table_name)}..."
+        run_aws('dynamodb', 'update-continuous-backups',
+                '--table-name', table_name,
+                '--point-in-time-recovery-specification', 'PointInTimeRecoveryEnabled=true')
+      end
+
+      def create_snapshot(table_name)
+        short = short_table_name(table_name)
+        backup_name = "#{short}-backup-#{@timestamp}"
+
+        output, status = Open3.capture2e(
+          'aws', 'dynamodb', 'create-backup',
+          '--table-name', table_name,
+          '--backup-name', backup_name
+        )
+
+        if status.success?
+          puts "    ✅ Snapshot: #{short}"
+        else
+          puts "    ⚠️  Snapshot failed for #{short}: #{output.strip}"
+        end
+      end
+
+      # ─── Cognito ─────────────────────────────────────────────────────
+
+      def backup_cognito
+        puts '  👥 Cognito backups...'
+        pool_id = resolve_cognito_pool_id
+
+        if pool_id.nil? || pool_id.empty?
+          @errors << 'Could not resolve Cognito user pool ID'
+          puts '  ⚠️  No Cognito pool ID found'
+          return
+        end
+
+        exports_done = []
+
+        if @config.cognito_exports.include?(:users)
+          export_cognito_users(pool_id)
+          exports_done << 'users'
+        end
+
+        if @config.cognito_exports.include?(:pool_config)
+          export_cognito_pool_config(pool_id)
+          exports_done << 'pool_config'
+        end
+
+        @summary << "Cognito: #{exports_done.join(', ')} exported" if exports_done.any?
+      end
+
+      def export_cognito_users(pool_id)
+        all_users = []
+        pagination_token = nil
+
+        loop do
+          args = ['aws', 'cognito-idp', 'list-users', '--user-pool-id', pool_id, '--max-results', '60']
+          args += ['--pagination-token', pagination_token] if pagination_token
+
+          output, status = Open3.capture2(*args)
+          break unless status.success?
+
+          data = JSON.parse(output)
+          all_users.concat(data['Users'] || [])
+
+          pagination_token = data['PaginationToken']
+          break if pagination_token.nil? || pagination_token.empty?
+
+          puts '    Fetching next page of users...'
+        end
+
+        # Upload to backup bucket
+        tmp_file = File.join(Dir.tmpdir, "cognito-users-#{@timestamp}.json")
+        File.write(tmp_file, JSON.pretty_generate(all_users))
+
+        run_aws('s3', 'cp', tmp_file, "s3://#{@backup_bucket}/cognito/users-#{@timestamp}.json", '--quiet')
+        FileUtils.rm_f(tmp_file)
+
+        puts "    ✅ Cognito users exported (#{all_users.size} users)"
+      end
+
+      def export_cognito_pool_config(pool_id)
+        output, status = Open3.capture2(
+          'aws', 'cognito-idp', 'describe-user-pool',
+          '--user-pool-id', pool_id
+        )
+
+        unless status.success?
+          @errors << 'Failed to export Cognito pool configuration'
+          return
+        end
+
+        tmp_file = File.join(Dir.tmpdir, "cognito-pool-config-#{@timestamp}.json")
+        File.write(tmp_file, output)
+
+        run_aws('s3', 'cp', tmp_file, "s3://#{@backup_bucket}/cognito/pool-config-#{@timestamp}.json", '--quiet')
+        FileUtils.rm_f(tmp_file)
+
+        puts '    ✅ Cognito pool configuration exported'
+      end
+
+      # ─── S3 Bucket Sync ──────────────────────────────────────────────
+
+      def backup_s3_buckets
+        puts '  📄 S3 bucket backups...'
+
+        @config.s3_buckets.each do |bucket_key|
+          bucket_name = resolve_s3_bucket_name(bucket_key)
+
+          if bucket_name.nil? || bucket_name.empty?
+            puts "    ⚠️  Could not resolve bucket name for :#{bucket_key}"
+            @errors << "Could not resolve S3 bucket: #{bucket_key}"
+            next
+          end
+
+          puts "    Syncing #{bucket_name}..."
+          output, status = Open3.capture2e(
+            'aws', 's3', 'sync',
+            "s3://#{bucket_name}",
+            "s3://#{@backup_bucket}/s3/#{bucket_key}/#{@timestamp}/",
+            '--quiet'
+          )
+
+          if status.success?
+            puts "    ✅ #{bucket_key} synced"
+          else
+            puts "    ⚠️  #{bucket_key} sync failed: #{output.strip}"
+            @errors << "S3 sync failed for #{bucket_key}"
+          end
+        end
+
+        @summary << "S3: #{@config.s3_buckets.size} bucket(s) synced" if @errors.none? { |e| e.include?('S3') }
+      end
+
+      # ─── Cleanup ─────────────────────────────────────────────────────
+
+      def cleanup_old_backups!
+        puts '  🧹 Cleaning up old backups...'
+        cleanup_cognito_backups if @config.cognito?
+        cleanup_s3_backups if @config.s3_buckets.any?
+        cleanup_dynamodb_snapshots if @config.dynamodb?
+      end
+
+      def cleanup_cognito_backups
+        max_keep = @config.retention[:cognito] || 10
+
+        %w[users pool-config].each do |prefix|
+          output, status = Open3.capture2(
+            'aws', 's3api', 'list-objects-v2',
+            '--bucket', @backup_bucket,
+            '--prefix', "cognito/#{prefix}-",
+            '--query', 'Contents[].Key',
+            '--output', 'json'
+          )
+          next unless status.success?
+
+          keys = begin
+            JSON.parse(output)
+          rescue StandardError
+            []
+          end
+          keys = Array(keys).compact.sort.reverse
+
+          next if keys.size <= max_keep
+
+          keys[max_keep..].each do |key|
+            Open3.capture2('aws', 's3', 'rm', "s3://#{@backup_bucket}/#{key}")
+          end
+        end
+      end
+
+      def cleanup_s3_backups
+        max_keep = @config.retention[:s3] || 10
+
+        @config.s3_buckets.each do |bucket_key|
+          output, status = Open3.capture2(
+            'aws', 's3api', 'list-objects-v2',
+            '--bucket', @backup_bucket,
+            '--prefix', "s3/#{bucket_key}/",
+            '--delimiter', '/',
+            '--query', 'CommonPrefixes[].Prefix',
+            '--output', 'json'
+          )
+          next unless status.success?
+
+          prefixes = begin
+            JSON.parse(output)
+          rescue StandardError
+            []
+          end
+          prefixes = Array(prefixes).compact.sort.reverse
+
+          next if prefixes.size <= max_keep
+
+          prefixes[max_keep..].each do |prefix|
+            Open3.capture2e('aws', 's3', 'rm', "s3://#{@backup_bucket}/#{prefix}", '--recursive')
+          end
+        end
+      end
+
+      def cleanup_dynamodb_snapshots
+        retention_days = @config.retention[:snapshots] || 90
+        cutoff = Time.now - (retention_days * 86_400)
+        cutoff_epoch = cutoff.to_i
+
+        table_names = resolve_table_names
+        table_names.each do |table_name|
+          output, status = Open3.capture2(
+            'aws', 'dynamodb', 'list-backups',
+            '--table-name', table_name,
+            '--time-range-upper-bound', cutoff_epoch.to_s
+          )
+          next unless status.success?
+
+          data = begin
+            JSON.parse(output)
+          rescue StandardError
+            {}
+          end
+          arns = (data['BackupSummaries'] || []).map { |b| b['BackupArn'] }.compact
+
+          arns.each do |arn|
+            Open3.capture2('aws', 'dynamodb', 'delete-backup', '--backup-arn', arn)
+          end
+        end
+      end
+
+      # ─── Terraform Output Resolution ─────────────────────────────────
+
+      def resolve_table_names
+        @resolve_table_names ||= fetch_table_names_from_terraform
+      end
+
+      def fetch_table_names_from_terraform
+        env_dir = File.join(@infra_dir, @env)
+        return [] unless Dir.exist?(env_dir)
+
+        Dir.chdir(env_dir) do
+          names = try_terraform_table_output
+          return names if names
+
+          fetch_table_names_from_all_outputs
+        end
+      end
+
+      def try_terraform_table_output
+        output, status = Open3.capture2('terraform', 'output', '-json', 'dynamodb_table_names')
+        return nil unless status.success?
+
+        names = begin
+          JSON.parse(output)
+        rescue StandardError
+          nil
+        end
+        names.is_a?(Array) && names.any? ? Array(names) : nil
+      end
+
+      def fetch_table_names_from_all_outputs
+        output, status = Open3.capture2('terraform', 'output', '-json')
+        return [] unless status.success?
+
+        all_outputs = begin
+          JSON.parse(output)
+        rescue StandardError
+          {}
+        end
+
+        # Look for dynamodb_table_names in output
+        if all_outputs['dynamodb_table_names']
+          val = all_outputs['dynamodb_table_names']['value']
+          return Array(val) if val
+        end
+
+        # Try to find any output that looks like table names
+        all_outputs.each do |key, data|
+          next unless key.include?('table')
+
+          val = data['value']
+          return Array(val) if val.is_a?(Array) && val.any?
+        end
+
+        []
+      end
+
+      def resolve_cognito_pool_id
+        env_dir = File.join(@infra_dir, @env)
+        return nil unless Dir.exist?(env_dir)
+
+        Dir.chdir(env_dir) do
+          output, status = Open3.capture2('terraform', 'output', '-json', 'user_pool_id')
+          if status.success?
+            val = begin
+              JSON.parse(output)
+            rescue StandardError
+              nil
+            end
+            return val if val.is_a?(String) && !val.empty?
+          end
+
+          # Fallback: search all outputs
+          output, status = Open3.capture2('terraform', 'output', '-json')
+          if status.success?
+            all_outputs = begin
+              JSON.parse(output)
+            rescue StandardError
+              {}
+            end
+            %w[user_pool_id cognito_user_pool_id cognito_pool_id].each do |key|
+              val = all_outputs.dig(key, 'value')
+              return val if val.is_a?(String) && !val.empty?
+            end
+          end
+        end
+
+        nil
+      end
+
+      def resolve_s3_bucket_name(bucket_key)
+        env_dir = File.join(@infra_dir, @env)
+        return nil unless Dir.exist?(env_dir)
+
+        name = Dir.chdir(env_dir) { fetch_s3_bucket_from_terraform(bucket_key) }
+        name || discover_bucket_by_prefix(bucket_key)
+      end
+
+      def fetch_s3_bucket_from_terraform(bucket_key)
+        key_name = "#{bucket_key}_bucket_name"
+        output, status = Open3.capture2('terraform', 'output', '-json', key_name)
+        if status.success?
+          val = begin
+            JSON.parse(output)
+          rescue StandardError
+            nil
+          end
+          return val if val.is_a?(String) && !val.empty?
+        end
+
+        fetch_s3_bucket_from_all_outputs(bucket_key)
+      end
+
+      def fetch_s3_bucket_from_all_outputs(bucket_key)
+        output, status = Open3.capture2('terraform', 'output', '-json')
+        return nil unless status.success?
+
+        all_outputs = begin
+          JSON.parse(output)
+        rescue StandardError
+          {}
+        end
+
+        [
+          "#{bucket_key}_bucket_name",
+          "#{bucket_key}_bucket",
+          bucket_key.to_s
+        ].each do |var|
+          val = all_outputs.dig(var, 'value')
+          return val if val.is_a?(String) && !val.empty?
+        end
+
+        nil
+      end
+
+      def discover_bucket_by_prefix(bucket_key)
+        prefix = "#{@app_name}-#{bucket_key.to_s.tr('_', '-')}-#{@env}"
+        output, status = Open3.capture2(
+          'aws', 's3api', 'list-buckets',
+          '--query', "Buckets[?starts_with(Name, '#{prefix}')].Name | [0]",
+          '--output', 'text'
+        )
+        return nil unless status.success?
+
+        val = output.strip
+        val == 'None' ? nil : val
+      end
+
+      # ─── Helpers ─────────────────────────────────────────────────────
+
+      def short_table_name(table_name)
+        table_name.sub(/\A#{Regexp.escape(@app_name)}-#{Regexp.escape(@env)}-/, '')
+      end
+
+      def detect_region
+        # Try AWS_DEFAULT_REGION, then AWS_REGION, then fall back to us-east-1
+        ENV['AWS_DEFAULT_REGION'] || ENV['AWS_REGION'] || 'us-east-1'
+      end
+
+      def run_aws(*args)
+        output, status = Open3.capture2e('aws', *args)
+        return if status.success?
+
+        @errors << "AWS CLI failed: aws #{args.join(' ')}"
+        puts "    ⚠️  #{output.strip}" unless output.strip.empty?
+      end
+
+      def print_summary
+        puts ''
+        if @summary.any?
+          puts "  ✅ Backup complete (#{@backup_bucket})"
+          @summary.each { |s| puts "     • #{s}" }
+        end
+        return unless @errors.any?
+
+        puts "  ⚠️  #{@errors.size} warning(s):"
+        @errors.each { |e| puts "     • #{e}" }
+      end
+    end
+  end
+end

data/lib/belt/cli/deploy_command.rb

@@ -6,6 +6,8 @@ require 'open3'
 require 'tmpdir'
 require_relative 'env_resolver'
 require_relative 'terraform_command'
+require_relative 'backup_config'
+require_relative 'backup_runner'
 
 module Belt
   module CLI
@@ -20,6 +22,8 @@ module Belt
 
         auto_approve = false
         rebuild = false
+        skip_backup = false
+        backup_only = false
         filtered_args = []
 
         args.each do |arg|
@@ -28,6 +32,10 @@ module Belt
             auto_approve = true
           when '--rebuild'
             rebuild = true
+          when '--skip-backup', '--no-backup'
+            skip_backup = true
+          when '--backup-only'
+            backup_only = true
           when '-h', '--help'
             puts help_text
             exit 0
@@ -61,10 +69,12 @@ module Belt
           end
         end
 
-        if rebuild
-          new(env, auto_approve: auto_approve, extra_args: filtered_args).run_rebuild
+        if backup_only
+          new(env, auto_approve: auto_approve, skip_backup: false, extra_args: filtered_args).run_backup_only
+        elsif rebuild
+          new(env, auto_approve: auto_approve, skip_backup: skip_backup, extra_args: filtered_args).run_rebuild
         else
-          new(env, auto_approve: auto_approve, extra_args: filtered_args).run
+          new(env, auto_approve: auto_approve, skip_backup: skip_backup, extra_args: filtered_args).run
         end
       end
 
@@ -77,34 +87,54 @@ module Belt
 
           This runs the full deployment lifecycle:
             1. Ensure Gemfile.lock is consistent (fix stale PATH refs)
-            2. terraform init    (initialize providers/modules)
-            3. terraform plan    (preview changes)
-            4. Prompt for confirmation (unless --auto)
-            5. terraform apply   (deploy changes)
+            2. Regenerate route manifests
+            3. Run pre-deploy backups (if configured in belt.rb)
+            4. terraform init    (initialize providers/modules)
+            5. terraform plan    (preview changes)
+            6. Prompt for confirmation (unless --auto)
+            7. terraform apply   (deploy changes)
 
           Options:
             --auto, --yes, -y    Skip confirmation prompt (auto-approve)
             --rebuild            Rebuild and push Lambda code directly (bypasses Terraform).
                                  Much faster for code-only changes — packages gems via Docker,
                                  zips, and pushes with `aws lambda update-function-code`.
+            --skip-backup        Skip pre-deploy backup phase
+            --no-backup          Alias for --skip-backup
+            --backup-only        Run backups only, do not deploy
             -h, --help           Show this help
 
           Environment:
             Defaults to BELT_ENV if set, otherwise the first available environment.
 
+          Backup Configuration:
+            Create `infrastructure/<env>/belt.rb` to configure backups:
+
+              Belt.configure do |config|
+                config.backups do
+                  dynamodb :all
+                  cognito :users, :pool_config
+                  s3 :legal_documents
+                  retention snapshots: 90, cognito: 10, s3: 10
+                end
+              end
+
           Examples:
             belt deploy                # Deploy dev (or BELT_ENV)
             belt deploy prod           # Deploy to prod
             belt deploy dev --auto     # Deploy without confirmation (CI mode)
             belt deploy --rebuild      # Fast code push to dev (no infra changes)
             belt deploy prod --rebuild # Fast code push to prod
+            belt deploy prod --skip-backup  # Skip backups for this deploy
+            belt deploy prod --backup-only  # Run backups without deploying
             belt deploy frontend dev   # Deploy frontend assets only
         HELP
       end
 
-      def initialize(env, auto_approve: false, extra_args: [])
+      def initialize(env, auto_approve: false, skip_backup: false, extra_args: [])
         @env = env
         @auto_approve = auto_approve
+        @skip_backup = skip_backup
         @extra_args = extra_args
         @infra_dir = TerraformCommand.find_infrastructure_dir
         @project_root = find_project_root
@@ -118,6 +148,7 @@ module Belt
 
         ensure_lockfile_consistent!
         generate_routes_if_needed
+        run_backups unless @skip_backup
 
         Dir.chdir(env_dir) do
           run_init
@@ -135,6 +166,28 @@ module Belt
         puts "\n   Run `belt server` to view your app locally (auto-connects to the deployed API)."
       end
 
+      def run_backup_only
+        validate!
+
+        puts "belt → running backups for #{@env}\n\n"
+
+        backup_config = load_backup_config
+        if backup_config.nil?
+          puts "No backup configuration found for #{@env}."
+          puts "Create infrastructure/#{@env}/belt.rb to configure backups."
+          puts "\nExample:"
+          puts '  Belt.configure do |config|'
+          puts '    config.backups do'
+          puts '      dynamodb :all'
+          puts '    end'
+          puts '  end'
+          exit 1
+        end
+
+        run_backup_phase(backup_config)
+        puts "\n✅ Backups complete for #{@env}!"
+      end
+
       def run_rebuild
         validate!
         validate_aws!
@@ -200,6 +253,42 @@ module Belt
         end
       end
 
+      # ─── Backup Phase ───────────────────────────────────────────────
+
+      def run_backups
+        backup_config = load_backup_config
+        return unless backup_config
+
+        run_backup_phase(backup_config)
+        puts ''
+      end
+
+      def load_backup_config
+        BackupConfig.load(@env, infra_dir: @infra_dir)
+      end
+
+      def run_backup_phase(backup_config)
+        puts '━━━ pre-deploy backup ━━━'
+        app_name = detect_app_name_for_backup
+        runner = BackupRunner.new(@env, backup_config, infra_dir: @infra_dir, app_name: app_name)
+        runner.run
+      end
+
+      def detect_app_name_for_backup
+        # Try terraform.tfvars first for the app name
+        tfvars_file = File.join(@infra_dir, @env, 'terraform.tfvars')
+        if File.exist?(tfvars_file)
+          match = File.read(tfvars_file).match(/^\s*app_name\s*=\s*"([^"]+)"/)
+          return match[1] if match
+
+          match = File.read(tfvars_file).match(/^\s*project\s*=\s*"([^"]+)"/)
+          return match[1] if match
+        end
+
+        # Fall back to directory name
+        File.basename(@project_root)
+      end
+
       # Ensure Gemfile.lock doesn't have stale PATH references that will break
       # the Docker-based gem build. If the Gemfile no longer uses `path:` but the
       # lockfile still references a PATH source, refresh the lock.