belt 0.4.4 → 0.4.6

data/lib/belt/cli/dynamo_copier.rb

@@ -3,23 +3,60 @@
 require 'json'
 require 'open3'
 require 'tempfile'
-require_relative 'nested_environment'
 
 module Belt
   module CLI
-    # Copies DynamoDB items from a parent environment into a nested child.
+    # Copies DynamoDB items between two sets of tables identified by name
+    # prefix (typically `<app>-<env>-`). Used both by the nested-environment
+    # (PR preview) deploy hook and the standalone `belt db:copy` command.
     #
-    # Copy is skipped when the child table already has any items, so a PR
-    # sync / second deploy will not clobber test data. If a copy fails the
-    # child table is wiped (it was empty when we started) so the next deploy
-    # retries.
+    # By default, copy is skipped when the destination table already has any
+    # items, so re-running against a live environment will not clobber data.
+    # Pass `force: true` to overwrite non-empty destination tables anyway.
+    # If a copy fails, the destination table is wiped back to its starting
+    # state (empty, or restored — best effort) so a retry starts clean.
     class DynamoCopier
       BATCH_SIZE = 25
       MAX_RETRIES = 8
 
-      def initialize(nested_env, app_name:)
-        @nested = nested_env
-        @app_name = app_name
+      # Belt's `cognito_authenticatable` convention: the identity table is
+      # `<app>-<env>-users`, its primary key attribute is `id` (the Cognito
+      # `sub`), and it carries an `email` attribute. A Cognito `sub` is unique
+      # *per user pool*, and each environment has its own pool — so the same
+      # human has a different `id` in every environment. Any row that
+      # references a user by sub therefore has a stale reference the moment it
+      # crosses an environment boundary.
+      #
+      # `cognito_sub` is the conventional foreign-key attribute name for that
+      # reference (see FeatureParity's Membership, Belt's invitation pattern).
+      # When a referencing row also carries an `email`, we can re-anchor it to
+      # the destination environment's user with the same email.
+      IDENTITY_TABLE_SUFFIX = 'users'
+      IDENTITY_ID_ATTR = 'id'
+      IDENTITY_EMAIL_ATTR = 'email'
+      IDENTITY_FK_ATTR = 'cognito_sub'
+
+      # from_prefixes / to_prefixes: array of candidate table-name prefixes
+      #   (the source/destination env's tables are matched against these).
+      # from_profile / to_profile: AWS_PROFILE to use when reading the source
+      #   / writing the destination, respectively (nil = use current
+      #   credentials / AWS_PROFILE already in the environment).
+      # label: short description used in log output (e.g. "dev01 → dev01-pr").
+      # remap_identity: when true, re-anchor Cognito-sub foreign keys to the
+      #   destination environment's users by email (see IDENTITY_* above). The
+      #   users table itself is left untouched so the destination keeps its own
+      #   pool's identities. Defaults to true — copying identity-referencing
+      #   rows verbatim across environments silently hides the data (the row
+      #   points at a sub that doesn't exist in the destination pool).
+      def initialize(from_prefixes:, to_prefixes:, from_profile: nil, to_profile: nil,
+                     force: false, label: nil, remap_identity: true)
+        @from_prefixes = Array(from_prefixes)
+        @to_prefixes = Array(to_prefixes)
+        @from_profile = from_profile
+        @to_profile = to_profile
+        @force = force
+        @label = label
+        @remap_identity = remap_identity
         @errors = []
       end
 
@@ -28,16 +65,17 @@ module Belt
         # rubocop:enable Naming/PredicateMethod
         pairs = table_pairs
         if pairs.empty?
-          puts '  ℹ  No parent DynamoDB tables found to copy'
+          puts '  ℹ  No matching DynamoDB tables found to copy'
           return true
         end
 
-        puts "  💾 Copying DynamoDB data from '#{@nested.parent}' (empty tables only)"
+        mode = @force ? 'overwriting existing data' : 'empty tables only'
+        puts "  💾 Copying DynamoDB data#{" (#{@label})" if @label} (#{mode})"
 
         copied = 0
         skipped = 0
-        pairs.each do |parent_table, child_table|
-          result = copy_pair(parent_table, child_table)
+        pairs.each do |source_table, dest_table|
+          result = copy_pair(source_table, dest_table)
           case result
           when :copied then copied += 1
           when :skipped then skipped += 1
@@ -51,55 +89,140 @@ module Belt
 
       private
 
-      def copy_pair(parent_table, child_table)
-        short = suffix_for(child_table, child_prefixes)
+      def copy_pair(source_table, dest_table)
+        short = suffix_for(dest_table, @to_prefixes)
 
-        unless table_exists?(child_table)
-          puts "    ⚠  #{short}: child table missing — skip"
+        # The identity (users) table is left alone when remapping: the
+        # destination environment's Cognito pool is authoritative for who its
+        # users are and what `sub` each one has. Overwriting it with the
+        # source pool's identities would create user rows that can never
+        # authenticate here (their sub belongs to the other pool).
+        if @remap_identity && identity_table?(short)
+          puts "    skip  #{short} (identity table — destination pool is authoritative)"
           return :skipped
         end
 
-        if table_has_items?(child_table)
+        unless table_exists?(dest_table, profile: @to_profile)
+          puts "    ⚠  #{short}: destination table missing — skip"
+          return :skipped
+        end
+
+        if !@force && table_has_items?(dest_table, profile: @to_profile)
           puts "    skip  #{short} (already has data)"
           return :skipped
         end
 
-        items = scan_items(parent_table)
+        items = scan_items(source_table, profile: @from_profile)
         if items.nil?
-          fail_table(short, "failed to scan parent #{parent_table}")
+          fail_table(short, "failed to scan source #{source_table}")
           return :failed
         end
 
         if items.empty?
-          puts "    skip  #{short} (parent empty)"
+          puts "    skip  #{short} (source empty)"
           return :skipped
         end
 
+        items, remapped, dropped = remap_identity_refs(items) if @remap_identity
+
         begin
-          write_items(child_table, items)
-          puts "    copy  #{short} (#{items.size} item#{'s' if items.size != 1})"
+          wipe_table(dest_table, profile: @to_profile) if @force
+          write_items(dest_table, items, profile: @to_profile)
+          suffix = ''
+          if @remap_identity && (remapped.to_i.positive? || dropped.to_i.positive?)
+            parts = []
+            parts << "#{remapped} re-anchored" if remapped.to_i.positive?
+            parts << "#{dropped} unmatched" if dropped.to_i.positive?
+            suffix = ", #{parts.join(', ')}"
+          end
+          puts "    copy  #{short} (#{items.size} item#{'s' if items.size != 1}#{suffix})"
           :copied
         rescue StandardError => e
-          wipe_table(child_table)
+          wipe_table(dest_table, profile: @to_profile)
           fail_table(short, e.message)
           :failed
         end
       end
 
+      def identity_table?(suffix)
+        suffix == IDENTITY_TABLE_SUFFIX
+      end
+
+      # Re-anchors Cognito-sub foreign keys to the destination environment's
+      # users. For each item that carries both an email and a `cognito_sub`,
+      # the sub is rewritten to the destination user with the same email. Items
+      # whose email has no destination user keep their attributes but have the
+      # stale sub cleared, so they surface as unclaimed (e.g. a pending
+      # invitation) rather than pointing at a nonexistent identity.
+      #
+      # Returns [items, remapped_count, dropped_count].
+      def remap_identity_refs(items)
+        map = destination_email_to_id
+        remapped = 0
+        dropped = 0
+
+        rewritten = items.map do |item|
+          fk = item[IDENTITY_FK_ATTR]
+          email_attr = item[IDENTITY_EMAIL_ATTR]
+          # Only touch rows that actually reference an identity by sub AND
+          # carry an email to re-anchor on. Everything else passes through.
+          next item unless fk.is_a?(Hash) && fk.key?('S') && !fk['S'].to_s.empty?
+          next item unless email_attr.is_a?(Hash) && !email_attr['S'].to_s.empty?
+
+          email = email_attr['S'].to_s.downcase
+          dest_id = map[email]
+
+          if dest_id
+            next item if dest_id == fk['S']
+
+            remapped += 1
+            item.merge(IDENTITY_FK_ATTR => { 'S' => dest_id })
+          else
+            dropped += 1
+            item.reject { |key, _| key == IDENTITY_FK_ATTR }
+          end
+        end
+
+        [rewritten, remapped, dropped]
+      end
+
+      # email (downcased) => destination user id (Cognito sub), built from the
+      # destination environment's users table as it exists right now.
+      def destination_email_to_id
+        @destination_email_to_id ||= begin
+          table = dest_identity_table
+          map = {}
+          if table
+            items = scan_items(table, profile: @to_profile) || []
+            items.each do |item|
+              email = item.dig(IDENTITY_EMAIL_ATTR, 'S')
+              id = item.dig(IDENTITY_ID_ATTR, 'S')
+              map[email.to_s.downcase] = id if email && id
+            end
+          end
+          map
+        end
+      end
+
+      def dest_identity_table
+        dest_tables = tables_with_prefixes(@to_prefixes, profile: @to_profile)
+        dest_tables.find { |name| suffix_for(name, @to_prefixes) == IDENTITY_TABLE_SUFFIX }
+      end
+
       def table_pairs
-        parent_tables = tables_with_prefixes(parent_prefixes)
-        child_tables = tables_with_prefixes(child_prefixes)
+        source_tables = tables_with_prefixes(@from_prefixes, profile: @from_profile)
+        dest_tables = tables_with_prefixes(@to_prefixes, profile: @to_profile)
         pairs = {}
 
-        parent_tables.each do |parent_table|
-          suffix = suffix_for(parent_table, parent_prefixes)
+        source_tables.each do |source_table|
+          suffix = suffix_for(source_table, @from_prefixes)
           next if suffix.empty?
 
-          child_prefixes.each do |prefix|
+          @to_prefixes.each do |prefix|
             candidate = "#{prefix}#{suffix}"
-            next unless child_tables.include?(candidate)
+            next unless dest_tables.include?(candidate)
 
-            pairs[parent_table] = candidate
+            pairs[source_table] = candidate
             break
           end
         end
@@ -107,20 +230,6 @@ module Belt
         pairs
       end
 
-      def parent_prefixes
-        @parent_prefixes ||= prefixes_for(@nested.parent)
-      end
-
-      def child_prefixes
-        @child_prefixes ||= prefixes_for(@nested.env)
-      end
-
-      def prefixes_for(env_name)
-        raw = "#{@app_name}-#{env_name}-"
-        sanitized = raw.tr('_', '-').downcase
-        [raw, sanitized].uniq
-      end
-
       def suffix_for(table_name, prefixes)
         prefixes.each do |prefix|
           return table_name.delete_prefix(prefix) if table_name.start_with?(prefix)
@@ -128,21 +237,22 @@ module Belt
         table_name
       end
 
-      def tables_with_prefixes(prefixes)
-        all_tables.select { |name| prefixes.any? { |prefix| name.start_with?(prefix) } }
+      def tables_with_prefixes(prefixes, profile:)
+        all_tables(profile: profile).select { |name| prefixes.any? { |prefix| name.start_with?(prefix) } }
       end
 
-      def all_tables
-        @all_tables ||= list_all_tables
+      def all_tables(profile:)
+        @all_tables ||= {}
+        @all_tables[profile] ||= list_all_tables(profile: profile)
       end
 
-      def list_all_tables
+      def list_all_tables(profile:)
         names = []
         start_name = nil
         loop do
           args = ['dynamodb', 'list-tables', '--output', 'json']
           args += ['--exclusive-start-table-name', start_name] if start_name
-          data = aws_json(*args)
+          data = aws_json(*args, profile: profile)
           return names if data.nil?
 
           names.concat(Array(data['TableNames']))
@@ -152,25 +262,25 @@ module Belt
         names
       end
 
-      def table_exists?(name)
-        all_tables.include?(name)
+      def table_exists?(name, profile:)
+        all_tables(profile: profile).include?(name)
       end
 
-      def table_has_items?(table_name)
+      def table_has_items?(table_name, profile:)
         data = aws_json('dynamodb', 'scan', '--table-name', table_name,
-                        '--select', 'COUNT', '--limit', '1', '--output', 'json')
+                        '--select', 'COUNT', '--limit', '1', '--output', 'json', profile: profile)
         return false if data.nil?
 
         data.fetch('Count', 0).to_i.positive?
       end
 
-      def scan_items(table_name)
+      def scan_items(table_name, profile:)
         items = []
         start_key = nil
         loop do
           args = ['dynamodb', 'scan', '--table-name', table_name, '--output', 'json']
           args += ['--exclusive-start-key', JSON.generate(start_key)] if start_key
-          data = aws_json(*args)
+          data = aws_json(*args, profile: profile)
           return nil if data.nil?
 
           items.concat(Array(data['Items']))
@@ -180,17 +290,17 @@ module Belt
         items
       end
 
-      def write_items(table_name, items)
+      def write_items(table_name, items, profile:)
         items.each_slice(BATCH_SIZE) do |batch|
           request = {
             table_name => batch.map { |item| { 'PutRequest' => { 'Item' => item } } }
           }
-          write_batch(request)
+          write_batch(request, profile: profile)
         end
       end
 
-      def write_batch(request_items, attempt = 0)
-        data = batch_write(request_items)
+      def write_batch(request_items, profile:, attempt: 0)
+        data = batch_write(request_items, profile: profile)
         raise "batch-write-item failed for #{request_items.keys.join(', ')}" if data.nil?
 
         unprocessed = data['UnprocessedItems']
@@ -198,24 +308,24 @@ module Belt
         raise "unprocessed items after #{MAX_RETRIES} retries" if attempt >= MAX_RETRIES
 
         sleep(0.2 * (2**attempt))
-        write_batch(unprocessed, attempt + 1)
+        write_batch(unprocessed, profile: profile, attempt: attempt + 1)
       end
 
-      def batch_write(request_items)
+      def batch_write(request_items, profile:)
         Tempfile.create(['belt-dynamo', '.json']) do |file|
           file.write(JSON.generate(request_items))
           file.flush
           aws_json('dynamodb', 'batch-write-item',
                    '--request-items', "file://#{file.path}",
-                   '--output', 'json')
+                   '--output', 'json', profile: profile)
         end
       end
 
-      def wipe_table(table_name)
-        items = scan_items(table_name)
+      def wipe_table(table_name, profile:)
+        items = scan_items(table_name, profile: profile)
         return if items.nil? || items.empty?
 
-        keys = key_attribute_names(table_name)
+        keys = key_attribute_names(table_name, profile: profile)
         return if keys.empty?
 
         items.each_slice(BATCH_SIZE) do |batch|
@@ -224,14 +334,14 @@ module Belt
               { 'DeleteRequest' => { 'Key' => item.slice(*keys) } }
             end
           }
-          batch_write(request)
+          batch_write(request, profile: profile)
         end
       rescue StandardError
         nil
       end
 
-      def key_attribute_names(table_name)
-        data = aws_json('dynamodb', 'describe-table', '--table-name', table_name, '--output', 'json')
+      def key_attribute_names(table_name, profile:)
+        data = aws_json('dynamodb', 'describe-table', '--table-name', table_name, '--output', 'json', profile: profile)
         return [] if data.nil?
 
         Array(data.dig('Table', 'KeySchema')).map { |key| key['AttributeName'] }.compact
@@ -239,11 +349,13 @@ module Belt
 
       def fail_table(short, message)
         @errors << "#{short}: #{message}"
-        puts "    ⚠  #{short}: #{message} (will retry on next deploy if table is empty)"
+        puts "    ⚠  #{short}: #{message}"
       end
 
-      def aws_json(*)
-        output, status = Open3.capture2('aws', *)
+      def aws_json(*args, profile: nil)
+        cmd = ['aws'] + args
+        cmd += ['--profile', profile] if profile && !profile.empty?
+        output, status = Open3.capture2(*cmd)
         return nil unless status.success?
 
         JSON.parse(output)

data/lib/belt/cli/explain_command.rb

@@ -44,7 +44,14 @@ module Belt
         'irb' => 'console',
         'repl' => 'console',
         'frontends' => 'frontend',
-        'spa' => 'frontend'
+        'spa' => 'frontend',
+        'seeds' => 'data_seeding',
+        'seed' => 'data_seeding',
+        'seeding' => 'data_seeding',
+        'db:seed' => 'data_seeding',
+        'db:copy' => 'data_seeding',
+        'copy' => 'data_seeding',
+        'db_copy' => 'data_seeding'
       }.freeze
 
       def self.run(args)
@@ -105,6 +112,8 @@ module Belt
             backup                    → backups
             plugin                    → plugins
             irb, repl                 → console
+            seeds, seed, db:seed      → data_seeding
+            db:copy, copy             → data_seeding
 
           Examples:
             belt explain routing

data/lib/belt/cli/frontend_deploy_command.rb

@@ -119,7 +119,7 @@ module Belt
 
       def sync_to_s3
         bucket = fetch_bucket_name
-        abort "Error: Could not determine S3 bucket. Run `belt apply #{@env}` first." unless bucket
+        abort(bucket_lookup_failure_message) unless bucket
 
         dist = @frontend.dist_dir
         unless Dir.exist?(dist)
@@ -161,6 +161,63 @@ module Belt
         fetch_tf_output(@frontend.bucket_output)
       end
 
+      # The bucket output came back nil. Figure out *why* instead of always
+      # blaming a missing apply. `terraform output` swallows its own stderr in
+      # fetch_tf_output, so re-run it once with stderr captured and translate the
+      # failure into something actionable:
+      #   - no state at all      → env was never applied (or wrong dir)
+      #   - credential/SSO error → the AWS profile/session is the problem
+      #   - output just missing  → applied, but this frontend's output isn't there
+      def bucket_lookup_failure_message
+        _out, err, _status = Open3.capture3(
+          'terraform', 'output', '-raw', @frontend.bucket_output.to_s,
+          chdir: @env_dir
+        )
+        stderr = err.to_s.strip
+        first_line = stderr.lines.first&.strip
+
+        if credential_error?(stderr)
+          [
+            "Error: Could not reach Terraform state for '#{@env}' — AWS credentials failed.",
+            "  #{first_line}",
+            "  Check the aws_profile in infrastructure/#{@env}/belt.rb and that its SSO " \
+            'session is active (`aws sso login --profile <profile>`).'
+          ].join("\n")
+        elsif no_state?(stderr) || !state_present?
+          [
+            "Error: No Terraform state for '#{@env}' yet — nothing to deploy the frontend against.",
+            "  Run `belt deploy #{@env}` to provision the backend first, then retry the frontend deploy."
+          ].join("\n")
+        else
+          lines = [
+            "Error: Terraform output `#{@frontend.bucket_output}` not found for '#{@env}'.",
+            "  The backend is applied but this frontend's bucket output is missing. " \
+            'Check config/frontends.yml and that the frontend module is included in terraform.'
+          ]
+          lines << "  terraform: #{first_line}" if first_line
+          lines.join("\n")
+        end
+      rescue Errno::ENOENT
+        "Error: `terraform` not found on PATH. Install Terraform, then run `belt deploy #{@env}` first."
+      end
+
+      def credential_error?(stderr)
+        stderr.match?(/credential|sso|token|AccessDenied|not authorized|403/i)
+      end
+
+      def no_state?(stderr)
+        stderr.match?(
+          /No state file|no outputs|state.*not.*found|Backend initialization required|not been initialized/i
+        )
+      end
+
+      # A locally-applied env has a terraform.tfstate; a remote-backed one has an
+      # initialized .terraform dir. Absence of both means it was never applied here.
+      def state_present?
+        File.exist?(File.join(@env_dir, 'terraform.tfstate')) ||
+          Dir.exist?(File.join(@env_dir, '.terraform'))
+      end
+
       def fetch_distribution_id
         if probe_distribution_output?
           id = fetch_tf_output(@frontend.distribution_output)

data/lib/belt/cli/new_command.rb

@@ -168,6 +168,7 @@ module Belt
           'lambda/lib/routes/routes.rb.erb' => "#{@app_name}/lambda/lib/routes/api_routes.rb",
           'config/routes.rb.erb' => "#{@app_name}/config/routes.rb",
           'config/contracts.rb.erb' => "#{@app_name}/config/contracts.rb",
+          'config/seeds.rb.erb' => "#{@app_name}/config/seeds.rb",
           'config/lambda/api.yml.erb' => "#{@app_name}/config/lambda/api.yml",
           'README.md.erb' => "#{@app_name}/README.md",
           'AGENTS.md.erb' => "#{@app_name}/AGENTS.md",

data/lib/belt/cli.rb

@@ -24,6 +24,8 @@ require_relative 'cli/contracts_command'
 require_relative 'cli/lambda_config_command'
 require_relative 'cli/tasks_command'
 require_relative 'cli/console_command'
+require_relative 'cli/db_copy_command'
+require_relative 'cli/db_seed_command'
 require_relative 'cli/logs_command'
 require_relative 'cli/doctor_command'
 require_relative 'cli/plugin_command'
@@ -39,6 +41,8 @@ module Belt
       'contracts' => Belt::CLI::ContractsCommand,
       'lambda-config' => Belt::CLI::LambdaConfigCommand,
       %w[console c] => Belt::CLI::ConsoleCommand,
+      'db:copy' => Belt::CLI::DbCopyCommand,
+      'db:seed' => Belt::CLI::DbSeedCommand,
       'logs' => Belt::CLI::LogsCommand,
       %w[tasks --tasks -T] => Belt::CLI::TasksCommand,
       'setup' => Belt::CLI::SetupCommand,
@@ -125,6 +129,8 @@ module Belt
 
           console                                     Start an interactive console (IRB)
           c                                           Alias for console
+          db:copy <from-env> <to-env> [--force]        Copy DynamoDB data between environments
+          db:seed [environment] [--force]              Run config/seeds.rb against an environment
           logs [lambda] [-f] [-s 5m] [-e env]         View Lambda function logs
           tasks [-g PATTERN] [-a]                     List available rake tasks
           -T [-g PATTERN] [-a]                        Alias for tasks
@@ -170,6 +176,10 @@ module Belt
           belt tasks                    # list all rake tasks
           belt lambda:build_layer       # run a rake task directly
           belt plugin new messaging     # scaffold a belt-messaging style plugin gem
+          belt db:copy prod dev         # copy prod's DynamoDB data into dev
+          belt db:copy prod dev --force # overwrite dev tables even if non-empty
+          belt db:seed                  # run config/seeds.rb against dev (or BELT_ENV)
+          belt db:seed dev01
       USAGE
     end