pgdump_scrambler 0.4.0 → 0.5.0

data/lib/pgdump_scrambler/config.rb

@@ -1,4 +1,5 @@
 # frozen_string_literal: true
+
 require 'yaml'
 require 'erb'
 require 'set'
@@ -19,14 +20,14 @@ module PgdumpScrambler
       'prefix' => 'YOUR_S3_PATH_PREFIX',
       'access_key_id' => "<%= ENV['AWS_ACCESS_KEY_ID'] %>",
       'secret_key' => "<%= ENV['AWS_SECRET_KEY'] %>"
-    }
+    }.freeze
     attr_reader :dump_path, :s3, :resolved_s3, :exclude_tables, :pgdump_args
 
-    def initialize(tables, dump_path, s3, exclude_tables, pgdump_args)
-      @table_hash = tables.sort_by(&:name).map { |table| [table.name, table] }.to_h
+    def initialize(tables, dump_path, s3, exclude_tables, pgdump_args) # rubocop:disable Naming/MethodParameterName
+      @table_hash = tables.sort_by(&:name).to_h { |table| [table.name, table] }
       @dump_path = dump_path
       @s3 = s3
-      @resolved_s3 = s3.map { |k, v| [k, ERB.new(v).result] }.to_h if s3
+      @resolved_s3 = s3.transform_values { |v| ERB.new(v).result } if s3
       @exclude_tables = exclude_tables
       @pgdump_args = pgdump_args
     end
@@ -45,7 +46,7 @@ module PgdumpScrambler
 
     def update_with(other)
       new_tables = @table_hash.map do |_, table|
-        if other_table = other.table(table.name)
+        if (other_table = other.table(table.name))
           table.update_with(other_table)
         else
           table
@@ -66,15 +67,15 @@ module PgdumpScrambler
       yml = {}
       yml[KEY_DUMP_PATH] = @dump_path
       yml[KEY_S3] = @s3 if @s3
-      yml[KEY_EXCLUDE_TABLES] = @exclude_tables if @exclude_tables.size > 0
+      yml[KEY_EXCLUDE_TABLES] = @exclude_tables if @exclude_tables.size.positive?
       yml[KEY_TABLES] = @table_hash.map do |_, table|
         columns = table.columns
-        unless columns.empty?
-          [
-            table.name,
-            columns.map { |column| [column.name, column.scramble_method] }.to_h,
-          ]
-        end
+        next if columns.empty?
+
+        [
+          table.name,
+          columns.to_h { |column| [column.name, column.scramble_method] }
+        ]
       end.compact.to_h
       YAML.dump(yml, io)
     end
@@ -92,21 +93,22 @@ module PgdumpScrambler
     class << self
       def read(io)
         yml = YAML.safe_load(io, permitted_classes: [], permitted_symbols: [], aliases: true)
-        if yml[KEY_TABLES]
-          tables = yml[KEY_TABLES].map do |table_name, columns|
-            Table.new(
-              table_name,
-              columns.map { |name, scramble_method| Column.new(name, scramble_method) }
-            )
+        tables =
+          if yml[KEY_TABLES]
+            yml[KEY_TABLES].map do |table_name, columns|
+              Table.new(
+                table_name,
+                columns.map { |name, scramble_method| Column.new(name, scramble_method) }
+              )
+            end
+          else
+            []
           end
-        else
-          tables = []
-        end
         Config.new(tables, yml[KEY_DUMP_PATH], yml[KEY_S3], yml[KEY_EXCLUDE_TABLES] || [], yml[KEY_PGDUMP_ARGS])
       end
 
       def read_file(path)
-        open(path, 'r') do |f|
+        File.open(path, 'r') do |f|
           read(f)
         end
       end
@@ -118,18 +120,17 @@ module PgdumpScrambler
           else
             Rails.application.eager_load!
           end
-          klasses_by_table = ActiveRecord::Base.descendants.map { |klass| [klass.table_name, klass] }.to_h
+          klasses_by_table = ActiveRecord::Base.descendants.to_h { |klass| [klass.table_name, klass] }
           table_names = ActiveRecord::Base.connection.tables.sort - IGNORED_ACTIVE_RECORD_TABLES
           tables = table_names.map do |table_name|
             klass = klasses_by_table[table_name]
-            if klass
-              columns = klass.columns.map(&:name).reject do |name|
-                IGNORED_ACTIVE_RECORD_COLUMNS.member?(name)
-              end.map do |name|
-                Column.new(name)
-              end
-              Table.new(table_name, columns)
+            next unless klass
+
+            column_names = klass.columns.map(&:name).reject do |name|
+              IGNORED_ACTIVE_RECORD_COLUMNS.member?(name)
             end
+            columns = column_names.map { |name| Column.new(name) }
+            Table.new(table_name, columns)
           end.compact
           Config.new(tables, 'scrambled.dump.gz', Config::DEFAULT_S3_PROPERTIES, [], nil)
         end

data/lib/pgdump_scrambler/dumper.rb

@@ -1,5 +1,8 @@
 # frozen_string_literal: true
+
+require 'pgdump_scrambler/utils'
 require 'open3'
+
 module PgdumpScrambler
   class Dumper
     def initialize(config, db_config = {})
@@ -9,48 +12,53 @@ module PgdumpScrambler
     end
 
     def run
-      puts "executing pg_dump..."
+      puts 'Executing pg_dump...'
       puts full_command
-      if system(full_command)
-        puts "done!"
-      else
-        raise "pg_dump failed!"
-      end
+      raise 'pg_dump failed!' unless system(env_vars, full_command)
+
+      puts 'Done!'
     end
 
     private
 
+    def env_vars
+      vars = {}
+      vars['PGPASSWORD'] = @db_config['password'] if @db_config['password']
+      vars
+    end
+
     def full_command
       [pgdump_command, obfuscator_command, 'gzip -c'].compact.join(' | ') + "> #{@output_path}"
     end
 
     def obfuscator_command
-      if options = @config.obfuscator_options
-        command = File.expand_path('../../../bin/pgdump-obfuscator', __FILE__)
-        "#{command} #{options}"
-      end
+      return unless (options = @config.obfuscator_options)
+
+      command = File.expand_path('../../bin/pgdump-obfuscator', __dir__)
+      "#{command} #{options}"
     end
 
     def pgdump_command
       command = []
-      command << "PGPASSWORD=#{Shellwords.escape(@db_config['password'])}" if @db_config['password']
       command << 'pg_dump'
       command << @config.pgdump_args if @config.pgdump_args
       command << "--username=#{Shellwords.escape(@db_config['username'])}" if @db_config['username']
       command << "--host='#{@db_config['host']}'" if @db_config['host']
       command << "--port='#{@db_config['port']}'" if @db_config['port']
-      command << @config.exclude_tables.map { |exclude_table| "--exclude-table-data=#{exclude_table}" }.join(' ') if @config.exclude_tables.present?
+      if @config.exclude_tables.present?
+        command << @config.exclude_tables.map do |exclude_table|
+          "--exclude-table-data=#{exclude_table}"
+        end.join(' ')
+      end
       command << @db_config['database']
       command.join(' ')
     end
 
     def load_database_yml
-      if defined?(Rails)
-        db_config = open(Rails.root.join('config', 'database.yml'), 'r') do |f|
-          YAML.safe_load(f, permitted_classes: [], permitted_symbols: [], aliases: true)
-        end
-        db_config[Rails.env]
-      end
+      return unless defined?(Rails)
+
+      db_config = Utils.load_yaml_with_erb(Rails.root.join('config', 'database.yml'))
+      db_config[Rails.env]
     end
   end
 end

data/lib/pgdump_scrambler/railtie.rb

@@ -1,8 +1,9 @@
 # frozen_string_literal: true
+
 module PgdumpScrambler
   class Railtie < ::Rails::Railtie
     rake_tasks do
-      load File.expand_path('../../tasks/pgdump_scrambler_tasks.rake', __FILE__)
+      load File.expand_path('../tasks/pgdump_scrambler_tasks.rake', __dir__)
     end
   end
 end

data/lib/pgdump_scrambler/s3_request.rb

@@ -1,11 +1,12 @@
 # frozen_string_literal: true
+
 require 'uri'
 require 'digest'
 require 'openssl'
 
 module PgdumpScrambler
   class S3Request
-    def initialize(s3_path:, verb:, region:, bucket:, access_key_id:, secret_key:, time: nil)
+    def initialize(s3_path:, verb:, region:, bucket:, access_key_id:, secret_key:, time: nil) # rubocop:disable Metrics/ParameterLists
       @s3_path = s3_path.start_with?('/') ? s3_path : "/#{s3_path}"
       @verb = verb
       @time = time || Time.now.utc
@@ -18,7 +19,7 @@ module PgdumpScrambler
     def canonical_request
       [
         @verb,
-        URI.encode(@s3_path),
+        self.class.uri_encode(@s3_path),
         canonical_query_string,
         "host:#{@bucket}.s3.amazonaws.com\n", # canonical headers
         'host', # signed headers
@@ -38,13 +39,15 @@ module PgdumpScrambler
     end
 
     def url
-      File.join("https://#{@bucket}.s3.amazonaws.com/", "#{@s3_path}?#{canonical_query_string}&X-Amz-Signature=#{signature}")
+      encoded_path = self.class.uri_encode(@s3_path)
+      File.join("https://#{@bucket}.s3.amazonaws.com/",
+        "#{encoded_path}?#{canonical_query_string}&X-Amz-Signature=#{signature}")
     end
 
     private
 
     def iso_time
-      @time.strftime("%Y%m%dT%H%M%SZ")
+      @time.strftime('%Y%m%dT%H%M%SZ')
     end
 
     def iso_date
@@ -52,15 +55,17 @@ module PgdumpScrambler
     end
 
     def hmac_sha256(key, message)
-      OpenSSL::HMAC.digest(OpenSSL::Digest::SHA256.new, key, message)
+      OpenSSL::HMAC.digest(OpenSSL::Digest.new('SHA256'), key, message)
     end
 
     def hmac_sha256_hex(key, message)
-      OpenSSL::HMAC.hexdigest(OpenSSL::Digest::SHA256.new, key, message)
+      OpenSSL::HMAC.hexdigest(OpenSSL::Digest.new('SHA256'), key, message)
     end
 
     def canonical_query_string
+      # rubocop:disable Layout/LineLength
       "X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=#{@access_key_id}%2F#{iso_date}%2F#{@region}%2Fs3%2Faws4_request&X-Amz-Date=#{iso_time}&X-Amz-Expires=86400&X-Amz-SignedHeaders=host"
+      # rubocop:enable Layout/LineLength
     end
 
     def string_to_sign
@@ -68,39 +73,29 @@ module PgdumpScrambler
         'AWS4-HMAC-SHA256',
         iso_time,
         "#{iso_date}/#{@region}/s3/aws4_request",
-        Digest::SHA256.hexdigest(canonical_request),
+        Digest::SHA256.hexdigest(canonical_request)
       ].join("\n")
     end
-  end
-end
-
-if $0 == __FILE__
-  # https://docs.aws.amazon.com/AmazonS3/latest/API/sigv4-query-string-auth.html
-  require "minitest/autorun"
-  class TestS3Request < Minitest::Test
-    def setup
-      @s3_request = PgdumpScrambler::S3Request.new(verb: 'GET', s3_path: '/test.txt', region: 'us-east-1', bucket: 'examplebucket', access_key_id: 'AKIAIOSFODNN7EXAMPLE', secret_key: 'wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY', time: Time.utc(2013, 5, 24, 0, 0, 0))
-    end
-
-    def test_canonical_request
-      assert_equal <<~EOS.chomp, @s3_request.canonical_request
-      GET
-      /test.txt
-      X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=AKIAIOSFODNN7EXAMPLE%2F20130524%2Fus-east-1%2Fs3%2Faws4_request&X-Amz-Date=20130524T000000Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host
-      host:examplebucket.s3.amazonaws.com
-      
-      host
-      UNSIGNED-PAYLOAD
-      EOS
-    end
-
-    def test_signature
-      assert_equal 'aeeed9bbccd4d02ee5c0109b86d86835f995330da4c265957d157751f604d404', @s3_request.signature
-    end
 
-    def test_url
-      exected_url = 'https://examplebucket.s3.amazonaws.com/test.txt?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=AKIAIOSFODNN7EXAMPLE%2F20130524%2Fus-east-1%2Fs3%2Faws4_request&X-Amz-Date=20130524T000000Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-Amz-Signature=aeeed9bbccd4d02ee5c0109b86d86835f995330da4c265957d157751f604d404'
-      assert_equal exected_url, @s3_request.url
+    class << self
+      # https://docs.aws.amazon.com/AmazonS3/latest/API/sigv4-query-string-auth.html
+      # * URI encode every byte except the unreserved characters: 'A'-'Z', 'a'-'z', '0'-'9', '-', '.', '_', and '~'.
+      # * The space character is a reserved character and must be encoded as "%20" (and not as "+").
+      # * Each URI encoded byte is formed by a '%' and the two-digit hexadecimal value of the byte.
+      # * Letters in the hexadecimal value must be uppercase, for example "%1A".
+      # * Encode the forward slash character, '/', everywhere except in the object key name.
+      #   For example, if the object key name is photos/Jan/sample.jpg,
+      #   the forward slash in the key name is not encoded.
+      def uri_encode(str)
+        str.gsub(%r{[^A-Za-z0-9\-._~/]}) do
+          us = Regexp.last_match(0)
+          tmp = +''
+          us.each_byte do |uc|
+            tmp << sprintf('%%%02X', uc)
+          end
+          tmp
+        end.force_encoding(Encoding::US_ASCII)
+      end
     end
   end
 end

data/lib/pgdump_scrambler/s3_uploader.rb

@@ -1,11 +1,13 @@
 # frozen_string_literal: true
+
 require 'net/http'
 require 'uri'
-require_relative './s3_request'
+require_relative 's3_request'
 
 module PgdumpScrambler
   class S3UploadError < StandardError
     attr_reader :response
+
     def initialize(response)
       @response = response
       super "S3 upload failed: #{response.body}"
@@ -13,17 +15,25 @@ module PgdumpScrambler
   end
 
   class S3Uploader
-    def initialize(s3_path:, local_path:, region:, bucket:, access_key_id:, secret_key:)
+    def initialize(s3_path:, local_path:, region:, bucket:, access_key_id:, secret_key:) # rubocop:disable Metrics/ParameterLists
       raise 'missing access_key_id' if access_key_id.nil? || access_key_id.empty?
       raise 'missing secret_key' if secret_key.nil? || secret_key.empty?
-      @s3_request = S3Request.new(s3_path: s3_path, verb: 'PUT', region: region, bucket: bucket, access_key_id: access_key_id, secret_key: secret_key)
+
+      @s3_request = S3Request.new(
+        s3_path: s3_path,
+        verb: 'PUT',
+        region: region,
+        bucket: bucket,
+        access_key_id: access_key_id,
+        secret_key: secret_key
+      )
       @local_path = local_path
     end
 
     def run
       uri = URI.parse(@s3_request.url)
-      puts "upload #{@local_path} to #{uri.host}#{uri.path}"
-      open(@local_path, 'r') do |io|
+      puts "Uploading #{@local_path} to #{uri.host}#{uri.path}"
+      File.open(@local_path, 'r') do |io|
         uri_path = uri.path
         uri_path += "?#{uri.query}" if uri.query
         req = Net::HTTP::Put.new(uri_path)
@@ -33,10 +43,9 @@ module PgdumpScrambler
         http = Net::HTTP.new(uri.host, uri.port)
         http.use_ssl = true
         res = http.request(req)
-        if res.code != '200'
-          raise S3UploadError.new(res)
-        end
+        raise S3UploadError, res if res.code != '200'
       end
+      puts 'Done.'
     end
   end
 end

data/lib/pgdump_scrambler/utils.rb

@@ -0,0 +1,21 @@
+# frozen_string_literal: true
+
+require 'yaml'
+require 'erb'
+
+module PgdumpScrambler
+  module Utils
+    module_function
+
+    def load_yaml_with_erb(path)
+      yaml_content = File.read(path)
+      resolved = ERB.new(yaml_content).result
+      YAML.safe_load(
+        resolved,
+        permitted_classes: [],
+        permitted_symbols: [],
+        aliases: true
+      )
+    end
+  end
+end

data/lib/pgdump_scrambler/version.rb

@@ -1,4 +1,5 @@
 # frozen_string_literal: true
+
 module PgdumpScrambler
-  VERSION = "0.4.0"
+  VERSION = '0.5.0'
 end

data/lib/pgdump_scrambler.rb

@@ -1,11 +1,10 @@
 # frozen_string_literal: true
-require "pgdump_scrambler/version"
-require "pgdump_scrambler/config"
-require "pgdump_scrambler/dumper"
-require "pgdump_scrambler/s3_uploader"
-if defined?(Rails)
-  require 'pgdump_scrambler/railtie'
-end
+
+require 'pgdump_scrambler/version'
+require 'pgdump_scrambler/config'
+require 'pgdump_scrambler/dumper'
+require 'pgdump_scrambler/s3_uploader'
+require 'pgdump_scrambler/railtie' if defined?(Rails)
 
 module PgdumpScrambler
 end

data/lib/tasks/pgdump_scrambler_tasks.rake

@@ -1,4 +1,5 @@
 # frozen_string_literal: true
+
 namespace :pgdump_scrambler do
   default_config_path = ENV['SCRAMBLER_CONFIG_PATH'] || 'config/pgdump_scrambler.yml'
 
@@ -6,15 +7,16 @@ namespace :pgdump_scrambler do
   task config_from_db: :environment do
     config =
       if File.exist?(default_config_path)
-        puts "#{default_config_path} found!\nmerge existing config with config from database"
+        puts "#{default_config_path} found!\nMerging existing config with config from database"
         PgdumpScrambler::Config
           .read_file(default_config_path)
           .update_with(PgdumpScrambler::Config.from_db)
       else
-        puts "craete config from database"
+        puts 'Creating a config file from database'
         PgdumpScrambler::Config.from_db
       end
     config.write_file(default_config_path)
+    puts "Wrote to #{default_config_path}"
   end
 
   desc 'check if new columns exist'
@@ -24,7 +26,7 @@ namespace :pgdump_scrambler do
       .update_with(PgdumpScrambler::Config.from_db)
     unspecified_columns = config.unspecified_columns
     count = unspecified_columns.sum { |_, columns| columns.size }
-    if count > 0
+    if count.positive?
       unspecified_columns.each_key do |table_name|
         puts "#{table_name}:"
         unspecified_columns[table_name].each do |column_name|
@@ -34,7 +36,7 @@ namespace :pgdump_scrambler do
       puts "#{count} unspecified columns found!"
       exit 1
     else
-      puts "No unspecified columns found."
+      puts 'No unspecified columns found.'
     end
   end
 
@@ -47,7 +49,7 @@ namespace :pgdump_scrambler do
   desc 'create scrambled dump'
   task clear_dump: :environment do
     config = PgdumpScrambler::Config.read_file(default_config_path)
-    if File.exists? config.dump_path
+    if File.exist? config.dump_path
       File.delete(config.dump_path)
       puts "Dump file #{config.dump_path} has been deleted."
     end
@@ -57,7 +59,7 @@ namespace :pgdump_scrambler do
   task s3_upload: :environment do
     config = PgdumpScrambler::Config.read_file(default_config_path)
     uploader = PgdumpScrambler::S3Uploader.new(
-      s3_path: File.join(config.resolved_s3['prefix'], File::basename(config.dump_path)),
+      s3_path: File.join(config.resolved_s3['prefix'], File.basename(config.dump_path)),
       local_path: config.dump_path,
       region: config.resolved_s3['region'],
       bucket: config.resolved_s3['bucket'],

data/pgdump_scrambler.gemspec

@@ -1,37 +1,30 @@
+# frozen_string_literal: true
 
-lib = File.expand_path("../lib", __FILE__)
+lib = File.expand_path('lib', __dir__)
 $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
-require "pgdump_scrambler/version"
+require 'pgdump_scrambler/version'
 
 Gem::Specification.new do |spec|
-  spec.name          = "pgdump_scrambler"
+  spec.name          = 'pgdump_scrambler'
   spec.version       = PgdumpScrambler::VERSION
-  spec.authors       = ["Shunichi Ikegami"]
-  spec.email         = ["sike.tm@gmail.com"]
+  spec.authors       = ['Shunichi Ikegami']
+  spec.email         = ['sike.tm@gmail.com']
 
-  spec.summary       = %q{scramble pg_dump columns}
-  spec.description   = %q{scramble pg_dump columns.}
+  spec.summary       = 'Scramble pg_dump columns'
+  spec.description   = 'Scramble pg_dump columns.'
   spec.homepage      = 'https://github.com/shunichi/pgdump_scrambler'
-  spec.license       = "MIT"
+  spec.license       = 'MIT'
+  spec.required_ruby_version = '>= 3.0'
 
-  # Prevent pushing this gem to RubyGems.org. To allow pushes either set the 'allowed_push_host'
-  # to allow pushing to a single host or delete this section to allow pushing to any host.
-  if spec.respond_to?(:metadata)
-    spec.metadata["allowed_push_host"] = 'https://rubygems.org'
-  else
-    raise "RubyGems 2.0 or newer is required to protect against " \
-      "public gem pushes."
-  end
+  spec.metadata['homepage_uri'] = spec.homepage
+  spec.metadata['source_code_uri'] = 'https://github.com/shunichi/pgdump_scrambler'
+  spec.metadata['changelog_uri'] = 'https://github.com/shunichi/pgdump_scrambler/blob/main/CHANGELOG.md'
+  spec.metadata['rubygems_mfa_required'] = 'true'
 
-  spec.files         = `git ls-files -z`.split("\x0").reject do |f|
+  spec.files = `git ls-files -z`.split("\x0").reject do |f|
     f.match(%r{^(test|spec|features)/})
   end
-  spec.bindir        = "exe"
+  spec.bindir        = 'exe'
   spec.executables   = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
-  spec.require_paths = ["lib"]
-
-  spec.add_development_dependency "rake", "~> 13.0"
-  spec.add_development_dependency "rspec", "~> 3.12"
-  spec.add_development_dependency "rails", "~> 7.0"
-  spec.add_development_dependency "rubocop"
+  spec.require_paths = ['lib']
 end