pgdice 0.4.2 → 2.0.0

data/examples/aws/lib/sqs_listener.rb

@@ -0,0 +1,47 @@
+# frozen_string_literal: true
+
+require 'aws-sdk-sqs'
+
+# READ_ONLY_SQS can be set to ensure we don't delete good messages
+class SqsListener
+  DEFAULT_VISIBILITY_TIMEOUT ||= 600
+  attr_reader :logger, :queue_url, :visibility_timeout
+
+  def initialize(opts = {})
+    @logger = opts[:logger] ||= Sidekiq.logger
+    @queue_url = opts[:queue_url] ||= ENV['SqsQueueUrl']
+    @sqs_client = opts[:sqs_client] ||= Aws::SQS::Client.new
+    @sqs_event_router = opts[:sqs_event_router] ||= SqsEventRouter.new(logger: logger)
+    increase_timeout_resolver = opts[:increase_timeout_resolver] ||= -> { ENV['READ_ONLY_SQS'].to_s == 'true' }
+    @visibility_timeout = calculate_visibility_timeout(increase_timeout_resolver.call)
+
+    logger.debug { "Running in environment: #{ENV['RAILS_ENV']} and using sqs queue: #{queue_url}" }
+  end
+
+  # http://docs.aws.amazon.com/sdk-for-ruby/v3/developer-guide/sqs-example-get-messages-with-long-polling.html
+  def call
+    # This uses long polling to retrieve sqs events so we can process them
+    response = @sqs_client.receive_message(queue_url: queue_url,
+                                           max_number_of_messages: 10,
+                                           wait_time_seconds: 20,
+                                           visibility_timeout: visibility_timeout)
+
+    if response.messages&.size&.positive?
+      logger.debug { "The number of messages received from the queue was: #{response.messages&.size}" }
+    end
+
+    # Iterate over all the messages in the response (Response is a Struct which acts like an object with methods)
+    response.messages&.each do |message|
+      @sqs_event_router.handle_message(message)
+    end
+  end
+
+  private
+
+  def calculate_visibility_timeout(increase_timeout)
+    visibility_timeout = increase_timeout ? DEFAULT_VISIBILITY_TIMEOUT * 4 : DEFAULT_VISIBILITY_TIMEOUT
+
+    logger.info { "Visibility timeout set to: #{visibility_timeout} seconds" }
+    visibility_timeout
+  end
+end

data/examples/aws/lib/sqs_message_deleter.rb

@@ -0,0 +1,32 @@
+# frozen_string_literal: true
+
+require 'aws-sdk-sqs'
+
+class SqsMessageDeleter
+  attr_reader :logger
+
+  def initialize(opts = {})
+    @logger = opts[:logger] ||= Sidekiq.logger
+    @queue_url = opts[:queue_url] ||= ENV['SqsQueueUrl']
+    @sqs_client = opts[:sqs_client] ||= Aws::SQS::Client.new
+    @skip_delete_predicate = opts[:skip_delete_predicate] ||= proc do
+      Rails.env != 'production' || ENV['READ_ONLY_SQS'].to_s == 'true'
+    end
+  end
+
+  def call(sqs_message_receipt_handle)
+    if @skip_delete_predicate.call
+      logger.info { "Not destroying sqs message because environment is not prod or READ_ONLY_SQS was set to 'true'" }
+      return false
+    end
+
+    logger.debug { "Destroying sqs message with handle: #{sqs_message_receipt_handle}" }
+
+    response = @sqs_client.delete_message(queue_url: @queue_url, receipt_handle: sqs_message_receipt_handle)
+    unless response.successful?
+      raise "Attempt to delete SQS message: #{sqs_message_receipt_handle} was not successful. Response: #{response}"
+    end
+
+    true
+  end
+end

data/examples/aws/lib/sqs_poller.rb

@@ -0,0 +1,67 @@
+# frozen_string_literal: true
+
+require 'aws-sdk-sqs'
+
+class SqsPoller
+  attr_reader :logger, :queue_url
+
+  MAX_RETRIES ||= 3
+  DEFAULT_WAIT_TIME ||= 5
+
+  def initialize(opts = {})
+    @logger = opts[:logger] ||= ActiveSupport::TaggedLogging.new(Logger.new(ENV['POLL_SQS_LOG_OUTPUT'] || STDOUT))
+    @max_retries = opts[:max_retries] ||= MAX_RETRIES
+    @sleep_seconds = opts[:sleep_seconds] ||= DEFAULT_WAIT_TIME
+    @error_sleep_seconds = opts[:error_sleep_seconds] ||= @sleep_seconds * 2
+    @sqs_listener = opts[:sqs_listener] ||= SqsListener.new(logger: logger)
+  end
+
+  def poll(iterations = Float::INFINITY)
+    logger.info { "Starting loop to #{iterations}, press Ctrl-C to exit" }
+
+    retries = 0
+    i = 0
+
+    while i < iterations
+      begin
+        i += 1
+        execute_loop
+      rescue StandardError => e
+        if retries < MAX_RETRIES
+          retries = handle_retry(retries, e)
+          retry
+        else
+          die(e)
+        end
+      rescue Exception => e
+        die(e)
+      end
+    end
+  end
+
+  private
+
+  def execute_loop
+    @sqs_listener.call
+    sleep @sleep_seconds
+  end
+
+  def handle_retry(retries, error)
+    logger.error do
+      "Polling loop encountered an error. Will retry in #{@error_sleep_seconds} seconds. "\
+        "Error: #{error}. Retries: #{retries}"
+    end
+    retries += 1
+
+    # Handle error with error tracking service
+    # @error_handler.call(error)
+
+    sleep @error_sleep_seconds
+    retries
+  end
+
+  def die(error)
+    logger.fatal { "Polling loop is stopping due to exception: #{error}" }
+    raise error
+  end
+end

data/examples/aws/tasks/poll_sqs.rake

@@ -0,0 +1,8 @@
+# frozen_string_literal: true
+
+# You can set READ_ONLY_SQS=true if you don't want to delete messages
+
+desc 'Poll SQS for any new test executions'
+task poll_sqs: :environment do
+  SqsPoller.new.poll
+end

data/examples/aws/workers/pg_dice_worker.rb

@@ -0,0 +1,54 @@
+# frozen_string_literal: true
+
+# https://github.com/mperham/sidekiq/wiki/Best-Practices
+# Sidekiq job parameters must be JSON serializable. That means Ruby symbols are
+# lost when they are sent through JSON!
+class PgdiceWorker
+  include Sidekiq::Worker
+  attr_reader :logger
+  sidekiq_options queue: :default, backtrace: true, retry: 5
+
+  def initialize(opts = {})
+    @pgdice = opts[:pgdice] ||= PgDice
+    @logger = opts[:logger] ||= Sidekiq.logger
+    @validator = opts[:validator] ||= lambda do |table_name, params|
+      @pgdice.public_send(:assert_tables, table_name, params)
+    end
+  end
+
+  def perform(method, params)
+    table_names = params.delete('table_names')
+    validate = params.delete('validate').present?
+    # Don't pass in params to PgDice if the hash is empty. PgDice will behave differently when params are passed.
+    pgdice_params = params.keys.size.zero? ? nil : handle_pgdice_params(params)
+
+    logger.debug { "PgdiceWorker called with method: #{method} and table_names: #{table_names}. Validate: #{validate}" }
+
+    [table_names].flatten.compact.each do |table_name|
+      @pgdice.public_send(method, table_name, pgdice_params)
+      @validator.call(table_name, pgdice_params) if validate
+    end
+  end
+
+  private
+
+  def handle_pgdice_params(pgdice_params)
+    convert_pgdice_param_values(pgdice_known_symbol_keys(pgdice_params))
+  end
+
+  def pgdice_known_symbol_keys(params)
+    convertable_keys = ['only']
+    params.keys.each do |key|
+      params[key.to_sym] = params.delete(key) if convertable_keys.include?(key)
+    end
+    params
+  end
+
+  def convert_pgdice_param_values(params)
+    symbolize_values_for_keys = [:only]
+    params.each do |key, value|
+      params[key] = value.to_sym if symbolize_values_for_keys.include?(key)
+    end
+    params
+  end
+end

data/lib/pgdice/approved_tables.rb

@@ -4,6 +4,7 @@ module PgDice
   # Hash-like object to contain approved tables. Adds some convenience validation and a simpleish interface.
   class ApprovedTables
     attr_reader :tables
+
     extend Forwardable
 
     def_delegators :@tables, :size, :empty?, :map, :each, :each_with_index, :to_a
@@ -11,12 +12,12 @@ module PgDice
     def initialize(*args)
       @tables = args.flatten.compact
 
-      raise ArgumentError, 'Objects must be a PgDice::Table!' unless tables.all? { |item| item.is_a?(PgDice::Table) }
+      raise ArgumentError, 'Objects must be a PgDice::Table!' unless tables.all?(PgDice::Table)
     end
 
     def [](arg)
       key = check_string_args(arg)
-      tables.select { |table| table.name == key }.first
+      tables.find { |table| table.name == key }
     end
 
     def include?(arg)

data/lib/pgdice/configuration.rb

@@ -14,10 +14,10 @@ module PgDice
 
   # Configuration class which holds all configurable values
   class Configuration
-    DEFAULT_VALUES ||= { logger_factory: proc { Logger.new(STDOUT) },
-                         database_url: nil,
-                         dry_run: false,
-                         batch_size: 7 }.freeze
+    DEFAULT_VALUES = { logger_factory: proc { Logger.new($stdout) },
+                       database_url: nil,
+                       dry_run: false,
+                       batch_size: 7 }.freeze
 
     attr_writer :logger,
                 :logger_factory,
@@ -63,7 +63,7 @@ module PgDice
         raise PgDice::InvalidConfigurationError, 'approved_tables must be an instance of PgDice::ApprovedTables!'
       end
 
-      if !config_file_loader.file_loaded? && config_file.present?
+      if !config_file_loader.file_loaded? && !config_file.nil?
         config_file_loader.load_file
         @approved_tables
       end

data/lib/pgdice/configuration_file_loader.rb

@@ -17,7 +17,7 @@ module PgDice
       end
       @config_loader = opts[:config_loader] ||= lambda do |file|
         logger.debug { "Loading PgDice configuration file: '#{config_file}'" }
-        YAML.safe_load(ERB.new(IO.read(file)).result)
+        YAML.safe_load(ERB.new(File.read(file)).result)
       end
       @file_loaded = opts[:file_loaded]
     end

data/lib/pgdice/database_connection_factory.rb

@@ -1,21 +1,20 @@
 # frozen_string_literal: true
 
-# Entry point for PartitionManager
+# Entry point
 module PgDice
-  #  PartitionListerFactory is a class used to build PartitionListers
+  #  DatabaseConnectionFactory is a class used to build DatabaseConnections
   class DatabaseConnectionFactory
     extend Forwardable
 
-    def_delegators :@configuration, :logger, :pg_connection, :dry_run
+    def_delegators :@configuration, :logger, :dry_run
 
     def initialize(configuration, opts = {})
       @configuration = configuration
-      @query_executor = opts[:query_executor] ||= PgDice::QueryExecutor.new(logger: logger,
-                                                                            connection_supplier: -> { pg_connection })
+      @query_executor_factory = opts[:query_executor_factory] ||= PgDice::QueryExecutorFactory.new(configuration, opts)
     end
 
     def call
-      PgDice::DatabaseConnection.new(logger: logger, query_executor: @query_executor, dry_run: dry_run)
+      PgDice::DatabaseConnection.new(logger: logger, query_executor: @query_executor_factory.call, dry_run: dry_run)
     end
   end
 end

data/lib/pgdice/date_helper.rb

@@ -8,9 +8,9 @@ module PgDice
 
       case numbers.size
       when 6
-        return numbers + '01'
+        "#{numbers}01"
       when 4
-        return numbers + '0101'
+        "#{numbers}0101"
       else
         raise ArgumentError, "Invalid date. Cannot parse date from #{numbers}"
       end

data/lib/pgdice/error.rb

@@ -25,7 +25,7 @@ module PgDice
   class InsufficientTablesError < Error
     def initialize(direction, table_name, expected, period, found_count)
       super("Insufficient #{direction} tables exist for table: #{table_name}. "\
-      "Expected: #{expected} having period of: #{period} but found: #{found_count}")
+            "Expected: #{expected} having period of: #{period} but found: #{found_count}")
     end
   end
 
@@ -51,7 +51,7 @@ module PgDice
   class NotConfiguredError < ConfigurationError
     def initialize(method_name)
       super("Cannot use #{method_name} before PgDice has been configured! "\
-          'See README.md for configuration help.')
+            'See README.md for configuration help.')
     end
   end
 

data/lib/pgdice/partition_dropper.rb

@@ -28,7 +28,7 @@ module PgDice
       sql_query = old_partitions.reduce("BEGIN;\n") do |sql, table_name|
         sql + "DROP TABLE IF EXISTS #{table_name} CASCADE;\n"
       end
-      sql_query + 'COMMIT;'
+      "#{sql_query}COMMIT;"
     end
   end
 end

data/lib/pgdice/partition_helper.rb

@@ -40,8 +40,8 @@ module PgDice
 
     def undo_partitioning(table_name)
       undo_partitioning!(table_name)
-    rescue PgDice::PgSliceError => error
-      logger.error { "Rescued PgSliceError: #{error}" }
+    rescue PgDice::PgSliceError => e
+      logger.error { "Rescued PgSliceError: #{e}" }
       false
     end
 

data/lib/pgdice/pg_slice_manager.rb

@@ -73,8 +73,8 @@ module PgDice
       table_name = params.fetch(:table_name)
 
       run_pgslice("unprep #{table_name}", params[:dry_run])
-    rescue PgSliceError => error
-      logger.error { "Rescued PgSliceError: #{error}" }
+    rescue PgSliceError => e
+      logger.error { "Rescued PgSliceError: #{e}" }
       false
     end
 
@@ -82,8 +82,8 @@ module PgDice
       table_name = params.fetch(:table_name)
 
       run_pgslice("unswap #{table_name}", params[:dry_run])
-    rescue PgSliceError => error
-      logger.error { "Rescued PgSliceError: #{error}" }
+    rescue PgSliceError => e
+      logger.error { "Rescued PgSliceError: #{e}" }
       false
     end
 
@@ -97,7 +97,7 @@ module PgDice
       if status.to_i.positive?
         raise PgDice::PgSliceError,
               "pgslice with arguments: '#{argument_string}' failed with status: '#{status}' "\
-                 "STDOUT: '#{stdout}' STDERR: '#{stderr}'"
+              "STDOUT: '#{stdout}' STDERR: '#{stderr}'"
       end
       true
     end

data/lib/pgdice/query_executor.rb

@@ -1,6 +1,6 @@
 # frozen_string_literal: true
 
-# Entry point for DatabaseConnection
+# Entry point
 module PgDice
   # Wrapper class around pg_connection to reset connection on PG errors
   class QueryExecutor
@@ -13,8 +13,8 @@ module PgDice
 
     def call(query)
       @connection_supplier.call.exec(query)
-    rescue PG::Error => error
-      logger.error { "Caught error: #{error}. Going to reset connection and try again" }
+    rescue PG::Error => e
+      logger.error { "Caught error: #{e}. Going to reset connection and try again" }
       @connection_supplier.call.reset
       @connection_supplier.call.exec(query)
     end

data/lib/pgdice/query_executor_factory.rb

@@ -0,0 +1,20 @@
+# frozen_string_literal: true
+
+# Entry point
+module PgDice
+  #  QueryExecutorFactory is a class used to build QueryExecutors
+  class QueryExecutorFactory
+    extend Forwardable
+
+    def_delegators :@configuration, :logger, :pg_connection
+
+    def initialize(configuration, opts = {})
+      @configuration = configuration
+      @connection_supplier = opts[:connection_supplier] ||= -> { pg_connection }
+    end
+
+    def call
+      PgDice::QueryExecutor.new(logger: logger, connection_supplier: @connection_supplier)
+    end
+  end
+end

data/lib/pgdice/table.rb

@@ -70,7 +70,7 @@ module PgDice
     end
 
     def self.from_hash(hash)
-      Table.new(**hash.each_with_object({}) { |(k, v), memo| memo[k.to_sym] = v; })
+      Table.new(**hash.transform_keys(&:to_sym))
     end
 
     private
@@ -79,7 +79,7 @@ module PgDice
       unless send(field).is_a?(expected_type)
         raise ArgumentError,
               "PgDice::Table: #{name} failed validation on field: #{field}. "\
-                "Expected type of: #{expected_type} but found #{send(field).class}"
+              "Expected type of: #{expected_type} but found #{send(field).class}"
       end
       true
     end

data/lib/pgdice/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module PgDice
-  VERSION = '0.4.2'
+  VERSION = '2.0.0'
 end

data/lib/pgdice.rb

@@ -43,6 +43,7 @@ require 'pgdice/partition_dropper'
 require 'pgdice/partition_dropper_factory'
 
 require 'pgdice/query_executor'
+require 'pgdice/query_executor_factory'
 
 require 'pgdice/database_connection'
 require 'pgdice/database_connection_factory'

data/pgdice.gemspec

@@ -5,39 +5,43 @@ $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
 require 'pgdice/version'
 
 Gem::Specification.new do |spec|
-  spec.name          = 'pgdice'
-  spec.version       = PgDice::VERSION
-  spec.authors       = ['Andrew Newell']
-  spec.email         = ['andrew@andrewcn.com illuminuslimited@gmail.com']
-
-  spec.summary       = 'Postgres table partitioning with a Ruby API!'
-  spec.description   = 'Postgres table partitioning with a Ruby API built on top of https://github.com/ankane/pgslice'
-  spec.homepage      = 'https://github.com/IlluminusLimited/pgdice'
-  spec.license       = 'MIT'
+  spec.name = 'pgdice'
+  spec.version = PgDice::VERSION
+  spec.authors = ['Andrew Newell']
+  spec.email = ['andrew@illuminusltd.com']
+  spec.summary = 'Postgres table partitioning with a Ruby API!'
+  spec.description = 'Postgres table partitioning with a Ruby API built on top of https://github.com/ankane/pgslice'
+  spec.homepage = 'https://github.com/IlluminusLimited/pgdice'
+  spec.license = 'MIT'
 
   # Specify which files should be added to the gem when it is released.
   # The `git ls-files -z` loads the files in the RubyGem that have been added into git.
-  spec.files         = Dir.chdir(File.expand_path(__dir__)) do
+  spec.files = Dir.chdir(File.expand_path(__dir__)) do
     `git ls-files -z`.split("\x0").reject { |f| f.match(%r{^(test|spec|features)/}) }
   end
-  spec.bindir        = 'exe'
-  spec.executables   = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
+  spec.bindir = 'exe'
+  spec.executables = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
   spec.require_paths = ['lib']
 
+  spec.required_ruby_version = '>= 3.0.0'
+
+  # It looks like there's a bug in pg 1.3+ that I don't have time to fix
+  spec.add_runtime_dependency 'pg', '~> 1.2.3', '>= 1.1.0'
   # Locked because we depend on internal behavior for table commenting
-  spec.add_runtime_dependency 'pg', '~> 1.1.0', '>= 1.1.0'
-  spec.add_runtime_dependency 'pgslice', '0.4.5'
+  spec.add_runtime_dependency 'pgslice', '0.4.7'
 
-  spec.add_development_dependency 'bundler', '~> 1.16', '>= 1.16'
-  spec.add_development_dependency 'coveralls', '~> 0.8.22', '>= 0.8.22'
-  spec.add_development_dependency 'guard', '~> 2.14.2', '>= 2.14.2'
+  spec.add_development_dependency 'bundler', '~> 2.3.6', '>= 1.16'
+  spec.add_development_dependency 'guard', '~> 2.18.0', '>= 2.14.2'
   spec.add_development_dependency 'guard-minitest', '~> 2.4.6', '>= 2.4.6'
-  spec.add_development_dependency 'guard-rubocop', '~> 1.3.0', '>= 1.3.0'
+  spec.add_development_dependency 'guard-rubocop', '~> 1.5.0', '>= 1.3.0'
   spec.add_development_dependency 'guard-shell', '~> 0.7.1', '>= 0.7.1'
   spec.add_development_dependency 'minitest', '~> 5.0', '>= 5.0'
   spec.add_development_dependency 'minitest-ci', '~> 3.4.0', '>= 3.4.0'
-  spec.add_development_dependency 'minitest-reporters', '~> 1.3.4', '>= 1.3.4'
-  spec.add_development_dependency 'rake', '~> 10.0', '>= 10.0'
-  spec.add_development_dependency 'rubocop', '0.59'
-  spec.add_development_dependency 'simplecov', '~> 0.16.1', '>= 0.16.1'
+  spec.add_development_dependency 'minitest-reporters', '~> 1.5.0', '>= 1.3.4'
+  spec.add_development_dependency 'rake', '~> 13.0.6', '>= 10.0'
+  spec.add_development_dependency 'rubocop', '1.25.1'
+  spec.add_development_dependency 'rubocop-performance', '~> 1.13.2', '>= 1.13.2'
+  spec.add_development_dependency 'rubocop-rake', '~> 0.6.0', '>= 0.6.0'
+  spec.add_development_dependency 'simplecov', '~> 0.21.2', '>= 0.16.1'
+  spec.metadata['rubygems_mfa_required'] = 'true'
 end