pgdice 0.1.0 → 0.2.0

data/lib/pgdice/partition_manager_factory.rb

@@ -0,0 +1,52 @@
+# frozen_string_literal: true
+
+# Entry point for PartitionManager
+module PgDice
+  #  PartitionManagerFactory is a class used to build PartitionManagers
+  class PartitionManagerFactory
+    def initialize(configuration, opts = {})
+      @configuration = configuration
+      initialize_simple_factories(opts)
+      initialize_complex_factories(opts)
+      initialize_values(opts)
+    end
+
+    def call
+      PgDice::PartitionManager.new(logger: @logger_factory.call,
+                                   batch_size: @batch_size_factory.call,
+                                   approved_tables: @approved_tables_factory.call,
+                                   validation: @validation_factory.call,
+                                   partition_adder: @partition_adder_factory.call,
+                                   partition_lister: @partition_lister_factory.call,
+                                   partition_dropper: @partition_dropper_factory.call,
+                                   current_date_provider: @current_date_provider)
+    end
+
+    private
+
+    def initialize_simple_factories(opts)
+      @logger_factory = opts[:logger_factory] ||= proc { @configuration.logger }
+      @batch_size_factory = opts[:batch_size_factory] ||= proc { @configuration.batch_size }
+      @approved_tables_factory = opts[:approved_tables_factory] ||= proc { @configuration.approved_tables }
+    end
+
+    def initialize_complex_factories(opts)
+      @validation_factory = opts[:validation_factory] ||= PgDice::ValidationFactory.new(@configuration)
+      @partition_adder_factory = opts[:partition_adder_factory] ||= partition_adder_factory
+      @partition_lister_factory = opts[:partition_lister_factory] ||= PgDice::PartitionListerFactory.new(@configuration)
+      @partition_dropper_factory =
+        opts[:partition_dropper_factory] ||= PgDice::PartitionDropperFactory.new(@configuration)
+    end
+
+    def initialize_values(opts)
+      @current_date_provider = opts[:current_date_provider] ||= proc { Time.now.utc.to_date }
+    end
+
+    def partition_adder_factory
+      proc do
+        pg_slice_manager = PgDice::PgSliceManagerFactory.new(@configuration).call
+        ->(all_params) { pg_slice_manager.add_partitions(all_params) }
+      end
+    end
+  end
+end

data/lib/pgdice/period_fetcher.rb

@@ -0,0 +1,31 @@
+# frozen_string_literal: true
+
+module PgDice
+  # Used to find the period of a postgres table using the comment on the table created by pgslice
+  class PeriodFetcher
+    def initialize(query_executor:)
+      @query_executor = query_executor
+    end
+
+    def call(params)
+      sql = build_table_comment_sql(params.fetch(:table_name), params.fetch(:schema))
+      values = @query_executor.call(sql)
+      convert_comment_to_hash(values.first)[:period]
+    end
+
+    private
+
+    def convert_comment_to_hash(comment)
+      return {} unless comment
+
+      comment.split(',').reduce({}) do |hash, key_value_pair|
+        key, value = key_value_pair.split(':')
+        hash.merge(key.to_sym => value)
+      end
+    end
+
+    def build_table_comment_sql(table_name, schema)
+      "SELECT obj_description('#{schema}.#{table_name}'::REGCLASS) AS comment"
+    end
+  end
+end

data/lib/pgdice/period_fetcher_factory.rb

@@ -0,0 +1,19 @@
+# frozen_string_literal: true
+
+module PgDice
+  # Factory for PeriodFetcher
+  class PeriodFetcherFactory
+    extend Forwardable
+
+    def_delegators :@configuration, :database_connection
+
+    def initialize(configuration, opts = {})
+      @configuration = configuration
+      @query_executor = opts[:query_executor] ||= ->(sql) { database_connection.execute(sql).values.flatten.compact }
+    end
+
+    def call
+      PgDice::PeriodFetcher.new(query_executor: @query_executor)
+    end
+  end
+end

data/lib/pgdice/pg_slice_manager.rb

@@ -4,18 +4,22 @@
 module PgDice
   # PgSliceManager is a wrapper around PgSlice
   class PgSliceManager
-    extend Forwardable
-    def_delegators :@configuration, :logger, :database_url, :dry_run
+    include PgDice::LogHelper
 
-    def initialize(configuration = PgDice::Configuration.new)
-      @configuration = configuration
+    attr_reader :logger, :database_url
+
+    def initialize(logger:, database_url:, pg_slice_executor:, dry_run: false)
+      @logger = logger
+      @database_url = database_url
+      @dry_run = dry_run
+      @pg_slice_executor = pg_slice_executor
     end
 
     def prep(params = {})
       table_name = params.fetch(:table_name)
       column_name = params.fetch(:column_name)
       period = params.fetch(:period)
-      run_pgslice("prep #{table_name} #{column_name} #{period}")
+      run_pgslice("prep #{table_name} #{column_name} #{period}", params[:dry_run])
     end
 
     def fill(params = {})
@@ -23,7 +27,7 @@ module PgDice
       swapped = params.fetch(:swapped, '')
       swapped = '--swapped' if swapped.to_s.casecmp('true').zero?
 
-      run_pgslice("fill #{table_name} #{swapped}")
+      run_pgslice("fill #{table_name} #{swapped}", params[:dry_run])
     end
 
     def analyze(params = {})
@@ -31,12 +35,12 @@ module PgDice
       swapped = params.fetch(:swapped, '')
       swapped = '--swapped' if swapped.to_s.casecmp('true').zero?
 
-      run_pgslice("analyze #{table_name} #{swapped}")
+      run_pgslice("analyze #{table_name} #{swapped}", params[:dry_run])
     end
 
     def swap(params = {})
       table_name = params.fetch(:table_name)
-      run_pgslice("swap #{table_name}")
+      run_pgslice("swap #{table_name}", params[:dry_run])
     end
 
     def add_partitions(params = {})
@@ -50,25 +54,25 @@ module PgDice
       intermediate = params.fetch(:intermediate, nil)
       intermediate = '--intermediate' if intermediate.to_s.casecmp('true').zero?
 
-      run_pgslice("add_partitions #{table_name} #{intermediate} #{future_tables} #{past_tables}")
+      run_pgslice("add_partitions #{table_name} #{intermediate} #{future_tables} #{past_tables}", params[:dry_run])
     end
 
     def unprep!(params = {})
       table_name = params.fetch(:table_name)
 
-      run_pgslice("unprep #{table_name}")
+      run_pgslice("unprep #{table_name}", params[:dry_run])
     end
 
     def unswap!(params = {})
       table_name = params.fetch(:table_name)
 
-      run_pgslice("unswap #{table_name}")
+      run_pgslice("unswap #{table_name}", params[:dry_run])
     end
 
     def unprep(params = {})
       table_name = params.fetch(:table_name)
 
-      run_pgslice("unprep #{table_name}")
+      run_pgslice("unprep #{table_name}", params[:dry_run])
     rescue PgSliceError => error
       logger.error { "Rescued PgSliceError: #{error}" }
       false
@@ -77,7 +81,7 @@ module PgDice
     def unswap(params = {})
       table_name = params.fetch(:table_name)
 
-      run_pgslice("unswap #{table_name}")
+      run_pgslice("unswap #{table_name}", params[:dry_run])
     rescue PgSliceError => error
       logger.error { "Rescued PgSliceError: #{error}" }
       false
@@ -85,35 +89,46 @@ module PgDice
 
     private
 
-    def run_pgslice(argument_string)
-      parameters = build_pg_slice_command(argument_string)
+    def run_pgslice(argument_string, dry_run)
+      command = build_pg_slice_command(argument_string, dry_run)
 
-      stdout, stderr, status = Open3.capture3(parameters)
-      log_result(stdout, stderr, status)
+      stdout, stderr, status = run_and_log(command)
 
-      if status.exitstatus.to_i.positive?
+      if status.to_i.positive?
         raise PgDice::PgSliceError,
-              "pgslice with arguments: '#{argument_string}' failed with status: '#{status.exitstatus}' "\
+              "pgslice with arguments: '#{argument_string}' failed with status: '#{status}' "\
                  "STDOUT: '#{stdout}' STDERR: '#{stderr}'"
       end
       true
     end
 
-    def build_pg_slice_command(argument_string)
+    def build_pg_slice_command(argument_string, dry_run)
       argument_string = argument_string.strip
-      logger.info { "Running pgslice command: '#{argument_string}'" }
       $stdout.flush
       $stderr.flush
-      command = "pgslice #{argument_string} "
-      command += '--dry-run true ' if dry_run
-      command += "--url #{database_url}"
-      command
+      command = "pgslice #{argument_string}"
+      command += ' --dry-run true' if @dry_run || dry_run
+      command = squish(command)
+      logger.info { "Running pgslice command: '#{command}'" }
+      command + " --url #{database_url}"
+    end
+
+    def log_result(stdout, stderr)
+      logger.debug { "pgslice STDERR: #{stderr}" } unless blank?(stderr)
+      logger.debug { "pgslice STDOUT: #{stdout}" } unless blank?(stdout)
     end
 
-    def log_result(stdout, stderr, status)
-      logger.debug { "pgslice STDERR: #{stderr}" } if stderr
-      logger.debug { "pgslice STDOUT: #{stdout}" } if stdout
-      logger.debug { "pgslice exit status: #{status.exitstatus}" } if status
+    def log_status(status)
+      logger.debug { "pgslice exit status: #{status}" } unless blank?(status) || status.to_i.zero?
+    end
+
+    def run_and_log(command)
+      PgDice::LogHelper.log_duration('PgSlice', logger) do
+        stdout, stderr, status = @pg_slice_executor.call(command)
+        log_result(stdout, stderr)
+        log_status(status)
+        [stdout, stderr, status]
+      end
     end
   end
 end

data/lib/pgdice/pg_slice_manager_factory.rb

@@ -0,0 +1,35 @@
+# frozen_string_literal: true
+
+# Entry point for PartitionManager
+module PgDice
+  #  PartitionManagerFactory is a class used to build PartitionManagers
+  class PgSliceManagerFactory
+    extend Forwardable
+    include PgDice::LogHelper
+
+    def_delegators :@configuration, :logger, :database_url, :dry_run
+
+    def initialize(configuration, opts = {})
+      @configuration = configuration
+      @pg_slice_executor = opts[:pg_slice_executor] ||= executor
+    end
+
+    def call
+      PgDice::PgSliceManager.new(logger: logger,
+                                 database_url: database_url,
+                                 pg_slice_executor: @pg_slice_executor,
+                                 dry_run: dry_run)
+    end
+
+    private
+
+    def executor
+      lambda do |command|
+        results = Open3.capture3(command)
+        stdout, stderr = results.first(2).map { |output| squish(output.to_s) }
+        status = results[2].exitstatus.to_s
+        [stdout, stderr, status]
+      end
+    end
+  end
+end

data/lib/pgdice/table.rb

@@ -0,0 +1,87 @@
+# frozen_string_literal: true
+
+module PgDice
+  # Object to represent a table's configuration in the context of PgDice.
+  class Table
+    include Comparable
+    attr_reader :table_name
+    attr_accessor :past, :future, :column_name, :period, :schema
+
+    def initialize(table_name:, past: 90, future: 7, column_name: 'created_at', period: 'day', schema: 'public')
+      raise ArgumentError, 'table_name must be a string' unless table_name.is_a?(String)
+
+      @table_name = table_name
+      @past = past
+      @future = future
+      @column_name = column_name
+      @period = period
+      @schema = schema
+    end
+
+    def validate!
+      check_type(:past, Integer)
+      check_type(:future, Integer)
+      check_type(:column_name, String)
+      check_type(:period, String)
+      check_type(:schema, String)
+      unless PgDice::SUPPORTED_PERIODS.include?(period)
+        raise ArgumentError,
+              "Period must be one of: #{PgDice::SUPPORTED_PERIODS.keys}. Value: #{period} is not valid."
+      end
+      true
+    end
+
+    def name
+      table_name
+    end
+
+    def full_name
+      "#{schema}.#{name}"
+    end
+
+    def to_h
+      { table_name: table_name,
+        past: past,
+        future: future,
+        column_name: column_name,
+        period: period,
+        schema: schema }
+    end
+
+    def to_s
+      "#{schema}.#{name}: <past: #{past}, future: #{future}, column_name: #{column_name}, period: #{period}>"
+    end
+
+    def smash(override_parameters)
+      to_h.merge!(override_parameters)
+    end
+
+    def ==(other)
+      to_h == other.to_h
+    end
+
+    def <=>(other)
+      table_name <=> other.table_name
+    end
+
+    # Get expected size of this configured table (past + present + future table counts)
+    def size
+      past + future + 1
+    end
+
+    def self.from_hash(hash)
+      Table.new(**hash.each_with_object({}) { |(k, v), memo| memo[k.to_sym] = v; })
+    end
+
+    private
+
+    def check_type(field, expected_type)
+      unless send(field).is_a?(expected_type)
+        raise ArgumentError,
+              "PgDice::Table: #{name} failed validation on field: #{field}. "\
+                "Expected type of: #{expected_type} but found #{send(field).class}"
+      end
+      true
+    end
+  end
+end

data/lib/pgdice/table_finder.rb

@@ -0,0 +1,41 @@
+# frozen_string_literal: true
+
+module PgDice
+  # Module which is a collection of methods used by PartitionManager to find and list tables
+  module TableFinder
+    include PgDice::DateHelper
+
+    def find_droppable_partitions(all_tables, older_than, minimum_tables, period)
+      tables_older_than = tables_older_than(all_tables, older_than, period)
+      tables_to_grab = tables_to_grab(tables_older_than.size, minimum_tables)
+      tables_older_than.first(tables_to_grab)
+    end
+
+    def tables_to_grab(eligible_tables, minimum_tables)
+      tables_to_grab = eligible_tables - minimum_tables
+      tables_to_grab.positive? ? tables_to_grab : 0
+    end
+
+    def batched_tables(tables, batch_size)
+      tables.first(batch_size)
+    end
+
+    def tables_older_than(tables, older_than, period)
+      table_tester(tables, lambda do |partition_created_at_time|
+        partition_created_at_time < truncate_date(older_than.to_date, period)
+      end)
+    end
+
+    def tables_newer_than(tables, newer_than, period)
+      table_tester(tables, lambda do |partition_created_at_time|
+        partition_created_at_time > truncate_date(newer_than.to_date, period)
+      end)
+    end
+
+    def table_tester(tables, predicate)
+      tables.select do |partition_name|
+        predicate.call(safe_date_builder(partition_name))
+      end
+    end
+  end
+end

data/lib/pgdice/validation.rb

@@ -3,23 +3,27 @@
 module PgDice
   # Collection of utilities that provide ways for users to ensure things are working properly
   class Validation
-    extend Forwardable
-    def_delegators :@configuration, :logger, :additional_validators, :database_connection, :approved_tables
+    include PgDice::TableFinder
 
-    def initialize(configuration = PgDice::Configuration.new)
-      @configuration = configuration
+    attr_reader :logger, :approved_tables
+
+    def initialize(logger:, partition_lister:, period_fetcher:, approved_tables:, current_date_provider:)
+      @logger = logger
+      @approved_tables = approved_tables
+      @partition_lister = partition_lister
+      @period_fetcher = period_fetcher
+      @current_date_provider = current_date_provider
     end
 
-    def assert_tables(params = {})
-      unless params[:future] || params[:past]
-        raise ArgumentError, 'You must provide either a future or past number of tables to assert on.'
-      end
+    def assert_tables(table_name, opts = nil)
+      table, period, all_params, params = filter_parameters(approved_tables.fetch(table_name), opts)
+      validate_parameters(all_params)
+      logger.debug { "Running asserts on table: #{table} with params: #{all_params}" }
 
-      table_name = validate_table_name(params)
-      period = resolve_period(params)
+      partitions = @partition_lister.call(all_params)
 
-      assert_future_tables(table_name, params[:future], period) if params[:future]
-      assert_past_tables(table_name, params[:past], period) if params[:past]
+      assert_future_tables(table_name, partitions, period, params[:future]) if params && params[:future]
+      assert_past_tables(table_name, partitions, period, params[:past]) if params && params[:past]
       true
     end
 
@@ -27,14 +31,46 @@ module PgDice
       validate_table_name(params)
       validate_period(params)
 
-      run_additional_validators(params)
       true
     end
 
     private
 
+    def filter_parameters(table, params)
+      if params.nil?
+        params = {}
+        params[:future] = table.future
+        params[:past] = table.past
+        period = table.period
+      else
+        period = resolve_period(schema: table.schema, table_name: table.name, **params)
+      end
+      all_params = table.smash(params.merge!(period: period))
+
+      [table, period, all_params, params]
+    end
+
+    def assert_future_tables(table_name, partitions, period, expected)
+      newer_tables = tables_newer_than(partitions, @current_date_provider.call, period).size
+      if newer_tables < expected
+        raise PgDice::InsufficientFutureTablesError.new(table_name, expected, period, newer_tables)
+      end
+
+      true
+    end
+
+    def assert_past_tables(table_name, partitions, period, expected)
+      older_tables = tables_older_than(partitions, @current_date_provider.call, period).size
+      if older_tables < expected
+        raise PgDice::InsufficientPastTablesError.new(table_name, expected, period, older_tables)
+      end
+
+      true
+    end
+
     def resolve_period(params)
-      period = validate_period(params) || fetch_period_from_table_comment(params.fetch(:table_name))
+      validate_period(params) if params[:period]
+      period = @period_fetcher.call(params)
 
       # If the user doesn't supply a period and we fail to find one on the table then it's a pretty good bet
       # this table is not partitioned at all.
@@ -42,17 +78,10 @@ module PgDice
         raise TableNotPartitionedError,
               "Table: #{params.fetch(:table_name)} is not partitioned! Cannot validate partitions that don't exist!"
       end
+      validate_period(period: period)
       period
     end
 
-    def run_additional_validators(params)
-      return true if additional_validators.all? { |validator| validator.call(params, logger) }
-
-      raise PgDice::CustomValidationError.new(params, additional_validators)
-    rescue StandardError => error
-      raise PgDice::CustomValidationError.new(params, additional_validators, error)
-    end
-
     def validate_table_name(params)
       table_name = params.fetch(:table_name)
       unless approved_tables.include?(table_name)
@@ -65,68 +94,12 @@ module PgDice
     def validate_period(params)
       return unless params[:period]
 
-      unless PgDice::SUPPORTED_PERIODS.include?(params[:period])
+      unless PgDice::SUPPORTED_PERIODS.include?(params[:period].to_s)
         raise ArgumentError,
               "Period must be one of: #{PgDice::SUPPORTED_PERIODS.keys}. Value: #{params[:period]} is not valid."
       end
 
       params[:period].to_sym
     end
-
-    def assert_future_tables(table_name, future, period)
-      sql = build_assert_sql(table_name, future, period, :future)
-
-      response = database_connection.execute(sql)
-
-      return true if response.values.size == 1
-
-      raise PgDice::InsufficientFutureTablesError.new(table_name, future, period)
-    end
-
-    def assert_past_tables(table_name, past, period)
-      sql = build_assert_sql(table_name, past, period, :past)
-
-      response = database_connection.execute(sql)
-
-      return true if response.values.size == 1
-
-      raise PgDice::InsufficientPastTablesError.new(table_name, past, period)
-    end
-
-    def build_assert_sql(table_name, table_count, period, direction)
-      add_or_subtract = { future: '+', past: '-' }.fetch(direction, '+')
-      <<~SQL
-        SELECT 1
-        FROM pg_catalog.pg_class pg_class
-        INNER JOIN pg_catalog.pg_namespace pg_namespace ON pg_namespace.oid = pg_class.relnamespace
-        WHERE pg_class.relkind = 'r'
-          AND pg_namespace.nspname = 'public'
-          AND pg_class.relname = '#{table_name}_' || to_char(NOW()
-            #{add_or_subtract} INTERVAL '#{table_count} #{period}', '#{SUPPORTED_PERIODS[period.to_sym]}')
-      SQL
-    end
-
-    def fetch_period_from_table_comment(table_name)
-      sql = build_table_comment_sql(table_name, 'public')
-      values = database_connection.execute(sql).values.flatten.compact
-      convert_comment_to_hash(values.first)[:period]
-    end
-
-    def convert_comment_to_hash(comment)
-      return {} unless comment
-
-      partition_template = {}
-      comment.split(',').each do |key_value_pair|
-        key, value = key_value_pair.split(':')
-        partition_template[key.to_sym] = value
-      end
-      partition_template
-    end
-
-    def build_table_comment_sql(table_name, schema)
-      <<~SQL
-        SELECT obj_description('#{schema}.#{table_name}'::REGCLASS) AS comment
-      SQL
-    end
   end
 end

data/lib/pgdice/validation_factory.rb

@@ -0,0 +1,21 @@
+# frozen_string_literal: true
+
+module PgDice
+  # Factory for PgDice::Validations
+  class ValidationFactory
+    def initialize(configuration, opts = {})
+      @configuration = configuration
+      @partition_lister_factory = opts[:partition_lister_factory] ||= PgDice::PartitionListerFactory.new(@configuration)
+      @period_fetcher_factory = opts[:period_fetcher_factory] ||= PgDice::PeriodFetcherFactory.new(@configuration)
+      @current_date_provider = opts[:current_date_provider] ||= proc { Time.now.utc.to_date }
+    end
+
+    def call
+      PgDice::Validation.new(logger: @configuration.logger,
+                             partition_lister: @partition_lister_factory.call,
+                             period_fetcher: @period_fetcher_factory.call,
+                             approved_tables: @configuration.approved_tables,
+                             current_date_provider: @current_date_provider)
+    end
+  end
+end

data/lib/pgdice/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module PgDice
-  VERSION = '0.1.0'
+  VERSION = '0.2.0'
 end