table_sync 2.3.0 → 4.0.0

data/lib/table_sync/publishing.rb

@@ -0,0 +1,11 @@
+# frozen_string_literal: true
+
+module TableSync
+  module Publishing
+    require_relative "publishing/base_publisher"
+    require_relative "publishing/publisher"
+    require_relative "publishing/batch_publisher"
+    require_relative "publishing/orm_adapter/active_record"
+    require_relative "publishing/orm_adapter/sequel"
+  end
+end

data/lib/table_sync/{base_publisher.rb → publishing/base_publisher.rb}

@@ -1,6 +1,6 @@
 # frozen_string_literal: true
 
-class TableSync::BasePublisher
+class TableSync::Publishing::BasePublisher
   include Memery
 
   BASE_SAFE_JSON_TYPES = [NilClass, String, TrueClass, FalseClass, Numeric, Symbol].freeze

data/lib/table_sync/{batch_publisher.rb → publishing/batch_publisher.rb}

@@ -1,6 +1,6 @@
 # frozen_string_literal: true
 
-class TableSync::BatchPublisher < TableSync::BasePublisher
+class TableSync::Publishing::BatchPublisher < TableSync::Publishing::BasePublisher
   def initialize(object_class, original_attributes_array, **options)
     @original_attributes_array = original_attributes_array.map do |hash|
       filter_safe_for_serialization(hash.deep_symbolize_keys)
@@ -22,7 +22,7 @@ class TableSync::BatchPublisher < TableSync::BasePublisher
     return unless need_publish?
     Rabbit.publish(params)
 
-    model_naming = TableSync.orm.model_naming(object_class)
+    model_naming = TableSync.publishing_adapter.model_naming(object_class)
     TableSync::Instrument.notify table: model_naming.table, schema: model_naming.schema,
                                  event: event,
                                  count: publishing_data[:attributes].size, direction: :publish
@@ -41,7 +41,7 @@ class TableSync::BatchPublisher < TableSync::BasePublisher
   end
 
   memoize def objects
-    needles.map { |needle| TableSync.orm.find(object_class, needle) }.compact
+    needles.map { |needle| TableSync.publishing_adapter.find(object_class, needle) }.compact
   end
 
   def job_callable
@@ -90,7 +90,7 @@ class TableSync::BatchPublisher < TableSync::BasePublisher
       if attributes_for_sync_defined?
         object.attributes_for_sync
       else
-        TableSync.orm.attributes(object)
+        TableSync.publishing_adapter.attributes(object)
       end
     end
   end

data/lib/table_sync/{orm_adapter → publishing/orm_adapter}/active_record.rb

@@ -1,13 +1,9 @@
 # frozen_string_literal: true
 
-module TableSync::ORMAdapter
+module TableSync::Publishing::ORMAdapter
   module ActiveRecord
     module_function
 
-    def model
-      ::TableSync::Model::ActiveRecord
-    end
-
     def model_naming(object)
       ::TableSync::NamingResolver::ActiveRecord.new(table_name: object.table_name)
     end
@@ -26,8 +22,8 @@ module TableSync::ORMAdapter
       klass.instance_exec do
         { create: :created, update: :updated, destroy: :destroyed }.each do |event, state|
           after_commit(on: event, **opts) do
-            TableSync::Publisher.new(self.class.name, attributes,
-                                     state: state, debounce_time: debounce_time).publish
+            TableSync::Publishing::Publisher.new(self.class.name, attributes,
+                                                 state: state, debounce_time: debounce_time).publish
           end
         end
       end

data/lib/table_sync/{orm_adapter → publishing/orm_adapter}/sequel.rb

@@ -1,13 +1,9 @@
 # frozen_string_literal: true
 
-module TableSync::ORMAdapter
+module TableSync::Publishing::ORMAdapter
   module Sequel
     module_function
 
-    def model
-      ::TableSync::Model::Sequel
-    end
-
     def model_naming(object)
       ::TableSync::NamingResolver::Sequel.new(table_name: object.table_name, db: object.db)
     end
@@ -44,7 +40,7 @@ module TableSync::ORMAdapter
         klass.send(:define_method, :"after_#{event}") do
           if instance_eval(&if_predicate) && !instance_eval(&unless_predicate)
             db.after_commit do
-              TableSync::Publisher.new(
+              TableSync::Publishing::Publisher.new(
                 self.class.name,
                 values,
                 state: state,

data/lib/table_sync/{publisher.rb → publishing/publisher.rb}

@@ -1,6 +1,6 @@
 # frozen_string_literal: true
 
-class TableSync::Publisher < TableSync::BasePublisher
+class TableSync::Publishing::Publisher < TableSync::Publishing::BasePublisher
   DEBOUNCE_TIME = 1.minute
 
   # 'original_attributes' are not published, they are used to resolve the routing key
@@ -34,7 +34,7 @@ class TableSync::Publisher < TableSync::BasePublisher
     return if !object && !destroyed?
 
     Rabbit.publish(params)
-    model_naming = TableSync.orm.model_naming(object_class)
+    model_naming = TableSync.publishing_adapter.model_naming(object_class)
     TableSync::Instrument.notify table: model_naming.table, schema: model_naming.schema,
                                  event: event, direction: :publish
   end
@@ -95,12 +95,12 @@ class TableSync::Publisher < TableSync::BasePublisher
     elsif attributes_for_sync_defined?
       object.attributes_for_sync
     else
-      TableSync.orm.attributes(object)
+      TableSync.publishing_adapter.attributes(object)
     end
   end
 
   memoize def object
-    TableSync.orm.find(object_class, needle)
+    TableSync.publishing_adapter.find(object_class, needle)
   end
 
   def event

data/lib/table_sync/receiving.rb

@@ -0,0 +1,14 @@
+# frozen_string_literal: true
+
+module TableSync
+  module Receiving
+    AVAILABLE_EVENTS = [:update, :destroy].freeze
+
+    require_relative "receiving/config"
+    require_relative "receiving/config_decorator"
+    require_relative "receiving/dsl"
+    require_relative "receiving/handler"
+    require_relative "receiving/model/active_record"
+    require_relative "receiving/model/sequel"
+  end
+end

data/lib/table_sync/receiving/config.rb

@@ -0,0 +1,218 @@
+# frozen_string_literal: true
+
+module TableSync::Receiving
+  class Config
+    attr_reader :model, :events
+
+    def initialize(model:, events: AVAILABLE_EVENTS)
+      @model = model
+
+      @events = [events].flatten.map(&:to_sym)
+
+      unless @events.all? { |event| AVAILABLE_EVENTS.include?(event) }
+        raise TableSync::UndefinedEvent.new(events)
+      end
+
+      self.class.default_values_for_options.each do |ivar, default_value_generator|
+        instance_variable_set(ivar, default_value_generator.call(self))
+      end
+    end
+
+    class << self
+      attr_reader :default_values_for_options
+
+      # In a configs this options are requested as they are
+      # config.option - get value
+      # config.option(args) - set static value
+      # config.option { ... } - set proc as value
+      #
+      # In `Receiving::Handler` or `Receiving::EventActions` this options are requested
+      # through `Receiving::ConfigDecorator#method_missing` which always executes `config.option`
+
+      def add_option(name, value_setter_wrapper:, value_as_proc_setter_wrapper:, default:)
+        ivar = "@#{name}".to_sym
+
+        @default_values_for_options ||= {}
+        @default_values_for_options[ivar] = default
+
+        define_method(name) do |*value, &value_as_proc|
+          return instance_variable_get(ivar) if value.empty? && value_as_proc.nil?
+
+          value = value.first if value.size == 1
+
+          if value_as_proc.present?
+            new_value = TableSync::Utils.proc_keywords_resolver(&value_as_proc)
+            setter_wrapper = value_as_proc_setter_wrapper
+          else
+            new_value = value
+            setter_wrapper = value_setter_wrapper
+          end
+
+          old_value = instance_variable_get(ivar)
+          result_value = instance_exec(name, new_value, old_value, &setter_wrapper)
+          instance_variable_set(ivar, result_value)
+        end
+      end
+    end
+
+    def allow_event?(name)
+      events.include?(name)
+    end
+  end
+end
+
+# Helpers:
+
+column_key = proc do |option_name, new_value|
+  unless model.columns.include?(new_value)
+    raise TableSync::WrongOptionValue.new(model, option_name, new_value)
+  end
+  new_value
+end
+
+exactly_symbol = proc do |option_name, new_value|
+  unless new_value.is_a? Symbol
+    raise TableSync::WrongOptionValue.new(model, option_name, new_value)
+  end
+  new_value
+end
+
+to_array = proc do |block|
+  proc do |option_name, new_value|
+    new_value = [new_value].flatten
+    new_value.map { |value| instance_exec(option_name, value, &block) }
+  end
+end
+
+exactly_not_array = proc do |block|
+  proc do |option_name, new_value|
+    if new_value.is_a? Array
+      raise TableSync::WrongOptionValue.new(model, option_name, new_value)
+    end
+    instance_exec(option_name, new_value, &block)
+  end
+end
+
+exactly_hash = proc do |block_for_keys, block_for_values|
+  proc do |option_name, new_value|
+    unless new_value.is_a? Hash
+      raise TableSync::WrongOptionValue.new(model, option_name, new_value)
+    end
+
+    new_value.to_a.map do |key, value|
+      [
+        instance_exec("#{option_name} keys", key, &block_for_keys),
+        instance_exec("#{option_name} values", value, &block_for_values),
+      ]
+    end.to_h
+  end
+end
+
+any_value = proc do |_option_name, new_value|
+  new_value
+end
+
+raise_error = proc do |option_name, new_value|
+  raise TableSync::WrongOptionValue.new(model, option_name, new_value)
+end
+
+exactly_boolean = proc do |option_name, new_value|
+  unless new_value.is_a?(TrueClass) || new_value.is_a?(FalseClass)
+    raise TableSync::WrongOptionValue.new(model, option_name, new_value)
+  end
+  new_value
+end
+
+allow_false = proc do |block|
+  proc do |option_name, new_value|
+    next false if new_value.is_a? FalseClass
+    instance_exec(option_name, new_value, &block)
+  end
+end
+
+proc_result = proc do |block|
+  proc do |option_name, new_value|
+    proc do |*args, &b|
+      result = new_value.call(*args, &b)
+      instance_exec(option_name, result, &block)
+    end
+  end
+end
+
+accumulate_procs = proc do |block|
+  proc do |option_name, new_value, old_value|
+    old_value.push(&instance_exec(option_name, new_value, &block))
+  end
+end
+
+# Options:
+
+# rubocop:disable Layout/ArgumentAlignment
+
+TableSync::Receiving::Config.add_option :only,
+  value_setter_wrapper: to_array[column_key],
+  value_as_proc_setter_wrapper: proc_result[to_array[column_key]],
+  default: proc { |config| config.model.columns }
+
+TableSync::Receiving::Config.add_option :target_keys,
+  value_setter_wrapper: to_array[column_key],
+  value_as_proc_setter_wrapper: proc_result[to_array[column_key]],
+  default: proc { |config| Array.wrap(config.model.primary_keys) }
+
+TableSync::Receiving::Config.add_option :rest_key,
+  value_setter_wrapper: exactly_not_array[allow_false[column_key]],
+  value_as_proc_setter_wrapper: proc_result[exactly_not_array[allow_false[column_key]]],
+  default: proc { :rest }
+
+TableSync::Receiving::Config.add_option :version_key,
+  value_setter_wrapper: exactly_not_array[column_key],
+  value_as_proc_setter_wrapper: proc_result[exactly_not_array[column_key]],
+  default: proc { :version }
+
+TableSync::Receiving::Config.add_option :except,
+  value_setter_wrapper: to_array[exactly_symbol],
+  value_as_proc_setter_wrapper: proc_result[to_array[exactly_symbol]],
+  default: proc { [] }
+
+TableSync::Receiving::Config.add_option :mapping_overrides,
+  value_setter_wrapper: exactly_hash[exactly_symbol, exactly_symbol],
+  value_as_proc_setter_wrapper: proc_result[exactly_hash[exactly_symbol, exactly_symbol]],
+  default: proc { {} }
+
+TableSync::Receiving::Config.add_option :additional_data,
+  value_setter_wrapper: exactly_hash[exactly_symbol, any_value],
+  value_as_proc_setter_wrapper: proc_result[exactly_hash[exactly_symbol, any_value]],
+  default: proc { {} }
+
+TableSync::Receiving::Config.add_option :default_values,
+  value_setter_wrapper: exactly_hash[exactly_symbol, any_value],
+  value_as_proc_setter_wrapper: proc_result[exactly_hash[exactly_symbol, any_value]],
+  default: proc { {} }
+
+TableSync::Receiving::Config.add_option :skip,
+  value_setter_wrapper: exactly_boolean,
+  value_as_proc_setter_wrapper: proc_result[exactly_boolean],
+  default: proc { false }
+
+TableSync::Receiving::Config.add_option :wrap_receiving,
+  value_setter_wrapper: raise_error,
+  value_as_proc_setter_wrapper: any_value,
+  default: proc { proc { |&block| block.call } }
+
+%i[
+  before_update
+  after_commit_on_update
+  before_destroy
+  after_commit_on_destroy
+].each do |option|
+  TableSync::Receiving::Config.add_option option,
+    value_setter_wrapper: raise_error,
+    value_as_proc_setter_wrapper: accumulate_procs[any_value],
+    default: (proc do |_config|
+      TableSync::Utils::ProcArray.new do |array, args, &block|
+        array.map { |pr| pr.call(*args, &block) }
+      end
+    end)
+end
+
+# rubocop:enable Layout/ArgumentAlignment

data/lib/table_sync/receiving/config_decorator.rb

@@ -0,0 +1,27 @@
+# frozen_string_literal: true
+
+module TableSync::Receiving
+  class ConfigDecorator
+    extend Forwardable
+
+    def_delegators :@config, :allow_event?
+    # rubocop:disable Metrics/ParameterLists
+    def initialize(config:, event:, model:, version:, project_id:, raw_data:)
+      @config = config
+
+      @default_params = {
+        event: event,
+        model: model,
+        version: version,
+        project_id: project_id,
+        raw_data: raw_data,
+      }
+    end
+    # rubocop:enable Metrics/ParameterLists
+
+    def method_missing(name, **additional_params, &block)
+      value = @config.send(name)
+      value.is_a?(Proc) ? value.call(@default_params.merge(additional_params), &block) : value
+    end
+  end
+end

data/lib/table_sync/receiving/dsl.rb

@@ -0,0 +1,28 @@
+# frozen_string_literal: true
+
+module TableSync::Receiving
+  module DSL
+    def inherited(klass)
+      klass.instance_variable_set(:@configs, configs.deep_dup)
+      super
+    end
+
+    def configs
+      @configs ||= Hash.new { |hash, key| hash[key] = [] }
+    end
+
+    def receive(source, to_table: nil, to_model: nil, events: [:update, :destroy], &block)
+      model = to_table ? TableSync.receiving_model.new(to_table) : to_model
+
+      TableSync::Utils::InterfaceChecker.new(model).implements(:receiving_model)
+
+      config = ::TableSync::Receiving::Config.new(model: model, events: events)
+
+      config.instance_exec(&block) if block
+
+      configs[source.to_s] << config
+
+      self
+    end
+  end
+end

data/lib/table_sync/receiving/handler.rb

@@ -0,0 +1,131 @@
+# frozen_string_literal: true
+
+class TableSync::Receiving::Handler < Rabbit::EventHandler
+  extend TableSync::Receiving::DSL
+
+  # Rabbit::EventHandler uses Tainbox and performs `handler.new(message).call`
+  attribute :event
+  attribute :model
+  attribute :version
+
+  def call
+    configs.each do |config|
+      next unless config.allow_event?(event)
+
+      data = processed_data(config)
+
+      next if data.empty?
+
+      version_key = config.version_key(data: data)
+      data.each { |row| row[version_key] = version }
+
+      target_keys = config.target_keys(data: data)
+
+      validate_data(data, target_keys: target_keys)
+
+      params = { data: data, target_keys: target_keys, version_key: version_key }
+
+      if event == :update
+        params[:default_values] = config.default_values(data: data)
+      end
+
+      config.wrap_receiving(**params) do
+        perform(config, params)
+      end
+    end
+  end
+
+  private
+
+  # redefine setter from Rabbit::EventHandler
+  def data=(data)
+    super(Array.wrap(data[:attributes]))
+  end
+
+  def event=(name)
+    name = name.to_sym
+    raise TableSync::UndefinedEvent.new(event) unless %i[update destroy].include?(name)
+    super(name)
+  end
+
+  def model=(name)
+    super(name.to_s)
+  end
+
+  def configs
+    @configs ||= self.class.configs[model]&.map do |config|
+      ::TableSync::Receiving::ConfigDecorator.new(
+        config: config,
+        # next parameters will be send to each proc-options from config
+        event: event,
+        model: model,
+        version: version,
+        project_id: project_id,
+        raw_data: data,
+      )
+    end
+  end
+
+  def processed_data(config)
+    data.map do |row|
+      next if config.skip(row: row)
+
+      row = row.dup
+
+      config.mapping_overrides(row: row).each do |before, after|
+        row[after] = row.delete(before)
+      end
+
+      config.except(row: row).each(&row.method(:delete))
+
+      row.merge!(config.additional_data(row: row))
+
+      only = config.only(row: row)
+      row, rest = row.partition { |key, _| key.in?(only) }.map(&:to_h)
+
+      rest_key = config.rest_key(row: row, rest: rest)
+      (row[rest_key] ||= {}).merge!(rest) if rest_key
+
+      row
+    end.compact
+  end
+
+  def validate_data(data, target_keys:)
+    data.each do |row|
+      next if target_keys.all?(&row.keys.method(:include?))
+      raise TableSync::DataError.new(
+        data, target_keys, "Some target keys not found in received attributes!"
+      )
+    end
+
+    if data.uniq { |row| row.slice(*target_keys) }.size != data.size
+      raise TableSync::DataError.new(
+        data, target_keys, "Duplicate rows found!"
+      )
+    end
+  end
+
+  def perform(config, params)
+    model = config.model
+
+    model.transaction do
+      if event == :update
+        config.before_update(**params)
+
+        results = model.upsert(**params)
+
+        model.after_commit do
+          config.after_commit_on_update(**params.merge(results: results))
+        end
+      else
+        config.before_destroy(**params)
+
+        results = model.destroy(**params)
+
+        model.after_commit do
+          config.after_commit_on_destroy(**params.merge(results: results))
+        end
+      end
+    end
+  end
+end