table_sync 1.13.1 → 4.0.0

data/lib/table_sync.rb

@@ -1,61 +1,51 @@
 # frozen_string_literal: true
 
 require "memery"
+require "self_data"
 require "rabbit_messaging"
 require "rabbit/event_handler" # NOTE: from rabbit_messaging"
 require "active_support/core_ext/object/blank"
 require "active_support/core_ext/numeric/time"
 
 module TableSync
-  require_relative "./table_sync/version"
-  require_relative "./table_sync/errors"
-  require_relative "./table_sync/event_actions"
-  require_relative "./table_sync/event_actions/data_wrapper"
-  require_relative "./table_sync/config"
-  require_relative "./table_sync/config/callback_registry"
-  require_relative "./table_sync/config_decorator"
-  require_relative "./table_sync/dsl"
-  require_relative "./table_sync/receiving_handler"
-  require_relative "./table_sync/base_publisher"
-  require_relative "./table_sync/publisher"
-  require_relative "./table_sync/batch_publisher"
-  require_relative "./table_sync/orm_adapter/active_record"
-  require_relative "./table_sync/orm_adapter/sequel"
-  require_relative "./table_sync/model/active_record"
-  require_relative "./table_sync/model/sequel"
-  require_relative "./table_sync/instrument"
-  require_relative "./table_sync/instrument_adapter/active_support"
-  require_relative "./table_sync/naming_resolver/active_record"
-  require_relative "./table_sync/naming_resolver/sequel"
+  require_relative "table_sync/utils"
+  require_relative "table_sync/version"
+  require_relative "table_sync/errors"
+  require_relative "table_sync/instrument"
+  require_relative "table_sync/instrument_adapter/active_support"
+  require_relative "table_sync/naming_resolver/active_record"
+  require_relative "table_sync/naming_resolver/sequel"
+  require_relative "table_sync/receiving"
+  require_relative "table_sync/publishing"
 
   class << self
-    include Memery
-
     attr_accessor :publishing_job_class_callable
     attr_accessor :batch_publishing_job_class_callable
     attr_accessor :routing_key_callable
     attr_accessor :exchange_name
     attr_accessor :routing_metadata_callable
     attr_accessor :notifier
+    attr_reader :orm
+    attr_reader :publishing_adapter
+    attr_reader :receiving_model
 
-    def sync(*args)
-      orm.setup_sync(*args)
+    def sync(klass, **opts)
+      publishing_adapter.setup_sync(klass, opts)
     end
 
     def orm=(val)
-      clear_memery_cache!
-      @orm = val
-    end
-
-    memoize def orm
-      case @orm
+      case val
       when :active_record
-        ORMAdapter::ActiveRecord
+        @publishing_adapter = Publishing::ORMAdapter::ActiveRecord
+        @receiving_model = Receiving::Model::ActiveRecord
       when :sequel
-        ORMAdapter::Sequel
+        @publishing_adapter = Publishing::ORMAdapter::Sequel
+        @receiving_model = Receiving::Model::Sequel
       else
-        raise "ORM not supported: #{@orm.inspect}"
+        raise ORMNotSupported.new(val.inspect)
       end
+
+      @orm = val
     end
   end
 end

data/lib/table_sync/errors.rb

@@ -4,38 +4,76 @@ module TableSync
   Error = Class.new(StandardError)
 
   class UpsertError < Error
-    def initialize(data:, target_keys:, version_key:, first_sync_time_key:, default_values:)
-      super <<~MSG
-        Upsert has changed more than 1 row;
-          data: #{data.inspect}
-          target_keys: #{target_keys.inspect}
-          version_key: #{version_key.inspect}
-          first_sync_time_key: #{first_sync_time_key.inspect}
-          default_values: #{default_values.inspect}
+    def initialize(data:, target_keys:, result:)
+      super("data: #{data.inspect}, target_keys: #{target_keys.inspect}, result: #{result.inspect}")
+    end
+  end
+
+  class DestroyError < Error
+    def initialize(data:, target_keys:, result:)
+      super("data: #{data.inspect}, target_keys: #{target_keys.inspect}, result: #{result.inspect}")
+    end
+  end
+
+  class DataError < Error
+    def initialize(data, target_keys, description)
+      super(<<~MSG.squish)
+        #{description}
+        target_keys: #{target_keys}
+        data: #{data}
       MSG
     end
   end
 
-  class UndefinedConfig < Error
-    def initialize(model)
-      super("Config not defined for model; model: #{model.inspect}")
+  # @api public
+  # @since 2.2.0
+  PluginError = Class.new(Error)
+
+  # @api public
+  # @since 2.2.0
+  class UnregisteredPluginError < PluginError
+    # @param plugin_name [Any]
+    def initialize(plugin_name)
+      super("#{plugin_name} plugin is not registered")
     end
   end
 
-  class DestroyError < Error
-    def initialize(data)
-      super("Destroy has changed more than 1 row; data: #{data.inspect}")
+  # @api public
+  # @since 2.2.0
+  class AlreadyRegisteredPluginError < PluginError
+    # @param plugin_name [Any]
+    def initialize(plugin_name)
+      super("#{plugin_name} plugin already exists")
     end
   end
 
-  class UnprovidedEventTargetKeysError < Error
-    # @param target_keys [Array<Symbol,String>]
-    # @param target_attributes [Hash<Symbol|String,Any>]
-    def initialize(target_keys, target_attributes)
-      super(<<~MSG.squish)
-        Some target keys not found in received attributes!
-        (Expects: #{target_keys}, Actual: #{target_attributes.keys})
-      MSG
+  class InterfaceError < Error
+    def initialize(object, method_name, parameters, description)
+      parameters = parameters.map do |parameter|
+        type, name = parameter
+
+        case type
+        when :req
+          name.to_s
+        when :keyreq
+          "#{name}:"
+        when :block
+          "&#{name}"
+        end
+      end
+
+      signature = "#{method_name}(#{parameters.join(", ")})"
+
+      super("#{object} has to implement method `#{signature}`\n#{description}")
+    end
+  end
+
+  UndefinedEvent = Class.new(Error)
+  ORMNotSupported = Class.new(Error)
+
+  class WrongOptionValue < Error
+    def initialize(model, option, value)
+      super("TableSync config for #{model.inspect} can't contain #{value.inspect} as #{option}")
     end
   end
 end

data/lib/table_sync/instrument.rb

@@ -3,7 +3,7 @@
 module TableSync::Instrument
   module_function
 
-  def notify(*args)
-    TableSync.notifier&.notify(*args)
+  def notify(**args)
+    TableSync.notifier&.notify(**args)
   end
 end

data/lib/table_sync/publishing.rb

@@ -0,0 +1,11 @@
+# frozen_string_literal: true
+
+module TableSync
+  module Publishing
+    require_relative "publishing/base_publisher"
+    require_relative "publishing/publisher"
+    require_relative "publishing/batch_publisher"
+    require_relative "publishing/orm_adapter/active_record"
+    require_relative "publishing/orm_adapter/sequel"
+  end
+end

data/lib/table_sync/{base_publisher.rb → publishing/base_publisher.rb}

@@ -1,6 +1,6 @@
 # frozen_string_literal: true
 
-class TableSync::BasePublisher
+class TableSync::Publishing::BasePublisher
   include Memery
 
   BASE_SAFE_JSON_TYPES = [NilClass, String, TrueClass, FalseClass, Numeric, Symbol].freeze

data/lib/table_sync/{batch_publisher.rb → publishing/batch_publisher.rb}

@@ -1,14 +1,17 @@
 # frozen_string_literal: true
 
-class TableSync::BatchPublisher < TableSync::BasePublisher
+class TableSync::Publishing::BatchPublisher < TableSync::Publishing::BasePublisher
   def initialize(object_class, original_attributes_array, **options)
-    @object_class = object_class.constantize
     @original_attributes_array = original_attributes_array.map do |hash|
       filter_safe_for_serialization(hash.deep_symbolize_keys)
     end
-    @confirm = options[:confirm] || true
-    @routing_key = options[:routing_key] || resolve_routing_key
+
+    @object_class             = object_class.constantize
+    @confirm                  = options[:confirm] || true
+    @routing_key              = options[:routing_key] || resolve_routing_key
     @push_original_attributes = options[:push_original_attributes] || false
+    @headers                  = options[:headers]
+    @event                    = options[:event] || :update
   end
 
   def publish
@@ -19,7 +22,7 @@ class TableSync::BatchPublisher < TableSync::BasePublisher
     return unless need_publish?
     Rabbit.publish(params)
 
-    model_naming = TableSync.orm.model_naming(object_class)
+    model_naming = TableSync.publishing_adapter.model_naming(object_class)
     TableSync::Instrument.notify table: model_naming.table, schema: model_naming.schema,
                                  event: event,
                                  count: publishing_data[:attributes].size, direction: :publish
@@ -27,7 +30,7 @@ class TableSync::BatchPublisher < TableSync::BasePublisher
 
   private
 
-  attr_reader :original_attributes_array, :routing_key
+  attr_reader :original_attributes_array, :routing_key, :headers, :event
 
   def push_original_attributes?
     @push_original_attributes
@@ -38,7 +41,7 @@ class TableSync::BatchPublisher < TableSync::BasePublisher
   end
 
   memoize def objects
-    needles.map { |needle| TableSync.orm.find(object_class, needle) }.compact
+    needles.map { |needle| TableSync.publishing_adapter.find(object_class, needle) }.compact
   end
 
   def job_callable
@@ -64,7 +67,7 @@ class TableSync::BatchPublisher < TableSync::BasePublisher
   def params
     {
       **super,
-      headers: nil,
+      headers: headers,
     }
   end
 
@@ -80,10 +83,6 @@ class TableSync::BatchPublisher < TableSync::BasePublisher
     }
   end
 
-  def event
-    :update
-  end
-
   def attributes_for_sync
     return original_attributes_array if push_original_attributes?
 
@@ -91,7 +90,7 @@ class TableSync::BatchPublisher < TableSync::BasePublisher
       if attributes_for_sync_defined?
         object.attributes_for_sync
       else
-        TableSync.orm.attributes(object)
+        TableSync.publishing_adapter.attributes(object)
       end
     end
   end

data/lib/table_sync/{orm_adapter → publishing/orm_adapter}/active_record.rb

@@ -1,13 +1,9 @@
 # frozen_string_literal: true
 
-module TableSync::ORMAdapter
+module TableSync::Publishing::ORMAdapter
   module ActiveRecord
     module_function
 
-    def model
-      ::TableSync::Model::ActiveRecord
-    end
-
     def model_naming(object)
       ::TableSync::NamingResolver::ActiveRecord.new(table_name: object.table_name)
     end
@@ -20,14 +16,14 @@ module TableSync::ORMAdapter
       object.attributes
     end
 
-    def setup_sync(klass, **opts)
+    def setup_sync(klass, opts)
       debounce_time = opts.delete(:debounce_time)
 
       klass.instance_exec do
         { create: :created, update: :updated, destroy: :destroyed }.each do |event, state|
           after_commit(on: event, **opts) do
-            TableSync::Publisher.new(self.class.name, attributes,
-                                     state: state, debounce_time: debounce_time).publish
+            TableSync::Publishing::Publisher.new(self.class.name, attributes,
+                                                 state: state, debounce_time: debounce_time).publish
           end
         end
       end

data/lib/table_sync/{orm_adapter → publishing/orm_adapter}/sequel.rb

@@ -1,13 +1,9 @@
 # frozen_string_literal: true
 
-module TableSync::ORMAdapter
+module TableSync::Publishing::ORMAdapter
   module Sequel
     module_function
 
-    def model
-      ::TableSync::Model::Sequel
-    end
-
     def model_naming(object)
       ::TableSync::NamingResolver::Sequel.new(table_name: object.table_name, db: object.db)
     end
@@ -20,7 +16,7 @@ module TableSync::ORMAdapter
       object.values
     end
 
-    def setup_sync(klass, **opts)
+    def setup_sync(klass, opts)
       if_predicate     = to_predicate(opts.delete(:if), true)
       unless_predicate = to_predicate(opts.delete(:unless), false)
       debounce_time    = opts.delete(:debounce_time)
@@ -40,24 +36,21 @@ module TableSync::ORMAdapter
     end
 
     def register_callbacks(klass, if_predicate, unless_predicate, debounce_time)
-      { create: :created, update: :updated }.each do |event, state|
+      { create: :created, update: :updated, destroy: :destroyed }.each do |event, state|
         klass.send(:define_method, :"after_#{event}") do
           if instance_eval(&if_predicate) && !instance_eval(&unless_predicate)
             db.after_commit do
-              TableSync::Publisher.new(self.class.name, values,
-                                       state: state, debounce_time: debounce_time).publish
+              TableSync::Publishing::Publisher.new(
+                self.class.name,
+                values,
+                state: state,
+                debounce_time: debounce_time,
+              ).publish
             end
           end
-          super()
-        end
-      end
 
-      klass.send(:define_method, :after_destroy) do
-        # publish anyway
-        db.after_commit do
-          TableSync::Publisher.new(self.class.name, values, state: :destroyed).publish
+          super()
         end
-        super()
       end
     end
   end

data/lib/table_sync/{publisher.rb → publishing/publisher.rb}

@@ -1,6 +1,6 @@
 # frozen_string_literal: true
 
-class TableSync::Publisher < TableSync::BasePublisher
+class TableSync::Publishing::Publisher < TableSync::Publishing::BasePublisher
   DEBOUNCE_TIME = 1.minute
 
   # 'original_attributes' are not published, they are used to resolve the routing key
@@ -34,7 +34,7 @@ class TableSync::Publisher < TableSync::BasePublisher
     return if !object && !destroyed?
 
     Rabbit.publish(params)
-    model_naming = TableSync.orm.model_naming(object_class)
+    model_naming = TableSync.publishing_adapter.model_naming(object_class)
     TableSync::Instrument.notify table: model_naming.table, schema: model_naming.schema,
                                  event: event, direction: :publish
   end
@@ -95,12 +95,12 @@ class TableSync::Publisher < TableSync::BasePublisher
     elsif attributes_for_sync_defined?
       object.attributes_for_sync
     else
-      TableSync.orm.attributes(object)
+      TableSync.publishing_adapter.attributes(object)
     end
   end
 
   memoize def object
-    TableSync.orm.find(object_class, needle)
+    TableSync.publishing_adapter.find(object_class, needle)
   end
 
   def event

data/lib/table_sync/receiving.rb

@@ -0,0 +1,14 @@
+# frozen_string_literal: true
+
+module TableSync
+  module Receiving
+    AVAILABLE_EVENTS = [:update, :destroy].freeze
+
+    require_relative "receiving/config"
+    require_relative "receiving/config_decorator"
+    require_relative "receiving/dsl"
+    require_relative "receiving/handler"
+    require_relative "receiving/model/active_record"
+    require_relative "receiving/model/sequel"
+  end
+end

data/lib/table_sync/receiving/config.rb

@@ -0,0 +1,218 @@
+# frozen_string_literal: true
+
+module TableSync::Receiving
+  class Config
+    attr_reader :model, :events
+
+    def initialize(model:, events: AVAILABLE_EVENTS)
+      @model = model
+
+      @events = [events].flatten.map(&:to_sym)
+
+      unless @events.all? { |event| AVAILABLE_EVENTS.include?(event) }
+        raise TableSync::UndefinedEvent.new(events)
+      end
+
+      self.class.default_values_for_options.each do |ivar, default_value_generator|
+        instance_variable_set(ivar, default_value_generator.call(self))
+      end
+    end
+
+    class << self
+      attr_reader :default_values_for_options
+
+      # In a configs this options are requested as they are
+      # config.option - get value
+      # config.option(args) - set static value
+      # config.option { ... } - set proc as value
+      #
+      # In `Receiving::Handler` or `Receiving::EventActions` this options are requested
+      # through `Receiving::ConfigDecorator#method_missing` which always executes `config.option`
+
+      def add_option(name, value_setter_wrapper:, value_as_proc_setter_wrapper:, default:)
+        ivar = "@#{name}".to_sym
+
+        @default_values_for_options ||= {}
+        @default_values_for_options[ivar] = default
+
+        define_method(name) do |*value, &value_as_proc|
+          return instance_variable_get(ivar) if value.empty? && value_as_proc.nil?
+
+          value = value.first if value.size == 1
+
+          if value_as_proc.present?
+            new_value = TableSync::Utils.proc_keywords_resolver(&value_as_proc)
+            setter_wrapper = value_as_proc_setter_wrapper
+          else
+            new_value = value
+            setter_wrapper = value_setter_wrapper
+          end
+
+          old_value = instance_variable_get(ivar)
+          result_value = instance_exec(name, new_value, old_value, &setter_wrapper)
+          instance_variable_set(ivar, result_value)
+        end
+      end
+    end
+
+    def allow_event?(name)
+      events.include?(name)
+    end
+  end
+end
+
+# Helpers:
+
+column_key = proc do |option_name, new_value|
+  unless model.columns.include?(new_value)
+    raise TableSync::WrongOptionValue.new(model, option_name, new_value)
+  end
+  new_value
+end
+
+exactly_symbol = proc do |option_name, new_value|
+  unless new_value.is_a? Symbol
+    raise TableSync::WrongOptionValue.new(model, option_name, new_value)
+  end
+  new_value
+end
+
+to_array = proc do |block|
+  proc do |option_name, new_value|
+    new_value = [new_value].flatten
+    new_value.map { |value| instance_exec(option_name, value, &block) }
+  end
+end
+
+exactly_not_array = proc do |block|
+  proc do |option_name, new_value|
+    if new_value.is_a? Array
+      raise TableSync::WrongOptionValue.new(model, option_name, new_value)
+    end
+    instance_exec(option_name, new_value, &block)
+  end
+end
+
+exactly_hash = proc do |block_for_keys, block_for_values|
+  proc do |option_name, new_value|
+    unless new_value.is_a? Hash
+      raise TableSync::WrongOptionValue.new(model, option_name, new_value)
+    end
+
+    new_value.to_a.map do |key, value|
+      [
+        instance_exec("#{option_name} keys", key, &block_for_keys),
+        instance_exec("#{option_name} values", value, &block_for_values),
+      ]
+    end.to_h
+  end
+end
+
+any_value = proc do |_option_name, new_value|
+  new_value
+end
+
+raise_error = proc do |option_name, new_value|
+  raise TableSync::WrongOptionValue.new(model, option_name, new_value)
+end
+
+exactly_boolean = proc do |option_name, new_value|
+  unless new_value.is_a?(TrueClass) || new_value.is_a?(FalseClass)
+    raise TableSync::WrongOptionValue.new(model, option_name, new_value)
+  end
+  new_value
+end
+
+allow_false = proc do |block|
+  proc do |option_name, new_value|
+    next false if new_value.is_a? FalseClass
+    instance_exec(option_name, new_value, &block)
+  end
+end
+
+proc_result = proc do |block|
+  proc do |option_name, new_value|
+    proc do |*args, &b|
+      result = new_value.call(*args, &b)
+      instance_exec(option_name, result, &block)
+    end
+  end
+end
+
+accumulate_procs = proc do |block|
+  proc do |option_name, new_value, old_value|
+    old_value.push(&instance_exec(option_name, new_value, &block))
+  end
+end
+
+# Options:
+
+# rubocop:disable Layout/ArgumentAlignment
+
+TableSync::Receiving::Config.add_option :only,
+  value_setter_wrapper: to_array[column_key],
+  value_as_proc_setter_wrapper: proc_result[to_array[column_key]],
+  default: proc { |config| config.model.columns }
+
+TableSync::Receiving::Config.add_option :target_keys,
+  value_setter_wrapper: to_array[column_key],
+  value_as_proc_setter_wrapper: proc_result[to_array[column_key]],
+  default: proc { |config| Array.wrap(config.model.primary_keys) }
+
+TableSync::Receiving::Config.add_option :rest_key,
+  value_setter_wrapper: exactly_not_array[allow_false[column_key]],
+  value_as_proc_setter_wrapper: proc_result[exactly_not_array[allow_false[column_key]]],
+  default: proc { :rest }
+
+TableSync::Receiving::Config.add_option :version_key,
+  value_setter_wrapper: exactly_not_array[column_key],
+  value_as_proc_setter_wrapper: proc_result[exactly_not_array[column_key]],
+  default: proc { :version }
+
+TableSync::Receiving::Config.add_option :except,
+  value_setter_wrapper: to_array[exactly_symbol],
+  value_as_proc_setter_wrapper: proc_result[to_array[exactly_symbol]],
+  default: proc { [] }
+
+TableSync::Receiving::Config.add_option :mapping_overrides,
+  value_setter_wrapper: exactly_hash[exactly_symbol, exactly_symbol],
+  value_as_proc_setter_wrapper: proc_result[exactly_hash[exactly_symbol, exactly_symbol]],
+  default: proc { {} }
+
+TableSync::Receiving::Config.add_option :additional_data,
+  value_setter_wrapper: exactly_hash[exactly_symbol, any_value],
+  value_as_proc_setter_wrapper: proc_result[exactly_hash[exactly_symbol, any_value]],
+  default: proc { {} }
+
+TableSync::Receiving::Config.add_option :default_values,
+  value_setter_wrapper: exactly_hash[exactly_symbol, any_value],
+  value_as_proc_setter_wrapper: proc_result[exactly_hash[exactly_symbol, any_value]],
+  default: proc { {} }
+
+TableSync::Receiving::Config.add_option :skip,
+  value_setter_wrapper: exactly_boolean,
+  value_as_proc_setter_wrapper: proc_result[exactly_boolean],
+  default: proc { false }
+
+TableSync::Receiving::Config.add_option :wrap_receiving,
+  value_setter_wrapper: raise_error,
+  value_as_proc_setter_wrapper: any_value,
+  default: proc { proc { |&block| block.call } }
+
+%i[
+  before_update
+  after_commit_on_update
+  before_destroy
+  after_commit_on_destroy
+].each do |option|
+  TableSync::Receiving::Config.add_option option,
+    value_setter_wrapper: raise_error,
+    value_as_proc_setter_wrapper: accumulate_procs[any_value],
+    default: (proc do |_config|
+      TableSync::Utils::ProcArray.new do |array, args, &block|
+        array.map { |pr| pr.call(*args, &block) }
+      end
+    end)
+end
+
+# rubocop:enable Layout/ArgumentAlignment