table_sync 1.13.1 → 4.0.0

data/lib/table_sync/receiving/config_decorator.rb

@@ -0,0 +1,27 @@
+# frozen_string_literal: true
+
+module TableSync::Receiving
+  class ConfigDecorator
+    extend Forwardable
+
+    def_delegators :@config, :allow_event?
+    # rubocop:disable Metrics/ParameterLists
+    def initialize(config:, event:, model:, version:, project_id:, raw_data:)
+      @config = config
+
+      @default_params = {
+        event: event,
+        model: model,
+        version: version,
+        project_id: project_id,
+        raw_data: raw_data,
+      }
+    end
+    # rubocop:enable Metrics/ParameterLists
+
+    def method_missing(name, **additional_params, &block)
+      value = @config.send(name)
+      value.is_a?(Proc) ? value.call(@default_params.merge(additional_params), &block) : value
+    end
+  end
+end

data/lib/table_sync/receiving/dsl.rb

@@ -0,0 +1,28 @@
+# frozen_string_literal: true
+
+module TableSync::Receiving
+  module DSL
+    def inherited(klass)
+      klass.instance_variable_set(:@configs, configs.deep_dup)
+      super
+    end
+
+    def configs
+      @configs ||= Hash.new { |hash, key| hash[key] = [] }
+    end
+
+    def receive(source, to_table: nil, to_model: nil, events: [:update, :destroy], &block)
+      model = to_table ? TableSync.receiving_model.new(to_table) : to_model
+
+      TableSync::Utils::InterfaceChecker.new(model).implements(:receiving_model)
+
+      config = ::TableSync::Receiving::Config.new(model: model, events: events)
+
+      config.instance_exec(&block) if block
+
+      configs[source.to_s] << config
+
+      self
+    end
+  end
+end

data/lib/table_sync/receiving/handler.rb

@@ -0,0 +1,131 @@
+# frozen_string_literal: true
+
+class TableSync::Receiving::Handler < Rabbit::EventHandler
+  extend TableSync::Receiving::DSL
+
+  # Rabbit::EventHandler uses Tainbox and performs `handler.new(message).call`
+  attribute :event
+  attribute :model
+  attribute :version
+
+  def call
+    configs.each do |config|
+      next unless config.allow_event?(event)
+
+      data = processed_data(config)
+
+      next if data.empty?
+
+      version_key = config.version_key(data: data)
+      data.each { |row| row[version_key] = version }
+
+      target_keys = config.target_keys(data: data)
+
+      validate_data(data, target_keys: target_keys)
+
+      params = { data: data, target_keys: target_keys, version_key: version_key }
+
+      if event == :update
+        params[:default_values] = config.default_values(data: data)
+      end
+
+      config.wrap_receiving(**params) do
+        perform(config, params)
+      end
+    end
+  end
+
+  private
+
+  # redefine setter from Rabbit::EventHandler
+  def data=(data)
+    super(Array.wrap(data[:attributes]))
+  end
+
+  def event=(name)
+    name = name.to_sym
+    raise TableSync::UndefinedEvent.new(event) unless %i[update destroy].include?(name)
+    super(name)
+  end
+
+  def model=(name)
+    super(name.to_s)
+  end
+
+  def configs
+    @configs ||= self.class.configs[model]&.map do |config|
+      ::TableSync::Receiving::ConfigDecorator.new(
+        config: config,
+        # next parameters will be send to each proc-options from config
+        event: event,
+        model: model,
+        version: version,
+        project_id: project_id,
+        raw_data: data,
+      )
+    end
+  end
+
+  def processed_data(config)
+    data.map do |row|
+      next if config.skip(row: row)
+
+      row = row.dup
+
+      config.mapping_overrides(row: row).each do |before, after|
+        row[after] = row.delete(before)
+      end
+
+      config.except(row: row).each(&row.method(:delete))
+
+      row.merge!(config.additional_data(row: row))
+
+      only = config.only(row: row)
+      row, rest = row.partition { |key, _| key.in?(only) }.map(&:to_h)
+
+      rest_key = config.rest_key(row: row, rest: rest)
+      (row[rest_key] ||= {}).merge!(rest) if rest_key
+
+      row
+    end.compact
+  end
+
+  def validate_data(data, target_keys:)
+    data.each do |row|
+      next if target_keys.all?(&row.keys.method(:include?))
+      raise TableSync::DataError.new(
+        data, target_keys, "Some target keys not found in received attributes!"
+      )
+    end
+
+    if data.uniq { |row| row.slice(*target_keys) }.size != data.size
+      raise TableSync::DataError.new(
+        data, target_keys, "Duplicate rows found!"
+      )
+    end
+  end
+
+  def perform(config, params)
+    model = config.model
+
+    model.transaction do
+      if event == :update
+        config.before_update(**params)
+
+        results = model.upsert(**params)
+
+        model.after_commit do
+          config.after_commit_on_update(**params.merge(results: results))
+        end
+      else
+        config.before_destroy(**params)
+
+        results = model.destroy(**params)
+
+        model.after_commit do
+          config.after_commit_on_destroy(**params.merge(results: results))
+        end
+      end
+    end
+  end
+end

data/lib/table_sync/{model → receiving/model}/active_record.rb

@@ -1,6 +1,6 @@
 # frozen_string_literal: true
 
-module TableSync::Model
+module TableSync::Receiving::Model
   class ActiveRecord
     class AfterCommitWrap
       def initialize(&block)
@@ -52,27 +52,29 @@ module TableSync::Model
       SQL
     end
 
-    def upsert(data:, target_keys:, version_key:, first_sync_time_key:, default_values:)
-      data = Array.wrap(data)
+    def upsert(data:, target_keys:, version_key:, default_values:)
+      result = data.map do |datum|
+        conditions = datum.select { |k| target_keys.include?(k) }
 
-      result = transaction do
-        data.map do |datum|
-          conditions = datum.select { |k| target_keys.include?(k) }
+        row = raw_model.lock("FOR NO KEY UPDATE").where(conditions)
 
-          row = raw_model.lock("FOR NO KEY UPDATE").find_by(conditions)
-          if row
-            next if datum[version_key] <= row[version_key]
+        if row.to_a.size > 1
+          raise TableSync::UpsertError.new(data: datum, target_keys: target_keys, result: row)
+        end
 
-            row.update!(datum)
-          else
-            create_data = default_values.merge(datum)
-            create_data[first_sync_time_key] = Time.current if first_sync_time_key
-            row = raw_model.create!(create_data)
-          end
+        row = row.first
 
-          row_to_hash(row)
-        end.compact
-      end
+        if row
+          next if datum[version_key] <= row[version_key]
+
+          row.update!(datum)
+        else
+          create_data = default_values.merge(datum)
+          row = raw_model.create!(create_data)
+        end
+
+        row_to_hash(row)
+      end.compact
 
       TableSync::Instrument.notify(table: model_naming.table, schema: model_naming.schema,
                                    event: :update, count: result.count, direction: :receive)
@@ -80,10 +82,22 @@ module TableSync::Model
       result
     end
 
-    def destroy(data)
-      result = transaction do
-        row = raw_model.lock("FOR UPDATE").find_by(data)&.destroy!
-        [row_to_hash(row)]
+    def destroy(data:, target_keys:, version_key:)
+      sanitized_data = data.map { |attr| attr.select { |key, _value| target_keys.include?(key) } }
+
+      query = nil
+      sanitized_data.each_with_index do |row, index|
+        if index == 0
+          query = raw_model.lock("FOR UPDATE").where(row)
+        else
+          query = query.or(raw_model.lock("FOR UPDATE").where(row))
+        end
+      end
+
+      result = query.destroy_all.map(&method(:row_to_hash))
+
+      if result.size > data.size
+        raise TableSync::DestroyError.new(data: data, target_keys: target_keys, result: result)
       end
 
       TableSync::Instrument.notify(
@@ -115,7 +129,7 @@ module TableSync::Model
     end
 
     def row_to_hash(row)
-      row.attributes.each_with_object({}) { |(k, v), o| o[k.to_sym] = v }
+      row.attributes.transform_keys(&:to_sym)
     end
   end
 end

data/lib/table_sync/{model → receiving/model}/sequel.rb

@@ -1,6 +1,6 @@
 # frozen_string_literal: true
 
-module TableSync::Model
+module TableSync::Receiving::Model
   class Sequel
     def initialize(table_name)
       @raw_model = Class.new(::Sequel::Model(table_name)).tap(&:unrestrict_primary_key)
@@ -14,8 +14,7 @@ module TableSync::Model
       [raw_model.primary_key].flatten
     end
 
-    def upsert(data:, target_keys:, version_key:, first_sync_time_key:, default_values:)
-      data = Array.wrap(data)
+    def upsert(data:, target_keys:, version_key:, default_values:)
       qualified_version = ::Sequel.qualify(table_name, version_key)
       version_condition = ::Sequel.function(:coalesce, qualified_version, 0) <
                           ::Sequel.qualify(:excluded, version_key)
@@ -24,9 +23,6 @@ module TableSync::Model
       data.map! { |d| default_values.merge(d) }
 
       insert_data = type_cast(data)
-      if first_sync_time_key
-        insert_data.each { |datum| datum[first_sync_time_key] = Time.current }
-      end
 
       result = dataset.returning
                       .insert_conflict(
@@ -38,14 +34,23 @@ module TableSync::Model
 
       TableSync::Instrument.notify table: model_naming.table, schema: model_naming.schema,
                                    count: result.count, event: :update, direction: :receive
+
       result
     end
 
-    def destroy(data)
-      result = dataset.returning.where(data).delete
+    def destroy(data:, target_keys:, version_key:)
+      sanitized_data = data.map { |attr| attr.select { |key, _value| target_keys.include?(key) } }
+      sanitized_data = type_cast(sanitized_data)
+      result = dataset.returning.where(::Sequel.|(*sanitized_data)).delete
+
+      if result.size > data.size
+        raise TableSync::DestroyError.new(data: data, target_keys: target_keys, result: result)
+      end
+
       TableSync::Instrument.notify table: model_naming.table, schema: model_naming.schema,
                                    count: result.count,
                                    event: :destroy, direction: :receive
+
       result
     end
 

data/lib/table_sync/utils.rb

@@ -0,0 +1,9 @@
+# frozen_string_literal: true
+
+module TableSync
+  module Utils
+    require_relative "utils/proc_array"
+    require_relative "utils/proc_keywords_resolver"
+    require_relative "utils/interface_checker"
+  end
+end

data/lib/table_sync/utils/interface_checker.rb

@@ -0,0 +1,97 @@
+# frozen_string_literal: true
+
+# Ruby does not support interfaces, and there is no way to implement them.
+# Interfaces check a methods of a class after the initialization of the class is complete.
+# But in Ruby, the initialization of a class cannot be completed.
+# In execution time we can open any class and add some methods (monkey patching).
+# Ruby has `define_method`, singleton methods, etc.
+#
+# Duck typing is a necessary measure, the only one available in the Ruby architecture.
+#
+# Interfaces can be implemented in particular cases with tests for example.
+# But this is not suitable for gems that are used by third-party code.
+#
+# So, we still want to check interfaces and have a nice error messages,
+# even if it will be duck typing.
+#
+# Next code do this.
+
+class TableSync::Utils::InterfaceChecker
+  INTERFACES = SelfData.load
+
+  attr_reader :object
+
+  def initialize(object)
+    @object = object
+  end
+
+  def implements(interface_name)
+    INTERFACES[interface_name].each do |method_name, options|
+      unless object.respond_to?(method_name)
+        raise_error(method_name, options)
+      end
+
+      unless include?(object.method(method_name).parameters, options[:parameters])
+        raise_error(method_name, options)
+      end
+    end
+    self
+  end
+
+  private
+
+  def include?(checked, expected)
+    (filter(expected) - filter(checked)).empty?
+  end
+
+  def raise_error(method_name, options)
+    raise TableSync::InterfaceError.new(
+      object,
+      method_name,
+      options[:parameters],
+      options[:description],
+    )
+  end
+
+  def filter(parameters)
+    # for req and block parameters types we can ignore names
+    parameters.map { |param| %i[req block].include?(param.first) ? [param.first] : param }
+  end
+end
+
+__END__
+:receiving_model:
+  :upsert:
+    :parameters:
+      - - :keyreq
+        - :data
+      - - :keyreq
+        - :target_keys
+      - - :keyreq
+        - :version_key
+      - - :keyreq
+        - :default_values
+    :description: "returns an array with updated rows"
+  :columns:
+    :parameters: []
+    :description: "returns all table columns"
+  :destroy:
+    :parameters:
+      - - :keyreq
+        - :data
+      - - :keyreq
+        - :target_keys
+    :description: "returns an array with destroyed rows"
+  :transaction:
+    :parameters:
+      - - :block
+        - :block
+    :description: "implements the database transaction"
+  :after_commit:
+    :parameters:
+      - - :block
+        - :block
+    :description: "executes the block after committing the transaction"
+  :primary_keys:
+    :parameters: []
+    :description: "returns an array with the primary_keys"

data/lib/table_sync/utils/proc_array.rb

@@ -0,0 +1,17 @@
+# frozen_string_literal: true
+
+class TableSync::Utils::ProcArray < Proc
+  def initialize(&block)
+    @array = []
+    super(&block)
+  end
+
+  def push(&block)
+    @array.push(block)
+    self
+  end
+
+  def call(*args, &block)
+    super(@array, args, &block)
+  end
+end