table_sync 2.3.0 → 4.2.0

data/lib/table_sync/receiving/config_decorator.rb

@@ -0,0 +1,27 @@
+# frozen_string_literal: true
+
+module TableSync::Receiving
+  class ConfigDecorator
+    extend Forwardable
+
+    def_delegators :@config, :allow_event?
+    # rubocop:disable Metrics/ParameterLists
+    def initialize(config:, event:, model:, version:, project_id:, raw_data:)
+      @config = config
+
+      @default_params = {
+        event: event,
+        model: model,
+        version: version,
+        project_id: project_id,
+        raw_data: raw_data,
+      }
+    end
+    # rubocop:enable Metrics/ParameterLists
+
+    def method_missing(name, **additional_params, &block)
+      value = @config.send(name)
+      value.is_a?(Proc) ? value.call(@default_params.merge(additional_params), &block) : value
+    end
+  end
+end

data/lib/table_sync/receiving/dsl.rb

@@ -0,0 +1,28 @@
+# frozen_string_literal: true
+
+module TableSync::Receiving
+  module DSL
+    def inherited(klass)
+      klass.instance_variable_set(:@configs, configs.deep_dup)
+      super
+    end
+
+    def configs
+      @configs ||= Hash.new { |hash, key| hash[key] = [] }
+    end
+
+    def receive(source, to_table: nil, to_model: nil, events: [:update, :destroy], &block)
+      model = to_table ? TableSync.receiving_model.new(to_table) : to_model
+
+      TableSync::Utils::InterfaceChecker.new(model).implements(:receiving_model)
+
+      config = ::TableSync::Receiving::Config.new(model: model, events: events)
+
+      config.instance_exec(&block) if block
+
+      configs[source.to_s] << config
+
+      self
+    end
+  end
+end

data/lib/table_sync/receiving/handler.rb

@@ -0,0 +1,136 @@
+# frozen_string_literal: true
+
+class TableSync::Receiving::Handler < Rabbit::EventHandler
+  extend TableSync::Receiving::DSL
+
+  # Rabbit::EventHandler uses Tainbox and performs `handler.new(message).call`
+  attribute :event
+  attribute :model
+  attribute :version
+
+  def call
+    configs.each do |config|
+      next unless config.allow_event?(event)
+
+      data = processed_data(config)
+
+      next if data.empty?
+
+      version_key = config.version_key(data: data)
+      data.each { |row| row[version_key] = version }
+
+      target_keys = config.target_keys(data: data)
+
+      validate_data(data, target_keys: target_keys)
+
+      data.sort_by! { |row| row.values_at(*target_keys).hash }
+
+      params = { data: data, target_keys: target_keys, version_key: version_key }
+
+      if event == :update
+        params[:default_values] = config.default_values(data: data)
+      end
+
+      config.wrap_receiving(**params) do
+        perform(config, params)
+      end
+    end
+  end
+
+  private
+
+  # redefine setter from Rabbit::EventHandler
+  def data=(data)
+    super(Array.wrap(data[:attributes]))
+  end
+
+  def event=(name)
+    name = name.to_sym
+    raise TableSync::UndefinedEvent.new(event) unless %i[update destroy].include?(name)
+    super(name)
+  end
+
+  def model=(name)
+    super(name.to_s)
+  end
+
+  def configs
+    @configs ||= begin
+      configs = self.class.configs[model]
+      configs = configs.sort_by { |config| "#{config.model.schema}.#{config.model.table}" }
+      configs.map do |config|
+        ::TableSync::Receiving::ConfigDecorator.new(
+          config: config,
+          # next parameters will be send to each proc-options from config
+          event: event,
+          model: model,
+          version: version,
+          project_id: project_id,
+          raw_data: data,
+        )
+      end
+    end
+  end
+
+  def processed_data(config)
+    data.map do |row|
+      next if config.skip(row: row)
+
+      row = row.dup
+
+      config.mapping_overrides(row: row).each do |before, after|
+        row[after] = row.delete(before)
+      end
+
+      config.except(row: row).each(&row.method(:delete))
+
+      row.merge!(config.additional_data(row: row))
+
+      only = config.only(row: row)
+      row, rest = row.partition { |key, _| key.in?(only) }.map(&:to_h)
+
+      rest_key = config.rest_key(row: row, rest: rest)
+      (row[rest_key] ||= {}).merge!(rest) if rest_key
+
+      row
+    end.compact
+  end
+
+  def validate_data(data, target_keys:)
+    data.each do |row|
+      next if target_keys.all?(&row.keys.method(:include?))
+      raise TableSync::DataError.new(
+        data, target_keys, "Some target keys not found in received attributes!"
+      )
+    end
+
+    if data.uniq { |row| row.slice(*target_keys) }.size != data.size
+      raise TableSync::DataError.new(data, target_keys, "Duplicate rows found!")
+    end
+  end
+
+  def perform(config, params)
+    model = config.model
+
+    model.transaction do
+      results = if event == :update
+                  config.before_update(**params)
+                  model.upsert(**params)
+                else
+                  config.before_destroy(**params)
+                  model.destroy(**params)
+                end
+
+      model.after_commit do
+        TableSync::Instrument.notify table: model.table, schema: model.schema,
+                                     count: results.count, event: event, direction: :receive
+      end
+
+      if event == :update
+        model.after_commit { config.after_commit_on_update(**params.merge(results: results)) }
+      else
+        model.after_commit { config.after_commit_on_destroy(**params.merge(results: results)) }
+      end
+    end
+  end
+end

data/lib/table_sync/{model → receiving/model}/active_record.rb

@@ -1,6 +1,6 @@
 # frozen_string_literal: true
 
-module TableSync::Model
+module TableSync::Receiving::Model
   class ActiveRecord
     class AfterCommitWrap
       def initialize(&block)
@@ -18,11 +18,18 @@ module TableSync::Model
       def trigger_transactional_callbacks?(*); end
     end
 
+    attr_reader :table, :schema
+
     def initialize(table_name)
       @raw_model = Class.new(::ActiveRecord::Base) do
         self.table_name = table_name
         self.inheritance_column = nil
       end
+
+      model_naming = ::TableSync::NamingResolver::ActiveRecord.new(table_name: table_name)
+
+      @table = model_naming.table.to_sym
+      @schema = model_naming.schema.to_sym
     end
 
     def columns
@@ -45,51 +52,55 @@ module TableSync::Model
           AND kcu.constraint_name = tc.constraint_name
         WHERE
           t.table_schema NOT IN ('pg_catalog', 'information_schema')
-          AND t.table_schema = '#{model_naming.schema}'
-          AND t.table_name = '#{model_naming.table}'
+          AND t.table_schema = '#{schema}'
+          AND t.table_name = '#{table}'
         ORDER BY
           kcu.ordinal_position
       SQL
     end
 
-    def upsert(data:, target_keys:, version_key:, first_sync_time_key:, default_values:)
-      data = Array.wrap(data)
+    def upsert(data:, target_keys:, version_key:, default_values:)
+      data.map do |datum|
+        conditions = datum.select { |k| target_keys.include?(k) }
 
-      result = transaction do
-        data.map do |datum|
-          conditions = datum.select { |k| target_keys.include?(k) }
+        row = raw_model.lock("FOR NO KEY UPDATE").where(conditions)
 
-          row = raw_model.lock("FOR NO KEY UPDATE").find_by(conditions)
-          if row
-            next if datum[version_key] <= row[version_key]
+        if row.to_a.size > 1
+          raise TableSync::UpsertError.new(data: datum, target_keys: target_keys, result: row)
+        end
 
-            row.update!(datum)
-          else
-            create_data = default_values.merge(datum)
-            create_data[first_sync_time_key] = Time.current if first_sync_time_key
-            row = raw_model.create!(create_data)
-          end
+        row = row.first
 
-          row_to_hash(row)
-        end.compact
-      end
+        if row
+          next if datum[version_key] <= row[version_key]
 
-      TableSync::Instrument.notify(table: model_naming.table, schema: model_naming.schema,
-                                   event: :update, count: result.count, direction: :receive)
+          row.update!(datum)
+        else
+          create_data = default_values.merge(datum)
+          row = raw_model.create!(create_data)
+        end
 
-      result
+        row_to_hash(row)
+      end.compact
     end
 
-    def destroy(data)
-      result = transaction do
-        row = raw_model.lock("FOR UPDATE").find_by(data)&.destroy!
-        [row_to_hash(row)]
+    def destroy(data:, target_keys:, version_key:)
+      sanitized_data = data.map { |attr| attr.select { |key, _value| target_keys.include?(key) } }
+
+      query = nil
+      sanitized_data.each_with_index do |row, index|
+        if index == 0
+          query = raw_model.lock("FOR UPDATE").where(row)
+        else
+          query = query.or(raw_model.lock("FOR UPDATE").where(row))
+        end
       end
 
-      TableSync::Instrument.notify(
-        table: model_naming.table, schema: model_naming.schema,
-        event: :destroy, count: result.count, direction: :receive
-      )
+      result = query.destroy_all.map(&method(:row_to_hash))
+
+      if result.size > data.size
+        raise TableSync::DestroyError.new(data: data, target_keys: target_keys, result: result)
+      end
 
       result
     end
@@ -107,11 +118,7 @@ module TableSync::Model
     attr_reader :raw_model
 
     def db
-      @raw_model.connection
-    end
-
-    def model_naming
-      ::TableSync::NamingResolver::ActiveRecord.new(table_name: raw_model.table_name)
+      raw_model.connection
     end
 
     def row_to_hash(row)

data/lib/table_sync/receiving/model/sequel.rb

@@ -0,0 +1,83 @@
+# frozen_string_literal: true
+
+module TableSync::Receiving::Model
+  class Sequel
+    attr_reader :table, :schema
+
+    def initialize(table_name)
+      @raw_model = Class.new(::Sequel::Model(table_name)).tap(&:unrestrict_primary_key)
+
+      model_naming = ::TableSync::NamingResolver::Sequel.new(
+        table_name: table_name,
+        db: @raw_model.db,
+      )
+
+      @table = model_naming.table.to_sym
+      @schema = model_naming.schema.to_sym
+    end
+
+    def columns
+      dataset.columns
+    end
+
+    def primary_keys
+      [raw_model.primary_key].flatten
+    end
+
+    def upsert(data:, target_keys:, version_key:, default_values:)
+      qualified_version = ::Sequel.qualify(raw_model.table_name, version_key)
+      version_condition = ::Sequel.function(:coalesce, qualified_version, 0) <
+                          ::Sequel.qualify(:excluded, version_key)
+
+      upd_spec = update_spec(data.first.keys - target_keys)
+      data.map! { |d| default_values.merge(d) }
+
+      insert_data = type_cast(data)
+
+      dataset
+        .returning
+        .insert_conflict(target: target_keys, update: upd_spec, update_where: version_condition)
+        .multi_insert(insert_data)
+    end
+
+    def destroy(data:, target_keys:, version_key:)
+      sanitized_data = data.map { |attr| attr.select { |key, _value| target_keys.include?(key) } }
+      sanitized_data = type_cast(sanitized_data)
+      result = dataset.returning.where(::Sequel.|(*sanitized_data)).delete
+
+      if result.size > data.size
+        raise TableSync::DestroyError.new(data: data, target_keys: target_keys, result: result)
+      end
+
+      result
+    end
+
+    def transaction(&block)
+      db.transaction(&block)
+    end
+
+    def after_commit(&block)
+      db.after_commit(&block)
+    end
+
+    private
+
+    attr_reader :raw_model
+
+    def dataset
+      raw_model.dataset
+    end
+
+    def db
+      dataset.db
+    end
+
+    def type_cast(data)
+      data.map { |d| raw_model.new(d).values.keep_if { |k| d.key?(k) } }
+    end
+
+    def update_spec(keys)
+      keys.map { |key| [key, ::Sequel[:excluded][key]] }.to_h
+    end
+  end
+end

data/lib/table_sync/utils.rb

@@ -0,0 +1,9 @@
+# frozen_string_literal: true
+
+module TableSync
+  module Utils
+    require_relative "utils/proc_array"
+    require_relative "utils/proc_keywords_resolver"
+    require_relative "utils/interface_checker"
+  end
+end

data/lib/table_sync/utils/interface_checker.rb

@@ -0,0 +1,103 @@
+# frozen_string_literal: true
+
+# Ruby does not support interfaces, and there is no way to implement them.
+# Interfaces check a methods of a class after the initialization of the class is complete.
+# But in Ruby, the initialization of a class cannot be completed.
+# In execution time we can open any class and add some methods (monkey patching).
+# Ruby has `define_method`, singleton methods, etc.
+#
+# Duck typing is a necessary measure, the only one available in the Ruby architecture.
+#
+# Interfaces can be implemented in particular cases with tests for example.
+# But this is not suitable for gems that are used by third-party code.
+#
+# So, we still want to check interfaces and have a nice error messages,
+# even if it will be duck typing.
+#
+# Next code do this.
+
+class TableSync::Utils::InterfaceChecker
+  INTERFACES = SelfData.load
+
+  attr_reader :object
+
+  def initialize(object)
+    @object = object
+  end
+
+  def implements(interface_name)
+    INTERFACES[interface_name].each do |method_name, options|
+      unless object.respond_to?(method_name)
+        raise_error(method_name, options)
+      end
+
+      unless include?(object.method(method_name).parameters, options[:parameters])
+        raise_error(method_name, options)
+      end
+    end
+    self
+  end
+
+  private
+
+  def include?(checked, expected)
+    (filter(expected) - filter(checked)).empty?
+  end
+
+  def raise_error(method_name, options)
+    raise TableSync::InterfaceError.new(
+      object,
+      method_name,
+      options[:parameters],
+      options[:description],
+    )
+  end
+
+  def filter(parameters)
+    # for req and block parameters types we can ignore names
+    parameters.map { |param| %i[req block].include?(param.first) ? [param.first] : param }
+  end
+end
+
+__END__
+:receiving_model:
+  :upsert:
+    :parameters:
+      - - :keyreq
+        - :data
+      - - :keyreq
+        - :target_keys
+      - - :keyreq
+        - :version_key
+      - - :keyreq
+        - :default_values
+    :description: "returns an array with updated rows"
+  :columns:
+    :parameters: []
+    :description: "returns all table columns"
+  :destroy:
+    :parameters:
+      - - :keyreq
+        - :data
+      - - :keyreq
+        - :target_keys
+    :description: "returns an array with destroyed rows"
+  :transaction:
+    :parameters:
+      - - :block
+        - :block
+    :description: "implements the database transaction"
+  :after_commit:
+    :parameters:
+      - - :block
+        - :block
+    :description: "executes the block after committing the transaction"
+  :primary_keys:
+    :parameters: []
+    :description: "returns an array with the primary_keys"
+  :table:
+    :parameters: []
+    :description: "returns an instance of Symbol"
+  :schema:
+    :parameters: []
+    :description: "returns an instance of Symbol"