table_sync 0.0.0 → 1.4.0

data/lib/table_sync/dsl.rb

@@ -0,0 +1,25 @@
+# frozen_string_literal: true
+
+module TableSync
+  module DSL
+    def inherited(klass)
+      klass.instance_variable_set(:@configs, configs.deep_dup)
+      super
+    end
+
+    def configs
+      @configs ||= Hash.new { |hash, key| hash[key] = [] }
+    end
+
+    def receive(source, to_table:, events: nil, &block)
+      config = ::TableSync::Config.new(
+        model: TableSync.orm.model.new(to_table),
+        events: events,
+      )
+
+      config.instance_exec(&block) if block
+
+      configs[source.to_s] << config
+    end
+  end
+end

data/lib/table_sync/errors.rb

@@ -0,0 +1,30 @@
+# frozen_string_literal: true
+
+module TableSync
+  Error = Class.new(StandardError)
+
+  class UpsertError < Error
+    def initialize(data:, target_keys:, version_key:, first_sync_time_key:, default_values:)
+      super <<~MSG
+        Upsert has changed more than 1 row;
+          data: #{data.inspect}
+          target_keys: #{target_keys.inspect}
+          version_key: #{version_key.inspect}
+          first_sync_time_key: #{first_sync_time_key.inspect}
+          default_values: #{default_values.inspect}
+      MSG
+    end
+  end
+
+  class DestroyError < Error
+    def initialize(data)
+      super("Destroy has changed more than 1 row; data: #{data.inspect}")
+    end
+  end
+
+  class UndefinedConfig < Error
+    def initialize(model)
+      super("Config not defined for model; model: #{model.inspect}")
+    end
+  end
+end

data/lib/table_sync/event_actions.rb

@@ -0,0 +1,60 @@
+# frozen_string_literal: true
+
+module TableSync
+  module EventActions
+    def update(data) # rubocop:disable Metrics/MethodLength
+      model.transaction do
+        args = {
+          data: data,
+          target_keys: target_keys,
+          version_key: version_key,
+          first_sync_time_key: first_sync_time_key,
+          default_values: default_values,
+        }
+
+        @config.callback_registry.get_callbacks(kind: :before_commit, event: :update).each do |cb|
+          cb[data.values.flatten]
+        end
+
+        results = data.reduce([]) do |upserts, (part_model, part_data)|
+          upserts + part_model.upsert(**args, data: part_data)
+        end
+
+        return if results.empty?
+        raise TableSync::UpsertError.new(**args) unless correct_keys?(results)
+
+        @config.model.after_commit do
+          @config.callback_registry.get_callbacks(kind: :after_commit, event: :update).each do |cb|
+            cb[results]
+          end
+        end
+      end
+    end
+
+    def destroy(data)
+      attributes = data.first || {}
+      target_attributes = attributes.select { |key| target_keys.include?(key) }
+
+      model.transaction do
+        @config.callback_registry.get_callbacks(kind: :before_commit, event: :destroy).each do |cb|
+          cb[attributes]
+        end
+
+        results = model.destroy(target_attributes)
+
+        return if results.empty?
+        raise TableSync::DestroyError.new(target_attributes) if results.size != 1
+
+        @config.model.after_commit do
+          @config.callback_registry.get_callbacks(kind: :after_commit, event: :destroy).each do |cb|
+            cb[results]
+          end
+        end
+      end
+    end
+
+    def correct_keys?(query_results)
+      query_results.uniq { |d| d.slice(*target_keys) }.size == query_results.size
+    end
+  end
+end

data/lib/table_sync/model/active_record.rb

@@ -0,0 +1,110 @@
+# frozen_string_literal: true
+
+module TableSync::Model
+  class ActiveRecord
+    class AfterCommitWrap
+      def initialize(&block)
+        @callback = block
+      end
+
+      def committed!(*)
+        @callback.call
+      end
+
+      def before_committed!(*); end
+
+      def rolledback!(*); end
+    end
+
+    def initialize(table_name)
+      @raw_model = Class.new(::ActiveRecord::Base) do
+        self.table_name = table_name
+        self.inheritance_column = nil
+      end
+    end
+
+    def columns
+      raw_model.column_names.map(&:to_sym)
+    end
+
+    def primary_keys
+      db.execute(<<~SQL).column_values(0).map(&:to_sym)
+        SELECT kcu.column_name
+        FROM INFORMATION_SCHEMA.TABLES t
+        LEFT JOIN INFORMATION_SCHEMA.TABLE_CONSTRAINTS tc
+          ON tc.table_catalog = t.table_catalog
+          AND tc.table_schema = t.table_schema
+          AND tc.table_name = t.table_name
+          AND tc.constraint_type = 'PRIMARY KEY'
+        LEFT JOIN INFORMATION_SCHEMA.KEY_COLUMN_USAGE kcu
+          ON kcu.table_catalog = tc.table_catalog
+          AND kcu.table_schema = tc.table_schema
+          AND kcu.table_name = tc.table_name
+          AND kcu.constraint_name = tc.constraint_name
+        WHERE
+          t.table_schema NOT IN ('pg_catalog', 'information_schema')
+          AND t.table_schema = '#{table_info[:schema]}'
+          AND t.table_name = '#{table_info[:name]}'
+        ORDER BY
+          kcu.ordinal_position
+      SQL
+    end
+
+    def upsert(data:, target_keys:, version_key:, first_sync_time_key:, default_values:)
+      data = Array.wrap(data)
+
+      transaction do
+        data.map do |datum|
+          conditions = datum.select { |k| target_keys.include?(k) }
+
+          row = raw_model.lock("FOR NO KEY UPDATE").find_by(conditions)
+          if row
+            next if datum[version_key] <= row[version_key]
+
+            row.update!(datum)
+          else
+            create_data = datum.merge(default_values)
+            create_data[first_sync_time_key] = Time.current if first_sync_time_key
+            row = raw_model.create!(create_data)
+          end
+
+          row_to_hash(row)
+        end.compact
+      end
+    end
+
+    def destroy(data)
+      transaction do
+        row = raw_model.lock("FOR UPDATE").find_by(data)&.destroy!
+        [row_to_hash(row)]
+      end
+    end
+
+    def transaction(&block)
+      ::ActiveRecord::Base.transaction(&block)
+    end
+
+    def after_commit(&block)
+      db.add_transaction_record(AfterCommitWrap.new(&block))
+    end
+
+    private
+
+    attr_reader :raw_model
+
+    def table_info
+      keys = raw_model.table_name.split(".")
+      name = keys[-1]
+      schema = keys[-2] || "public"
+      { schema: schema, name: name }
+    end
+
+    def db
+      @raw_model.connection
+    end
+
+    def row_to_hash(row)
+      row.attributes.each_with_object({}) { |(k, v), o| o[k.to_sym] = v }
+    end
+  end
+end

data/lib/table_sync/model/sequel.rb

@@ -0,0 +1,76 @@
+# frozen_string_literal: true
+
+module TableSync::Model
+  class Sequel
+    def initialize(table_name)
+      @raw_model = Class.new(::Sequel::Model(table_name)).tap(&:unrestrict_primary_key)
+    end
+
+    def columns
+      dataset.columns
+    end
+
+    def primary_keys
+      [raw_model.primary_key].flatten
+    end
+
+    def upsert(data:, target_keys:, version_key:, first_sync_time_key:, default_values:)
+      data = Array.wrap(data)
+      qualified_version = ::Sequel.qualify(table_name, version_key)
+      version_condition = ::Sequel.function(:coalesce, qualified_version, 0) <
+                          ::Sequel.qualify(:excluded, version_key)
+
+      upd_spec = update_spec(data.first.keys - target_keys)
+      data.map! { |d| d.merge(default_values) }
+
+      insert_data = type_cast(data)
+      if first_sync_time_key
+        insert_data.each { |datum| datum[first_sync_time_key] = Time.current }
+      end
+
+      dataset.returning
+             .insert_conflict(
+               target: target_keys,
+               update: upd_spec,
+               update_where: version_condition,
+             )
+             .multi_insert(insert_data)
+    end
+
+    def destroy(data)
+      dataset.returning.where(data).delete
+    end
+
+    def transaction(&block)
+      db.transaction(&block)
+    end
+
+    def after_commit(&block)
+      db.after_commit(&block)
+    end
+
+    private
+
+    attr_reader :raw_model
+
+    def table_name
+      raw_model.table_name
+    end
+
+    def dataset
+      raw_model.dataset
+    end
+
+    def db
+      dataset.db
+    end
+
+    def type_cast(data)
+      data.map { |d| raw_model.new(d).values.keep_if { |k| d.key?(k) } }
+    end
+
+    def update_spec(keys)
+      keys.map { |key| [key, ::Sequel[:excluded][key]] }.to_h
+    end
+  end
+end

data/lib/table_sync/orm_adapter/active_record.rb

@@ -0,0 +1,29 @@
+# frozen_string_literal: true
+
+module TableSync::ORMAdapter
+  module ActiveRecord
+    module_function
+
+    def model
+      ::TableSync::Model::ActiveRecord
+    end
+
+    def find(dataset, conditions)
+      dataset.find_by(conditions)
+    end
+
+    def attributes(object)
+      object.attributes
+    end
+
+    def setup_sync(klass, **opts)
+      klass.instance_exec do
+        { create: :created, update: :updated, destroy: :destroyed }.each do |event, state|
+          after_commit(on: event, **opts) do
+            TableSync::Publisher.new(self.class.name, attributes, state: state).publish
+          end
+        end
+      end
+    end
+  end
+end

data/lib/table_sync/orm_adapter/sequel.rb

@@ -0,0 +1,50 @@
+# frozen_string_literal: true
+
+module TableSync::ORMAdapter
+  module Sequel
+    module_function
+
+    def model
+      ::TableSync::Model::Sequel
+    end
+
+    def find(dataset, conditions)
+      dataset.find(conditions)
+    end
+
+    def attributes(object)
+      object.values
+    end
+
+    def setup_sync(klass, **opts)
+      if_predicate     = to_predicate(opts.delete(:if), true)
+      unless_predicate = to_predicate(opts.delete(:unless), false)
+      raise "Only :if and :unless options are currently supported for Sequel hooks" if opts.any?
+
+      { create: :created, update: :updated }.each do |event, state|
+        klass.send(:define_method, :"after_#{event}") do
+          if instance_eval(&if_predicate) && !instance_eval(&unless_predicate)
+            db.after_commit do
+              TableSync::Publisher.new(self.class.name, values, state: state).publish
+            end
+          end
+          super()
+        end
+      end
+
+      klass.send(:define_method, :after_destroy) do
+        # publish anyway
+        db.after_commit do
+          TableSync::Publisher.new(self.class.name, values, state: :destroyed).publish
+        end
+        super()
+      end
+    end
+
+    def to_predicate(val, default)
+      return val.to_proc if val.respond_to?(:to_proc)
+
+      -> (*) { default }
+    end
+  end
+end

data/lib/table_sync/publisher.rb

@@ -0,0 +1,123 @@
+# frozen_string_literal: true
+
+class TableSync::Publisher < TableSync::BasePublisher
+  DEBOUNCE_TIME = 1.minute
+
+  # 'original_attributes' are not published, they are used to resolve the routing key
+  def initialize(object_class, original_attributes, destroyed: nil, confirm: true, state: :updated)
+    @object_class = object_class.constantize
+    @original_attributes = filter_safe_for_serialization(original_attributes.deep_symbolize_keys)
+    @confirm = confirm
+
+    if destroyed.nil?
+      @state = validate_state(state)
+    else
+      # TODO Legacy job support, remove
+      @state = destroyed ? :destroyed : :updated
+    end
+  end
+
+  def publish
+    return enqueue_job if destroyed?
+
+    sync_time = Rails.cache.read(cache_key) || current_time - DEBOUNCE_TIME - 1.second
+    return if sync_time > current_time
+
+    next_sync_time = sync_time + DEBOUNCE_TIME
+    next_sync_time <= current_time ? enqueue_job : enqueue_job(next_sync_time)
+  end
+
+  def publish_now
+    # Update request and object does not exist -> skip publishing
+    return if !object && !destroyed?
+
+    Rabbit.publish(params)
+  end
+
+  private
+
+  attr_reader :original_attributes
+  attr_reader :state
+
+  def attrs_for_callables
+    original_attributes
+  end
+
+  def attrs_for_routing_key
+    return object.attrs_for_routing_key if attrs_for_routing_key_defined?
+    attrs_for_callables
+  end
+
+  def attrs_for_metadata
+    return object.attrs_for_metadata if attrs_for_metadata_defined?
+    attrs_for_callables
+  end
+
+  def job_callable
+    TableSync.publishing_job_class_callable
+  end
+
+  def job_callable_error_message
+    "Can't publish, set TableSync.publishing_job_class_callable"
+  end
+
+  def enqueue_job(perform_at = current_time)
+    job = job_class.set(wait_until: perform_at)
+    job.perform_later(object_class.name, original_attributes, state: state.to_s, confirm: confirm?)
+    Rails.cache.write(cache_key, perform_at)
+  end
+
+  def routing_key
+    resolve_routing_key
+  end
+
+  def publishing_data
+    {
+      **super,
+      event: (destroyed? ? :destroy : :update),
+      metadata: { created: created? },
+    }
+  end
+
+  def attributes_for_sync
+    if destroyed?
+      if object_class.respond_to?(:table_sync_destroy_attributes)
+        object_class.table_sync_destroy_attributes(original_attributes)
+      else
+        needle
+      end
+    elsif attributes_for_sync_defined?
+      object.attributes_for_sync
+    else
+      TableSync.orm.attributes(object)
+    end
+  end
+
+  memoize def object
+    TableSync.orm.find(object_class, needle)
+  end
+
+  def needle
+    original_attributes.slice(*primary_keys)
+  end
+
+  def cache_key
+    "#{object_class}/#{needle}_table_sync_time".delete(" ")
+  end
+
+  def destroyed?
+    state == :destroyed
+  end
+
+  def created?
+    state == :created
+  end
+
+  def validate_state(state)
+    if %i[created updated destroyed].include?(state&.to_sym)
+      state.to_sym
+    else
+      raise "Unknown state: #{state.inspect}"
+    end
+  end
+end

data/lib/table_sync/receiving_handler.rb

@@ -0,0 +1,76 @@
+# frozen_string_literal: true
+
+class TableSync::ReceivingHandler < Rabbit::EventHandler
+  extend TableSync::DSL
+
+  attribute :event
+  attribute :model
+  attribute :version
+
+  def call
+    raise TableSync::UndefinedConfig.new(model) if configs.blank?
+
+    configs.each do |config|
+      next unless config.allow_event?(event)
+
+      data = processed_data(config)
+      next if data.empty?
+
+      case event
+      when :update
+        config.model.transaction do
+          config.update(data)
+        end
+      when :destroy
+        config.destroy(data.values.first)
+      else
+        raise "Unknown event: #{event}"
+      end
+    end
+  end
+
+  private
+
+  def data=(data)
+    @data = data[:attributes]
+  end
+
+  def event=(name)
+    super(name.to_sym)
+  end
+
+  def model=(name)
+    super(name.to_s)
+  end
+
+  def configs
+    @configs ||= self.class.configs[model]
+                     &.map { |c| ::TableSync::ConfigDecorator.new(c, self) }
+  end
+
+  def processed_data(config)
+    parts = config.partitions&.transform_keys { |k| config.model.class.new(k) } ||
+            { config.model => Array.wrap(data) }
+
+    parts.transform_values! do |data_part|
+      data_part.map do |row|
+        original_row_for_data = row.dup
+        row = row.dup
+
+        config.mapping_overrides.each do |before, after|
+          row[after] = row.delete(before)
+        end
+
+        only = config.only
+        row, missed = row.partition { |key, _| key.in?(only) }.map(&:to_h)
+
+        row.deep_merge!(config.rest_key => missed) if config.rest_key
+        row[config.version_key] = version
+
+        row.merge!(config.additional_data(original_row_for_data))
+
+        row unless config.skip(original_row_for_data)
+      end.compact.presence
+    end.compact
+  end
+end

data/lib/table_sync/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module TableSync
-  VERSION = "0.0.0"
+  VERSION = "1.4.0"
 end

data/lib/table_sync.rb

@@ -1,5 +1,55 @@
 # frozen_string_literal: true
 
+require "memery"
+require "rabbit_messaging"
+require "rabbit/event_handler" # NOTE: from rabbit_messaging"
+require "active_support/core_ext/object/blank"
+require "active_support/core_ext/numeric/time"
+
 module TableSync
-  require "table_sync/version"
+  require_relative "./table_sync/version"
+  require_relative "./table_sync/errors"
+  require_relative "./table_sync/event_actions"
+  require_relative "./table_sync/config"
+  require_relative "./table_sync/config/callback_registry"
+  require_relative "./table_sync/config_decorator"
+  require_relative "./table_sync/dsl"
+  require_relative "./table_sync/receiving_handler"
+  require_relative "./table_sync/base_publisher"
+  require_relative "./table_sync/publisher"
+  require_relative "./table_sync/batch_publisher"
+  require_relative "./table_sync/orm_adapter/active_record"
+  require_relative "./table_sync/orm_adapter/sequel"
+  require_relative "./table_sync/model/active_record"
+  require_relative "./table_sync/model/sequel"
+
+  class << self
+    include Memery
+
+    attr_accessor :publishing_job_class_callable
+    attr_accessor :batch_publishing_job_class_callable
+    attr_accessor :routing_key_callable
+    attr_accessor :exchange_name
+    attr_accessor :routing_metadata_callable
+
+    def sync(*args)
+      orm.setup_sync(*args)
+    end
+
+    def orm=(val)
+      clear_memery_cache!
+      @orm = val
+    end
+
+    memoize def orm
+      case @orm
+      when :active_record
+        ORMAdapter::ActiveRecord
+      when :sequel
+        ORMAdapter::Sequel
+      else
+        raise "ORM not supported: #{@orm.inspect}"
+      end
+    end
+  end
 end

data/table_sync.gemspec

@@ -11,8 +11,10 @@ Gem::Specification.new do |spec|
   spec.version     = TableSync::VERSION
   spec.authors     = ["Umbrellio"]
   spec.email       = ["oss@umbrellio.biz"]
-  spec.summary     = "Coming soon"
-  spec.description = "Coming soon"
+  spec.summary     = "DB Table synchronization between microservices " \
+                     "based on Model's event system and RabbitMQ messaging"
+  spec.description = "DB Table synchronization between microservices " \
+                     "based on Model's event system and RabbitMQ messaging"
   spec.homepage    = "https://github.com/umbrellio/table_sync"
   spec.license     = "MIT"
 
@@ -24,12 +26,23 @@ Gem::Specification.new do |spec|
     f.match(%r{^(test|spec|features)/})
   end
 
+  spec.add_runtime_dependency "memery"
+  spec.add_runtime_dependency "rabbit_messaging", "~> 0.3"
+  spec.add_runtime_dependency "rails"
+
   spec.add_development_dependency "coveralls", "~> 0.8"
   spec.add_development_dependency "rspec", "~> 3.8"
   spec.add_development_dependency "rubocop-config-umbrellio", "~> 0.70"
   spec.add_development_dependency "simplecov", "~> 0.16"
 
+  spec.add_development_dependency "activejob"
+  spec.add_development_dependency "activerecord"
+  spec.add_development_dependency "pg", "~> 0.18"
+  spec.add_development_dependency "sequel"
+  spec.add_development_dependency "timecop"
+
   spec.add_development_dependency "bundler"
+  spec.add_development_dependency "bundler-audit"
   spec.add_development_dependency "pry"
   spec.add_development_dependency "rake"
 end