table_sync 0.0.0 → 1.4.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 415c63b858b66466938250116f5119c6e1c1889bd66e013d80f5c129348325cf
-  data.tar.gz: 8f721b8f02f89361d76c363374cdf3879d860a0e6604286d89d994ba0463f707
+  metadata.gz: 837e9058c5d54a0c4fe0e99d91289997f50014b54829bef3a88cb8a690b84306
+  data.tar.gz: 73cee0ef5ebb441170a3170ae95591ffe8bdadd0b145cbe35ab9203ba71ca126
 SHA512:
-  metadata.gz: 879f1e9daf37a2047cacbb7dc4d6ae0131bd87dba2468206882c3793e4d2f0ffe65cb48cc600e9cd97478b18d3bbe97d0ccfedf9f90555dba9ebc366edc1b059
-  data.tar.gz: 55646503747e75b0acf98cab6c6274cd54b1c662523342c58680282258970986b696ff4bdd2c7279409dff904f36760df7e7a871f3b8d55fdfb4eecdd3974b80
+  metadata.gz: 7225eac461cbe364aa098ccadc77ddd710d529dccc5464d1d27818de98ece66ec65389ff4a5f2f8ae05164130d4afe1607f7288e76cc4b7adc8e0d9db3cd8b39
+  data.tar.gz: a145ad745bb10f14b6964b94403cb901d536a0f82c091b3bb54dd776989808343acc9ca901b27063680e8f86f8d368e061ced59f838a049fbbdf3592c0de55f3

data/.gitignore

@@ -8,3 +8,4 @@
 /tmp/
 .ruby-version
 Gemfile.lock
+/log/

data/.rspec

@@ -0,0 +1,3 @@
+--color
+--format progress
+--require spec_helper

data/.travis.yml

@@ -10,8 +10,18 @@ matrix:
   allow_failures:
   - rvm: ruby-head
 sudo: false
+dist: xenial
 cache: bundler
+services:
+  - postgresql
+addons:
+  postgresql: "10"
 before_install: gem install bundler
+before_script:
+  - psql -c 'create database table_sync_test;' -U postgres
 script:
+- mkdir log
+- bundle exec rake bundle:audit
 - bundle exec rubocop
 - bundle exec rspec
+

data/README.md

@@ -1,5 +1,7 @@
 # TableSync · [![Gem Version](https://badge.fury.io/rb/table_sync.svg)](https://badge.fury.io/rb/table_sync) [![Build Status](https://travis-ci.org/umbrellio/table_sync.svg?branch=master)](https://travis-ci.org/umbrellio/table_sync) [![Coverage Status](https://coveralls.io/repos/github/umbrellio/table_sync/badge.svg?branch=master)](https://coveralls.io/github/umbrellio/table_sync?branch=master)
 
+DB Table synchronization between microservices based on Model's event system and RabbitMQ messaging
+
 ## Instalation
 
 ```ruby
@@ -14,7 +16,7 @@ $ gem install table_sync
 
 ## Usage
 
-Coming soon...
+- [Documentation](docs/synopsis.md)
 
 ## Contributing
 

data/Rakefile

@@ -1,6 +1,7 @@
 # frozen_string_literal: true
 
 require "bundler/gem_tasks"
+require "bundler/audit/task"
 require "rspec/core/rake_task"
 require "rubocop"
 require "rubocop-rspec"
@@ -16,5 +17,6 @@ RuboCop::RakeTask.new(:rubocop) do |t|
 end
 
 RSpec::Core::RakeTask.new(:rspec)
+Bundler::Audit::Task.new
 
 task default: :rspec

data/docs/synopsis.md

@@ -0,0 +1,246 @@
+# TableSync
+
+Table synchronization via RabbitMQ
+
+# Publishing changes
+
+Include `TableSync.sync(self)` into a Sequel or ActiveRecord model. `:if` and `:unless` are
+supported for Sequel and ActiveRecord
+
+Functioning `Rails.cache` is required
+
+Example:
+
+```ruby
+class SomeModel < Sequel::Model
+    TableSync.sync(self, { if: -> (*) { some_code } })
+end
+```
+
+#### #attributes_for_sync
+
+Models can implement `#attributes_for_sync` to override which attributes are published. If not
+present, all attributes are published
+
+#### #attrs_for_routing_key
+
+Models can implement `#attrs_for_routing_key` to override which attributes are given to routing_key_callable. If not present, default attributes are given
+
+#### #attrs_for_metadata
+
+Models can implement `#attrs_for_metadata` to override which attributes are given to metadata_callable. If not present, default attributes are given
+
+#### .table_sync_model_name
+
+Models can implement `.table_sync_model_name` class method to override the model name used for
+publishing events. Default is model class name
+
+#### .table_sync_destroy_attributes(original_attributes)
+
+Models can implement `.table_sync_destroy_attributes` class method to override the attributes
+used for publishing destroy events. Default is object's primary key
+
+## Configuration
+
+- `TableSync.publishing_job_class_callable` is a callable which should resolve to a ActiveJob
+subclass that calls TableSync back to actually publish changes (required)
+
+Example:
+
+```ruby
+class TableSync::Job < ActiveJob::Base
+  def perform(*args)
+    TableSync::Publisher.new(*args).publish_now
+  end
+end
+```
+
+- `TableSync.batch_publishing_job_class_callable` is a callable which should resolve to a ActiveJob
+subclass that calls TableSync batch publisher back to actually publish changes (required for batch publisher)
+
+- `TableSync.routing_key_callable` is a callable which resolves which routing key to use when
+publishing changes. It receives object class and attributes (required)
+
+Example:
+
+```ruby
+TableSync.routing_key_callable = -> (klass, attributes) { klass.gsub('::', '_').tableize }
+```
+
+- `TableSync.routing_metadata_callable` is a callable that adds RabbitMQ headers which can be
+used in routing (optional). One possible way of using it is defining a headers exchange and
+routing rules based on key-value pairs (which correspond to sent headers)
+
+Example:
+
+```ruby
+TableSync.routing_metadata_callable = -> (klass, attributes) { attributes.slice("project_id") }
+```
+
+- `TableSync.exchange_name` defines the exchange name used for publishing (optional, falls back
+to default Rabbit gem configuration).
+
+# Manual publishing
+
+`TableSync::Publisher.new(object_class, original_attributes, confirm: true, state: :updated)` where
+state is one of `:created / :updated / :destroyed` and `confirm` is Rabbit's confirm delivery flag
+
+# Manual publishing with batches
+
+You can use `TableSync::BatchPublisher` to publish changes in batches (array of hashes in `attributes`).
+For now, only the following changes in the table can be published: `create` and` update`.
+
+When using `TableSync::BatchPublisher`,` TableSync.routing_key_callable` is called as follows:
+`TableSync.routing_key_callable.call(klass, {})`, i.e. empty hash is passed instead of attributes.
+And `TableSync.routing_metadata_callable` is not called at all: header value is set to empty hash.
+
+`TableSync::BatchPublisher.new(object_class, original_attributes_array, **options)`, where `original_attributes_array` is an array with hash of attributes of published objects and `options` is a hash of options.
+
+`options` consists of:
+- `confirm`, which is a flag for RabbitMQ, `true` by default
+- `routing_key`, which is a custom key used (if given) to override one from `TableSync.routing_key_callable`, `nil` by default
+- `push_original_attributes` (default value is `false`), if this option is set to `true`,
+original_attributes_array will be pushed to Rabbit instead of fetching records from database and sending their mapped attributes.
+
+Example:
+
+```ruby
+TableSync::BatchPublisher.new("SomeClass", [{ id: 1 }, { id: 2 }], confirm: false, routing_key: "custom_routing_key")
+```
+
+# Manual publishing with batches (Russian)
+
+С помощью класса `TableSync::BatchPublisher` вы можете опубликовать изменения батчами (массивом в `attributes`).
+Пока можно публиковать только следующие изменения в таблице: `создание записи` и `обновление записи`.
+
+При использовании `TableSync::BatchPublisher`, `TableSync.routing_key_callable` вызывается следующим образом:
+`TableSync.routing_key_callable.call(klass, {})`, то есть вместо аттрибутов передается пустой хэш.
+А `TableSync.routing_metadata_callable` не вызывается вовсе: в хидерах устанавливается пустой хэш.
+
+`TableSync::BatchPublisher.new(object_class, original_attributes_array, **options)`, где `original_attributes_array` - массив с аттрибутами публикуемых объектов и `options`- это хэш с дополнительными опциями.
+
+`options` состоит из:
+- `confirm`, флаг для RabbitMQ, по умолчанию - `true`
+- `routing_key`, ключ, который (если указан) замещает ключ, получаемый из `TableSync.routing_key_callable`, по умолчанию - `nil`
+- `push_original_attributes` (значение по умолчанию `false`), если для этой опции задано значение true, в Rabbit будут отправлены original_attributes_array, вместо получения значений записей из базы непосредственно перед отправкой.
+
+Example:
+
+```ruby
+TableSync::BatchPublisher.new("SomeClass", [{ id: 1 }, { id: 2 }], confirm: false, routing_key: "custom_routing_key")
+```
+
+# Receiving changes
+
+Naming convention for receiving handlers is `Rabbit::Handler::GROUP_ID::TableSync`,
+where `GROUP_ID` represents first part of source exchange name.
+Define handler class inherited from `TableSync::ReceivingHandler`
+and named according to described convention.
+You should use DSL inside the class.
+Suppose we will synchronize models {Project, News, User} project {MainProject}, then:
+
+```ruby
+class Rabbit::Handler::MainProject::TableSync < TableSync::ReceivingHandler
+  queue_as :custom_queue
+
+  receive "Project", to_table: :projects
+
+  receive "News", to_table: :news, events: :update do
+    after_commit on: :update do
+      NewsCache.reload
+    end
+  end
+
+  receive "User", to_table: :clients, events: %i[update destroy] do
+    mapping_overrides email: :project_user_email, id: :project_user_id
+
+    only :project_user_email, :project_user_id
+    target_keys :project_id, :project_user_id
+    rest_key :project_user_rest
+    version_key :project_user_version
+
+    additional_data do |project_id:|
+      { project_id: project_id }
+    end
+
+    default_values do
+      { created_at: Time.current }
+    end
+  end
+
+  receive "User", to_table: :users do
+    rest_key nil
+  end
+end
+```
+
+### Handler class (`Rabbit::Handler::MainProject::TableSync`)
+
+In this case:
+- `TableSync` - RabbitMQ event type.
+- `MainProject` - event source.
+- `Rabbit::Handler` - module for our handlers of events from RabbitMQ (there might be others)
+
+Method `queue_as` allow you to set custom queue.
+
+### Recieving handler batch processing
+
+Receiving handler supports array of attributes in a single update event. Corresponding
+upsert-style logic in ActiveRecord and Sequel orm handlers is provided.
+
+### Config DSL
+```ruby
+receive source, to_table:, [events:, &block]
+```
+
+The method receives following arguments
+- `source` - string, name of source model (required)
+- `to_table` - destination_table hash (required)
+- `events` - array of supported events (optional)
+- `block` - configuration block (optional)
+
+This method implements logic of mapping `source` to `to_table` and allows customizing the event handling logic with provided block.
+You can use one `source` for a lot of `to_table`.
+
+The following options are available inside the block:
+- `only` - whitelist for receiving attributes
+- `skip` - return truthy value to skip the row
+- `target_keys` - primary keys or unique keys
+- `rest_key` - name of jsonb column for attributes which are not included in the whitelist. You can set the `rest_key(false)` or `rest_key(nil)` if you won't need the rest data.
+- `version_key` - name of version column
+- `first_sync_time_key` - name of the column where the time of first record synchronization should be stored. Disabled by default.
+- `mapping_overrides` - map for overriding receiving columns
+- `additional_data` - additional data for insert or update (e.g. `project_id`)
+- `default_values` - values for insert if a row is not found
+- `partitions` - proc that is used to obtain partitioned data to support table partitioning. Must return a hash which
+ keys are names of partitions of partitioned table and values - arrays of attributes to be inserted into particular
+ partition `{ measurements_2018_01: [ { attrs }, ... ], measurements_2018_02: [ { attrs }, ... ], ...}`.
+ While the proc is called inside an upsert transaction it is suitable place for creating partitions for new data.
+ Note that transaction of proc is a TableSynk.orm transaction.
+
+```ruby
+partitions do |data:|
+  data.group_by { |d| "measurements_#{d[:time].year}_#{d[:time].month}" }
+      .tap { |data| data.keys.each { |table| DB.run("CREATE TABLE IF NOT EXISTS #{table} PARTITION OF measurements") } }
+end
+```
+
+Each of options can receive static value or code block which will be called for each event with the following arguments:
+- `event` - type of event (`:update` or `:destroy`)
+- `model` - source model (`Project`, `News`, `User` in example)
+- `version` - version of the data
+- `project_id` - id of project which is used in RabbitMQ
+- `data` - raw data from event (before applying `mapping_overrides`, `only`, etc.)
+
+Also, the `additional_data`, `skip` has a `current_row` field, which gives you a hash of all parameters of the current row (useful when receiving changes in batches).
+
+Block can receive any number of parameters from the list.
+
+### Callbacks
+You can set callbacks like this:
+```ruby
+before_commit on: event, &block
+after_commit on: event, &block
+```
+TableSync performs this callbacks after transaction commit as to avoid side effects. Block receives array of
+record attributes.

data/lib/table_sync/base_publisher.rb

@@ -0,0 +1,103 @@
+# frozen_string_literal: true
+
+class TableSync::BasePublisher
+  include Memery
+
+  BASE_SAFE_JSON_TYPES = [NilClass, String, TrueClass, FalseClass, Numeric, Symbol].freeze
+  NOT_MAPPED = Object.new
+
+  private
+
+  attr_accessor :object_class
+
+  # @!method job_callable
+  # @!method job_callable_error_message
+  # @!method attrs_for_callables
+  # @!method attrs_for_routing_key
+  # @!method attrs_for_metadata
+  # @!method attributes_for_sync
+
+  memoize def current_time
+    Time.current
+  end
+
+  memoize def primary_keys
+    Array(object_class.primary_key).map(&:to_sym)
+  end
+
+  memoize def attributes_for_sync_defined?
+    object_class.method_defined?(:attributes_for_sync)
+  end
+
+  memoize def attrs_for_routing_key_defined?
+    object_class.method_defined?(:attrs_for_routing_key)
+  end
+
+  memoize def attrs_for_metadata_defined?
+    object_class.method_defined?(:attrs_for_metadata)
+  end
+
+  def resolve_routing_key
+    routing_key_callable.call(object_class.name, attrs_for_routing_key)
+  end
+
+  def metadata
+    TableSync.routing_metadata_callable&.call(object_class.name, attrs_for_metadata)
+  end
+
+  def confirm?
+    @confirm
+  end
+
+  def routing_key_callable
+    return TableSync.routing_key_callable if TableSync.routing_key_callable
+    raise "Can't publish, set TableSync.routing_key_callable"
+  end
+
+  def filter_safe_for_serialization(object)
+    case object
+    when Array
+      object.map(&method(:filter_safe_for_serialization)).select(&method(:object_mapped?))
+    when Hash
+      object
+        .transform_keys(&method(:filter_safe_for_serialization))
+        .transform_values(&method(:filter_safe_for_serialization))
+        .select { |*objects| objects.all?(&method(:object_mapped?)) }
+    when *BASE_SAFE_JSON_TYPES
+      object
+    else
+      NOT_MAPPED
+    end
+  end
+
+  def object_mapped?(object)
+    object != NOT_MAPPED
+  end
+
+  def job_class
+    job_callable ? job_callable.call : raise(job_callable_error_message)
+  end
+
+  def publishing_data
+    {
+      model: object_class.try(:table_sync_model_name) || object_class.name,
+      attributes: attributes_for_sync,
+      version: current_time.to_f,
+    }
+  end
+
+  def params
+    params = {
+      event: :table_sync,
+      data: publishing_data,
+      confirm_select: confirm?,
+      routing_key: routing_key,
+      realtime: true,
+      headers: metadata,
+    }
+
+    params[:exchange_name] = TableSync.exchange_name if TableSync.exchange_name
+
+    params
+  end
+end

data/lib/table_sync/batch_publisher.rb

@@ -0,0 +1,102 @@
+# frozen_string_literal: true
+
+class TableSync::BatchPublisher < TableSync::BasePublisher
+  def initialize(object_class, original_attributes_array, **options)
+    @object_class = object_class.constantize
+    @original_attributes_array = original_attributes_array.map do |hash|
+      filter_safe_for_serialization(hash.deep_symbolize_keys)
+    end
+    @confirm = options[:confirm] || true
+    @routing_key = options[:routing_key] || resolve_routing_key
+    @push_original_attributes = options[:push_original_attributes] || false
+  end
+
+  def publish
+    enqueue_job
+  end
+
+  def publish_now
+    return unless need_publish?
+
+    Rabbit.publish(params)
+  end
+
+  private
+
+  attr_reader :original_attributes_array, :routing_key
+
+  def push_original_attributes?
+    @push_original_attributes
+  end
+
+  def need_publish?
+    (push_original_attributes? && original_attributes_array.present?) || objects.present?
+  end
+
+  memoize def objects
+    needles.map { |needle| TableSync.orm.find(object_class, needle) }.compact
+  end
+
+  def job_callable
+    TableSync.batch_publishing_job_class_callable
+  end
+
+  def job_callable_error_message
+    "Can't publish, set TableSync.batch_publishing_job_class_callable"
+  end
+
+  def attrs_for_callables
+    {}
+  end
+
+  def attrs_for_routing_key
+    {}
+  end
+
+  def attr_for_metadata
+    {}
+  end
+
+  def params
+    {
+      **super,
+      headers: nil,
+    }
+  end
+
+  def needles
+    original_attributes_array.map { |original_attributes| original_attributes.slice(*primary_keys) }
+  end
+
+  def publishing_data
+    {
+      **super,
+      event: :update,
+      metadata: {},
+    }
+  end
+
+  def attributes_for_sync
+    return original_attributes_array if push_original_attributes?
+
+    objects.map do |object|
+      if attributes_for_sync_defined?
+        object.attributes_for_sync
+      else
+        TableSync.orm.attributes(object)
+      end
+    end
+  end
+
+  def enqueue_job
+    job_class.perform_later(
+      object_class.name,
+      original_attributes_array,
+      enqueue_additional_options,
+    )
+  end
+
+  def enqueue_additional_options
+    { confirm: confirm?, push_original_attributes: push_original_attributes? }
+  end
+end

data/lib/table_sync/config/callback_registry.rb

@@ -0,0 +1,53 @@
+# frozen_string_literal: true
+
+class TableSync::Config::CallbackRegistry
+  CALLBACK_KINDS = %i[after_commit before_commit].freeze
+  EVENTS = %i[create update destroy].freeze
+
+  InvalidCallbackKindError = Class.new(ArgumentError)
+  InvalidEventError = Class.new(ArgumentError)
+
+  def initialize
+    @callbacks = CALLBACK_KINDS.map { |event| [event, make_event_hash] }.to_h
+  end
+
+  def register_callback(callback, kind:, event:)
+    validate_callback_kind!(kind)
+    validate_event!(event)
+
+    callbacks.fetch(kind)[event] << callback
+  end
+
+  def get_callbacks(kind:, event:)
+    validate_callback_kind!(kind)
+    validate_event!(event)
+
+    callbacks.fetch(kind).fetch(event, [])
+  end
+
+  private
+
+  attr_reader :callbacks
+
+  def make_event_hash
+    Hash.new { |hsh, key| hsh[key] = [] }
+  end
+
+  def validate_callback_kind!(kind)
+    unless CALLBACK_KINDS.include?(kind)
+      raise(
+        InvalidCallbackKindError,
+        "Invalid callback kind: #{kind.inspect}. Valid kinds are #{CALLBACK_KINDS.inspect}",
+      )
+    end
+  end
+
+  def validate_event!(event)
+    unless EVENTS.include?(event)
+      raise(
+        InvalidEventError,
+        "Invalid event: #{event.inspect}. Valid events are #{EVENTS.inspect}",
+      )
+    end
+  end
+end

data/lib/table_sync/config.rb

@@ -0,0 +1,84 @@
+# frozen_string_literal: true
+
+module TableSync
+  class Config
+    attr_reader :model, :events, :callback_registry
+
+    def initialize(model:, events: nil)
+      @model = model
+      @events = events.nil? ? nil : [events].flatten.map(&:to_sym)
+
+      @callback_registry = CallbackRegistry.new
+
+      only(model.columns)
+      mapping_overrides({})
+      additional_data({})
+      default_values({})
+      @rest_key = :rest
+      @version_key = :version
+      @first_sync_time_key = nil
+      target_keys(model.primary_keys)
+    end
+
+    # add_option implements next logic
+    # config.option - get value
+    # config.option(args) - set static value
+    # config.option { ... } - set proc as value
+    def self.add_option(name, &option_block)
+      ivar = "@#{name}".to_sym
+
+      option_block ||= proc { |value| value }
+
+      define_method(name) do |*args, &block|
+        if args.empty? && block.nil?
+          instance_variable_get(ivar)
+        elsif block
+          params = block.parameters.map { |param| param[0] == :keyreq ? param[1] : nil }.compact
+          unified_block = proc { |hash = {}| block.call(hash.slice(*params)) }
+          instance_variable_set(ivar, unified_block)
+        else
+          instance_variable_set(ivar, instance_exec(*args, &option_block))
+        end
+      end
+    end
+
+    def allow_event?(name)
+      return true if events.nil?
+      events.include?(name)
+    end
+
+    def before_commit(on:, &block)
+      callback_registry.register_callback(block, kind: :before_commit, event: on.to_sym)
+    end
+
+    def after_commit(on:, &block)
+      callback_registry.register_callback(block, kind: :after_commit, event: on.to_sym)
+    end
+
+    check_and_set_column_key = proc do |key|
+      key = key.to_sym
+      raise "#{model.inspect} doesn't have key: #{key}" unless model.columns.include?(key)
+      key
+    end
+
+    set_column_keys = proc do |*keys|
+      [keys].flatten.map { |k| instance_exec(k, &check_and_set_column_key) }
+    end
+
+    add_option(:only, &set_column_keys)
+    add_option(:target_keys, &set_column_keys)
+    add_option(:rest_key) do |value|
+      value ? instance_exec(value, &check_and_set_column_key) : nil
+    end
+    add_option(:version_key, &check_and_set_column_key)
+    add_option(:first_sync_time_key) do |value|
+      value ? instance_exec(value, &check_and_set_column_key) : nil
+    end
+
+    add_option(:mapping_overrides)
+    add_option(:additional_data)
+    add_option(:default_values)
+    add_option(:partitions)
+    add_option(:skip)
+  end
+end

data/lib/table_sync/config_decorator.rb

@@ -0,0 +1,35 @@
+# frozen_string_literal: true
+
+module TableSync
+  # rubocop:disable Style/MissingRespondToMissing, Style/MethodMissingSuper
+  class ConfigDecorator
+    include ::TableSync::EventActions
+
+    def initialize(config, handler)
+      @config = config
+      @handler = handler
+    end
+
+    def method_missing(name, *args)
+      value = @config.send(name)
+
+      if value.is_a?(Proc)
+        value.call(
+          event: @handler.event,
+          model: @handler.model,
+          version: @handler.version,
+          project_id: @handler.project_id,
+          data: @handler.data,
+          current_row: args.first,
+        )
+      else
+        value
+      end
+    end
+
+    def allow_event?(event)
+      @config.allow_event?(event)
+    end
+  end
+  # rubocop:enable Style/MissingRespondToMissing, Style/MethodMissingSuper
+end