table_sync 2.1.0 → 4.1.1

data/lib/table_sync/utils/proc_array.rb

@@ -0,0 +1,17 @@
+# frozen_string_literal: true
+
+class TableSync::Utils::ProcArray < Proc
+  def initialize(&block)
+    @array = []
+    super(&block)
+  end
+
+  def push(&block)
+    @array.push(block)
+    self
+  end
+
+  def call(*args, &block)
+    super(@array, args, &block)
+  end
+end

data/lib/table_sync/utils/proc_keywords_resolver.rb

@@ -0,0 +1,46 @@
+# frozen_string_literal: true
+
+# Problem:
+#
+# > fn = proc { |first| puts first  }
+# > fn.call(:first, :second, :third)
+# first
+#
+# :second and :third was ignored. It's ok.
+#
+# > fn = proc { puts "test"  }
+# > fn.call(first: :first, second: :second, third: :third)
+# test
+#
+# And it's ok.
+#
+# > fn = proc { |&block| block.call }
+# > fn.call(first: :first, second: :second, third: :third) { puts "test" }
+# test
+#
+# And this is ok too.
+#
+# > fn = proc { |first:| puts first  }
+# > fn.call(first: :first, second: :second, third: :third)
+# ArgumentError (unknown keywords: :second, :third)
+#
+# ¯\_(ツ)_/¯
+#
+# ❤ Ruby ❤
+#
+# Next code solve this problem for procs without word arguments,
+# only keywords and block.
+
+module TableSync::Utils
+  module_function
+
+  def proc_keywords_resolver(&proc_for_wrap)
+    available_keywords = proc_for_wrap.parameters
+      .select { |type, _name| type == :keyreq }
+      .map { |_type, name| name }
+
+    proc do |keywords = {}, &block|
+      proc_for_wrap.call(**keywords.slice(*available_keywords), &block)
+    end
+  end
+end

data/lib/table_sync/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module TableSync
-  VERSION = "2.1.0"
+  VERSION = "4.1.1"
 end

data/table_sync.gemspec

@@ -5,7 +5,7 @@ $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
 require "table_sync/version"
 
 Gem::Specification.new do |spec|
-  spec.required_ruby_version = ">= 2.3.8"
+  spec.required_ruby_version = ">= 2.5.6"
 
   spec.name        = "table_sync"
   spec.version     = TableSync::VERSION
@@ -29,14 +29,15 @@ Gem::Specification.new do |spec|
   spec.add_runtime_dependency "memery"
   spec.add_runtime_dependency "rabbit_messaging", "~> 0.3"
   spec.add_runtime_dependency "rails"
+  spec.add_runtime_dependency "self_data"
 
   spec.add_development_dependency "coveralls", "~> 0.8"
   spec.add_development_dependency "rspec", "~> 3.8"
-  spec.add_development_dependency "rubocop-config-umbrellio", "~> 0.74"
+  spec.add_development_dependency "rubocop-config-umbrellio"
   spec.add_development_dependency "simplecov", "~> 0.16"
 
-  spec.add_development_dependency "activejob", ">= 4.2.11"
-  spec.add_development_dependency "activerecord"
+  spec.add_development_dependency "activejob", ">= 6.0"
+  spec.add_development_dependency "activerecord", ">= 6.0"
   spec.add_development_dependency "pg", "~> 0.18"
   spec.add_development_dependency "sequel"
   spec.add_development_dependency "timecop"

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: table_sync
 version: !ruby/object:Gem::Version
-  version: 2.1.0
+  version: 4.1.1
 platform: ruby
 authors:
 - Umbrellio
-autorequire: 
+autorequire:
 bindir: bin
 cert_chain: []
-date: 2020-04-09 00:00:00.000000000 Z
+date: 2020-11-06 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: memery
@@ -52,6 +52,20 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  name: self_data
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: coveralls
   requirement: !ruby/object:Gem::Requirement
@@ -84,16 +98,16 @@ dependencies:
   name: rubocop-config-umbrellio
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
-        version: '0.74'
+        version: '0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
-        version: '0.74'
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: simplecov
   requirement: !ruby/object:Gem::Requirement
@@ -114,28 +128,28 @@ dependencies:
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 4.2.11
+        version: '6.0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 4.2.11
+        version: '6.0'
 - !ruby/object:Gem::Dependency
   name: activerecord
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: '0'
+        version: '6.0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: '0'
+        version: '6.0'
 - !ruby/object:Gem::Dependency
   name: pg
   requirement: !ruby/object:Gem::Requirement
@@ -248,35 +262,39 @@ files:
 - ".travis.yml"
 - CHANGELOG.md
 - Gemfile
+- Gemfile.lock
 - LICENSE.md
 - README.md
 - Rakefile
 - bin/console
 - bin/setup
-- docs/synopsis.md
+- docs/message_protocol.md
+- docs/notifications.md
+- docs/publishing.md
+- docs/receiving.md
 - lib/table_sync.rb
-- lib/table_sync/base_publisher.rb
-- lib/table_sync/batch_publisher.rb
-- lib/table_sync/config.rb
-- lib/table_sync/config/callback_registry.rb
-- lib/table_sync/config_decorator.rb
-- lib/table_sync/dsl.rb
 - lib/table_sync/errors.rb
-- lib/table_sync/event_actions.rb
-- lib/table_sync/event_actions/data_wrapper.rb
-- lib/table_sync/event_actions/data_wrapper/base.rb
-- lib/table_sync/event_actions/data_wrapper/destroy.rb
-- lib/table_sync/event_actions/data_wrapper/update.rb
 - lib/table_sync/instrument.rb
 - lib/table_sync/instrument_adapter/active_support.rb
-- lib/table_sync/model/active_record.rb
-- lib/table_sync/model/sequel.rb
 - lib/table_sync/naming_resolver/active_record.rb
 - lib/table_sync/naming_resolver/sequel.rb
-- lib/table_sync/orm_adapter/active_record.rb
-- lib/table_sync/orm_adapter/sequel.rb
-- lib/table_sync/publisher.rb
-- lib/table_sync/receiving_handler.rb
+- lib/table_sync/publishing.rb
+- lib/table_sync/publishing/base_publisher.rb
+- lib/table_sync/publishing/batch_publisher.rb
+- lib/table_sync/publishing/orm_adapter/active_record.rb
+- lib/table_sync/publishing/orm_adapter/sequel.rb
+- lib/table_sync/publishing/publisher.rb
+- lib/table_sync/receiving.rb
+- lib/table_sync/receiving/config.rb
+- lib/table_sync/receiving/config_decorator.rb
+- lib/table_sync/receiving/dsl.rb
+- lib/table_sync/receiving/handler.rb
+- lib/table_sync/receiving/model/active_record.rb
+- lib/table_sync/receiving/model/sequel.rb
+- lib/table_sync/utils.rb
+- lib/table_sync/utils/interface_checker.rb
+- lib/table_sync/utils/proc_array.rb
+- lib/table_sync/utils/proc_keywords_resolver.rb
 - lib/table_sync/version.rb
 - log/.keep
 - table_sync.gemspec
@@ -284,7 +302,7 @@ homepage: https://github.com/umbrellio/table_sync
 licenses:
 - MIT
 metadata: {}
-post_install_message: 
+post_install_message:
 rdoc_options: []
 require_paths:
 - lib
@@ -292,15 +310,15 @@ required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
-      version: 2.3.8
+      version: 2.5.6
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.1.2
-signing_key: 
+rubygems_version: 3.2.0.rc.1
+signing_key:
 specification_version: 4
 summary: DB Table synchronization between microservices based on Model's event system
   and RabbitMQ messaging

data/docs/synopsis.md

@@ -1,338 +0,0 @@
-# TableSync
-
-Table synchronization via RabbitMQ
-
-# Publishing changes
-
-Include `TableSync.sync(self)` into a Sequel or ActiveRecord model. `:if` and `:unless` are
-supported for Sequel and ActiveRecord
-
-Functioning `Rails.cache` is required
-
-Example:
-
-```ruby
-class SomeModel < Sequel::Model
-    TableSync.sync(self, { if: -> (*) { some_code } })
-end
-```
-
-#### #attributes_for_sync
-
-Models can implement `#attributes_for_sync` to override which attributes are published. If not
-present, all attributes are published
-
-#### #attrs_for_routing_key
-
-Models can implement `#attrs_for_routing_key` to override which attributes are given to routing_key_callable. If not present, default attributes are given
-
-#### #attrs_for_metadata
-
-Models can implement `#attrs_for_metadata` to override which attributes are given to metadata_callable. If not present, default attributes are given
-
-#### .table_sync_model_name
-
-Models can implement `.table_sync_model_name` class method to override the model name used for
-publishing events. Default is model class name
-
-#### .table_sync_destroy_attributes(original_attributes)
-
-Models can implement `.table_sync_destroy_attributes` class method to override the attributes
-used for publishing destroy events. Default is object's primary key
-
-## Configuration
-
-- `TableSync.publishing_job_class_callable` is a callable which should resolve to a ActiveJob
-subclass that calls TableSync back to actually publish changes (required)
-
-Example:
-
-```ruby
-class TableSync::Job < ActiveJob::Base
-  def perform(*args)
-    TableSync::Publisher.new(*args).publish_now
-  end
-end
-```
-
-- `TableSync.batch_publishing_job_class_callable` is a callable which should resolve to a ActiveJob
-subclass that calls TableSync batch publisher back to actually publish changes (required for batch publisher)
-
-- `TableSync.routing_key_callable` is a callable which resolves which routing key to use when
-publishing changes. It receives object class and attributes (required)
-
-Example:
-
-```ruby
-TableSync.routing_key_callable = -> (klass, attributes) { klass.gsub('::', '_').tableize }
-```
-
-- `TableSync.routing_metadata_callable` is a callable that adds RabbitMQ headers which can be
-used in routing (optional). One possible way of using it is defining a headers exchange and
-routing rules based on key-value pairs (which correspond to sent headers)
-
-Example:
-
-```ruby
-TableSync.routing_metadata_callable = -> (klass, attributes) { attributes.slice("project_id") }
-```
-
-- `TableSync.exchange_name` defines the exchange name used for publishing (optional, falls back
-to default Rabbit gem configuration).
-
-- `TableSync.notifier` is a module that provides publish and recieve notifications.
-
-# Manual publishing
-
-`TableSync::Publisher.new(object_class, original_attributes, confirm: true, state: :updated, debounce_time: 45)`
-where state is one of `:created / :updated / :destroyed` and `confirm` is Rabbit's confirm delivery flag and optional param `debounce_time` determines debounce time in seconds, 1 minute by default.
-
-# Manual publishing with batches
-
-You can use `TableSync::BatchPublisher` to publish changes in batches (array of hashes in `attributes`).
-For now, only the following changes in the table can be published: `create` and` update`.
-
-When using `TableSync::BatchPublisher`,` TableSync.routing_key_callable` is called as follows:
-`TableSync.routing_key_callable.call(klass, {})`, i.e. empty hash is passed instead of attributes.
-And `TableSync.routing_metadata_callable` is not called at all: header value is set to empty hash.
-
-`TableSync::BatchPublisher.new(object_class, original_attributes_array, **options)`, where `original_attributes_array` is an array with hash of attributes of published objects and `options` is a hash of options.
-
-`options` consists of:
-- `confirm`, which is a flag for RabbitMQ, `true` by default
-- `routing_key`, which is a custom key used (if given) to override one from `TableSync.routing_key_callable`, `nil` by default
-- `push_original_attributes` (default value is `false`), if this option is set to `true`,
-original_attributes_array will be pushed to Rabbit instead of fetching records from database and sending their mapped attributes.
-- `headers`, which is an option for custom headers (can be used for headers exchanges routes), `nil` by default
-- `event`, which is an option for event specification (`:destroy` or `:update`), `:update` by default
-
-Example:
-
-```ruby
-TableSync::BatchPublisher.new(
-  "SomeClass",
-  [{ id: 1 }, { id: 2 }],
-  confirm: false,
-  routing_key: "custom_routing_key",
-  push_original_attributes: true,
-  headers: { key: :value },
-  event: :destroy,
-)
-```
-
-# Manual publishing with batches (Russian)
-
-С помощью класса `TableSync::BatchPublisher` вы можете опубликовать изменения батчами (массивом в `attributes`).
-Пока можно публиковать только следующие изменения в таблице: `создание записи` и `обновление записи`.
-
-При использовании `TableSync::BatchPublisher`, `TableSync.routing_key_callable` вызывается следующим образом:
-`TableSync.routing_key_callable.call(klass, {})`, то есть вместо аттрибутов передается пустой хэш.
-А `TableSync.routing_metadata_callable` не вызывается вовсе: в хидерах устанавливается пустой хэш.
-
-`TableSync::BatchPublisher.new(object_class, original_attributes_array, **options)`, где `original_attributes_array` - массив с аттрибутами публикуемых объектов и `options`- это хэш с дополнительными опциями.
-
-`options` состоит из:
-- `confirm`, флаг для RabbitMQ, по умолчанию - `true`
-- `routing_key`, ключ, который (если указан) замещает ключ, получаемый из `TableSync.routing_key_callable`, по умолчанию - `nil`
-- `push_original_attributes` (значение по умолчанию `false`), если для этой опции задано значение true, в Rabbit будут отправлены original_attributes_array, вместо получения значений записей из базы непосредственно перед отправкой.
-- `headers`, опция для задания headers (можно использовать для задания маршрутов в headers exchange'ах), `nil` по умолчанию
-- `event`, опция для указания типа события (`:destroy` или `:update`), `:update` по умолчанию
-
-Example:
-
-```ruby
-TableSync::BatchPublisher.new(
-  "SomeClass",
-  [{ id: 1 }, { id: 2 }],
-  confirm: false,
-  routing_key: "custom_routing_key",
-  push_original_attributes: true,
-  headers: { key: :value },
-  event: :destroy,
-)
-```
-
-# Receiving changes
-
-Naming convention for receiving handlers is `Rabbit::Handler::GROUP_ID::TableSync`,
-where `GROUP_ID` represents first part of source exchange name.
-Define handler class inherited from `TableSync::ReceivingHandler`
-and named according to described convention.
-You should use DSL inside the class.
-Suppose we will synchronize models {Project, News, User} project {MainProject}, then:
-
-```ruby
-class Rabbit::Handler::MainProject::TableSync < TableSync::ReceivingHandler
-  queue_as :custom_queue
-
-  receive "Project", to_table: :projects
-
-  receive "News", to_table: :news, events: :update do
-    after_commit on: :update do
-      NewsCache.reload
-    end
-  end
-
-  receive "User", to_table: :clients, events: %i[update destroy] do
-    mapping_overrides email: :project_user_email, id: :project_user_id
-
-    only :project_user_email, :project_user_id
-    target_keys :project_id, :project_user_id
-    rest_key :project_user_rest
-    version_key :project_user_version
-
-    additional_data do |project_id:|
-      { project_id: project_id }
-    end
-
-    default_values do
-      { created_at: Time.current }
-    end
-  end
-
-  receive "User", to_table: :users do
-    rest_key nil
-  end
-end
-```
-
-### Handler class (`Rabbit::Handler::MainProject::TableSync`)
-
-In this case:
-- `TableSync` - RabbitMQ event type.
-- `MainProject` - event source.
-- `Rabbit::Handler` - module for our handlers of events from RabbitMQ (there might be others)
-
-Method `queue_as` allow you to set custom queue.
-
-### Recieving handler batch processing
-
-Receiving handler supports array of attributes in a single update event. Corresponding
-upsert-style logic in ActiveRecord and Sequel orm handlers is provided.
-
-### Config DSL
-```ruby
-receive source, to_table:, [events:, &block]
-```
-
-The method receives following arguments
-- `source` - string, name of source model (required)
-- `to_table` - destination_table hash (required)
-- `events` - array of supported events (optional)
-- `block` - configuration block (optional)
-
-This method implements logic of mapping `source` to `to_table` and allows customizing the event handling logic with provided block.
-You can use one `source` for a lot of `to_table`.
-
-The following options are available inside the block:
-- `on_destroy` - defines a custom logic and behavior for `destroy` event:
-  - definition:
-    ```ruby
-      on_destroy do |attributes:, target_keys:|
-        # your code here
-      end
-    ```
-  - `target_keys:` - primary keys or unique keys;
-  - `attributes:` - received model attributes;
-- `only` - whitelist for receiving attributes
-- `skip` - return truthy value to skip the row
-- `target_keys` - primary keys or unique keys
-- `rest_key` - name of jsonb column for attributes which are not included in the whitelist. You can set the `rest_key(false)` or `rest_key(nil)` if you won't need the rest data.
-- `version_key` - name of version column
-- `first_sync_time_key` - name of the column where the time of first record synchronization should be stored. Disabled by default.
-- `mapping_overrides` - map for overriding receiving columns
-- `additional_data` - additional data for insert or update (e.g. `project_id`)
-- `default_values` - values for insert if a row is not found
-- `partitions` - proc that is used to obtain partitioned data to support table partitioning. Must return a hash which
-  keys are names of partitions of partitioned table and values - arrays of attributes to be inserted into particular
-  partition `{ measurements_2018_01: [ { attrs }, ... ], measurements_2018_02: [ { attrs }, ... ], ...}`.
-  While the proc is called inside an upsert transaction it is suitable place for creating partitions for new data.
-  Note that transaction of proc is a TableSynk.orm transaction.
-  ```ruby
-  partitions do |data:|
-    data.group_by { |d| "measurements_#{d[:time].year}_#{d[:time].month}" }
-        .tap { |data| data.keys.each { |table| DB.run("CREATE TABLE IF NOT EXISTS #{table} PARTITION OF measurements") } }
-  end
-  ```
-- `wrap_reciving` - proc that is used to wrap the receiving logic by custom block of code. Receives `data` and `receiving` attributes
-  (received event data and receiving logic proc respectively). `receiving.call` runs receiving process (you should use it manually).
-    - example (concurrent receiving):
-    ```ruby
-      wrap_receiving do |data, receiving|
-        Locking.acquire("some-lock-key") { receiving.call }
-      end
-    ```
-  - `data` attribute:
-    - for `destroy` event - an instance of `TableSync::EventActions::DataWrapper::Destroy`;
-    - for `update` event - an instance of `TableSync::EventActions::DataWrapper::Update`;
-    - `#event_data` - raw recevied event data:
-      - for `destroy` event - simple `Hash`;
-      - for `update` event - `Hash` with `Hash<ModelKlass, Array<Hash<Symbol, Object>>>` signature;
-    - `#destroy?` / `#update?` - corresponding predicates;
-    - `#type` - indicates a type of data (`:destroy` and `:update` respectively);
-    - `#each` - iterates over `#event_data` elements (acts like an iteration over an array of elements);
-
-Each of options can receive static value or code block which will be called for each event with the following arguments:
-- `event` - type of event (`:update` or `:destroy`)
-- `model` - source model (`Project`, `News`, `User` in example)
-- `version` - version of the data
-- `project_id` - id of project which is used in RabbitMQ
-- `data` - raw data from event (before applying `mapping_overrides`, `only`, etc.)
-
-Also, the `additional_data`, `skip` has a `current_row` field, which gives you a hash of all parameters of the current row (useful when receiving changes in batches).
-
-Block can receive any number of parameters from the list.
-
-### Callbacks
-You can set callbacks like this:
-```ruby
-before_commit on: event, &block
-after_commit on: event, &block
-```
-TableSync performs this callbacks after transaction commit as to avoid side effects. Block receives array of record attributes.
-
-### Notifications
-
-#### ActiveSupport adapter
-
-You can use an already existing ActiveSupport adapter:
-```ruby
-  TableSync.notifier = TableSync::InstrumentAdapter::ActiveSupport
-```
-
-This instrumentation API is provided by Active Support. It allows to subscribe to notifications:
-
-```ruby
-ActiveSupport::Notifications.subscribe(/tablesync/) do |name, start, finish, id, payload|
-  # do something
-end
-```
-
-Types of events available:
-`"tablesync.receive.update"`, `"tablesync.receive.destroy"`, `"tablesync.publish.update"`
-and `"tablesync.publish.destroy"`.
-
-You have access to the payload, which contains  `event`, `direction`, `table`, `schema` and `count`.
-
-```
-{
-  :event => :update,       # one of update / destroy
-  :direction => :publish,  # one of publish / receive
-  :table => "users",
-  :schema => "public",
-  :count => 1
-}
-```
-
- See more at https://guides.rubyonrails.org/active_support_instrumentation.html
-
-
-#### Custom adapters
-
-You can also create a custom adapter. It is expected to respond to the following method:
-
-```ruby
-  def notify(table:, event:, direction:, count:)
-    # processes data about table_sync event
-  end
-```