table_sync 2.3.0 → 4.0.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: f0b124973e82ee0bf943d61fb64b4ef0eb2798d1bca0b9c6396a597b2bf00ca5
-  data.tar.gz: d119902d368a1af85d954f586036b4f2e12b727f82bf4901b1f852b86e047627
+  metadata.gz: 63d56abd7c8729d7301e24bf4da9b6e1b8f4a44c387f3041a746398ed1bb381e
+  data.tar.gz: 10344d248c85c9effc5f25ef1442b176a2c90f2135efea3dfaf653df2c53d408
 SHA512:
-  metadata.gz: 653ffcb319c370378c8e861b0dd0c3468970dc10c8736a570c9f6fabbb1af05ade6a0506a744617bb9b9c2b07f218405ab17ecf969c9e7fd584ef96468d8e990
-  data.tar.gz: 0d080baf2462565addf4cfada754a5ba1ef67772a5e155d3958a354b938e190a99eb2d961dc56c70f26fa903cdc1ee76d0f4761999a13caf05c7e93c351ac22c
+  metadata.gz: 1c2dc57d1e62c294ffab4e2ec32361b1bcecdc06f3037f6944fb044e19f3fe6f9efe1409b086ab6b8bbe0f5ac801cc609896a5750733bc4536a0f4c11806a008
+  data.tar.gz: edb939c55898245a92ac762adbbf933cfd4fb7fc6ebf8e4c0bc1a18398ecd2d354f7604f851206c0c1c7a6b1f2eab4af987a0505a204a6cbdff2966e37257800

data/.rubocop.yml

@@ -14,3 +14,24 @@ AllCops:
 Style/Alias:
   Enabled: true
   EnforcedStyle: prefer_alias_method
+
+RSpec/ExampleLength:
+  Enabled: false
+
+Lint/UnusedBlockArgument:
+  AllowUnusedKeywordArguments: true
+
+Lint/UnusedMethodArgument:
+  AllowUnusedKeywordArguments: true
+
+Style/AccessorGrouping:
+  Enabled: false
+
+Style/NumericPredicate:
+  Enabled: false
+
+Lint/MissingSuper:
+  Enabled: false
+
+Style/MissingRespondToMissing:
+  Enabled: false

data/CHANGELOG.md

@@ -1,6 +1,46 @@
 # Changelog
 All notable changes to this project will be documented in this file.
 
+## [4.0.0] - 2020-10-23
+### Returned
+- config inheritance
+
+### Removed
+- TableSync::Plugins
+
+## [3.0.0] - 2020-09-05
+### Added
+- option `except`
+- `to_model` in receive method
+- TableSync::Utils::InterfaceChecker
+
+### Changed
+- .rubocop.yml
+- documentation
+- modules hierarchy (split receiving and publishing)  
+  `TableSync::Publisher` -> `TableSync::Publishing::Publisher`  
+  `TableSync::BatchPublisher` -> `TableSync::Publishing::BatchPublisher`  
+  `TableSync::ReceivingHandler` -> `TableSync::Receiving::Handler`
+- made data batches processing as native
+- implemented callbacks as options
+- implemented `wrap_receiving` as option
+- type checking in options
+- `before_commit on: event, &block` -> `before_update(&block)` or `before_destroy(&block)`
+- `after_commit on: event, &block` -> `after_commit_on_update(&block)` or `after_commit_on_destroy(&block)`
+- changed parameters in some options:  
+  add `raw_data`  
+  `current_row` -> `row`  
+  ...  
+  see documents for details
+
+### Removed
+- TableSync::Config::CallbackRegistry
+- TableSync::EventActions
+- TableSync::EventActions::DataWrapper
+- config option `on_destroy`
+- config option `partitions`
+- config option `first_sync_time_key`
+
 ## [2.3.0] - 2020-07-22
 ### Added
 - ruby 2.7 in Travis

data/Gemfile.lock

@@ -1,64 +1,65 @@
 PATH
   remote: .
   specs:
-    table_sync (2.2.0)
+    table_sync (4.0.0)
       memery
       rabbit_messaging (~> 0.3)
       rails
+      self_data
 
 GEM
   remote: https://rubygems.org/
   specs:
-    actioncable (6.0.3.2)
-      actionpack (= 6.0.3.2)
+    actioncable (6.0.3.4)
+      actionpack (= 6.0.3.4)
       nio4r (~> 2.0)
       websocket-driver (>= 0.6.1)
-    actionmailbox (6.0.3.2)
-      actionpack (= 6.0.3.2)
-      activejob (= 6.0.3.2)
-      activerecord (= 6.0.3.2)
-      activestorage (= 6.0.3.2)
-      activesupport (= 6.0.3.2)
+    actionmailbox (6.0.3.4)
+      actionpack (= 6.0.3.4)
+      activejob (= 6.0.3.4)
+      activerecord (= 6.0.3.4)
+      activestorage (= 6.0.3.4)
+      activesupport (= 6.0.3.4)
       mail (>= 2.7.1)
-    actionmailer (6.0.3.2)
-      actionpack (= 6.0.3.2)
-      actionview (= 6.0.3.2)
-      activejob (= 6.0.3.2)
+    actionmailer (6.0.3.4)
+      actionpack (= 6.0.3.4)
+      actionview (= 6.0.3.4)
+      activejob (= 6.0.3.4)
       mail (~> 2.5, >= 2.5.4)
       rails-dom-testing (~> 2.0)
-    actionpack (6.0.3.2)
-      actionview (= 6.0.3.2)
-      activesupport (= 6.0.3.2)
+    actionpack (6.0.3.4)
+      actionview (= 6.0.3.4)
+      activesupport (= 6.0.3.4)
       rack (~> 2.0, >= 2.0.8)
       rack-test (>= 0.6.3)
       rails-dom-testing (~> 2.0)
       rails-html-sanitizer (~> 1.0, >= 1.2.0)
-    actiontext (6.0.3.2)
-      actionpack (= 6.0.3.2)
-      activerecord (= 6.0.3.2)
-      activestorage (= 6.0.3.2)
-      activesupport (= 6.0.3.2)
+    actiontext (6.0.3.4)
+      actionpack (= 6.0.3.4)
+      activerecord (= 6.0.3.4)
+      activestorage (= 6.0.3.4)
+      activesupport (= 6.0.3.4)
       nokogiri (>= 1.8.5)
-    actionview (6.0.3.2)
-      activesupport (= 6.0.3.2)
+    actionview (6.0.3.4)
+      activesupport (= 6.0.3.4)
       builder (~> 3.1)
       erubi (~> 1.4)
       rails-dom-testing (~> 2.0)
       rails-html-sanitizer (~> 1.1, >= 1.2.0)
-    activejob (6.0.3.2)
-      activesupport (= 6.0.3.2)
+    activejob (6.0.3.4)
+      activesupport (= 6.0.3.4)
       globalid (>= 0.3.6)
-    activemodel (6.0.3.2)
-      activesupport (= 6.0.3.2)
-    activerecord (6.0.3.2)
-      activemodel (= 6.0.3.2)
-      activesupport (= 6.0.3.2)
-    activestorage (6.0.3.2)
-      actionpack (= 6.0.3.2)
-      activejob (= 6.0.3.2)
-      activerecord (= 6.0.3.2)
+    activemodel (6.0.3.4)
+      activesupport (= 6.0.3.4)
+    activerecord (6.0.3.4)
+      activemodel (= 6.0.3.4)
+      activesupport (= 6.0.3.4)
+    activestorage (6.0.3.4)
+      actionpack (= 6.0.3.4)
+      activejob (= 6.0.3.4)
+      activerecord (= 6.0.3.4)
       marcel (~> 0.3.1)
-    activesupport (6.0.3.2)
+    activesupport (6.0.3.4)
       concurrent-ruby (~> 1.0, >= 1.0.2)
       i18n (>= 0.7, < 2)
       minitest (~> 5.1)
@@ -70,10 +71,10 @@ GEM
     bundler-audit (0.7.0.1)
       bundler (>= 1.2.0, < 3)
       thor (>= 0.18, < 2)
-    bunny (2.15.0)
+    bunny (2.17.0)
       amq-protocol (~> 2.3, >= 2.3.1)
     coderay (1.1.3)
-    concurrent-ruby (1.1.6)
+    concurrent-ruby (1.1.7)
     coveralls (0.8.23)
       json (>= 1.8, < 3)
       simplecov (~> 0.16.1)
@@ -89,13 +90,12 @@ GEM
       activesupport (>= 4.0, < 7)
     globalid (0.4.2)
       activesupport (>= 4.2.0)
-    i18n (1.8.4)
+    i18n (1.8.5)
       concurrent-ruby (~> 1.0)
-    jaro_winkler (1.5.4)
     json (2.3.1)
     lamian (1.2.0)
       rails (>= 4.2)
-    loofah (2.6.0)
+    loofah (2.7.0)
       crass (~> 1.0.2)
       nokogiri (>= 1.5.9)
     mail (2.7.1)
@@ -108,12 +108,12 @@ GEM
     mimemagic (0.3.5)
     mini_mime (1.0.2)
     mini_portile2 (2.4.0)
-    minitest (5.14.1)
-    nio4r (2.5.2)
+    minitest (5.14.2)
+    nio4r (2.5.4)
     nokogiri (1.10.10)
       mini_portile2 (~> 2.4.0)
     parallel (1.19.2)
-    parser (2.7.1.4)
+    parser (2.7.1.5)
       ast (~> 2.4.1)
     pg (0.21.0)
     pry (0.13.1)
@@ -129,34 +129,35 @@ GEM
     rack (2.2.3)
     rack-test (1.1.0)
       rack (>= 1.0, < 3)
-    rails (6.0.3.2)
-      actioncable (= 6.0.3.2)
-      actionmailbox (= 6.0.3.2)
-      actionmailer (= 6.0.3.2)
-      actionpack (= 6.0.3.2)
-      actiontext (= 6.0.3.2)
-      actionview (= 6.0.3.2)
-      activejob (= 6.0.3.2)
-      activemodel (= 6.0.3.2)
-      activerecord (= 6.0.3.2)
-      activestorage (= 6.0.3.2)
-      activesupport (= 6.0.3.2)
+    rails (6.0.3.4)
+      actioncable (= 6.0.3.4)
+      actionmailbox (= 6.0.3.4)
+      actionmailer (= 6.0.3.4)
+      actionpack (= 6.0.3.4)
+      actiontext (= 6.0.3.4)
+      actionview (= 6.0.3.4)
+      activejob (= 6.0.3.4)
+      activemodel (= 6.0.3.4)
+      activerecord (= 6.0.3.4)
+      activestorage (= 6.0.3.4)
+      activesupport (= 6.0.3.4)
       bundler (>= 1.3.0)
-      railties (= 6.0.3.2)
+      railties (= 6.0.3.4)
       sprockets-rails (>= 2.0.0)
     rails-dom-testing (2.0.3)
       activesupport (>= 4.2.0)
       nokogiri (>= 1.6)
     rails-html-sanitizer (1.3.0)
       loofah (~> 2.3)
-    railties (6.0.3.2)
-      actionpack (= 6.0.3.2)
-      activesupport (= 6.0.3.2)
+    railties (6.0.3.4)
+      actionpack (= 6.0.3.4)
+      activesupport (= 6.0.3.4)
       method_source
       rake (>= 0.8.7)
       thor (>= 0.20.3, < 2.0)
     rainbow (3.0.0)
     rake (13.0.1)
+    regexp_parser (1.8.1)
     rexml (3.2.4)
     rspec (3.9.0)
       rspec-core (~> 3.9.0)
@@ -171,30 +172,34 @@ GEM
       diff-lcs (>= 1.2.0, < 2.0)
       rspec-support (~> 3.9.0)
     rspec-support (3.9.3)
-    rubocop (0.81.0)
-      jaro_winkler (~> 1.5.1)
+    rubocop (0.90.0)
       parallel (~> 1.10)
-      parser (>= 2.7.0.1)
+      parser (>= 2.7.1.1)
       rainbow (>= 2.2.2, < 4.0)
+      regexp_parser (>= 1.7)
       rexml
+      rubocop-ast (>= 0.3.0, < 1.0)
       ruby-progressbar (~> 1.7)
       unicode-display_width (>= 1.4.0, < 2.0)
-    rubocop-config-umbrellio (0.81.0.78)
-      rubocop (= 0.81.0)
-      rubocop-performance (= 1.5.2)
-      rubocop-rails (= 2.5.0)
-      rubocop-rspec (= 1.38.1)
-    rubocop-performance (1.5.2)
-      rubocop (>= 0.71.0)
-    rubocop-rails (2.5.0)
-      activesupport
+    rubocop-ast (0.7.1)
+      parser (>= 2.7.1.5)
+    rubocop-config-umbrellio (0.90.0.93)
+      rubocop (= 0.90.0)
+      rubocop-performance (= 1.7.1)
+      rubocop-rails (= 2.7.0)
+      rubocop-rspec (= 1.42.0)
+    rubocop-performance (1.7.1)
+      rubocop (>= 0.82.0)
+    rubocop-rails (2.7.0)
+      activesupport (>= 4.2.0)
       rack (>= 1.1)
-      rubocop (>= 0.72.0)
-    rubocop-rspec (1.38.1)
-      rubocop (>= 0.68.1)
+      rubocop (>= 0.87.0)
+    rubocop-rspec (1.42.0)
+      rubocop (>= 0.87.0)
     ruby-progressbar (1.10.1)
     ruby2_keywords (0.0.2)
-    sequel (5.34.0)
+    self_data (1.2.1)
+    sequel (5.37.0)
     serverengine (2.0.7)
       sigdump (~> 0.2.2)
     sigdump (0.2.4)
@@ -212,7 +217,7 @@ GEM
     sprockets (4.0.2)
       concurrent-ruby (~> 1.0)
       rack (> 1, < 3)
-    sprockets-rails (3.2.1)
+    sprockets-rails (3.2.2)
       actionpack (>= 4.0)
       activesupport (>= 4.0)
       sprockets (>= 3.0.0)
@@ -247,7 +252,7 @@ DEPENDENCIES
   pry
   rake
   rspec (~> 3.8)
-  rubocop-config-umbrellio (~> 0.81)
+  rubocop-config-umbrellio
   sequel
   simplecov (~> 0.16)
   table_sync!

data/README.md

@@ -20,8 +20,10 @@ require 'table_sync'
 
 ## Usage
 
-- [Documentation](docs/synopsis.md)
-- [Development](docs/development.md)
+- [Message protocol](docs/message_protocol.md)
+- [Publising](docs/publishing.md)
+- [Receiving](docs/receiving.md)
+- [Notifications](docs/notifications.md)
 
 ## Contributing
 

data/docs/message_protocol.md

@@ -0,0 +1,24 @@
+# Messages protocol
+
+```
+{
+  project_id: "pid",
+  data: {
+    "event": "update",
+    "model": "User",
+    "version": 1.23,
+    "attributes": [
+      {
+        "id": "1",
+        "name": "user1",
+        "email": "user1@example.com"
+      },
+      {
+        "id": "2",
+        "name": "user2",
+        "email": "user2@example.com"
+      }
+    ]
+  }
+}
+```

data/docs/notifications.md

@@ -0,0 +1,45 @@
+### Notifications
+
+#### ActiveSupport adapter
+
+You can use an already existing ActiveSupport adapter:
+```ruby
+  TableSync.notifier = TableSync::InstrumentAdapter::ActiveSupport
+```
+
+This instrumentation API is provided by Active Support. It allows to subscribe to notifications:
+
+```ruby
+ActiveSupport::Notifications.subscribe(/tablesync/) do |name, start, finish, id, payload|
+  # do something
+end
+```
+
+Types of events available:
+`"tablesync.receive.update"`, `"tablesync.receive.destroy"`, `"tablesync.publish.update"`
+and `"tablesync.publish.destroy"`.
+
+You have access to the payload, which contains  `event`, `direction`, `table`, `schema` and `count`.
+
+```
+{
+  :event => :update,       # one of update / destroy
+  :direction => :publish,  # one of publish / receive
+  :table => "users",
+  :schema => "public",
+  :count => 1
+}
+```
+
+ See more at https://guides.rubyonrails.org/active_support_instrumentation.html
+
+
+#### Custom adapters
+
+You can also create a custom adapter. It is expected to respond to the following method:
+
+```ruby
+  def notify(table:, event:, direction:, count:)
+    # processes data about table_sync event
+  end
+```

data/docs/publishing.md

@@ -0,0 +1,147 @@
+# Publishing changes
+
+Include `TableSync.sync(self)` into a Sequel or ActiveRecord model. `:if` and `:unless` are
+supported for Sequel and ActiveRecord
+
+Functioning `Rails.cache` is required
+
+Example:
+
+```ruby
+class SomeModel < Sequel::Model
+    TableSync.sync(self, { if: -> (*) { some_code } })
+end
+```
+
+#### #attributes_for_sync
+
+Models can implement `#attributes_for_sync` to override which attributes are published. If not
+present, all attributes are published
+
+#### #attrs_for_routing_key
+
+Models can implement `#attrs_for_routing_key` to override which attributes are given to routing_key_callable. If not present, default attributes are given
+
+#### #attrs_for_metadata
+
+Models can implement `#attrs_for_metadata` to override which attributes are given to metadata_callable. If not present, default attributes are given
+
+#### .table_sync_model_name
+
+Models can implement `.table_sync_model_name` class method to override the model name used for
+publishing events. Default is model class name
+
+#### .table_sync_destroy_attributes(original_attributes)
+
+Models can implement `.table_sync_destroy_attributes` class method to override the attributes
+used for publishing destroy events. Default is object's primary key
+
+## Configuration
+
+- `TableSync.publishing_job_class_callable` is a callable which should resolve to a ActiveJob
+subclass that calls TableSync back to actually publish changes (required)
+
+Example:
+
+```ruby
+class TableSync::Job < ActiveJob::Base
+  def perform(*args)
+    TableSync::Publisher.new(*args).publish_now
+  end
+end
+```
+
+- `TableSync.batch_publishing_job_class_callable` is a callable which should resolve to a ActiveJob
+subclass that calls TableSync batch publisher back to actually publish changes (required for batch publisher)
+
+- `TableSync.routing_key_callable` is a callable which resolves which routing key to use when
+publishing changes. It receives object class and attributes (required)
+
+Example:
+
+```ruby
+TableSync.routing_key_callable = -> (klass, attributes) { klass.gsub('::', '_').tableize }
+```
+
+- `TableSync.routing_metadata_callable` is a callable that adds RabbitMQ headers which can be
+used in routing (optional). One possible way of using it is defining a headers exchange and
+routing rules based on key-value pairs (which correspond to sent headers)
+
+Example:
+
+```ruby
+TableSync.routing_metadata_callable = -> (klass, attributes) { attributes.slice("project_id") }
+```
+
+- `TableSync.exchange_name` defines the exchange name used for publishing (optional, falls back
+to default Rabbit gem configuration).
+
+- `TableSync.notifier` is a module that provides publish and recieve notifications.
+
+# Manual publishing
+
+`TableSync::Publisher.new(object_class, original_attributes, confirm: true, state: :updated, debounce_time: 45)`
+where state is one of `:created / :updated / :destroyed` and `confirm` is Rabbit's confirm delivery flag and optional param `debounce_time` determines debounce time in seconds, 1 minute by default.
+
+# Manual publishing with batches
+
+You can use `TableSync::BatchPublisher` to publish changes in batches (array of hashes in `attributes`).
+
+When using `TableSync::BatchPublisher`,` TableSync.routing_key_callable` is called as follows:
+`TableSync.routing_key_callable.call(klass, {})`, i.e. empty hash is passed instead of attributes.
+And `TableSync.routing_metadata_callable` is not called at all: metadata is set to empty hash.
+
+`TableSync::BatchPublisher.new(object_class, original_attributes_array, **options)`, where `original_attributes_array` is an array with hash of attributes of published objects and `options` is a hash of options.
+
+`options` consists of:
+- `confirm`, which is a flag for RabbitMQ, `true` by default
+- `routing_key`, which is a custom key used (if given) to override one from `TableSync.routing_key_callable`, `nil` by default
+- `push_original_attributes` (default value is `false`), if this option is set to `true`,
+original_attributes_array will be pushed to Rabbit instead of fetching records from database and sending their mapped attributes.
+- `headers`, which is an option for custom headers (can be used for headers exchanges routes), `nil` by default
+- `event`, which is an option for event specification (`:destroy` or `:update`), `:update` by default
+
+Example:
+
+```ruby
+TableSync::BatchPublisher.new(
+  "SomeClass",
+  [{ id: 1 }, { id: 2 }],
+  confirm: false,
+  routing_key: "custom_routing_key",
+  push_original_attributes: true,
+  headers: { key: :value },
+  event: :destroy,
+)
+```
+
+# Manual publishing with batches (Russian)
+
+С помощью класса `TableSync::BatchPublisher` вы можете опубликовать изменения батчами (массивом в `attributes`).
+
+При использовании `TableSync::BatchPublisher`, `TableSync.routing_key_callable` вызывается следующим образом:
+`TableSync.routing_key_callable.call(klass, {})`, то есть вместо аттрибутов передается пустой хэш.
+А `TableSync.routing_metadata_callable` не вызывается вовсе: в метадате устанавливается пустой хэш.
+
+`TableSync::BatchPublisher.new(object_class, original_attributes_array, **options)`, где `original_attributes_array` - массив с аттрибутами публикуемых объектов и `options`- это хэш с дополнительными опциями.
+
+`options` состоит из:
+- `confirm`, флаг для RabbitMQ, по умолчанию - `true`
+- `routing_key`, ключ, который (если указан) замещает ключ, получаемый из `TableSync.routing_key_callable`, по умолчанию - `nil`
+- `push_original_attributes` (значение по умолчанию `false`), если для этой опции задано значение true, в Rabbit будут отправлены original_attributes_array, вместо получения значений записей из базы непосредственно перед отправкой.
+- `headers`, опция для задания headers (можно использовать для задания маршрутов в headers exchange'ах), `nil` по умолчанию
+- `event`, опция для указания типа события (`:destroy` или `:update`), `:update` по умолчанию
+
+Example:
+
+```ruby
+TableSync::BatchPublisher.new(
+  "SomeClass",
+  [{ id: 1 }, { id: 2 }],
+  confirm: false,
+  routing_key: "custom_routing_key",
+  push_original_attributes: true,
+  headers: { key: :value },
+  event: :destroy,
+)
+```