table_sync 5.1.0 → 6.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (44) hide show
  1. checksums.yaml +4 -4
  2. data/CHANGELOG.md +42 -0
  3. data/Gemfile.lock +3 -3
  4. data/README.md +3 -0
  5. data/docs/publishing/configuration.md +143 -0
  6. data/docs/publishing/manual.md +155 -0
  7. data/docs/publishing/publishers.md +162 -0
  8. data/docs/publishing.md +35 -105
  9. data/lib/table_sync/errors.rb +31 -22
  10. data/lib/table_sync/event.rb +35 -0
  11. data/lib/table_sync/orm_adapter/active_record.rb +25 -0
  12. data/lib/table_sync/orm_adapter/base.rb +92 -0
  13. data/lib/table_sync/orm_adapter/sequel.rb +29 -0
  14. data/lib/table_sync/publishing/batch.rb +53 -0
  15. data/lib/table_sync/publishing/data/objects.rb +50 -0
  16. data/lib/table_sync/publishing/data/raw.rb +27 -0
  17. data/lib/table_sync/publishing/helpers/attributes.rb +63 -0
  18. data/lib/table_sync/publishing/helpers/debounce.rb +134 -0
  19. data/lib/table_sync/publishing/helpers/objects.rb +39 -0
  20. data/lib/table_sync/publishing/message/base.rb +73 -0
  21. data/lib/table_sync/publishing/message/batch.rb +14 -0
  22. data/lib/table_sync/publishing/message/raw.rb +54 -0
  23. data/lib/table_sync/publishing/message/single.rb +13 -0
  24. data/lib/table_sync/publishing/params/base.rb +66 -0
  25. data/lib/table_sync/publishing/params/batch.rb +23 -0
  26. data/lib/table_sync/publishing/params/raw.rb +7 -0
  27. data/lib/table_sync/publishing/params/single.rb +31 -0
  28. data/lib/table_sync/publishing/raw.rb +21 -0
  29. data/lib/table_sync/publishing/single.rb +65 -0
  30. data/lib/table_sync/publishing.rb +20 -5
  31. data/lib/table_sync/receiving/config.rb +6 -4
  32. data/lib/table_sync/receiving/handler.rb +10 -6
  33. data/lib/table_sync/receiving.rb +0 -2
  34. data/lib/table_sync/setup/active_record.rb +22 -0
  35. data/lib/table_sync/setup/base.rb +67 -0
  36. data/lib/table_sync/setup/sequel.rb +26 -0
  37. data/lib/table_sync/version.rb +1 -1
  38. data/lib/table_sync.rb +31 -8
  39. metadata +28 -7
  40. data/lib/table_sync/publishing/base_publisher.rb +0 -110
  41. data/lib/table_sync/publishing/batch_publisher.rb +0 -109
  42. data/lib/table_sync/publishing/orm_adapter/active_record.rb +0 -32
  43. data/lib/table_sync/publishing/orm_adapter/sequel.rb +0 -57
  44. data/lib/table_sync/publishing/publisher.rb +0 -129
data/docs/publishing.md CHANGED
@@ -1,133 +1,63 @@
1
- # Publishing changes
1
+ # Publishing
2
2
 
3
- Include `TableSync.sync(self)` into a Sequel or ActiveRecord model. `:if` and `:unless` are supported for Sequel and ActiveRecord
3
+ TableSync can be used to send data using RabbitMQ.
4
4
 
5
- Functioning `Rails.cache` is required
5
+ You can do in two ways. Automatic and manual.
6
+ Each one has its own pros and cons.
6
7
 
7
- Example:
8
-
9
- ```ruby
10
- class SomeModel < Sequel::Model
11
- TableSync.sync(self, { if: -> (*) { some_code } })
12
- end
13
- ```
14
-
15
- #### #attributes_for_sync
16
-
17
- Models can implement `#attributes_for_sync` to override which attributes are published. If not present, all attributes are published
8
+ Automatic is used to publish changes in realtime, as soon as the tracked entity changes.
9
+ Usually syncs one entity at a time.
18
10
 
19
- #### #attrs_for_routing_key
11
+ Manual allows to sync a lot of entities per message.
12
+ But demands greater amount of work and data preparation.
20
13
 
21
- Models can implement `#attrs_for_routing_key` to override which attributes are given to `routing_key_callable`. If not present, published attributes are given
14
+ ## Automatic
22
15
 
23
- #### #attrs_for_metadata
16
+ Include `TableSync.sync(self)` into a Sequel or ActiveRecord model.
24
17
 
25
- Models can implement `#attrs_for_metadata` to override which attributes are given to `metadata_callable`. If not present, published attributes are given
18
+ Options:
26
19
 
27
- #### .table_sync_model_name
20
+ - `if:` and `unless:` - Runs given proc in the scope of an instance. Skips sync on `false` for `if:` and on `true` for `unless:`.
21
+ - `on:` - specify events (`create`, `update`, `destroy`) to trigger sync on. Triggered for all of them without this option.
22
+ - `debounce_time` - min time period allowed between synchronizations.
28
23
 
29
- Models can implement `.table_sync_model_name` class method to override the model name used for publishing events. Default is model class name
24
+ Functioning `Rails.cache` is required.
30
25
 
31
- #### .table_sync_destroy_attributes(original_attributes)
26
+ How it works:
32
27
 
33
- Models can implement `.table_sync_destroy_attributes` class method to override the attributes used for publishing destroy events. Default is object's original attributes
28
+ - `TableSync.sync(self)` - registers new callbacks (for `create`, `update`, `destroy`) for ActiveRecord model, and defines `after_create`, `after_update` and `after_destroy` callback methods for Sequel model.
34
29
 
35
- ## Configuration
36
-
37
- - `TableSync.publishing_job_class_callable` is a callable which should resolve to a ActiveJob subclass that calls TableSync back to actually publish changes (required)
30
+ - Callbacks call `TableSync::Publishing::Single#publish_later` with given options and object attributes. It enqueues a job which then publishes a message.
38
31
 
39
32
  Example:
40
33
 
41
34
  ```ruby
42
- class TableSync::Job < ActiveJob::Base
43
- def perform(*args)
44
- TableSync::Publishing::Publisher.new(*args).publish_now
45
- end
35
+ class SomeModel < Sequel::Model
36
+ TableSync.sync(self, { if: -> (*) { some_code }, unless: -> (*) { some_code }, on: [:create, :update] })
46
37
  end
47
- ```
48
-
49
- - `TableSync.batch_publishing_job_class_callable` is a callable which should resolve to a ActiveJob subclass that calls TableSync batch publisher back to actually publish changes (required for batch publisher)
50
-
51
- - `TableSync.routing_key_callable` is a callable which resolves which routing key to use when publishing changes. It receives object class and published attributes (required)
52
-
53
- Example:
54
-
55
- ```ruby
56
- TableSync.routing_key_callable = -> (klass, attributes) { klass.gsub('::', '_').tableize }
57
- ```
58
-
59
- - `TableSync.routing_metadata_callable` is a callable that adds RabbitMQ headers which can be used in routing (optional). It receives object class and published attributes. One possible way of using it is defining a headers exchange and routing rules based on key-value pairs (which correspond to sent headers)
60
38
 
61
- Example:
62
-
63
- ```ruby
64
- TableSync.routing_metadata_callable = -> (klass, attributes) { attributes.slice("project_id") }
39
+ class SomeOtherModel < Sequel::Model
40
+ TableSync.sync(self)
41
+ end
65
42
  ```
66
43
 
67
- - `TableSync.exchange_name` defines the exchange name used for publishing (optional, falls back to default Rabbit gem configuration).
68
-
69
- - `TableSync.notifier` is a module that provides publish and recieve notifications.
70
-
71
- # Manual publishing
72
-
73
- `TableSync::Publishing::Publisher.new(object_class, original_attributes, confirm: true, state: :updated, debounce_time: 45)`
74
- where state is one of `:created / :updated / :destroyed` and `confirm` is Rabbit's confirm delivery flag and optional param `debounce_time` determines debounce time in seconds, 1 minute by default.
75
-
76
- # Manual publishing with batches
77
-
78
- You can use `TableSync::Publishing::BatchPublisher` to publish changes in batches (array of hashes in `attributes`).
79
-
80
- When using `TableSync::Publishing::BatchPublisher`,` TableSync.routing_key_callable` is called as follows: `TableSync.routing_key_callable.call(klass, {})`, i.e. empty hash is passed instead of attributes. And `TableSync.routing_metadata_callable` is not called at all: metadata is set to empty hash.
44
+ ## Manual
81
45
 
82
- `TableSync::Publishing::BatchPublisher.new(object_class, original_attributes_array, **options)`, where `original_attributes_array` is an array with hash of attributes of published objects and `options` is a hash of options.
83
-
84
- `options` consists of:
85
- - `confirm`, which is a flag for RabbitMQ, `true` by default
86
- - `routing_key`, which is a custom key used (if given) to override one from `TableSync.routing_key_callable`, `nil` by default
87
- - `push_original_attributes` (default value is `false`), if this option is set to `true`,
88
- original_attributes_array will be pushed to Rabbit instead of fetching records from database and sending their mapped attributes.
89
- - `headers`, which is an option for custom headers (can be used for headers exchanges routes), `nil` by default
90
- - `event`, which is an option for event specification (`:destroy` or `:update`), `:update` by default
46
+ Directly call one of the publishers. It's the best if you need to sync a lot of data.
47
+ This way you don't even need for the changes to occur.
91
48
 
92
49
  Example:
93
50
 
94
51
  ```ruby
95
- TableSync::Publishing::BatchPublisher.new(
96
- "SomeClass",
97
- [{ id: 1 }, { id: 2 }],
98
- confirm: false,
99
- routing_key: "custom_routing_key",
100
- push_original_attributes: true,
101
- headers: { key: :value },
102
- event: :destroy,
103
- )
52
+ TableSync::Publishing::Batch.new(
53
+ object_class: "User",
54
+ original_attributes: [{ id: 1 }, { id: 2 }],
55
+ event: :update,
56
+ ).publish_now
104
57
  ```
105
58
 
106
- # Manual publishing with batches (Russian)
107
-
108
- С помощью класса `TableSync::Publishing::BatchPublisher` вы можете опубликовать изменения батчами (массивом в `attributes`).
109
-
110
- При использовании `TableSync::Publishing::BatchPublisher`, `TableSync.routing_key_callable` вызывается следующим образом: `TableSync.routing_key_callable.call(klass, {})`, то есть вместо аттрибутов передается пустой хэш. А `TableSync.routing_metadata_callable` не вызывается вовсе: в метадате устанавливается пустой хэш.
111
-
112
- `TableSync::Publishing::BatchPublisher.new(object_class, original_attributes_array, **options)`, где `original_attributes_array` - массив с аттрибутами публикуемых объектов и `options`- это хэш с дополнительными опциями.
113
-
114
- `options` состоит из:
115
- - `confirm`, флаг для RabbitMQ, по умолчанию - `true`
116
- - `routing_key`, ключ, который (если указан) замещает ключ, получаемый из `TableSync.routing_key_callable`, по умолчанию - `nil`
117
- - `push_original_attributes` (значение по умолчанию `false`), если для этой опции задано значение true, в Rabbit будут отправлены original_attributes_array, вместо получения значений записей из базы непосредственно перед отправкой.
118
- - `headers`, опция для задания headers (можно использовать для задания маршрутов в headers exchange'ах), `nil` по умолчанию
119
- - `event`, опция для указания типа события (`:destroy` или `:update`), `:update` по умолчанию
59
+ ## Read More
120
60
 
121
- Example:
122
-
123
- ```ruby
124
- TableSync::Publishing::BatchPublisher.new(
125
- "SomeClass",
126
- [{ id: 1 }, { id: 2 }],
127
- confirm: false,
128
- routing_key: "custom_routing_key",
129
- push_original_attributes: true,
130
- headers: { key: :value },
131
- event: :destroy,
132
- )
133
- ```
61
+ - [Publishers](publishing/publishers.md)
62
+ - [Configuration](publishing/configuration.md)
63
+ - [Manual Sync (examples)](publishing/manual.md)
@@ -3,6 +3,35 @@
3
3
  module TableSync
4
4
  Error = Class.new(StandardError)
5
5
 
6
+ NoObjectsForSyncError = Class.new(Error)
7
+
8
+ class EventError < Error
9
+ def initialize(event)
10
+ super(<<~MSG.squish)
11
+ Event #{event.inspect} is invalid.#{' '}
12
+ Expected: #{TableSync::Event::VALID_RAW_EVENTS.inspect}.
13
+ MSG
14
+ end
15
+ end
16
+
17
+ class NoPrimaryKeyError < Error
18
+ def initialize(object_class, object_data, primary_key_columns)
19
+ super(<<~MSG.squish)
20
+ Can't find or init an object of #{object_class} with #{object_data.inspect}.
21
+ Incomplete primary key! object_data must contain: #{primary_key_columns.inspect}.
22
+ MSG
23
+ end
24
+ end
25
+
26
+ class NoCallableError < Error
27
+ def initialize(type)
28
+ super(<<~MSG.squish)
29
+ Can't find callable for #{type}!
30
+ Please initialize TableSync.#{type}_callable with the correct proc!
31
+ MSG
32
+ end
33
+ end
34
+
6
35
  class UpsertError < Error
7
36
  def initialize(data:, target_keys:, result:)
8
37
  super("data: #{data.inspect}, target_keys: #{target_keys.inspect}, result: #{result.inspect}")
@@ -25,28 +54,6 @@ module TableSync
25
54
  end
26
55
  end
27
56
 
28
- # @api public
29
- # @since 2.2.0
30
- PluginError = Class.new(Error)
31
-
32
- # @api public
33
- # @since 2.2.0
34
- class UnregisteredPluginError < PluginError
35
- # @param plugin_name [Any]
36
- def initialize(plugin_name)
37
- super("#{plugin_name} plugin is not registered")
38
- end
39
- end
40
-
41
- # @api public
42
- # @since 2.2.0
43
- class AlreadyRegisteredPluginError < PluginError
44
- # @param plugin_name [Any]
45
- def initialize(plugin_name)
46
- super("#{plugin_name} plugin already exists")
47
- end
48
- end
49
-
50
57
  class InterfaceError < Error
51
58
  def initialize(object, method_name, parameters, description)
52
59
  parameters = parameters.map do |parameter|
@@ -54,7 +61,9 @@ module TableSync
54
61
 
55
62
  case type
56
63
  when :req
64
+ #:nocov:
57
65
  name.to_s
66
+ #:nocov:
58
67
  when :keyreq
59
68
  "#{name}:"
60
69
  when :block
@@ -0,0 +1,35 @@
1
+ # frozen_string_literal: true
2
+
3
+ class TableSync::Event
4
+ attr_reader :event
5
+
6
+ UPSERT_EVENTS = %i[create update].freeze
7
+ VALID_RESOLVED_EVENTS = %i[update destroy].freeze
8
+ VALID_RAW_EVENTS = %i[create update destroy].freeze
9
+
10
+ def initialize(event)
11
+ @event = event
12
+
13
+ validate!
14
+ end
15
+
16
+ def validate!
17
+ raise TableSync::EventError.new(event) unless event.in?(VALID_RAW_EVENTS)
18
+ end
19
+
20
+ def resolve
21
+ destroy? ? :destroy : :update
22
+ end
23
+
24
+ def metadata
25
+ { created: event == :create }
26
+ end
27
+
28
+ def destroy?
29
+ event == :destroy
30
+ end
31
+
32
+ def upsert?
33
+ event.in?(UPSERT_EVENTS)
34
+ end
35
+ end
@@ -0,0 +1,25 @@
1
+ # frozen_string_literal: true
2
+
3
+ module TableSync::ORMAdapter
4
+ class ActiveRecord < Base
5
+ def find
6
+ @object = object_class.find_by(needle)
7
+
8
+ super
9
+ end
10
+
11
+ def init
12
+ @object = object_class.new(object_data)
13
+
14
+ super
15
+ end
16
+
17
+ def attributes
18
+ object.attributes.symbolize_keys
19
+ end
20
+
21
+ def self.model_naming(object_class)
22
+ TableSync::NamingResolver::ActiveRecord.new(table_name: object_class.table_name)
23
+ end
24
+ end
25
+ end
@@ -0,0 +1,92 @@
1
+ # frozen_string_literal: true
2
+
3
+ module TableSync::ORMAdapter
4
+ class Base
5
+ attr_reader :object, :object_class, :object_data
6
+
7
+ def initialize(object_class, object_data)
8
+ @object_class = object_class
9
+ @object_data = object_data.symbolize_keys
10
+
11
+ validate!
12
+ end
13
+
14
+ # VALIDATE
15
+
16
+ def validate!
17
+ if (primary_key_columns - object_data.keys).any?
18
+ raise TableSync::NoPrimaryKeyError.new(object_class, object_data, primary_key_columns)
19
+ end
20
+ end
21
+
22
+ # FIND OR INIT OBJECT
23
+
24
+ def init
25
+ self
26
+ end
27
+
28
+ def find
29
+ self
30
+ end
31
+
32
+ def needle
33
+ object_data.slice(*primary_key_columns)
34
+ end
35
+
36
+ # ATTRIBUTES
37
+
38
+ def attributes_for_update
39
+ if object.respond_to?(:attributes_for_sync)
40
+ object.attributes_for_sync
41
+ else
42
+ attributes
43
+ end
44
+ end
45
+
46
+ def attributes_for_destroy
47
+ if object.respond_to?(:attributes_for_destroy)
48
+ object.attributes_for_destroy
49
+ else
50
+ needle
51
+ end
52
+ end
53
+
54
+ def attributes_for_routing_key
55
+ if object.respond_to?(:attributes_for_routing_key)
56
+ object.attributes_for_routing_key
57
+ else
58
+ attributes
59
+ end
60
+ end
61
+
62
+ def attributes_for_headers
63
+ if object.respond_to?(:attributes_for_headers)
64
+ object.attributes_for_headers
65
+ else
66
+ attributes
67
+ end
68
+ end
69
+
70
+ def primary_key_columns
71
+ Array.wrap(object_class.primary_key).map(&:to_sym)
72
+ end
73
+
74
+ # MISC
75
+
76
+ def empty?
77
+ object.nil?
78
+ end
79
+
80
+ # NOT IMPLEMENTED
81
+
82
+ # :nocov:
83
+ def attributes
84
+ raise NotImplementedError
85
+ end
86
+
87
+ def self.model_naming
88
+ raise NotImplementedError
89
+ end
90
+ # :nocov:
91
+ end
92
+ end
@@ -0,0 +1,29 @@
1
+ # frozen_string_literal: true
2
+
3
+ module TableSync::ORMAdapter
4
+ class Sequel < Base
5
+ def attributes
6
+ object.values
7
+ end
8
+
9
+ def init
10
+ @object = object_class.new(object_data.except(*primary_key_columns))
11
+
12
+ @object.set_fields(needle, needle.keys)
13
+
14
+ super
15
+ end
16
+
17
+ def find
18
+ @object = object_class.find(needle)
19
+
20
+ super
21
+ end
22
+
23
+ def self.model_naming(object_class)
24
+ TableSync::NamingResolver::Sequel.new(
25
+ table_name: object_class.table_name, db: object_class.db,
26
+ )
27
+ end
28
+ end
29
+ end
@@ -0,0 +1,53 @@
1
+ # frozen_string_literal: true
2
+
3
+ class TableSync::Publishing::Batch
4
+ include Tainbox
5
+
6
+ attribute :object_class
7
+ attribute :original_attributes
8
+
9
+ attribute :routing_key
10
+ attribute :headers
11
+
12
+ attribute :event, default: :update
13
+
14
+ def publish_later
15
+ job.perform_later(job_attributes)
16
+ end
17
+
18
+ def publish_now
19
+ message.publish
20
+ end
21
+
22
+ def message
23
+ TableSync::Publishing::Message::Batch.new(attributes)
24
+ end
25
+
26
+ alias_method :publish_async, :publish_later
27
+
28
+ private
29
+
30
+ # JOB
31
+
32
+ def job
33
+ if TableSync.batch_publishing_job_class_callable
34
+ TableSync.batch_publishing_job_class_callable.call
35
+ else
36
+ raise TableSync::NoCallableError.new("batch_publishing_job_class")
37
+ end
38
+ end
39
+
40
+ def job_attributes
41
+ attributes.merge(
42
+ original_attributes: serialized_original_attributes,
43
+ )
44
+ end
45
+
46
+ def serialized_original_attributes
47
+ original_attributes.map do |set_of_attributes|
48
+ TableSync::Publishing::Helpers::Attributes
49
+ .new(set_of_attributes)
50
+ .serialize
51
+ end
52
+ end
53
+ end
@@ -0,0 +1,50 @@
1
+ # frozen_string_literal: true
2
+
3
+ module TableSync::Publishing::Data
4
+ class Objects
5
+ attr_reader :objects, :event
6
+
7
+ def initialize(objects:, event:)
8
+ @objects = objects
9
+ @event = TableSync::Event.new(event)
10
+ end
11
+
12
+ def construct
13
+ {
14
+ model: model,
15
+ attributes: attributes_for_sync,
16
+ version: version,
17
+ event: event.resolve,
18
+ metadata: event.metadata,
19
+ }
20
+ end
21
+
22
+ private
23
+
24
+ def model
25
+ if object_class.respond_to?(:table_sync_model_name)
26
+ object_class.table_sync_model_name
27
+ else
28
+ object_class.name
29
+ end
30
+ end
31
+
32
+ def version
33
+ Time.current.to_f
34
+ end
35
+
36
+ def object_class
37
+ objects.first.object_class
38
+ end
39
+
40
+ def attributes_for_sync
41
+ objects.map do |object|
42
+ if event.destroy?
43
+ object.attributes_for_destroy
44
+ else
45
+ object.attributes_for_update
46
+ end
47
+ end
48
+ end
49
+ end
50
+ end
@@ -0,0 +1,27 @@
1
+ # frozen_string_literal: true
2
+
3
+ module TableSync::Publishing::Data
4
+ class Raw
5
+ attr_reader :object_class, :attributes_for_sync, :event
6
+
7
+ def initialize(object_class:, attributes_for_sync:, event:)
8
+ @object_class = object_class
9
+ @attributes_for_sync = attributes_for_sync
10
+ @event = TableSync::Event.new(event)
11
+ end
12
+
13
+ def construct
14
+ {
15
+ model: object_class,
16
+ attributes: attributes_for_sync,
17
+ version: version,
18
+ event: event.resolve,
19
+ metadata: event.metadata,
20
+ }
21
+ end
22
+
23
+ def version
24
+ Time.current.to_f
25
+ end
26
+ end
27
+ end
@@ -0,0 +1,63 @@
1
+ # frozen_string_literal: true
2
+
3
+ module TableSync::Publishing::Helpers
4
+ class Attributes
5
+ BASE_SAFE_JSON_TYPES = [
6
+ NilClass,
7
+ String,
8
+ TrueClass,
9
+ FalseClass,
10
+ Numeric,
11
+ Symbol,
12
+ ].freeze
13
+
14
+ # add custom seializables?
15
+
16
+ NOT_MAPPED = Object.new
17
+
18
+ attr_reader :attributes
19
+
20
+ def initialize(attributes)
21
+ @attributes = attributes.deep_symbolize_keys
22
+ end
23
+
24
+ def serialize
25
+ filter_safe_for_serialization(attributes)
26
+ end
27
+
28
+ def filter_safe_for_serialization(object)
29
+ case object
30
+ when Array
31
+ object.each_with_object([]) do |value, memo|
32
+ value = filter_safe_for_serialization(value)
33
+ memo << value if object_mapped?(value)
34
+ end
35
+ when Hash
36
+ object.each_with_object({}) do |(key, value), memo|
37
+ key = filter_safe_for_serialization(key)
38
+ value = filter_safe_hash_values(value)
39
+ memo[key] = value if object_mapped?(key) && object_mapped?(value)
40
+ end
41
+ when Float::INFINITY
42
+ NOT_MAPPED
43
+ when *BASE_SAFE_JSON_TYPES
44
+ object
45
+ else # rubocop:disable Lint/DuplicateBranch
46
+ NOT_MAPPED
47
+ end
48
+ end
49
+
50
+ def filter_safe_hash_values(value)
51
+ case value
52
+ when Symbol
53
+ value.to_s # why?
54
+ else
55
+ filter_safe_for_serialization(value)
56
+ end
57
+ end
58
+
59
+ def object_mapped?(object)
60
+ object != NOT_MAPPED
61
+ end
62
+ end
63
+ end