saseo 0.4.3 → 0.5.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/README.md +5 -0
- data/lib/saseo/config.rb +0 -4
- data/lib/saseo/config/defaults.rb +1 -0
- data/lib/saseo/models/base.rb +36 -1
- data/lib/saseo/models/source/version.rb +0 -8
- data/lib/saseo/models/version.rb +0 -8
- data/lib/saseo/persistence/consumer.rb +1 -2
- data/lib/saseo/persistence/message_filter.rb +54 -0
- data/lib/saseo/persistence/persistor.rb +37 -19
- data/lib/saseo/version.rb +1 -1
- data/sample_saseo_setup.png +0 -0
- data/saseo.gemspec +2 -0
- metadata +32 -2
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: b329bf7dd271331dda62ef081a05d9f7e0c64af9
|
4
|
+
data.tar.gz: ed09bc9155ee23828beee19b3fa32e0c781307e5
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 993d46fb9c8d728324a334c627bdfefae21d53668d9c431bc615bd343370d04ea3c0636e83ecb1cc15d6014232498acaf5e26765c4e3410b67334e670e5ace9f
|
7
|
+
data.tar.gz: 8c270a1382502571d728940c10509d00f5f8a18c4d0bcb181b11088fe98180d08d259d112b3ebbab4be463491b6ac9ae526235e846b814ac34bc97a9f23db8ce
|
data/README.md
CHANGED
@@ -5,6 +5,9 @@ RabbitMQ based PaperTrail replacement
|
|
5
5
|
[![Dependency Status](https://gemnasium.com/avantcredit/saseo.svg)](https://gemnasium.com/avantcredit/saseo)
|
6
6
|
[![Code Climate](https://codeclimate.com/github/avantcredit/saseo/badges/gpa.svg)](https://codeclimate.com/github/avantcredit/saseo)
|
7
7
|
[![Test Coverage](https://codeclimate.com/github/avantcredit/saseo/badges/coverage.svg)](https://codeclimate.com/github/avantcredit/saseo)
|
8
|
+
[![Gem Version](http://img.shields.io/gem/v/saseo.svg)](https://rubygems.org/gems/saseo)
|
9
|
+
[![License](http://img.shields.io/:license-mit-blue.svg)](http://doge.mit-license.org)
|
10
|
+
|
8
11
|
|
9
12
|
## Installation
|
10
13
|
|
@@ -26,6 +29,8 @@ Or install it yourself as:
|
|
26
29
|
|
27
30
|
TODO: Write usage instructions here
|
28
31
|
|
32
|
+
## Sample Setup
|
33
|
+
![](https://raw.githubusercontent.com/avantcredit/saseo/master/sample_saseo_setup.png)
|
29
34
|
## Development
|
30
35
|
|
31
36
|
After checking out the repo, run `bin/setup` to install dependencies. Then, run `rake spec` to run the tests. You can also run `bin/console` for an interactive prompt that will allow you to experiment.
|
data/lib/saseo/config.rb
CHANGED
@@ -30,10 +30,6 @@ module Saseo
|
|
30
30
|
load_config(config[env])
|
31
31
|
end
|
32
32
|
|
33
|
-
def channels
|
34
|
-
@channels ||= _load_json_config :channels
|
35
|
-
end
|
36
|
-
|
37
33
|
def consumer_philotic_subscription
|
38
34
|
@consumer_philotic_subscription ||= _load_json_config :consumer_philotic_subscription
|
39
35
|
end
|
@@ -11,6 +11,7 @@ module Saseo
|
|
11
11
|
SOURCE_TABLE_SCHEMA = 'saseo'
|
12
12
|
SOURCE_TABLE_NAME = 'saseo_source_versions'
|
13
13
|
CONSUMER_PHILOTIC_SUBSCRIPTION = 'saseo_audit'
|
14
|
+
IGNORE_FIELDS_CONFIG_PATH = nil
|
14
15
|
|
15
16
|
def defaults
|
16
17
|
@defaults ||= Hash[Saseo::Config::Defaults.constants.map do |c|
|
data/lib/saseo/models/base.rb
CHANGED
@@ -32,7 +32,7 @@ module Saseo
|
|
32
32
|
end
|
33
33
|
|
34
34
|
def database_config_from_load_path
|
35
|
-
config
|
35
|
+
config = nil
|
36
36
|
config_path = database_config_path
|
37
37
|
$:.each do |load_path|
|
38
38
|
begin
|
@@ -43,9 +43,44 @@ module Saseo
|
|
43
43
|
end
|
44
44
|
config
|
45
45
|
end
|
46
|
+
|
47
|
+
def reconnect!
|
48
|
+
ActiveRecord::Base.clear_all_connections!
|
49
|
+
ActiveRecord::Base.establish_connection
|
50
|
+
end
|
46
51
|
end
|
47
52
|
|
53
|
+
validates :transaction_id, presence: true
|
54
|
+
validates :table_name, presence: true
|
55
|
+
validates :action_timestamp, presence: true
|
56
|
+
validates :action, presence: true, inclusion: {in: %w[INSERT UPDATE DELETE]}
|
57
|
+
validates :whodunnit, presence: true
|
58
|
+
validates :old_data, presence: true, unless: ->(version) { version.new_data.present? }
|
59
|
+
validates :new_data, presence: true, unless: ->(version) { version.old_data.present? }
|
60
|
+
|
61
|
+
validate :validate_has_changes
|
62
|
+
|
48
63
|
establish_connection database_config
|
64
|
+
|
65
|
+
def changes
|
66
|
+
old_keys = old_data ? old_data.keys : []
|
67
|
+
new_keys = new_data ? new_data.keys : []
|
68
|
+
(old_keys | new_keys).reduce({}) do |changes, key|
|
69
|
+
old_value = old_data ? old_data[key] : nil
|
70
|
+
new_value = new_data ? new_data[key] : nil
|
71
|
+
changes[key] = [old_value, new_value] if old_value != new_value
|
72
|
+
changes
|
73
|
+
end
|
74
|
+
end
|
75
|
+
|
76
|
+
def has_changes?
|
77
|
+
changes.any?
|
78
|
+
end
|
79
|
+
|
80
|
+
private
|
81
|
+
def validate_has_changes
|
82
|
+
errors.add(:new_data, 'can not be the same as old_data') unless has_changes?
|
83
|
+
end
|
49
84
|
end
|
50
85
|
end
|
51
86
|
end
|
@@ -6,14 +6,6 @@ module Saseo
|
|
6
6
|
module Source
|
7
7
|
class Version < Saseo::Models::Source::Base
|
8
8
|
self.table_name = "#{Saseo.config.source_table_schema}.#{Saseo.config.source_table_name}"
|
9
|
-
|
10
|
-
validates :transaction_id, presence: true
|
11
|
-
validates :table_name, presence: true
|
12
|
-
validates :action_timestamp, presence: true
|
13
|
-
validates :action, presence: true, inclusion: {in: %w[INSERT UPDATE DELETE]}
|
14
|
-
validates :whodunnit, presence: true
|
15
|
-
validates :old_data, presence: true, unless: ->(version) { version.new_data.present? }
|
16
|
-
validates :new_data, presence: true, unless: ->(version) { version.old_data.present? }
|
17
9
|
end
|
18
10
|
end
|
19
11
|
end
|
data/lib/saseo/models/version.rb
CHANGED
@@ -5,14 +5,6 @@ module Saseo
|
|
5
5
|
module Models
|
6
6
|
class Version < Saseo::Models::Base
|
7
7
|
self.table_name = Saseo.config.table_name
|
8
|
-
|
9
|
-
validates :transaction_id, presence: true
|
10
|
-
validates :table_name, presence: true
|
11
|
-
validates :action_timestamp, presence: true
|
12
|
-
validates :action, presence: true, inclusion: {in: %w[INSERT UPDATE DELETE]}
|
13
|
-
validates :whodunnit, presence: true
|
14
|
-
validates :old_data, presence: true, unless: ->(version) { version.new_data.present? }
|
15
|
-
validates :new_data, presence: true, unless: ->(version) { version.old_data.present? }
|
16
8
|
end
|
17
9
|
end
|
18
10
|
end
|
@@ -5,7 +5,6 @@ require 'saseo/persistence/persistor'
|
|
5
5
|
module Saseo
|
6
6
|
module Persistence
|
7
7
|
class Consumer < Philotic::Consumer
|
8
|
-
# subscribe_to philotic_message_type: :'saseo.record_audit'
|
9
8
|
subscribe_to Saseo.config.consumer_philotic_subscription
|
10
9
|
|
11
10
|
manually_acknowledge
|
@@ -13,7 +12,7 @@ module Saseo
|
|
13
12
|
def consume(message)
|
14
13
|
begin
|
15
14
|
logger.debug { "received source version: #{message.id}" }
|
16
|
-
Saseo::Persistence::Persistor.
|
15
|
+
Saseo::Persistence::Persistor.process message
|
17
16
|
|
18
17
|
acknowledge message
|
19
18
|
rescue => e
|
@@ -0,0 +1,54 @@
|
|
1
|
+
require 'oj'
|
2
|
+
require 'saseo/models/version'
|
3
|
+
require 'active_support/core_ext/hash/deep_merge'
|
4
|
+
|
5
|
+
module Saseo
|
6
|
+
module Persistence
|
7
|
+
module MessageFilter
|
8
|
+
extend self
|
9
|
+
|
10
|
+
WILD_CARD = '*'
|
11
|
+
|
12
|
+
def filter_message!(message)
|
13
|
+
return message unless ignored_fields_config && filter_match?(ignored_fields_config.keys, message.table_name)
|
14
|
+
|
15
|
+
table_config(message.table_name).each_pair do |fields, ignored_actions|
|
16
|
+
if filter_match?(ignored_actions, message.action)
|
17
|
+
fields_to_remove(fields, message).each do |field|
|
18
|
+
remove_field!(field, message)
|
19
|
+
end
|
20
|
+
end
|
21
|
+
end
|
22
|
+
|
23
|
+
return nil if message.old_data.blank? && message.new_data.blank?
|
24
|
+
message
|
25
|
+
end
|
26
|
+
|
27
|
+
def remove_field!(field, message)
|
28
|
+
message.old_data && message.old_data.delete(field)
|
29
|
+
message.new_data && message.new_data.delete(field)
|
30
|
+
end
|
31
|
+
|
32
|
+
def fields_to_remove(fields, message)
|
33
|
+
fields = Array(fields).map { |f| f.to_s.downcase }
|
34
|
+
fields = message.old_data.keys | message.new_data.keys if fields.include?(WILD_CARD)
|
35
|
+
fields
|
36
|
+
end
|
37
|
+
|
38
|
+
def ignored_fields_config
|
39
|
+
unless defined?(@ignored_fields_config)
|
40
|
+
@ignored_fields_config = Saseo.config.ignore_fields_config_path ? YAML::load(File.open(File.expand_path Saseo.config.ignore_fields_config_path))[Saseo.config.env] : nil
|
41
|
+
end
|
42
|
+
@ignored_fields_config
|
43
|
+
end
|
44
|
+
|
45
|
+
def filter_match?(filters, value)
|
46
|
+
(Array(filters).map { |f| f.to_s.downcase } & [WILD_CARD, value.to_s.downcase]).any?
|
47
|
+
end
|
48
|
+
|
49
|
+
def table_config(table_name)
|
50
|
+
(ignored_fields_config && ignored_fields_config[WILD_CARD] || {}).deep_merge(ignored_fields_config && ignored_fields_config[table_name] || {})
|
51
|
+
end
|
52
|
+
end
|
53
|
+
end
|
54
|
+
end
|
@@ -1,5 +1,6 @@
|
|
1
1
|
require 'oj'
|
2
2
|
require 'saseo/models/version'
|
3
|
+
require 'saseo/persistence/message_filter'
|
3
4
|
|
4
5
|
module Saseo
|
5
6
|
module Persistence
|
@@ -25,30 +26,47 @@ module Saseo
|
|
25
26
|
@logger ||= Logger.new(STDOUT)
|
26
27
|
end
|
27
28
|
|
28
|
-
def
|
29
|
+
def process(message)
|
29
30
|
logger.debug { "received source version for persistence: #{message.id}" }
|
30
|
-
version = Saseo::Models::Version.new
|
31
31
|
|
32
|
-
|
33
|
-
|
32
|
+
message = Saseo::Persistence::MessageFilter.filter_message!(message)
|
33
|
+
persist_message!(message) if message
|
34
|
+
end
|
35
|
+
|
36
|
+
def persist_message!(message)
|
37
|
+
save_version!(build_version(message))
|
38
|
+
end
|
39
|
+
|
40
|
+
def build_version(message)
|
41
|
+
Saseo::Models::Version.new.tap do |version|
|
42
|
+
|
43
|
+
version_attributes.each do |attr|
|
44
|
+
version.send("#{attr}=", message.send(attr))
|
45
|
+
end
|
46
|
+
|
47
|
+
# ActiveRecord 3 doesn't handle jsonb columns properly
|
48
|
+
version.old_data = message.old_data && Oj.dump(message.old_data)
|
49
|
+
version.new_data = message.new_data && Oj.dump(message.new_data)
|
34
50
|
end
|
51
|
+
end
|
52
|
+
|
53
|
+
def save_version!(version)
|
54
|
+
if version.has_changes?
|
55
|
+
begin
|
56
|
+
version.save!
|
57
|
+
logger.debug { "saved version: #{version.id}" }
|
35
58
|
|
36
|
-
|
37
|
-
|
38
|
-
|
39
|
-
|
40
|
-
|
41
|
-
|
42
|
-
|
43
|
-
|
44
|
-
Philotic::Message.publish({philotic_message_type: :'saseo.error.duplicate_record'}, version.attributes)
|
45
|
-
rescue ActiveRecord::StatementInvalid => e
|
46
|
-
if e.message.match /PG::ConnectionBad/
|
47
|
-
logger.warn { 'Lost connection to DB, attempting to reconnect.' }
|
48
|
-
ActiveRecord::Base.clear_all_connections!
|
49
|
-
ActiveRecord::Base.establish_connection
|
50
|
-
retry
|
59
|
+
rescue ActiveRecord::RecordNotUnique => e
|
60
|
+
logger.warn "Duplicate version: #{version.attributes}"
|
61
|
+
rescue ActiveRecord::StatementInvalid => e
|
62
|
+
if e.message.match /PG::ConnectionBad/
|
63
|
+
logger.warn { 'Lost connection to DB, attempting to reconnect.' }
|
64
|
+
Saseo::Models::Version.reconnect!
|
65
|
+
retry
|
66
|
+
end
|
51
67
|
end
|
68
|
+
else
|
69
|
+
logger.info { "Skipped version with no changes: #{version.attributes}" }
|
52
70
|
end
|
53
71
|
end
|
54
72
|
end
|
data/lib/saseo/version.rb
CHANGED
Binary file
|
data/saseo.gemspec
CHANGED
@@ -25,6 +25,8 @@ Gem::Specification.new do |spec|
|
|
25
25
|
spec.add_development_dependency 'generator_spec'
|
26
26
|
spec.add_development_dependency 'rake', '~> 10.0'
|
27
27
|
spec.add_development_dependency 'rspec'
|
28
|
+
spec.add_development_dependency 'shoulda-matchers', '~> 2.5'
|
29
|
+
spec.add_development_dependency 'simplecov', '= 0.10'
|
28
30
|
|
29
31
|
spec.add_dependency 'oj'
|
30
32
|
spec.add_dependency 'pg'
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: saseo
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.
|
4
|
+
version: 0.5.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Nathan Keyes
|
8
8
|
autorequire:
|
9
9
|
bindir: exe
|
10
10
|
cert_chain: []
|
11
|
-
date: 2015-11-
|
11
|
+
date: 2015-11-30 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: awesome_print
|
@@ -108,6 +108,34 @@ dependencies:
|
|
108
108
|
- - ">="
|
109
109
|
- !ruby/object:Gem::Version
|
110
110
|
version: '0'
|
111
|
+
- !ruby/object:Gem::Dependency
|
112
|
+
name: shoulda-matchers
|
113
|
+
requirement: !ruby/object:Gem::Requirement
|
114
|
+
requirements:
|
115
|
+
- - "~>"
|
116
|
+
- !ruby/object:Gem::Version
|
117
|
+
version: '2.5'
|
118
|
+
type: :development
|
119
|
+
prerelease: false
|
120
|
+
version_requirements: !ruby/object:Gem::Requirement
|
121
|
+
requirements:
|
122
|
+
- - "~>"
|
123
|
+
- !ruby/object:Gem::Version
|
124
|
+
version: '2.5'
|
125
|
+
- !ruby/object:Gem::Dependency
|
126
|
+
name: simplecov
|
127
|
+
requirement: !ruby/object:Gem::Requirement
|
128
|
+
requirements:
|
129
|
+
- - '='
|
130
|
+
- !ruby/object:Gem::Version
|
131
|
+
version: '0.10'
|
132
|
+
type: :development
|
133
|
+
prerelease: false
|
134
|
+
version_requirements: !ruby/object:Gem::Requirement
|
135
|
+
requirements:
|
136
|
+
- - '='
|
137
|
+
- !ruby/object:Gem::Version
|
138
|
+
version: '0.10'
|
111
139
|
- !ruby/object:Gem::Dependency
|
112
140
|
name: oj
|
113
141
|
requirement: !ruby/object:Gem::Requirement
|
@@ -227,12 +255,14 @@ files:
|
|
227
255
|
- lib/saseo/models/version.rb
|
228
256
|
- lib/saseo/persistence.rb
|
229
257
|
- lib/saseo/persistence/consumer.rb
|
258
|
+
- lib/saseo/persistence/message_filter.rb
|
230
259
|
- lib/saseo/persistence/persistor.rb
|
231
260
|
- lib/saseo/publishing.rb
|
232
261
|
- lib/saseo/publishing/data_change_message.rb
|
233
262
|
- lib/saseo/publishing/publisher.rb
|
234
263
|
- lib/saseo/version.rb
|
235
264
|
- lib/saseo/whodunnit.rb
|
265
|
+
- sample_saseo_setup.png
|
236
266
|
- saseo.example.yml
|
237
267
|
- saseo.gemspec
|
238
268
|
- tasks/bump.rake
|