saseo 0.3.0 → 0.4.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA1:
3
- metadata.gz: 870d97ea47c29c3c88bc485148418518159c1223
4
- data.tar.gz: b53a277d7db97fec52f4844d298af35bc3b01abf
3
+ metadata.gz: 8d3227155157a0d8e3df890b3e066aea67e73d68
4
+ data.tar.gz: 6673eb5e80dca1a684f97708e9f40bfced88b188
5
5
  SHA512:
6
- metadata.gz: dbdef412fd9e9b1d0a539178016a6687efd44293e560fb05fb12955819ffa36e25d91e97617491b382649aa3d05a3b448276f525b4f8e08764200198b3ae1f2f
7
- data.tar.gz: 83ee5c66b3a9d72cf594072999efc04de69ebc5434b04a3dcaefac23ecdadc8f844c4098d9f156f0c3c1d0e6621f6db920fc957fa845a19b2f0582755c52ff8f
6
+ metadata.gz: 1f86a332905806cee8e96e3f00104a87fa48de5b1c391e7f9dcaeceaf9d17260e968da2ae1675f31d3fb59e0e5b14418937d55f80d5c1752875c4bd080503a2f
7
+ data.tar.gz: 68206c9fa409dbdb4430ea92dab8824b7f7d3b9929b89d257506b20ec8f3783bc9317c31fa4ef082b3e21d807dc11f98fa5f951fb5b1cf04c835c2b1391b1b9e
data/exe/saseo_publisher CHANGED
@@ -4,4 +4,8 @@ $:.unshift File.expand_path('../../lib', __FILE__)
4
4
 
5
5
  require 'saseo/publishing/publisher'
6
6
  publisher = Saseo::Publishing::Publisher.new
7
- publisher.run
7
+
8
+
9
+ loop do
10
+ publisher.publish_batch
11
+ end
@@ -4,7 +4,8 @@ class AddSaseoTriggerFunction < ActiveRecord::Migration
4
4
  uuid_extension_sql = 'CREATE EXTENSION IF NOT EXISTS "uuid-ossp"'
5
5
 
6
6
  audit_table_sql = <<SQL
7
- CREATE schema saseo;
7
+ CREATE schema IF NOT EXISTS saseo;
8
+ DROP TABLE IF EXISTS saseo.saseo_source_versions;
8
9
 
9
10
  CREATE TABLE saseo.saseo_source_versions (
10
11
  id uuid PRIMARY KEY NOT NULL DEFAULT uuid_generate_v4(),
@@ -41,34 +42,28 @@ DECLARE
41
42
  v_old_data json;
42
43
  v_new_data json;
43
44
  v_whodunnit text;
44
- v_channel text;
45
45
  v_audit_uuid uuid;
46
46
  BEGIN
47
47
  v_whodunnit := saseo.whodunnit();
48
48
  v_audit_uuid := uuid_generate_v4();
49
49
 
50
- v_channel := upper('SASEO' || '_' || TG_TABLE_NAME::TEXT || '_' || TG_OP || '_AUDIT');
51
-
52
50
  IF (TG_OP = 'UPDATE') THEN
53
51
  v_old_data := row_to_json(OLD);
54
52
  v_new_data := row_to_json(NEW);
55
53
 
56
54
  INSERT INTO saseo.saseo_source_versions VALUES(v_audit_uuid, txid_current(), TG_TABLE_NAME, DEFAULT, TG_OP, v_whodunnit, v_old_data::JSONB, v_new_data::JSONB);
57
- PERFORM pg_notify(v_channel, v_audit_uuid::TEXT);
58
55
 
59
56
  RETURN NEW;
60
57
  ELSIF (TG_OP = 'DELETE') THEN
61
58
  v_old_data := row_to_json(OLD);
62
59
 
63
60
  INSERT INTO saseo.saseo_source_versions VALUES(v_audit_uuid, txid_current(), TG_TABLE_NAME, DEFAULT, TG_OP, v_whodunnit, v_old_data::JSONB, NULL);
64
- PERFORM pg_notify(v_channel, v_audit_uuid::TEXT);
65
61
 
66
62
  RETURN OLD;
67
63
  ELSIF (TG_OP = 'INSERT') THEN
68
64
  v_new_data := row_to_json(NEW);
69
65
 
70
66
  INSERT INTO saseo.saseo_source_versions VALUES(v_audit_uuid, txid_current(), TG_TABLE_NAME, DEFAULT, TG_OP, v_whodunnit, NULL, v_new_data::JSONB);
71
- PERFORM pg_notify(v_channel, v_audit_uuid::TEXT);
72
67
 
73
68
  RETURN NEW;
74
69
  ELSE
@@ -2,7 +2,7 @@ module Saseo
2
2
  class Config
3
3
  module Defaults
4
4
 
5
- CHANNELS = :all
5
+ PUBLISHER_BATCH_SIZE = 1000
6
6
  SOURCE_DATABASE_URL = nil
7
7
  SOURCE_DATABASE_CONFIG_PATH = 'config/database.yml'
8
8
  DATABASE_URL = nil
@@ -27,6 +27,8 @@ module Saseo
27
27
  Saseo::Config::Defaults.constants.each do |c|
28
28
  attr_symbol = c.downcase.to_sym
29
29
  base.send(:attr_writer, attr_symbol)
30
+
31
+ #TODO base.send(:define_method)
30
32
  base.class_eval %Q{
31
33
  def #{attr_symbol}
32
34
  unless defined? @#{attr_symbol}
data/lib/saseo/config.rb CHANGED
@@ -21,7 +21,7 @@ module Saseo
21
21
  def load_config(config)
22
22
  config.each do |k, v|
23
23
  mutator = "#{k}="
24
- send(mutator, v) if respond_to? mutator
24
+ public_send(mutator, v) if respond_to? mutator
25
25
  end
26
26
  end
27
27
 
@@ -1,4 +1,5 @@
1
1
  require 'active_record/base'
2
+ require 'saseo/whodunnit'
2
3
 
3
4
  module Saseo
4
5
  module Extensions
@@ -10,45 +11,6 @@ module Saseo
10
11
  execute 'BEGIN'
11
12
  Saseo::Whodunnit.set_db_whodunnit
12
13
  end
13
-
14
- def notify(channel, payload)
15
- conn = instance_variable_get(:@connection)
16
- conn.async_exec "SELECT pg_notify('#{channel}', '#{payload.to_s}')"
17
- end
18
-
19
- # Heavily cribbed from Sequel:
20
- # https://github.com/jeremyevans/sequel/blob/c6678741ce34aac52cff966ff6a6288ccb8d5b75/lib/sequel/adapters/postgres.rb#L440
21
- def listen(channels:, after_listen: nil, timeout: nil, loop: false, &block)
22
- if loop && !block
23
- raise ArgumentError, 'calling #listen with :loop requires a block'
24
- end
25
- channels = Array(channels)
26
-
27
- timeout_block = timeout.respond_to?(:call) ? timeout : proc { timeout }
28
- loop_callable = loop.respond_to?(:call)
29
- begin
30
- conn = instance_variable_get(:@connection)
31
- channels.each do |channel|
32
- conn.async_exec "LISTEN \"#{channel}\""
33
- ::ActiveRecord::Base.logger.try(:debug) { "listening to #{channel}..." }
34
-
35
- end
36
- after_listen.call(conn) if after_listen
37
-
38
- if loop
39
- catch(:stop) do
40
- loop do
41
- conn.wait_for_notify(timeout_block.call, &block)
42
- loop.call(conn) if loop_callable
43
- end
44
- end
45
- else
46
- conn.wait_for_notify(timeout_block.call, &block)
47
- end
48
- ensure
49
- conn.async_exec 'UNLISTEN *'
50
- end
51
- end
52
14
  end
53
15
  end
54
16
  end
@@ -12,7 +12,7 @@ module Saseo
12
12
 
13
13
  def consume(message)
14
14
  begin
15
- logger.debug { "recieved version: #{message.id}" }
15
+ logger.debug { "received version: #{message.id}" }
16
16
  Saseo::Persistence::Persistor.persist! message
17
17
 
18
18
  acknowledge message
@@ -26,7 +26,7 @@ module Saseo
26
26
  end
27
27
 
28
28
  def persist!(message)
29
- logger.debug { "recieved source version for persistence: #{message.id}" }
29
+ logger.debug { "received source version for persistence: #{message.id}" }
30
30
  version = Saseo::Models::Version.new
31
31
 
32
32
  version_attributes.each do |attr|
@@ -37,8 +37,13 @@ module Saseo
37
37
  version.old_data = message.old_data && Oj.dump(message.old_data)
38
38
  version.new_data = message.new_data && Oj.dump(message.new_data)
39
39
 
40
- version.save!
41
- logger.debug { "saved version: #{message.id}" }
40
+ begin
41
+ logger.debug { version.attributes }
42
+ version.save!
43
+ logger.debug { "saved version: #{message.id}" }
44
+ rescue ActiveRecord::RecordNotUnique => e
45
+ Philotic::Message.publish({philotic_message_type: :'saseo.error.duplicate_record'}, version.attributes)
46
+ end
42
47
  end
43
48
  end
44
49
  end
@@ -1,5 +1,3 @@
1
- require 'active_record'
2
- require 'saseo/extensions/active_record'
3
1
  require 'saseo/config'
4
2
  require 'saseo/publishing/data_change_message'
5
3
  require 'saseo/models/source/version'
@@ -9,47 +7,32 @@ module Saseo
9
7
  module Publishing
10
8
  class Publisher
11
9
 
12
- attr_reader :channels
10
+ attr_accessor :batch_size
13
11
  attr_writer :logger
14
12
 
15
- def initialize(channels: nil)
16
- self.channels = channels
13
+ def initialize(batch_size: nil)
14
+ batch_size ||= Saseo.config.publisher_batch_size
15
+ self.batch_size = batch_size
17
16
  end
18
17
 
19
18
  def logger
20
19
  @logger ||= Logger.new(STDOUT)
21
20
  end
22
21
 
23
- def channels=(val)
24
- if val == :all
25
- tables = db_connection.tables
26
-
27
- @channels = tables.reduce([]) do |channels, table|
28
- %w[insert update delete].each do |operation|
29
- channels << "SASEO_#{table}_#{operation}_AUDIT".upcase
30
- end
31
- channels
22
+ def publish_batch
23
+ Saseo::Models::Source::Version.transaction do
24
+ ids = []
25
+ Saseo::Models::Source::Version.lock.limit(batch_size).each do |record|
26
+ Saseo::Publishing::DataChangeMessage.publish(record)
27
+ logger.debug { "published source version: #{record.id}" }
28
+ ids << record.id
32
29
  end
33
- else
34
- @channels = Array(val).flatten
35
- end
36
- end
30
+ Saseo::Models::Source::Version.delete_all(id: ids)
31
+ logger.debug { "deleted source versions: #{ids.count}" } if ids.count > 0
37
32
 
38
- def run(channels: :all, loop: true, timeout: nil)
39
- self.channels = channels
40
- db_connection.listen channels: self.channels, loop: loop, timeout: timeout do |channel, pid, payload|
41
- uuid = payload
42
- logger.debug { "received NOTIFY for source version: #{uuid}" }
43
-
44
- record = Saseo::Models::Source::Version.find(uuid)
45
- Saseo::Publishing::DataChangeMessage.publish(record)
46
- record.delete
47
- logger.debug { "deleted source version: #{uuid}" }
48
33
  end
49
- end
50
-
51
- def db_connection
52
- Saseo::Models::Source::Base.connection
34
+ rescue => e
35
+ logger.error { "Saseo publishing error: #{e.class} - #{e.message} at #{e.backtrace}" }
53
36
  end
54
37
  end
55
38
  end
data/lib/saseo/version.rb CHANGED
@@ -1,3 +1,3 @@
1
1
  module Saseo
2
- VERSION = "0.3.0"
2
+ VERSION = "0.4.0"
3
3
  end
data/saseo.example.yml CHANGED
@@ -1,6 +1,12 @@
1
1
  defaults: &defaults
2
- channels:
3
- - SASEO_AUDIT
2
+ publisher_batch_size: 1000
3
+ source_database_url: postgres://user:pass@host:port/source
4
+ source_database_config_path: /path/to/source/db/config
5
+ database_url: postgres://user:pass@host:port/saseo
6
+ source_database_config_path: /path/to/saseo/db/config
7
+ table_name: saseo_audit_versions
8
+ source_table_schema: audits
9
+ source_table_name: audits
4
10
 
5
11
  development:
6
12
  <<: *defaults
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: saseo
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.3.0
4
+ version: 0.4.0
5
5
  platform: ruby
6
6
  authors:
7
7
  - Nathan Keyes
8
8
  autorequire:
9
9
  bindir: exe
10
10
  cert_chain: []
11
- date: 2015-11-09 00:00:00.000000000 Z
11
+ date: 2015-11-11 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  name: awesome_print