saseo 0.3.0 → 0.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/exe/saseo_publisher +5 -1
- data/lib/generators/saseo/templates/add_saseo_trigger_function.rb +2 -7
- data/lib/saseo/config/defaults.rb +3 -1
- data/lib/saseo/config.rb +1 -1
- data/lib/saseo/extensions/active_record/adapter_mixin.rb +1 -39
- data/lib/saseo/persistence/consumer.rb +1 -1
- data/lib/saseo/persistence/persistor.rb +8 -3
- data/lib/saseo/publishing/publisher.rb +15 -32
- data/lib/saseo/version.rb +1 -1
- data/saseo.example.yml +8 -2
- metadata +2 -2
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 8d3227155157a0d8e3df890b3e066aea67e73d68
|
4
|
+
data.tar.gz: 6673eb5e80dca1a684f97708e9f40bfced88b188
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 1f86a332905806cee8e96e3f00104a87fa48de5b1c391e7f9dcaeceaf9d17260e968da2ae1675f31d3fb59e0e5b14418937d55f80d5c1752875c4bd080503a2f
|
7
|
+
data.tar.gz: 68206c9fa409dbdb4430ea92dab8824b7f7d3b9929b89d257506b20ec8f3783bc9317c31fa4ef082b3e21d807dc11f98fa5f951fb5b1cf04c835c2b1391b1b9e
|
data/exe/saseo_publisher
CHANGED
@@ -4,7 +4,8 @@ class AddSaseoTriggerFunction < ActiveRecord::Migration
|
|
4
4
|
uuid_extension_sql = 'CREATE EXTENSION IF NOT EXISTS "uuid-ossp"'
|
5
5
|
|
6
6
|
audit_table_sql = <<SQL
|
7
|
-
CREATE schema saseo;
|
7
|
+
CREATE schema IF NOT EXISTS saseo;
|
8
|
+
DROP TABLE IF EXISTS saseo.saseo_source_versions;
|
8
9
|
|
9
10
|
CREATE TABLE saseo.saseo_source_versions (
|
10
11
|
id uuid PRIMARY KEY NOT NULL DEFAULT uuid_generate_v4(),
|
@@ -41,34 +42,28 @@ DECLARE
|
|
41
42
|
v_old_data json;
|
42
43
|
v_new_data json;
|
43
44
|
v_whodunnit text;
|
44
|
-
v_channel text;
|
45
45
|
v_audit_uuid uuid;
|
46
46
|
BEGIN
|
47
47
|
v_whodunnit := saseo.whodunnit();
|
48
48
|
v_audit_uuid := uuid_generate_v4();
|
49
49
|
|
50
|
-
v_channel := upper('SASEO' || '_' || TG_TABLE_NAME::TEXT || '_' || TG_OP || '_AUDIT');
|
51
|
-
|
52
50
|
IF (TG_OP = 'UPDATE') THEN
|
53
51
|
v_old_data := row_to_json(OLD);
|
54
52
|
v_new_data := row_to_json(NEW);
|
55
53
|
|
56
54
|
INSERT INTO saseo.saseo_source_versions VALUES(v_audit_uuid, txid_current(), TG_TABLE_NAME, DEFAULT, TG_OP, v_whodunnit, v_old_data::JSONB, v_new_data::JSONB);
|
57
|
-
PERFORM pg_notify(v_channel, v_audit_uuid::TEXT);
|
58
55
|
|
59
56
|
RETURN NEW;
|
60
57
|
ELSIF (TG_OP = 'DELETE') THEN
|
61
58
|
v_old_data := row_to_json(OLD);
|
62
59
|
|
63
60
|
INSERT INTO saseo.saseo_source_versions VALUES(v_audit_uuid, txid_current(), TG_TABLE_NAME, DEFAULT, TG_OP, v_whodunnit, v_old_data::JSONB, NULL);
|
64
|
-
PERFORM pg_notify(v_channel, v_audit_uuid::TEXT);
|
65
61
|
|
66
62
|
RETURN OLD;
|
67
63
|
ELSIF (TG_OP = 'INSERT') THEN
|
68
64
|
v_new_data := row_to_json(NEW);
|
69
65
|
|
70
66
|
INSERT INTO saseo.saseo_source_versions VALUES(v_audit_uuid, txid_current(), TG_TABLE_NAME, DEFAULT, TG_OP, v_whodunnit, NULL, v_new_data::JSONB);
|
71
|
-
PERFORM pg_notify(v_channel, v_audit_uuid::TEXT);
|
72
67
|
|
73
68
|
RETURN NEW;
|
74
69
|
ELSE
|
@@ -2,7 +2,7 @@ module Saseo
|
|
2
2
|
class Config
|
3
3
|
module Defaults
|
4
4
|
|
5
|
-
|
5
|
+
PUBLISHER_BATCH_SIZE = 1000
|
6
6
|
SOURCE_DATABASE_URL = nil
|
7
7
|
SOURCE_DATABASE_CONFIG_PATH = 'config/database.yml'
|
8
8
|
DATABASE_URL = nil
|
@@ -27,6 +27,8 @@ module Saseo
|
|
27
27
|
Saseo::Config::Defaults.constants.each do |c|
|
28
28
|
attr_symbol = c.downcase.to_sym
|
29
29
|
base.send(:attr_writer, attr_symbol)
|
30
|
+
|
31
|
+
#TODO base.send(:define_method)
|
30
32
|
base.class_eval %Q{
|
31
33
|
def #{attr_symbol}
|
32
34
|
unless defined? @#{attr_symbol}
|
data/lib/saseo/config.rb
CHANGED
@@ -1,4 +1,5 @@
|
|
1
1
|
require 'active_record/base'
|
2
|
+
require 'saseo/whodunnit'
|
2
3
|
|
3
4
|
module Saseo
|
4
5
|
module Extensions
|
@@ -10,45 +11,6 @@ module Saseo
|
|
10
11
|
execute 'BEGIN'
|
11
12
|
Saseo::Whodunnit.set_db_whodunnit
|
12
13
|
end
|
13
|
-
|
14
|
-
def notify(channel, payload)
|
15
|
-
conn = instance_variable_get(:@connection)
|
16
|
-
conn.async_exec "SELECT pg_notify('#{channel}', '#{payload.to_s}')"
|
17
|
-
end
|
18
|
-
|
19
|
-
# Heavily cribbed from Sequel:
|
20
|
-
# https://github.com/jeremyevans/sequel/blob/c6678741ce34aac52cff966ff6a6288ccb8d5b75/lib/sequel/adapters/postgres.rb#L440
|
21
|
-
def listen(channels:, after_listen: nil, timeout: nil, loop: false, &block)
|
22
|
-
if loop && !block
|
23
|
-
raise ArgumentError, 'calling #listen with :loop requires a block'
|
24
|
-
end
|
25
|
-
channels = Array(channels)
|
26
|
-
|
27
|
-
timeout_block = timeout.respond_to?(:call) ? timeout : proc { timeout }
|
28
|
-
loop_callable = loop.respond_to?(:call)
|
29
|
-
begin
|
30
|
-
conn = instance_variable_get(:@connection)
|
31
|
-
channels.each do |channel|
|
32
|
-
conn.async_exec "LISTEN \"#{channel}\""
|
33
|
-
::ActiveRecord::Base.logger.try(:debug) { "listening to #{channel}..." }
|
34
|
-
|
35
|
-
end
|
36
|
-
after_listen.call(conn) if after_listen
|
37
|
-
|
38
|
-
if loop
|
39
|
-
catch(:stop) do
|
40
|
-
loop do
|
41
|
-
conn.wait_for_notify(timeout_block.call, &block)
|
42
|
-
loop.call(conn) if loop_callable
|
43
|
-
end
|
44
|
-
end
|
45
|
-
else
|
46
|
-
conn.wait_for_notify(timeout_block.call, &block)
|
47
|
-
end
|
48
|
-
ensure
|
49
|
-
conn.async_exec 'UNLISTEN *'
|
50
|
-
end
|
51
|
-
end
|
52
14
|
end
|
53
15
|
end
|
54
16
|
end
|
@@ -26,7 +26,7 @@ module Saseo
|
|
26
26
|
end
|
27
27
|
|
28
28
|
def persist!(message)
|
29
|
-
logger.debug { "
|
29
|
+
logger.debug { "received source version for persistence: #{message.id}" }
|
30
30
|
version = Saseo::Models::Version.new
|
31
31
|
|
32
32
|
version_attributes.each do |attr|
|
@@ -37,8 +37,13 @@ module Saseo
|
|
37
37
|
version.old_data = message.old_data && Oj.dump(message.old_data)
|
38
38
|
version.new_data = message.new_data && Oj.dump(message.new_data)
|
39
39
|
|
40
|
-
|
41
|
-
|
40
|
+
begin
|
41
|
+
logger.debug { version.attributes }
|
42
|
+
version.save!
|
43
|
+
logger.debug { "saved version: #{message.id}" }
|
44
|
+
rescue ActiveRecord::RecordNotUnique => e
|
45
|
+
Philotic::Message.publish({philotic_message_type: :'saseo.error.duplicate_record'}, version.attributes)
|
46
|
+
end
|
42
47
|
end
|
43
48
|
end
|
44
49
|
end
|
@@ -1,5 +1,3 @@
|
|
1
|
-
require 'active_record'
|
2
|
-
require 'saseo/extensions/active_record'
|
3
1
|
require 'saseo/config'
|
4
2
|
require 'saseo/publishing/data_change_message'
|
5
3
|
require 'saseo/models/source/version'
|
@@ -9,47 +7,32 @@ module Saseo
|
|
9
7
|
module Publishing
|
10
8
|
class Publisher
|
11
9
|
|
12
|
-
|
10
|
+
attr_accessor :batch_size
|
13
11
|
attr_writer :logger
|
14
12
|
|
15
|
-
def initialize(
|
16
|
-
|
13
|
+
def initialize(batch_size: nil)
|
14
|
+
batch_size ||= Saseo.config.publisher_batch_size
|
15
|
+
self.batch_size = batch_size
|
17
16
|
end
|
18
17
|
|
19
18
|
def logger
|
20
19
|
@logger ||= Logger.new(STDOUT)
|
21
20
|
end
|
22
21
|
|
23
|
-
def
|
24
|
-
|
25
|
-
|
26
|
-
|
27
|
-
|
28
|
-
|
29
|
-
|
30
|
-
end
|
31
|
-
channels
|
22
|
+
def publish_batch
|
23
|
+
Saseo::Models::Source::Version.transaction do
|
24
|
+
ids = []
|
25
|
+
Saseo::Models::Source::Version.lock.limit(batch_size).each do |record|
|
26
|
+
Saseo::Publishing::DataChangeMessage.publish(record)
|
27
|
+
logger.debug { "published source version: #{record.id}" }
|
28
|
+
ids << record.id
|
32
29
|
end
|
33
|
-
|
34
|
-
|
35
|
-
end
|
36
|
-
end
|
30
|
+
Saseo::Models::Source::Version.delete_all(id: ids)
|
31
|
+
logger.debug { "deleted source versions: #{ids.count}" } if ids.count > 0
|
37
32
|
|
38
|
-
def run(channels: :all, loop: true, timeout: nil)
|
39
|
-
self.channels = channels
|
40
|
-
db_connection.listen channels: self.channels, loop: loop, timeout: timeout do |channel, pid, payload|
|
41
|
-
uuid = payload
|
42
|
-
logger.debug { "received NOTIFY for source version: #{uuid}" }
|
43
|
-
|
44
|
-
record = Saseo::Models::Source::Version.find(uuid)
|
45
|
-
Saseo::Publishing::DataChangeMessage.publish(record)
|
46
|
-
record.delete
|
47
|
-
logger.debug { "deleted source version: #{uuid}" }
|
48
33
|
end
|
49
|
-
|
50
|
-
|
51
|
-
def db_connection
|
52
|
-
Saseo::Models::Source::Base.connection
|
34
|
+
rescue => e
|
35
|
+
logger.error { "Saseo publishing error: #{e.class} - #{e.message} at #{e.backtrace}" }
|
53
36
|
end
|
54
37
|
end
|
55
38
|
end
|
data/lib/saseo/version.rb
CHANGED
data/saseo.example.yml
CHANGED
@@ -1,6 +1,12 @@
|
|
1
1
|
defaults: &defaults
|
2
|
-
|
3
|
-
|
2
|
+
publisher_batch_size: 1000
|
3
|
+
source_database_url: postgres://user:pass@host:port/source
|
4
|
+
source_database_config_path: /path/to/source/db/config
|
5
|
+
database_url: postgres://user:pass@host:port/saseo
|
6
|
+
source_database_config_path: /path/to/saseo/db/config
|
7
|
+
table_name: saseo_audit_versions
|
8
|
+
source_table_schema: audits
|
9
|
+
source_table_name: audits
|
4
10
|
|
5
11
|
development:
|
6
12
|
<<: *defaults
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: saseo
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.
|
4
|
+
version: 0.4.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Nathan Keyes
|
8
8
|
autorequire:
|
9
9
|
bindir: exe
|
10
10
|
cert_chain: []
|
11
|
-
date: 2015-11-
|
11
|
+
date: 2015-11-11 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: awesome_print
|