saseo 0.3.0 → 0.4.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 870d97ea47c29c3c88bc485148418518159c1223
-  data.tar.gz: b53a277d7db97fec52f4844d298af35bc3b01abf
+  metadata.gz: 8d3227155157a0d8e3df890b3e066aea67e73d68
+  data.tar.gz: 6673eb5e80dca1a684f97708e9f40bfced88b188
 SHA512:
-  metadata.gz: dbdef412fd9e9b1d0a539178016a6687efd44293e560fb05fb12955819ffa36e25d91e97617491b382649aa3d05a3b448276f525b4f8e08764200198b3ae1f2f
-  data.tar.gz: 83ee5c66b3a9d72cf594072999efc04de69ebc5434b04a3dcaefac23ecdadc8f844c4098d9f156f0c3c1d0e6621f6db920fc957fa845a19b2f0582755c52ff8f
+  metadata.gz: 1f86a332905806cee8e96e3f00104a87fa48de5b1c391e7f9dcaeceaf9d17260e968da2ae1675f31d3fb59e0e5b14418937d55f80d5c1752875c4bd080503a2f
+  data.tar.gz: 68206c9fa409dbdb4430ea92dab8824b7f7d3b9929b89d257506b20ec8f3783bc9317c31fa4ef082b3e21d807dc11f98fa5f951fb5b1cf04c835c2b1391b1b9e

data/exe/saseo_publisher

@@ -4,4 +4,8 @@ $:.unshift File.expand_path('../../lib', __FILE__)
 
 require 'saseo/publishing/publisher'
 publisher = Saseo::Publishing::Publisher.new
-publisher.run
+
+
+loop do
+  publisher.publish_batch
+end

data/lib/generators/saseo/templates/add_saseo_trigger_function.rb

@@ -4,7 +4,8 @@ class AddSaseoTriggerFunction < ActiveRecord::Migration
     uuid_extension_sql = 'CREATE EXTENSION IF NOT EXISTS "uuid-ossp"'
 
     audit_table_sql = <<SQL
-CREATE schema saseo;
+CREATE schema IF NOT EXISTS saseo;
+DROP TABLE IF EXISTS saseo.saseo_source_versions;
 
 CREATE TABLE saseo.saseo_source_versions (
   id uuid PRIMARY KEY NOT NULL DEFAULT uuid_generate_v4(),
@@ -41,34 +42,28 @@ DECLARE
   v_old_data json;
   v_new_data json;
   v_whodunnit text;
-  v_channel text;
   v_audit_uuid uuid;
 BEGIN
    v_whodunnit := saseo.whodunnit();
    v_audit_uuid := uuid_generate_v4();
 
-  v_channel := upper('SASEO' || '_' || TG_TABLE_NAME::TEXT || '_' || TG_OP || '_AUDIT');
-
   IF (TG_OP = 'UPDATE') THEN
     v_old_data := row_to_json(OLD);
     v_new_data := row_to_json(NEW);
 
     INSERT INTO saseo.saseo_source_versions VALUES(v_audit_uuid, txid_current(), TG_TABLE_NAME, DEFAULT, TG_OP, v_whodunnit, v_old_data::JSONB, v_new_data::JSONB);
-    PERFORM pg_notify(v_channel, v_audit_uuid::TEXT);
 
     RETURN NEW;
   ELSIF (TG_OP = 'DELETE') THEN
     v_old_data := row_to_json(OLD);
 
     INSERT INTO saseo.saseo_source_versions VALUES(v_audit_uuid, txid_current(), TG_TABLE_NAME, DEFAULT, TG_OP, v_whodunnit, v_old_data::JSONB, NULL);
-    PERFORM pg_notify(v_channel, v_audit_uuid::TEXT);
 
     RETURN OLD;
   ELSIF (TG_OP = 'INSERT') THEN
     v_new_data := row_to_json(NEW);
 
     INSERT INTO saseo.saseo_source_versions VALUES(v_audit_uuid, txid_current(), TG_TABLE_NAME, DEFAULT, TG_OP, v_whodunnit, NULL, v_new_data::JSONB);
-    PERFORM pg_notify(v_channel, v_audit_uuid::TEXT);
 
     RETURN NEW;
   ELSE

data/lib/saseo/config/defaults.rb

@@ -2,7 +2,7 @@ module Saseo
   class Config
     module Defaults
 
-      CHANNELS                       = :all
+      PUBLISHER_BATCH_SIZE           = 1000
       SOURCE_DATABASE_URL            = nil
       SOURCE_DATABASE_CONFIG_PATH    = 'config/database.yml'
       DATABASE_URL                   = nil
@@ -27,6 +27,8 @@ module Saseo
         Saseo::Config::Defaults.constants.each do |c|
           attr_symbol = c.downcase.to_sym
           base.send(:attr_writer, attr_symbol)
+
+          #TODO base.send(:define_method)
           base.class_eval %Q{
           def #{attr_symbol}
             unless defined? @#{attr_symbol}

data/lib/saseo/config.rb

@@ -21,7 +21,7 @@ module Saseo
     def load_config(config)
       config.each do |k, v|
         mutator = "#{k}="
-        send(mutator, v) if respond_to? mutator
+        public_send(mutator, v) if respond_to? mutator
       end
     end
 

data/lib/saseo/extensions/active_record/adapter_mixin.rb

@@ -1,4 +1,5 @@
 require 'active_record/base'
+require 'saseo/whodunnit'
 
 module Saseo
   module Extensions
@@ -10,45 +11,6 @@ module Saseo
           execute 'BEGIN'
           Saseo::Whodunnit.set_db_whodunnit
         end
-
-        def notify(channel, payload)
-            conn = instance_variable_get(:@connection)
-            conn.async_exec "SELECT pg_notify('#{channel}', '#{payload.to_s}')"
-        end
-
-        # Heavily cribbed from Sequel:
-        # https://github.com/jeremyevans/sequel/blob/c6678741ce34aac52cff966ff6a6288ccb8d5b75/lib/sequel/adapters/postgres.rb#L440
-        def listen(channels:, after_listen: nil, timeout: nil, loop: false, &block)
-          if loop && !block
-            raise ArgumentError, 'calling #listen with :loop requires a block'
-          end
-          channels = Array(channels)
-
-          timeout_block = timeout.respond_to?(:call) ? timeout : proc { timeout }
-          loop_callable = loop.respond_to?(:call)
-            begin
-              conn = instance_variable_get(:@connection)
-              channels.each do |channel|
-                conn.async_exec "LISTEN \"#{channel}\""
-                ::ActiveRecord::Base.logger.try(:debug) { "listening to #{channel}..." }
-
-              end
-              after_listen.call(conn) if after_listen
-
-              if loop
-                catch(:stop) do
-                  loop do
-                    conn.wait_for_notify(timeout_block.call, &block)
-                    loop.call(conn) if loop_callable
-                  end
-                end
-              else
-                conn.wait_for_notify(timeout_block.call, &block)
-              end
-            ensure
-              conn.async_exec 'UNLISTEN *'
-            end
-          end
       end
     end
   end

data/lib/saseo/persistence/consumer.rb

@@ -12,7 +12,7 @@ module Saseo
 
       def consume(message)
         begin
-          logger.debug { "recieved version: #{message.id}" }
+          logger.debug { "received version: #{message.id}" }
           Saseo::Persistence::Persistor.persist! message
 
           acknowledge message

data/lib/saseo/persistence/persistor.rb

@@ -26,7 +26,7 @@ module Saseo
       end
 
       def persist!(message)
-        logger.debug { "recieved source version for persistence: #{message.id}" }
+        logger.debug { "received source version for persistence: #{message.id}" }
         version = Saseo::Models::Version.new
 
         version_attributes.each do |attr|
@@ -37,8 +37,13 @@ module Saseo
         version.old_data = message.old_data && Oj.dump(message.old_data)
         version.new_data = message.new_data && Oj.dump(message.new_data)
 
-        version.save!
-        logger.debug { "saved version: #{message.id}" }
+        begin
+          logger.debug { version.attributes }
+          version.save!
+          logger.debug { "saved version: #{message.id}" }
+        rescue ActiveRecord::RecordNotUnique => e
+          Philotic::Message.publish({philotic_message_type: :'saseo.error.duplicate_record'}, version.attributes)
+        end
       end
     end
   end

data/lib/saseo/publishing/publisher.rb

@@ -1,5 +1,3 @@
-require 'active_record'
-require 'saseo/extensions/active_record'
 require 'saseo/config'
 require 'saseo/publishing/data_change_message'
 require 'saseo/models/source/version'
@@ -9,47 +7,32 @@ module Saseo
   module Publishing
     class Publisher
 
-      attr_reader :channels
+      attr_accessor :batch_size
       attr_writer :logger
 
-      def initialize(channels: nil)
-        self.channels = channels
+      def initialize(batch_size: nil)
+        batch_size      ||= Saseo.config.publisher_batch_size
+        self.batch_size = batch_size
       end
 
       def logger
         @logger ||= Logger.new(STDOUT)
       end
 
-      def channels=(val)
-        if val == :all
-          tables = db_connection.tables
-
-          @channels = tables.reduce([]) do |channels, table|
-            %w[insert update delete].each do |operation|
-              channels << "SASEO_#{table}_#{operation}_AUDIT".upcase
-            end
-            channels
+      def publish_batch
+        Saseo::Models::Source::Version.transaction do
+          ids = []
+          Saseo::Models::Source::Version.lock.limit(batch_size).each do |record|
+            Saseo::Publishing::DataChangeMessage.publish(record)
+            logger.debug { "published source version: #{record.id}" }
+            ids << record.id
           end
-        else
-          @channels = Array(val).flatten
-        end
-      end
+          Saseo::Models::Source::Version.delete_all(id: ids)
+          logger.debug { "deleted source versions: #{ids.count}" } if ids.count > 0
 
-      def run(channels: :all, loop: true, timeout: nil)
-        self.channels = channels
-        db_connection.listen channels: self.channels, loop: loop, timeout: timeout do |channel, pid, payload|
-          uuid = payload
-          logger.debug { "received NOTIFY for source version: #{uuid}" }
-
-          record = Saseo::Models::Source::Version.find(uuid)
-          Saseo::Publishing::DataChangeMessage.publish(record)
-          record.delete
-          logger.debug { "deleted source version: #{uuid}" }
         end
-      end
-
-      def db_connection
-        Saseo::Models::Source::Base.connection
+      rescue => e
+        logger.error { "Saseo publishing error: #{e.class} - #{e.message} at #{e.backtrace}" }
       end
     end
   end

data/lib/saseo/version.rb

@@ -1,3 +1,3 @@
 module Saseo
-  VERSION = "0.3.0"
+  VERSION = "0.4.0"
 end

data/saseo.example.yml

@@ -1,6 +1,12 @@
 defaults: &defaults
-  channels:
-  - SASEO_AUDIT
+  publisher_batch_size: 1000
+  source_database_url: postgres://user:pass@host:port/source
+  source_database_config_path: /path/to/source/db/config
+  database_url: postgres://user:pass@host:port/saseo
+  source_database_config_path: /path/to/saseo/db/config
+  table_name: saseo_audit_versions
+  source_table_schema: audits
+  source_table_name: audits
 
 development:
   <<: *defaults

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: saseo
 version: !ruby/object:Gem::Version
-  version: 0.3.0
+  version: 0.4.0
 platform: ruby
 authors:
 - Nathan Keyes
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2015-11-09 00:00:00.000000000 Z
+date: 2015-11-11 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: awesome_print