bricolage-streamingload 0.1.0 → 0.3.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: ed2342a1e0f9db4cbe53fbb97be7b72dcd362def
-  data.tar.gz: 1d1705ad9811bb4becee2de4168bb7f40f9a0b63
+  metadata.gz: d81ff86cb9addccb7ca9db4d240218679b1f72f9
+  data.tar.gz: f2ec045c994f1c6b619695f74a13aebcc9318722
 SHA512:
-  metadata.gz: 9a09e1ec19569e03a13e01e8d2aa767de54bd21ac22cb455c9ccb92798a410f655f3435b4b0e7ff4e8bde429dcf2fd904a75f5f3d303f7bf815cfa0ca08b5631
-  data.tar.gz: 3cfb076a0144c04592db35309236e2bfce5021525e34fe26b32e5f61ef000572d8d6641bdbee078f741a0f79a85d9ef9fd4cc47ce167338cec1ad4577c051bd2
+  metadata.gz: a78c3b8f35f8d10cbc3da301667ea811874ccadb7653b02388990d63afc2183ee3d0c4e357dccc7799721e1dcd8e33b88e9b3d66147903c32ff109595bced0f6
+  data.tar.gz: a3632f13d3ea039aa690deca646659e0d89ad636b2af55abf5f7011f54cfecddcef53d57152de513bcbaf5c03671af7a512a10781e7846efc60af9c752b8c364

data/README.md

@@ -11,7 +11,7 @@ See LICENSES file for details.
 
 ## Running Test
 
-% rake test
+% bundle exec rake test
 
 ## Author
 

data/lib/bricolage/snsdatasource.rb

@@ -0,0 +1,40 @@
+require 'bricolage/datasource'
+require 'aws-sdk'
+require 'json'
+require 'time'
+
+module Bricolage
+
+  class SNSTopicDataSource < DataSource
+
+    declare_type 'sns'
+
+    def initialize(region: 'ap-northeast-1', topic_arn:, access_key_id:, secret_access_key:)
+      @region = region
+      @topic_arn = topic_arn
+      @access_key_id = access_key_id
+      @secret_access_key = secret_access_key
+      @client = Aws::SNS::Client.new(region: region, access_key_id: access_key_id, secret_access_key: secret_access_key)
+      @topic = Aws::SNS::Topic.new(topic_arn, client: @client)
+    end
+
+    attr_reader :region
+    attr_reader :client, :topic
+
+    def publish(message)
+      @topic.publish(build_message(message))
+    end
+
+    alias write publish
+
+    def close
+      # do nothing
+    end
+
+    def build_message(message)
+      {message: message}
+    end
+
+  end # SNSDataSource
+
+end   # module Bricolage

data/lib/bricolage/sqsdatasource.rb

@@ -12,14 +12,13 @@ module Bricolage
     declare_type 'sqs'
 
     def initialize(region: 'ap-northeast-1', url:, access_key_id:, secret_access_key:,
-        visibility_timeout:, max_number_of_messages: 10, max_delete_batch_size: 10, wait_time_seconds: 20, noop: false)
+        visibility_timeout:, max_number_of_messages: 10, wait_time_seconds: 20, noop: false)
       @region = region
       @url = url
       @access_key_id = access_key_id
       @secret_access_key = secret_access_key
       @visibility_timeout = visibility_timeout
       @max_number_of_messages = max_number_of_messages
-      @max_delete_batch_size = max_delete_batch_size
       @wait_time_seconds = wait_time_seconds
       @noop = noop
     end
@@ -40,20 +39,21 @@ module Bricolage
     # High-Level Polling Interface
     #
 
-    def main_handler_loop(handlers)
+    def main_handler_loop(handlers:, message_class:)
       trap_signals
 
       n_zero = 0
       until terminating?
         insert_handler_wait(n_zero)
-        n_msg = handle_messages(handlers)
+        n_msg = handle_messages(handlers: handlers, message_class: message_class)
         if n_msg == 0
           n_zero += 1
         else
           n_zero = 0
         end
+        delete_message_buffer.flush
       end
-      @delete_message_buffer.flush if @delete_message_buffer
+      delete_message_buffer.flush_force
       logger.info "shutdown gracefully"
     end
 
@@ -115,8 +115,6 @@ module Bricolage
     def receive_messages
       result = client.receive_message(
         queue_url: @url,
-        attribute_names: ["All"],
-        message_attribute_names: ["All"],
         max_number_of_messages: @max_number_of_messages,
         visibility_timeout: @visibility_timeout,
         wait_time_seconds: @wait_time_seconds
@@ -125,19 +123,18 @@ module Bricolage
     end
 
     def delete_message(msg)
-      # TODO: use batch request?
       client.delete_message(
         queue_url: @url,
         receipt_handle: msg.receipt_handle
       )
     end
 
-    def buffered_delete_message(msg)
+    def delete_message_async(msg)
       delete_message_buffer.put(msg)
     end
 
     def delete_message_buffer
-      @delete_message_buffer ||= DeleteMessageBuffer.new(client, @url, @max_delete_batch_size, logger)
+      @delete_message_buffer ||= DeleteMessageBuffer.new(client, @url, logger)
     end
 
     def put(msg)
@@ -154,60 +151,107 @@ module Bricolage
 
     class DeleteMessageBuffer
 
-      def initialize(sqs_client, url, max_buffer_size, logger)
+      BATCH_SIZE_MAX = 10   # SQS system limit
+      MAX_RETRY_COUNT = 3
+
+      def initialize(sqs_client, url, logger)
         @sqs_client = sqs_client
         @url = url
-        @max_buffer_size = max_buffer_size
         @logger = logger
         @buf = {}
-        @retry_counts = Hash.new(0)
       end
 
-      MAX_RETRY_COUNT = 3
-
       def put(msg)
-        @buf[SecureRandom.uuid] = msg
-        flush if size >= @max_buffer_size
+        ent = Entry.new(msg)
+        @buf[ent.id] = ent
+        flush if full?
+      end
+
+      def empty?
+        @buf.empty?
+      end
+
+      def full?
+        @buf.size >= BATCH_SIZE_MAX
       end
 
       def size
         @buf.size
       end
 
-      def flush
-        return unless size > 0
-        response = @sqs_client.delete_message_batch({
-          queue_url: @url,
-          entries: @buf.to_a.map {|item| {id: item[0], receipt_handle: item[1].receipt_handle} }
-        })
-        clear_successes(response.successful)
-        retry_failures(response.failed)
-        @logger.debug "DeleteMessageBatch executed: #{response.successful.size} succeeded, #{response.failed.size} failed."
+      # Flushes all delayed delete requests, including pending requests
+      def flush_force
+        # retry continues in only 2m, now+1h must be after than all @next_issue_time
+        flush(Time.now + 3600)
       end
 
-      private
-
-      def clear_successes(successes)
-        successes.each do |s|
-          @buf.delete s.id
+      def flush(now = Time.now)
+        entries = @buf.values.select {|ent| ent.issuable?(now) }
+        return if entries.empty?
+        @logger.info "flushing async delete requests"
+        entries.each_slice(BATCH_SIZE_MAX) do |ents|
+          res = @sqs_client.delete_message_batch(queue_url: @url, entries: ents.map(&:request_params))
+          @logger.info "DeleteMessageBatch executed: #{res.successful.size} succeeded, #{res.failed.size} failed"
+          issued_time = Time.now
+          res.successful.each do |s|
+            @buf.delete s.id
+          end
+          res.failed.each do |f|
+            ent = @buf[f.id]
+            unless ent
+              @logger.error "[BUG] no corrensponding DeleteMessageBuffer entry: id=#{f.id}"
+              next
+            end
+            ent.failed!(issued_time)
+            if ent.too_many_failure?
+              @logger.warn "DeleteMessage failure count exceeded the limit; give up: message_id=#{ent.message.message_id}, receipt_handle=#{ent.message.receipt_handle}"
+              @buf.delete f.id
+              next
+            end
+            @logger.info "DeleteMessageBatch partially failed (#{ent.n_failure} times): sender_fault=#{f.sender_fault}, code=#{f.code}, message=#{f.message}"
+          end
         end
       end
 
-      def retry_failures(failures)
-        return unless failures.size > 0
-        failures.each do |f|
-          @logger.info "DeleteMessageBatch failed to retry for: id=#{f.id}, sender_fault=#{f.sender_fault}, code=#{f.code}, message=#{f.message}"
+      class Entry
+        def initialize(msg)
+          @message = msg
+          @id = SecureRandom.uuid
+          @n_failure = 0
+          @last_issued_time = nil
+          @next_issue_time = nil
         end
-        flush
-        @buf.keys.map {|k| @retry_counts[k] += 1 }
-        @retry_counts.select {|k, v| v >= MAX_RETRY_COUNT }.each do |k, v|
-          @logger.warn "DeleteMessageBatch failed #{MAX_RETRY_COUNT} times for: message_id=#{@buf[k].message_id}, receipt_handle=#{@buf[k].receipt_handle}"
-          @buf.delete k
-          @retry_counts.delete k
+
+        attr_reader :id
+        attr_reader :message
+        attr_reader :n_failure
+
+        def issuable?(now)
+          @n_failure == 0 or now > @next_issue_time
+        end
+
+        def failed!(issued_time = Time.now)
+          @n_failure += 1
+          @last_issued_time = issued_time
+          @next_issue_time = @last_issued_time + next_retry_interval
+        end
+
+        def next_retry_interval
+          # 16s, 32s, 64s -> total 2m
+          2 ** (3 + @n_failure)
+        end
+
+        def too_many_failure?
+          # (first request) + (3 retry requests) = (4 requests)
+          @n_failure > MAX_RETRY_COUNT
+        end
+
+        def request_params
+          { id: @id, receipt_handle: @message.receipt_handle }
         end
       end
 
-    end # DeleteMessageBuffer
+    end # class DeleteMessageBuffer
 
   end # class SQSDataSource
 

data/lib/bricolage/streamingload/alertinglogger.rb

@@ -0,0 +1,19 @@
+module Bricolage
+  module StreamingLoad
+    class AlertingLogger
+      extend Forwardable
+
+      def initialize(logger: , sns_datasource: , alert_level: 'warn')
+        @logger = logger
+        @sns_logger = Bricolage::Logger.new(device: sns_datasource)
+        @sns_logger.level = Kernel.const_get("Logger").const_get(alert_level.upcase)
+      end
+
+      %w(log debug info warn error fatal unknown).each do |m|
+        define_method(m) do |*args|
+          [@logger, @sns_logger].map {|t| t.send(m, *args) }
+        end
+      end
+    end
+  end
+end

data/lib/bricolage/streamingload/dispatcher.rb

@@ -5,6 +5,7 @@ require 'bricolage/logger'
 require 'bricolage/streamingload/event'
 require 'bricolage/streamingload/objectbuffer'
 require 'bricolage/streamingload/urlpatterns'
+require 'bricolage/streamingload/alertinglogger'
 require 'aws-sdk'
 require 'yaml'
 require 'optparse'
@@ -29,10 +30,15 @@ module Bricolage
         ctx = Context.for_application('.', environment: opts.environment, logger: logger)
         event_queue = ctx.get_data_source('sqs', config.fetch('event-queue-ds'))
         task_queue = ctx.get_data_source('sqs', config.fetch('task-queue-ds'))
+        alert_logger = AlertingLogger.new(
+          logger: ctx.logger,
+          sns_datasource: ctx.get_data_source('sns', config.fetch('sns-ds')),
+          alert_level: config.fetch('alert-level', 'warn')
+        )
 
         object_buffer = ObjectBuffer.new(
           control_data_source: ctx.get_data_source('sql', config.fetch('ctl-postgres-ds')),
-          logger: ctx.logger
+          logger: alert_logger
         )
 
         url_patterns = URLPatterns.for_config(config.fetch('url_patterns'))
@@ -43,7 +49,7 @@ module Bricolage
           object_buffer: object_buffer,
           url_patterns: url_patterns,
           dispatch_interval: 60,
-          logger: ctx.logger
+          logger: alert_logger
         )
 
         Process.daemon(true) if opts.daemon?
@@ -84,17 +90,18 @@ module Bricolage
 
       def handle_shutdown(e)
         @event_queue.initiate_terminate
+        # Delete this event immediately
         @event_queue.delete_message(e)
       end
 
       def handle_data(e)
         unless e.created?
-          @event_queue.delete_message(e)
+          @event_queue.delete_message_async(e)
           return
         end
         obj = e.loadable_object(@url_patterns)
         @object_buffer.put(obj)
-        @event_queue.buffered_delete_message(e)
+        @event_queue.delete_message_async(e)
       end
 
       def handle_dispatch(e)
@@ -103,18 +110,13 @@ module Bricolage
           tasks.each {|task| @task_queue.put task }
           set_dispatch_timer
         end
+        # Delete this event immediately
         @event_queue.delete_message(e)
       end
 
       def set_dispatch_timer
-        resp = @event_queue.send_message DispatchEvent.create(delay_seconds: @dispatch_interval)
-        @dispatch_message_id = resp.message_id
-      end
-
-      def delete_events(events)
-        events.each do |e|
-          @event_queue.delete_message(e)
-        end
+        res = @event_queue.send_message(DispatchEvent.create(delay_seconds: @dispatch_interval))
+        @dispatch_message_id = res.message_id
       end
 
     end

data/lib/bricolage/streamingload/loaderservice.rb

@@ -1,6 +1,7 @@
 require 'bricolage/sqsdatasource'
 require 'bricolage/streamingload/task'
 require 'bricolage/streamingload/loader'
+require 'bricolage/streamingload/alertinglogger'
 require 'bricolage/logger'
 require 'bricolage/exception'
 require 'bricolage/version'
@@ -25,13 +26,18 @@ module Bricolage
         ctx = Context.for_application('.', environment: opts.environment, logger: logger)
         redshift_ds = ctx.get_data_source('sql', config.fetch('redshift-ds'))
         task_queue = ctx.get_data_source('sqs', config.fetch('task-queue-ds'))
+        alert_logger = AlertingLogger.new(
+          logger: ctx.logger,
+          sns_datasource: ctx.get_data_source('sns', config.fetch('sns-ds')),
+          alert_level: config.fetch('alert-level', 'warn')
+        )
 
         service = new(
           context: ctx,
           control_data_source: ctx.get_data_source('sql', config.fetch('ctl-postgres-ds')),
           data_source: redshift_ds,
           task_queue: task_queue,
-          logger: ctx.logger
+          logger: alert_logger
         )
 
         if opts.task_id
@@ -89,6 +95,7 @@ module Bricolage
         loadtask = load_task(task.id, force: task.force)
         return if loadtask.disabled # skip if disabled, but don't delete sqs msg
         execute_task(loadtask)
+        # Delete load task immediately (do not use async delete)
         @task_queue.delete_message(task)
       end
 

data/lib/bricolage/streamingload/version.rb

@@ -1,5 +1,5 @@
 module Bricolage
   module StreamingLoad
-    VERSION = '0.1.0'
+    VERSION = '0.3.0'
   end
 end

data/test/streamingload/test_event.rb

@@ -5,12 +5,13 @@ module Bricolage::StreamingLoad
 
   class TestEvent < Test::Unit::TestCase
 
-    def new_s3event(message_id: nil, receipt_handle: nil, name: nil, time: nil, region: nil, bucket: nil, key: nil, size: nil)
+    def new_s3event(message_id: nil, receipt_handle: nil, name: nil, time: nil, source: nil, region: nil, bucket: nil, key: nil, size: nil)
       S3ObjectEvent.new(
         message_id: message_id,
         receipt_handle: receipt_handle,
         name: name,
         time: time,
+        source: source,
         region: region,
         bucket: bucket,
         key: key,

data/test/test_sqsdatasource.rb

@@ -0,0 +1,113 @@
+require 'test/unit'
+require 'bricolage/streamingload/event'
+require 'bricolage/logger'
+
+module Bricolage
+
+  class TestSQSDataSource < Test::Unit::TestCase
+
+    def new_sqs_ds(mock_client = nil)
+      SQSDataSource.new(
+        url: 'http://sqs/000000000000/queue-name',
+        access_key_id: 'access_key_id_1',
+        secret_access_key: 'secret_access_key_1',
+        visibility_timeout: 30
+      ).tap {|ds|
+        logger = NullLogger.new
+        #logger = Bricolage::Logger.default
+        ds.__send__(:initialize_base, 'name', nil, logger)
+        ds.instance_variable_set(:@client, mock_client) if mock_client
+      }
+    end
+
+    class MockSQSClient
+      def initialize(&block)
+        @handler = block
+      end
+
+      def delete_message_batch(**args)
+        @handler.call(args)
+      end
+    end
+
+    class NullLogger
+      def debug(*args) end
+      def info(*args) end
+      def warn(*args) end
+      def error(*args) end
+      def exception(*args) end
+      def with_elapsed_time(*args) yield end
+      def elapsed_time(*args) yield end
+    end
+
+    def sqs_message(seq)
+      MockSQSMessage.new("message_id_#{seq}", "receipt_handle_#{seq}")
+    end
+
+    MockSQSMessage = Struct.new(:message_id, :receipt_handle)
+
+    class MockSQSResponse
+      def initialize(successful: [], failed: [])
+        @successful = successful
+        @failed = failed
+      end
+
+      attr_reader :successful
+      attr_reader :failed
+
+      Success = Struct.new(:id)
+      Failure = Struct.new(:id, :sender_fault, :code, :message)
+
+      def add_success_for(ent)
+        @successful.push Success.new(ent[:id])
+      end
+
+      def add_failure_for(ent)
+        @failed.push Failure.new(ent[:id], true, '400', 'some reason')
+      end
+    end
+
+    test "#delete_message_async" do
+      messages = [sqs_message(0), sqs_message(1), sqs_message(2)]
+      mock = MockSQSClient.new {|args|
+        entries = args[:entries]
+        if entries.size == 3
+          # first time
+          assert_equal messages[0].receipt_handle, entries[0][:receipt_handle]
+          assert_equal messages[1].receipt_handle, entries[1][:receipt_handle]
+          assert_equal messages[2].receipt_handle, entries[2][:receipt_handle]
+          MockSQSResponse.new.tap {|res|
+            res.add_success_for(entries[0])
+            res.add_failure_for(entries[1])
+            res.add_success_for(entries[2])
+          }
+        else
+          # second time
+          MockSQSResponse.new.tap {|res|
+            res.add_success_for(entries[0])
+          }
+        end
+      }
+      ds = new_sqs_ds(mock)
+      ds.delete_message_async(messages[0])
+      ds.delete_message_async(messages[1])
+      ds.delete_message_async(messages[2])
+
+      # first flush
+      flush_time = Time.now
+      ds.delete_message_buffer.flush(flush_time)
+      assert_equal 1, ds.delete_message_buffer.size
+      bufent = ds.delete_message_buffer.instance_variable_get(:@buf).values.first
+      assert_equal 'receipt_handle_1', bufent.message.receipt_handle
+      assert_equal 1, bufent.n_failure
+      assert_false bufent.issuable?(flush_time)
+      assert_true bufent.issuable?(flush_time + 180)
+
+      # second flush
+      ds.delete_message_buffer.flush(flush_time + 180)
+      assert_true ds.delete_message_buffer.empty?
+    end
+
+  end
+
+end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: bricolage-streamingload
 version: !ruby/object:Gem::Version
-  version: 0.1.0
+  version: 0.3.0
 platform: ruby
 authors:
 - Minero Aoki
@@ -107,8 +107,10 @@ files:
 - README.md
 - bin/bricolage-streaming-dispatcher
 - bin/bricolage-streaming-loader
+- lib/bricolage/snsdatasource.rb
 - lib/bricolage/sqsdatasource.rb
 - lib/bricolage/sqswrapper.rb
+- lib/bricolage/streamingload/alertinglogger.rb
 - lib/bricolage/streamingload/dispatcher.rb
 - lib/bricolage/streamingload/event.rb
 - lib/bricolage/streamingload/loader.rb
@@ -121,6 +123,7 @@ files:
 - lib/bricolage/streamingload/version.rb
 - test/all.rb
 - test/streamingload/test_event.rb
+- test/test_sqsdatasource.rb
 homepage: https://github.com/aamine/bricolage-streamingload
 licenses:
 - MIT