bricolage-streamingload 0.6.0 → 0.6.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 17aa54eda4d063cb571f3a7671a4e6413ea079e1
-  data.tar.gz: 1e879a10e505c01a9f66393a079e18de997a3478
+  metadata.gz: b2043736baf77c16a2df73b7233e7f1839ea8c44
+  data.tar.gz: 4418e6433eaee7eca223f073a8d684b6325ac2c8
 SHA512:
-  metadata.gz: 5f5778b2ecff8e2bf8d06e1ab00e8e1ee394772a24e4812b74b921c8303566638a212632c896b9b46c7ed5918357db89c65116595664628dba1c2e3cfbef3375
-  data.tar.gz: ebb53db3b87334f5c725e28665f9716f2246d1aef4389d635627e31b07c356e0f9a9edea451162ca39505da82dba22f5fd710d8bbdcb24f8e0fd583c2c734955
+  metadata.gz: 078a0cb67b3d26a3f7dceedeedfb6e7631d39199d2d7dd762067be06e29dd1b26eb696d7d1f0fef305415b156b4b4507855170c065a7316e1d5325ee0bfb3608
+  data.tar.gz: 239686023066c23a915d68e313a7ed49fe015d2671c33f3e7a51f48ac1ed14d1354c6a117177dc66986da6d1a5ff20e65e34660f54b6bdc50edc203ed47295db

data/lib/bricolage/sqsdatasource.rb

@@ -349,7 +349,7 @@ module Bricolage
     def SQSMessage.for_sqs_result(result)
       result.messages.flat_map {|msg|
         body = JSON.parse(msg.body)
-        records = body['Records'] or next []
+        records = body['Records'] or next [UnknownSQSMessage.for_sqs_record(msg, nil)]
         records.map {|rec| get_concrete_class(msg, rec).for_sqs_record(msg, rec) }
       }
     end
@@ -361,17 +361,21 @@ module Bricolage
     end
 
     def SQSMessage.parse_sqs_record(msg, rec)
-      time_str = rec['eventTime']
-      tm = time_str ? (Time.parse(time_str) rescue nil) : nil
       {
         message_id: msg.message_id,
         receipt_handle: msg.receipt_handle,
-        name: rec['eventName'],
-        time: tm,
-        source: rec['eventSource']
+        name: (rec ? rec['eventName'] : nil),
+        time: get_event_time(rec),
+        source: (rec ? rec['eventSource'] : nil)
       }
     end
 
+    def SQSMessage.get_event_time(rec)
+      return nil unless rec
+      str = rec['eventTime'] or return nil
+      Time.parse(str) rescue nil
+    end
+
     def initialize(name:, time:, source:,
         message_id: nil, receipt_handle: nil, delay_seconds: nil,
         **message_params)
@@ -416,4 +420,23 @@ module Bricolage
 
   end   # class SQSMessage
 
+
+  class UnknownSQSMessage < SQSMessage
+
+    def UnknownSQSMessage.parse_sqs_record(msg, _rec)
+      { message_body: msg.body }
+    end
+
+    def message_type
+      'unknown'
+    end
+
+    def init_message(message_body:)
+      @message_body = message_body
+    end
+
+    attr_reader :message_body
+
+  end   # class UnknownSQSMessage
+
 end   # module Bricolage

data/lib/bricolage/streamingload/dispatcher.rb

@@ -27,19 +27,22 @@ module Bricolage
         end
         config_path, * = opts.rest_arguments
         config = YAML.load(File.read(config_path))
-        logger = opts.log_file_path ? new_logger(opts.log_file_path, config) : nil
-        ctx = Context.for_application('.', environment: opts.environment, logger: logger)
+        log = opts.log_file_path ? new_logger(opts.log_file_path, config) : nil
+        ctx = Context.for_application('.', environment: opts.environment, logger: log)
+        logger = raw_logger = ctx.logger
         event_queue = ctx.get_data_source('sqs', config.fetch('event-queue-ds', 'sqs_event'))
         task_queue = ctx.get_data_source('sqs', config.fetch('task-queue-ds', 'sqs_task'))
-        alert_logger = AlertingLogger.new(
-          logger: ctx.logger,
-          sns_datasource: ctx.get_data_source('sns', config.fetch('sns-ds', 'sns')),
-          alert_level: config.fetch('alert-level', 'warn')
-        )
+        if config['alert-level']
+          logger = AlertingLogger.new(
+            logger: raw_logger,
+            sns_datasource: ctx.get_data_source('sns', config.fetch('sns-ds', 'sns')),
+            alert_level: config.fetch('alert-level', 'warn')
+          )
+        end
 
         object_buffer = ObjectBuffer.new(
-          control_data_source: ctx.get_data_source('sql', config.fetch('ctl-postgres-ds', 'db_data')),
-          logger: alert_logger
+          control_data_source: ctx.get_data_source('sql', config.fetch('ctl-postgres-ds', 'db_ctl')),
+          logger: logger
         )
 
         url_patterns = URLPatterns.for_config(config.fetch('url_patterns'))
@@ -50,14 +53,15 @@ module Bricolage
           object_buffer: object_buffer,
           url_patterns: url_patterns,
           dispatch_interval: 60,
-          logger: alert_logger
+          logger: logger
         )
 
         Process.daemon(true) if opts.daemon?
         create_pid_file opts.pid_file_path if opts.pid_file_path
         dispatcher.event_loop
       rescue Exception => e
-        alert_logger.error e.message
+        logger.exception e
+        logger.error "dispatcher abort: pid=#{$$}"
         raise
       end
 
@@ -92,11 +96,11 @@ module Bricolage
       attr_reader :logger
 
       def event_loop
-        logger.info "dispatcher started"
+        logger.info "*** dispatcher started: pid=#{$$}"
         set_dispatch_timer
         @event_queue.handle_messages(handler: self, message_class: Event)
         @event_queue.process_async_delete_force
-        logger.info "shutdown gracefully"
+        logger.info "*** shutdown gracefully: pid=#{$$}"
       end
 
       # override
@@ -105,6 +109,7 @@ module Bricolage
         @event_queue.process_async_delete
 
         if @dispatch_requested
+          logger.info "*** dispatch requested"
           dispatch_tasks
           @dispatch_requested = false
         end
@@ -115,7 +120,13 @@ module Bricolage
         end
       end
 
+      def handle_unknown(e)
+        logger.warn "unknown event: #{e.message_body}"
+        @event_queue.delete_message_async(e)
+      end
+
       def handle_shutdown(e)
+        logger.info "*** shutdown requested"
         @event_queue.initiate_terminate
         # Delete this event immediately
         @event_queue.delete_message(e)
@@ -130,7 +141,7 @@ module Bricolage
       end
 
       def create_checkpoint
-        logger.info "*** Creating checkpoint requested ***"
+        logger.info "*** checkpoint requested"
         logger.info "Force-flushing all objects..."
         tasks = @object_buffer.flush_tasks_force
         send_tasks tasks
@@ -149,7 +160,6 @@ module Bricolage
       end
 
       def handle_dispatch(e)
-        logger.info "dispatching tasks requested"
         # Dispatching tasks may takes 10 minutes or more, it can exceeds visibility timeout.
         # To avoid this, delay dispatching until all events of current message batch are processed.
         if @dispatch_message_id == e.message_id
@@ -170,7 +180,7 @@ module Bricolage
       end
 
       def handle_flushtable(e)
-        logger.info "flushing #{e.table_name} requested"
+        logger.info "*** flushtable requested: table=#{e.table_name}"
         tasks = @object_buffer.flush_table_force(e.table_name)
         send_tasks tasks
         # Delete this event immediately

data/lib/bricolage/streamingload/event.rb

@@ -12,10 +12,8 @@ module Bricolage
         when rec['eventName'] == 'dispatch' then DispatchEvent
         when rec['eventName'] == 'flushtable' then FlushTableEvent
         when rec['eventName'] == 'checkpoint' then CheckPointEvent
-        when rec['eventSource'] == 'aws:s3'
-          S3ObjectEvent
-        else
-          raise "[FATAL] unknown SQS message record: eventSource=#{rec['eventSource']} event=#{rec['eventName']} message_id=#{msg.message_id}"
+        when !!rec['s3'] then S3ObjectEvent
+        else UnknownSQSMessage
         end
       end
 

data/lib/bricolage/streamingload/loaderservice.rb

@@ -105,6 +105,12 @@ module Bricolage
         @ctl_ds.open {|conn| LoadTask.load(conn, task_id, force: force) }
       end
 
+      # message handler
+      def handle_unknown(task)
+        @logger.warn "unknown task: #{task.message_body}"
+        @task_queue.delete_message task
+      end
+
       # message handler
       def handle_streaming_load_v3(task)
         Dir.chdir(@working_dir) {

data/lib/bricolage/streamingload/task.rb

@@ -10,8 +10,7 @@ module Bricolage
       def Task.get_concrete_class(msg, rec)
         case
         when rec['eventName'] == 'streaming_load_v3' then LoadTask
-        else
-          raise "[FATAL] unknown SQS message record: eventSource=#{rec['eventSource']} event=#{rec['eventName']} message_id=#{msg.message_id}"
+        else UnknownSQSMessage
         end
       end
 

data/lib/bricolage/streamingload/version.rb

@@ -1,5 +1,5 @@
 module Bricolage
   module StreamingLoad
-    VERSION = '0.6.0'
+    VERSION = '0.6.1'
   end
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: bricolage-streamingload
 version: !ruby/object:Gem::Version
-  version: 0.6.0
+  version: 0.6.1
 platform: ruby
 authors:
 - Minero Aoki