bricolage-streamingload 0.4.0 → 0.5.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 848b815669c6580505119917a72a4e97833064c2
-  data.tar.gz: 1809410699822e2a60a21407cad1b5814551adcd
+  metadata.gz: 41d99ce0dce7affca77b305ce527c5ec53e733b1
+  data.tar.gz: bd2254dab228018edf50debcf4627deef3bbdbd0
 SHA512:
-  metadata.gz: 84ecbe1e548953cc1f4889eb07e9289101e1ade28e89370dae39462b61d193521ceda25f59d8a0b0f1760b8555cf254abab50cf7ed3d11e70d81c422af2f8b82
-  data.tar.gz: b367bb4faa24e9755bee5dbeaaaf937a317826e8e562466e3264a19563bbfcc877ddba9801ffda4875d21e39feddfcc8e87df81a6433b9cd1c3a2cabcefc0754
+  metadata.gz: 66855f5839f86deb3c6af0500090bed718bd6860b50a83e6a11f1c79e822b81b3d7888ff5417391f4f1b0a669242ed5de3d9a9f4c3fe4e500d88ad77acfbe7fa
+  data.tar.gz: c18b88f15e4cd75095f5cf72863921a43ee467bb1b03f5b3481bf63a09e1a0e319f3afcd4e79e5902bdb6f62695528edd962ed561c29bfc5f6dc173ed029eb89

data/lib/bricolage/streamingload/alertinglogger.rb

@@ -9,11 +9,14 @@ module Bricolage
         @sns_logger.level = Kernel.const_get("Logger").const_get(alert_level.upcase)
       end
 
+      def_delegators '@logger', :level, :level=, :debug?, :info?, :warn?, :error?, :fatal?, :unknown?
+
       %w(log debug info warn error fatal unknown).each do |m|
         define_method(m) do |*args|
           [@logger, @sns_logger].map {|t| t.send(m, *args) }
         end
       end
+
     end
   end
 end

data/lib/bricolage/streamingload/dispatcher.rb

@@ -56,6 +56,9 @@ module Bricolage
         Process.daemon(true) if opts.daemon?
         create_pid_file opts.pid_file_path if opts.pid_file_path
         dispatcher.event_loop
+      rescue Exception => e
+        alert_logger.error e.message
+        raise
       end
 
       def Dispatcher.new_logger(path, config)
@@ -119,18 +122,12 @@ module Bricolage
       def create_checkpoint
         logger.info "*** Creating checkpoint requested ***"
         logger.info "Force-flushing all objects..."
-        flush_all_tasks_immediately
+        tasks = @object_buffer.flush_tasks_force
+        send_tasks tasks
         logger.info "All objects flushed; shutting down..."
         @event_queue.initiate_terminate
       end
 
-      def flush_all_tasks_immediately
-        tasks = @object_buffer.flush_tasks_force
-        tasks.each do |task|
-          @task_queue.put task
-        end
-      end
-
       def handle_data(e)
         unless e.created?
           @event_queue.delete_message_async(e)
@@ -144,7 +141,7 @@ module Bricolage
       def handle_dispatch(e)
         if @dispatch_message_id == e.message_id
           tasks = @object_buffer.flush_tasks
-          tasks.each {|task| @task_queue.put task }
+          send_tasks tasks
           set_dispatch_timer
         end
         # Delete this event immediately
@@ -156,6 +153,20 @@ module Bricolage
         @dispatch_message_id = res.message_id
       end
 
+      def handle_flushtable(e)
+        logger.info "flushing #{e.table_name} requested"
+        tasks = @object_buffer.flush_table_force(e.table_name)
+        send_tasks tasks
+        # Delete this event immediately
+        @event_queue.delete_message(e)
+      end
+
+      def send_tasks(tasks)
+        tasks.each do |task|
+          @task_queue.put task
+        end
+      end
+
     end
 
 

data/lib/bricolage/streamingload/event.rb

@@ -10,7 +10,7 @@ module Bricolage
         case
         when rec['eventName'] == 'shutdown' then ShutdownEvent
         when rec['eventName'] == 'dispatch' then DispatchEvent
-        when rec['eventName'] == 'flush' then FlushEvent
+        when rec['eventName'] == 'flushtable' then FlushTableEvent
         when rec['eventName'] == 'checkpoint' then CheckPointEvent
         when rec['eventSource'] == 'aws:s3'
           S3ObjectEvent
@@ -67,13 +67,13 @@ module Bricolage
     end
 
 
-    class FlushEvent < Event
+    class FlushTableEvent < Event
 
-      def FlushEvent.create(delay_seconds:, table_name:)
-        super name: 'flush', delay_seconds: delay_seconds, table_name: table_name
+      def FlushTableEvent.create(table_name:)
+        super name: 'flushtable', table_name: table_name
       end
 
-      def FlushEvent.parse_sqs_record(msg, rec)
+      def FlushTableEvent.parse_sqs_record(msg, rec)
         {
           table_name: rec['tableName']
         }

data/lib/bricolage/streamingload/loader.rb

@@ -120,6 +120,7 @@ module Bricolage
 
       def write_job_error(status, message)
         @end_time = Time.now
+        @logger.warn message.lines.first
         write_job_result status, message.lines.first.strip[0, MAX_MESSAGE_LENGTH]
       end
 

data/lib/bricolage/streamingload/loaderservice.rb

@@ -49,6 +49,9 @@ module Bricolage
           create_pid_file opts.pid_file_path if opts.pid_file_path
           service.event_loop
         end
+      rescue Exception => e
+        alert_logger.error e.message
+        raise
       end
 
       def LoaderService.new_logger(path, config)
@@ -75,6 +78,8 @@ module Bricolage
         @logger = logger
       end
 
+      attr_reader :logger
+
       def event_loop
         @task_queue.handle_messages(handler: self, message_class: Task)
         @logger.info "shutdown gracefully"

data/lib/bricolage/streamingload/objectbuffer.rb

@@ -55,6 +55,7 @@ module Bricolage
         }
       end
 
+      # Flushes multiple tables periodically
       def flush_tasks
         task_ids = nil
         @ctl_ds.open {|conn|
@@ -84,6 +85,24 @@ module Bricolage
         return task_ids.map {|id| LoadTask.create(task_id: id) }
       end
 
+      # Flushes the all objects of the specified table immediately
+      # with no additional conditions, to create "table checkpoint".
+      def flush_table_force(table_name)
+        task_ids  = []
+        @ctl_ds.open {|conn|
+          conn.transaction {|txn|
+            # insert_task_object_mappings may not consume all saved objects
+            # (e.g. there are too many objects for one table), we must create
+            # tasks repeatedly until there are no unassigned objects.
+            until (ids = insert_table_task_force(conn, table_name)).empty?
+              insert_task_object_mappings(conn)
+              task_ids.concat ids
+            end
+          }
+        }
+        return task_ids.map {|id| LoadTask.create(task_id: id) }
+      end
+
       private
 
       def insert_object(conn, obj)
@@ -185,6 +204,60 @@ module Bricolage
         task_ids
       end
 
+      def insert_table_task_force(conn, table_name)
+        task_ids = conn.query_values(<<-EndSQL)
+            insert into strload_tasks
+                ( task_class
+                , schema_name
+                , table_name
+                , submit_time
+                )
+            select
+                'streaming_load_v3'
+                , tbl.schema_name
+                , tbl.table_name
+                , current_timestamp
+            from
+                strload_tables tbl
+
+                -- The number of objects for each tables, which is not assigned to any task (> 0).
+                -- This subquery is covered by the index.
+                inner join (
+                    select
+                        data_source_id
+                        , count(*) as object_count
+                    from
+                        (
+                            select
+                                min(object_id) as object_id
+                                , object_url
+                            from
+                                strload_objects
+                            where
+                                data_source_id = #{s table_name}
+                            group by
+                                object_url
+                        ) uniq_objects
+                        inner join strload_objects using (object_id)
+                        left outer join strload_task_objects using (object_id)
+                    where
+                        task_id is null -- not assigned to a task
+                    group by
+                        data_source_id
+                ) obj
+                using (data_source_id)
+            where
+                -- does not check disabled
+                data_source_id = #{s table_name}
+            returning task_id
+            ;
+        EndSQL
+
+        # It must be 1
+        @logger.info "Number of task created: #{task_ids.size}"
+        task_ids
+      end
+
       def insert_task_object_mappings(conn)
         conn.update(<<-EndSQL)
             insert into strload_task_objects

data/lib/bricolage/streamingload/version.rb

@@ -1,5 +1,5 @@
 module Bricolage
   module StreamingLoad
-    VERSION = '0.4.0'
+    VERSION = '0.5.0'
   end
 end

data/test/streamingload/test_dispatcher.rb

@@ -69,6 +69,7 @@ module Bricolage
 
         # Event Queue Call Sequence
         hst = event_queue.client.call_history
+        assert_equal 6, hst.size
         assert_equal :send_message, hst[0].name      # start flush timer
         assert_equal :receive_message, hst[1].name
         assert_equal :delete_message_batch, hst[2].name
@@ -78,6 +79,7 @@ module Bricolage
 
         # Task Queue Call Sequence
         hst = task_queue.client.call_history
+        assert_equal 1, hst.size
         assert_equal :send_message, hst[0].name
         assert(/streaming_load_v3/ =~ hst[0].args[:message_body])
         task_id = JSON.load(hst[0].args[:message_body])['Records'][0]['taskId'].to_i
@@ -105,6 +107,100 @@ module Bricolage
         }
       end
 
+      test "flushtable event" do
+        ctx = Context.for_application('.', environment: 'test', logger: NullLogger.new)
+        ctl_ds = ctx.get_data_source('sql', 'dwhctl')
+
+        event_queue = SQSDataSource.new_mock(queue: [
+          # 1st ReceiveMessage
+          [
+            SQSMock::Message.s3_object_created_event('s3://test-bucket/testschema.aaa/datafile-0001.json.gz'),
+            SQSMock::Message.s3_object_created_event('s3://test-bucket/testschema.bbb/datafile-0001.json.gz'),
+            SQSMock::Message.s3_object_created_event('s3://test-bucket/testschema.ccc/datafile-0002.json.gz'),
+            SQSMock::Message.s3_object_created_event('s3://test-bucket/testschema.aaa/datafile-0002.json.gz'),
+            SQSMock::Message.s3_object_created_event('s3://test-bucket/testschema.bbb/datafile-0003.json.gz'),
+            SQSMock::Message.s3_object_created_event('s3://test-bucket/testschema.ccc/datafile-0003.json.gz'),
+            SQSMock::Message.new(body: {eventSource: 'bricolage:system', eventName: 'flushtable', tableName: 'testschema.bbb'}),
+            SQSMock::Message.new(body: {eventSource: 'bricolage:system', eventName: 'shutdown'})
+          ]
+        ])
+
+        task_queue = SQSDataSource.new_mock
+
+        object_buffer = ObjectBuffer.new(
+          control_data_source: ctl_ds,
+          logger: ctx.logger
+        )
+
+        url_patterns = URLPatterns.for_config([
+          {
+            "url" => %r<\As3://test-bucket/testschema\.(?<table>\w+)/datafile-\d{4}\.json\.gz>.source,
+            "schema" => 'testschema',
+            "table" => '%table'
+          }
+        ])
+
+        dispatcher = Dispatcher.new(
+          event_queue: event_queue,
+          task_queue: task_queue,
+          object_buffer: object_buffer,
+          url_patterns: url_patterns,
+          dispatch_interval: 600,
+          logger: ctx.logger
+        )
+
+        # FIXME: database cleaner
+        ctl_ds.open {|conn|
+          conn.update("truncate strload_tables")
+          conn.update("truncate strload_objects")
+          conn.update("truncate strload_task_objects")
+          conn.update("truncate strload_tasks")
+          conn.update("insert into strload_tables values ('testschema', 'aaa', 'testschema.aaa', 100, 1800, false)")
+          conn.update("insert into strload_tables values ('testschema', 'bbb', 'testschema.bbb', 100, 1800, false)")
+          conn.update("insert into strload_tables values ('testschema', 'ccc', 'testschema.ccc', 100, 1800, false)")
+        }
+        dispatcher.event_loop
+
+        # Event Queue Call Sequence
+        hst = event_queue.client.call_history
+        assert_equal 5, hst.size
+        assert_equal :send_message, hst[0].name      # start dispatch timer
+        assert_equal :receive_message, hst[1].name
+        assert_equal :delete_message, hst[2].name    # delete flushtable event
+        assert_equal :delete_message, hst[3].name    # delete shutdown event
+        assert_equal :delete_message_batch, hst[4].name
+
+        # Task Queue Call Sequence
+        hst = task_queue.client.call_history
+        assert_equal 1, hst.size
+        assert_equal :send_message, hst[0].name
+        assert(/streaming_load_v3/ =~ hst[0].args[:message_body])
+        task_id = JSON.load(hst[0].args[:message_body])['Records'][0]['taskId'].to_i
+        assert_not_equal 0, task_id
+
+        # Object Buffer
+        assert_equal [], unassigned_table_objects(ctl_ds, 'testschema.bbb')
+        task = ctl_ds.open {|conn| LoadTask.load(conn, task_id) }
+        assert_equal 'testschema', task.schema
+        assert_equal 'bbb', task.table
+        assert_equal 2, task.object_urls.size
+      end
+
+      def unassigned_table_objects(ctl_ds, table_name)
+        ctl_ds.open {|conn|
+          conn.query_values(<<-EndSQL)
+              select
+                  object_url
+              from
+                  strload_objects
+              where
+                  data_source_id = '#{table_name}'
+                  and object_id not in (select object_id from strload_task_objects)
+              ;
+          EndSQL
+        }
+      end
+
     end
 
   end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: bricolage-streamingload
 version: !ruby/object:Gem::Version
-  version: 0.4.0
+  version: 0.5.0
 platform: ruby
 authors:
 - Minero Aoki
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2016-08-12 00:00:00.000000000 Z
+date: 2016-08-24 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bricolage