bricolage-streamingload 0.13.0 → 0.14.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 1aacc6209260b4c74e823fc9ea903bdd811c4d2b
-  data.tar.gz: c0b869e51e67a22708dd36b22afefb275e9a3169
+  metadata.gz: 523da5342b4a48ba2c46e2a2565bf1561570e7a7
+  data.tar.gz: '02229582ac7ee8ca27a86eaef69a4c4824735e8e'
 SHA512:
-  metadata.gz: b6721bcb5c9dd506c801e71970e450e6a9318eac5171df099906ef91236e516a14f155cdd952240a534b80ab82771a88e22db49b309c8ab9cb210e1ae0747f1a
-  data.tar.gz: 95469bdc023a41ab08f4b0d6df776b95ef733949a87fb5e053f9bf0b1d74c75f35cde9b89335fe12ea486a214364cddcb9c263c7340bbdbdbd5f16f541f66653
+  metadata.gz: 17e440e8400bf237cc4d19bd01c6f16641e6e49a595a757e2e2925de9ae5645796f29acda03904230e0bf591311b44267f2cd77ecec03245c8a553bba94e4b93
+  data.tar.gz: 0ffbc7699e531268e471d769024b64281111fa44c5fb4d3e94245202e0ee6504996db814a29eb497d339f2c20f38af63ec37de5731bc23e272191d04dc296a4c

data/lib/bricolage/streamingload/dispatcher.rb

@@ -7,6 +7,7 @@ require 'bricolage/streamingload/dispatchermessage'
 require 'bricolage/streamingload/loadermessage'
 require 'bricolage/streamingload/chunkrouter'
 require 'bricolage/streamingload/chunkbuffer'
+require 'bricolage/streamingload/loadtasklogger'
 require 'bricolage/streamingload/alertinglogger'
 require 'aws-sdk'
 require 'yaml'
@@ -48,11 +49,16 @@ module Bricolage
 
         chunk_router = ChunkRouter.for_config(config.fetch('url_patterns'))
 
+        task_logger = LoadTaskLogger.new(
+          ctx.get_data_source('s3', config.fetch('ctl-s3-ds', 's3_ctl'))
+        )
+
         dispatcher = Dispatcher.new(
           event_queue: event_queue,
           task_queue: task_queue,
           chunk_router: chunk_router,
           chunk_buffer: chunk_buffer,
+          task_logger: task_logger,
           dispatch_interval: config.fetch('dispatch-interval', 60),
           logger: logger
         )
@@ -60,13 +66,13 @@ module Bricolage
         Process.daemon(true) if opts.daemon?
         create_pid_file opts.pid_file_path if opts.pid_file_path
         Dir.chdir '/'
-        dispatcher.event_loop
-      rescue SystemExit
-        ;
-      rescue Exception => e
-        logger.exception e
-        logger.error "dispatcher abort: pid=#{$$}"
-        raise
+        begin
+          dispatcher.event_loop
+        rescue Exception => e
+          logger.exception e
+          logger.error "dispatcher abort: pid=#{$$}"
+          raise
+        end
       end
 
       def Dispatcher.new_logger(path, config)
@@ -85,11 +91,12 @@ module Bricolage
         # ignore
       end
 
-      def initialize(event_queue:, task_queue:, chunk_router:, chunk_buffer:, dispatch_interval:, logger:)
+      def initialize(event_queue:, task_queue:, chunk_router:, chunk_buffer:, task_logger:, dispatch_interval:, logger:)
         @event_queue = event_queue
         @task_queue = task_queue
         @chunk_router = chunk_router
         @chunk_buffer = chunk_buffer
+        @task_logger = task_logger
         @dispatch_interval = dispatch_interval
         @dispatch_message_id = nil
         @logger = logger
@@ -198,6 +205,7 @@ module Bricolage
         tasks.each do |task|
           msg = StreamingLoadV3LoaderMessage.for_load_task(task)
           @task_queue.put msg
+          @task_logger.log task
         end
       end
 

data/lib/bricolage/streamingload/loadtasklogger.rb

@@ -0,0 +1,37 @@
+require 'stringio'
+
+module Bricolage
+
+  module StreamingLoad
+
+    class LoadTaskLogger
+
+      def initialize(s3ds)
+        @s3ds = s3ds
+      end
+
+      def log(task)
+        csv = content(task)
+        key = key(task)
+        @logger.info "s3: put: #{@s3ds.url(key)}"
+        @s3ds.object(key).put(body: csv)
+      end
+
+      def key(task)
+        now = Time.now
+        "task/#{now.strftime('%Y/%m/%d')}/task-#{task.id}.csv"
+      end
+
+      def content(task)
+        buf = StringIO.new
+        tasks.chunks.each do |chunk|
+          buf.puts %Q("#{task.id}","#{chunk.id}","#{chunk.url}")
+        end
+        buf.string
+      end
+
+    end
+
+  end
+
+end

data/lib/bricolage/streamingload/version.rb

@@ -1,5 +1,5 @@
 module Bricolage
   module StreamingLoad
-    VERSION = '0.13.0'
+    VERSION = '0.14.0'
   end
 end

data/test/streamingload/test_dispatcher.rb

@@ -50,11 +50,14 @@ module Bricolage
           }
         ])
 
+        task_logger = DummyLoadTaskLogger.new
+
         dispatcher = Dispatcher.new(
           event_queue: event_queue,
           task_queue: task_queue,
           chunk_buffer: chunk_buffer,
           chunk_router: chunk_router,
+          task_logger: task_logger,
           dispatch_interval: 600,
           logger: ctx.logger
         )
@@ -93,6 +96,10 @@ module Bricolage
         assert_equal 'testschema', task.schema_name
         assert_equal 'desttable', task.table_name
         assert_equal 10, task.object_urls.size
+
+        # Task Logger
+        assert_not_nil task_logger.last_task
+        assert_equal task_id, task_logger.last_task.id.to_i
       end
 
       def unassigned_objects(ctl_ds)
@@ -142,11 +149,14 @@ module Bricolage
           }
         ])
 
+        task_logger = DummyLoadTaskLogger.new
+
         dispatcher = Dispatcher.new(
           event_queue: event_queue,
           task_queue: task_queue,
           chunk_buffer: chunk_buffer,
           chunk_router: chunk_router,
+          task_logger: task_logger,
           dispatch_interval: 600,
           logger: ctx.logger
         )
@@ -186,6 +196,10 @@ module Bricolage
         assert_equal 'testschema', task.schema_name
         assert_equal 'bbb', task.table_name
         assert_equal 2, task.object_urls.size
+
+        # Task Logger
+        assert_not_nil task_logger.last_task
+        assert_equal task_id, task_logger.last_task.id.to_i
       end
 
       def unassigned_table_objects(ctl_ds, table_name)
@@ -205,5 +219,23 @@ module Bricolage
 
     end
 
+
+    class DummyLoadTaskLogger
+
+      def initialize
+        @task = nil
+      end
+
+      def log(task)
+        @task = task
+        nil
+      end
+
+      def last_task
+        @task
+      end
+
+    end
+
   end
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: bricolage-streamingload
 version: !ruby/object:Gem::Version
-  version: 0.13.0
+  version: 0.14.0
 platform: ruby
 authors:
 - Minero Aoki
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2018-01-05 00:00:00.000000000 Z
+date: 2018-01-09 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bricolage
@@ -130,6 +130,7 @@ files:
 - lib/bricolage/streamingload/jobparams.rb
 - lib/bricolage/streamingload/loadermessage.rb
 - lib/bricolage/streamingload/loadtask.rb
+- lib/bricolage/streamingload/loadtasklogger.rb
 - lib/bricolage/streamingload/manifest.rb
 - lib/bricolage/streamingload/taskhandler.rb
 - lib/bricolage/streamingload/version.rb