bricolage-streamingload 0.12.0 → 0.13.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: c8c9c822c1c2d207827cc5d0b6573140254b1638
-  data.tar.gz: 166188b17014074c82a963aa9aecaff796ced70d
+  metadata.gz: 1aacc6209260b4c74e823fc9ea903bdd811c4d2b
+  data.tar.gz: c0b869e51e67a22708dd36b22afefb275e9a3169
 SHA512:
-  metadata.gz: 17e35b2d5d49312d92a4f9f7e232cd6ff9daeb7975d7a1383936a935d12dd22c89550e1fece87920c36cac46fea6aa5517431c7af56e741cb5fe774a21cd7825
-  data.tar.gz: 7556f6eb45b74d041891f3d9254aee4bf32c30d12e9aa61093351f5a08af4c7f6a04926b75a37d8cfbde4c359a50e5efc610cd6ceda605f63ca147e3d96d3285
+  metadata.gz: b6721bcb5c9dd506c801e71970e450e6a9318eac5171df099906ef91236e516a14f155cdd952240a534b80ab82771a88e22db49b309c8ab9cb210e1ae0747f1a
+  data.tar.gz: 95469bdc023a41ab08f4b0d6df776b95ef733949a87fb5e053f9bf0b1d74c75f35cde9b89335fe12ea486a214364cddcb9c263c7340bbdbdbd5f16f541f66653

data/lib/bricolage/streamingload/chunk.rb

@@ -0,0 +1,19 @@
+module Bricolage
+
+  module StreamingLoad
+
+    class Chunk
+      def initialize(id:, url:, size: nil)
+        @id = id
+        @url = url
+        @size = size
+      end
+
+      attr_reader :id
+      attr_reader :url
+      attr_reader :size
+    end
+
+  end
+
+end

data/lib/bricolage/streamingload/{objectbuffer.rb → chunkbuffer.rb}

@@ -1,43 +1,12 @@
-require 'bricolage/streamingload/task'
+require 'bricolage/streamingload/loadtask'
+require 'bricolage/streamingload/chunk'
 require 'bricolage/sqlutils'
-require 'forwardable'
 
 module Bricolage
 
   module StreamingLoad
 
-    class LoadableObject
-
-      extend Forwardable
-
-      def initialize(event, components)
-        @event = event
-        @components = components
-      end
-
-      attr_reader :event
-
-      def_delegator '@event', :url
-      def_delegator '@event', :size
-      def_delegator '@event', :message_id
-      def_delegator '@event', :receipt_handle
-      def_delegator '@components', :schema_name
-      def_delegator '@components', :table_name
-
-      def data_source_id
-        "#{schema_name}.#{table_name}"
-      end
-
-      alias qualified_name data_source_id
-
-      def event_time
-        @event.time
-      end
-
-    end
-
-
-    class ObjectBuffer
+    class ChunkBuffer
 
       TASK_GENERATION_TIME_LIMIT = 30 #sec
 
@@ -49,74 +18,83 @@ module Bricolage
         @task_generation_time_limit = TASK_GENERATION_TIME_LIMIT
       end
 
-      def put(obj)
+      # chunk :: IncomingChunk
+      def save(chunk)
         @ctl_ds.open {|conn|
           suppress_sql_logging {
             conn.transaction {
-              object_id = insert_object(conn, obj)
+              object_id = insert_object(conn, chunk)
               if object_id
                 insert_task_objects(conn, object_id)
               else
-                insert_dup_object(conn, obj)
+                @logger.info "Duplicated object recieved: url=#{chunk.url}"
+                insert_dup_object(conn, chunk)
               end
             }
           }
         }
       end
 
-      # Flushes multiple tables periodically
-      def flush_tasks
-        task_ids = []
-        warn_slow_task_generation {
-          @ctl_ds.open {|conn|
+      # Flushes chunks of multiple streams, which are met conditions.
+      def flush_partial
+        task_ids = nil
+        tasks = nil
+
+        @ctl_ds.open {|conn|
+          warn_slow_task_generation {
             conn.transaction {|txn|
               task_ids = insert_tasks(conn)
-              unless task_ids.empty?
-                update_task_object_mappings(conn, task_ids)
-                log_mapped_object_num(conn, task_ids)
-              end
+              update_task_objects(conn, task_ids) unless task_ids.empty?
             }
           }
+          log_task_ids(task_ids)
+          tasks = load_tasks(conn, task_ids)
         }
-        return task_ids.map {|id| LoadTask.create(task_id: id) }
+        tasks
       end
 
-      # Flushes all objects of all tables immediately with no
-      # additional conditions, to create "stream checkpoint".
-      def flush_tasks_force
-        task_ids  = []
+      # Flushes all chunks of all stream with no additional conditions,
+      # to create "system checkpoint".
+      def flush_all
+        all_task_ids = []
+        tasks = nil
+
         @ctl_ds.open {|conn|
           conn.transaction {|txn|
-            # update_task_object_mappings may not consume all saved objects
+            # update_task_objects may not consume all saved objects
             # (e.g. there are too many objects for one table), we must create
-            # tasks repeatedly until there are no unassigned objects.
-            until (ids = insert_tasks_force(conn)).empty?
-              update_task_object_mappings(conn, ids)
-              log_mapped_object_num(conn, ids)
-              task_ids.concat ids
+            # tasks repeatedly until all objects are flushed.
+            until (task_ids = insert_tasks(conn, force: true)).empty?
+              update_task_objects(conn, task_ids)
+              all_task_ids.concat task_ids
             end
           }
+          log_task_ids(all_task_ids)
+          tasks = load_tasks(conn, all_task_ids)
         }
-        return task_ids.map {|id| LoadTask.create(task_id: id) }
+        tasks
       end
 
-      # Flushes the all objects of the specified table immediately
-      # with no additional conditions, to create "table checkpoint".
-      def flush_table_force(table_name)
-        task_ids  = []
+      # Flushes all chunks of the specified stream with no additional conditions,
+      # to create "stream checkpoint".
+      def flush_stream(stream_name)
+        all_task_ids = []
+        tasks = nil
+
         @ctl_ds.open {|conn|
           conn.transaction {|txn|
-            # update_task_object_mappings may not consume all saved objects
+            # update_task_objects may not consume all saved objects
             # (e.g. there are too many objects for one table), we must create
-            # tasks repeatedly until there are no unassigned objects.
-            until (ids = insert_table_task_force(conn, table_name)).empty?
-              update_task_object_mappings(conn, ids)
-              log_mapped_object_num(conn, ids)
-              task_ids.concat ids
+            # tasks repeatedly until all objects are flushed.
+            until (task_ids = insert_tasks_for_stream(conn, stream_name)).empty?
+              update_task_objects(conn, task_ids)
+              all_task_ids.concat task_ids
             end
           }
+          log_task_ids(all_task_ids)
+          tasks = load_tasks(conn, all_task_ids)
         }
-        return task_ids.map {|id| LoadTask.create(task_id: id) }
+        tasks
       end
 
       private
@@ -134,7 +112,7 @@ module Bricolage
             values
                 ( #{s obj.url}
                 , #{obj.size}
-                , #{s obj.data_source_id}
+                , #{s obj.stream_name}
                 , #{s obj.message_id}
                 , '#{obj.event_time}' AT TIME ZONE 'JST'
                 , current_timestamp
@@ -148,7 +126,6 @@ module Bricolage
       end
 
       def insert_dup_object(conn, obj)
-        @logger.info "Duplicated object recieved: object_url=#{obj.url}"
         conn.update(<<-EndSQL)
             insert into strload_dup_objects
                 ( object_url
@@ -161,7 +138,7 @@ module Bricolage
             values
                 ( #{s obj.url}
                 , #{obj.size}
-                , #{s obj.data_source_id}
+                , #{s obj.stream_name}
                 , #{s obj.message_id}
                 , '#{obj.event_time}' AT TIME ZONE 'JST'
                 , current_timestamp
@@ -184,10 +161,6 @@ module Bricolage
         EndSQL
       end
 
-      def insert_tasks_force(conn)
-        insert_tasks(conn, force: true)
-      end
-
       def insert_tasks(conn, force: false)
         task_ids = conn.query_values(<<-EndSQL)
             insert into strload_tasks
@@ -239,11 +212,10 @@ module Bricolage
             ;
         EndSQL
 
-        log_created_tasks task_ids
         task_ids
       end
 
-      def insert_table_task_force(conn, table_name)
+      def insert_tasks_for_stream(conn, stream_name)
         task_ids = conn.query_values(<<-EndSQL)
             insert into strload_tasks
                 ( task_class
@@ -273,17 +245,15 @@ module Bricolage
                 using (data_source_id)
             where
                 -- does not check disabled
-                data_source_id = #{s table_name}
+                data_source_id = #{s stream_name}
             returning task_id
             ;
         EndSQL
 
-        # It must be 1
-        log_created_tasks(task_ids)
         task_ids
       end
 
-      def update_task_object_mappings(conn, task_ids)
+      def update_task_objects(conn, task_ids)
         conn.update(<<-EndSQL)
             update strload_task_objects dst
             set
@@ -309,25 +279,36 @@ module Bricolage
                 and tsk_obj.object_seq <= tables.load_batch_size
             ;
         EndSQL
+        # UPDATE statement cannot return values
+        nil
       end
 
-      def log_mapped_object_num(conn, task_ids)
-        # This method is required since UPDATE does not "returning" multiple values
-        rows = conn.query_values(<<-EndSQL)
-            select
-                task_id
-                , count(*)
-            from
-                strload_task_objects
-            where
-                task_id in (#{task_ids.join(',')})
-            group by
-                task_id
-            ;
-        EndSQL
-        rows.each_slice(2) do |task_id, object_count|
-          @logger.info "Number of objects assigned to task: task_id=#{task_id} object_count=#{object_count}"
-        end
+      def load_tasks(conn, task_ids)
+        return [] if task_ids.empty?
+
+        records = suppress_sql_logging {
+          conn.query_rows(<<-EndSQL)
+              select
+                  t.task_id
+                  , t.object_id
+                  , o.object_url
+                  , o.object_size
+              from
+                  strload_task_objects t
+                  inner join strload_objects o using (object_id)
+              where
+                  task_id in (#{task_ids.join(',')})
+              ;
+          EndSQL
+        }
+
+        records.group_by {|row| row['task_id'] }.map {|task_id, rows|
+          chunks = rows.map {|row|
+            id, url, size = row.values_at('object_id', 'object_url', 'object_size')
+            Chunk.new(id: id, url: url, size: size)
+          }
+          LoadTask.new(id: task_id, chunks: chunks)
+        }
       end
 
       def suppress_sql_logging
@@ -341,7 +322,7 @@ module Bricolage
         end
       end
 
-      def log_created_tasks(task_ids)
+      def log_task_ids(task_ids)
         created_task_num = task_ids.size
         @logger.info "Number of task created: #{created_task_num}"
         @logger.info "Created task ids: #{task_ids}" if created_task_num > 0

data/lib/bricolage/streamingload/chunkrouter.rb

@@ -0,0 +1,57 @@
+require 'bricolage/streamingload/incomingchunk'
+
+module Bricolage
+
+  module StreamingLoad
+
+    class ChunkRoutingFailed < StandardError; end
+
+
+    class ChunkRouter
+
+      def ChunkRouter.for_config(configs)
+        new(configs.map {|c|
+          Route.new(url: c.fetch('url'), schema: c.fetch('schema'), table: c.fetch('table'))
+        })
+      end
+
+      def initialize(routes)
+        @routes = routes
+      end
+
+      def route(msg)
+        @routes.each do |route|
+          stream_name = route.match(msg.url)
+          return IncomingChunk.new(msg, stream_name) if stream_name
+        end
+        raise ChunkRoutingFailed, "could not detect stream name: #{url.inspect}"
+      end
+
+      class Route
+        def initialize(url:, schema:, table:)
+          @url_pattern = /\A#{url}\z/
+          @schema = schema
+          @table = table
+        end
+
+        def match(url)
+          m = @url_pattern.match(url) or return nil
+          c1 = get_component(m, @schema)
+          c2 = get_component(m, @table)
+          "#{c1}.#{c2}"
+        end
+
+        def get_component(m, label)
+          if /\A%/ =~ label
+            m[label[1..-1]]
+          else
+            label
+          end
+        end
+      end
+
+    end
+
+  end   # module StreamingLoad
+
+end   # module Bricolage

data/lib/bricolage/streamingload/dispatcher.rb

@@ -3,9 +3,10 @@ require 'bricolage/exception'
 require 'bricolage/version'
 require 'bricolage/sqsdatasource'
 require 'bricolage/logger'
-require 'bricolage/streamingload/event'
-require 'bricolage/streamingload/objectbuffer'
-require 'bricolage/streamingload/urlpatterns'
+require 'bricolage/streamingload/dispatchermessage'
+require 'bricolage/streamingload/loadermessage'
+require 'bricolage/streamingload/chunkrouter'
+require 'bricolage/streamingload/chunkbuffer'
 require 'bricolage/streamingload/alertinglogger'
 require 'aws-sdk'
 require 'yaml'
@@ -40,18 +41,18 @@ module Bricolage
           )
         end
 
-        object_buffer = ObjectBuffer.new(
+        chunk_buffer = ChunkBuffer.new(
           control_data_source: ctx.get_data_source('sql', config.fetch('ctl-postgres-ds', 'db_ctl')),
           logger: logger
         )
 
-        url_patterns = URLPatterns.for_config(config.fetch('url_patterns'))
+        chunk_router = ChunkRouter.for_config(config.fetch('url_patterns'))
 
         dispatcher = Dispatcher.new(
           event_queue: event_queue,
           task_queue: task_queue,
-          object_buffer: object_buffer,
-          url_patterns: url_patterns,
+          chunk_router: chunk_router,
+          chunk_buffer: chunk_buffer,
           dispatch_interval: config.fetch('dispatch-interval', 60),
           logger: logger
         )
@@ -60,6 +61,8 @@ module Bricolage
         create_pid_file opts.pid_file_path if opts.pid_file_path
         Dir.chdir '/'
         dispatcher.event_loop
+      rescue SystemExit
+        ;
       rescue Exception => e
         logger.exception e
         logger.error "dispatcher abort: pid=#{$$}"
@@ -82,11 +85,11 @@ module Bricolage
         # ignore
       end
 
-      def initialize(event_queue:, task_queue:, object_buffer:, url_patterns:, dispatch_interval:, logger:)
+      def initialize(event_queue:, task_queue:, chunk_router:, chunk_buffer:, dispatch_interval:, logger:)
         @event_queue = event_queue
         @task_queue = task_queue
-        @object_buffer = object_buffer
-        @url_patterns = url_patterns
+        @chunk_router = chunk_router
+        @chunk_buffer = chunk_buffer
         @dispatch_interval = dispatch_interval
         @dispatch_message_id = nil
         @logger = logger
@@ -99,7 +102,7 @@ module Bricolage
       def event_loop
         logger.info "*** dispatcher started: pid=#{$$}"
         set_dispatch_timer
-        @event_queue.handle_messages(handler: self, message_class: Event)
+        @event_queue.handle_messages(handler: self, message_class: DispatcherMessage)
         @event_queue.process_async_delete_force
         logger.info "*** shutdown gracefully: pid=#{$$}"
       end
@@ -111,86 +114,90 @@ module Bricolage
 
         if @dispatch_requested
           logger.info "*** dispatch requested"
-          dispatch_tasks
+          do_handle_dispatch
           @dispatch_requested = false
         end
 
         if @checkpoint_requested
-          create_checkpoint
+          do_handle_checkpoint
           @checkpoint_requested = false   # is needless, but reset it just in case
         end
       end
 
-      def handle_unknown(e)
-        logger.warn "unknown event: #{e.message_body}"
-        @event_queue.delete_message_async(e)
+      def handle_unknown(msg)
+        logger.warn "unknown event: #{msg.message_body}"
+        @event_queue.delete_message_async(msg)
       end
 
-      def handle_shutdown(e)
+      def handle_shutdown(msg)
         logger.info "*** shutdown requested"
         @event_queue.initiate_terminate
         # Delete this event immediately
-        @event_queue.delete_message(e)
+        @event_queue.delete_message(msg)
       end
 
-      def handle_checkpoint(e)
+      def handle_checkpoint(msg)
         # Delay creating CHECKPOINT after the current message batch,
         # because any other extra events are already received.
         @checkpoint_requested = true
         # Delete this event immediately
-        @event_queue.delete_message(e)
+        @event_queue.delete_message(msg)
       end
 
-      def create_checkpoint
+      def do_handle_checkpoint
         logger.info "*** checkpoint requested"
         logger.info "Force-flushing all objects..."
-        tasks = @object_buffer.flush_tasks_force
-        send_tasks tasks
+        tasks = @chunk_buffer.flush_all
+        dispatch_tasks tasks
         logger.info "All objects flushed; shutting down..."
         @event_queue.initiate_terminate
       end
 
-      def handle_data(e)
-        unless e.created?
-          @event_queue.delete_message_async(e)
+      def handle_data(msg)
+        unless msg.created_event?
+          @event_queue.delete_message_async(msg)
           return
         end
-        obj = e.loadable_object(@url_patterns)
-        @object_buffer.put(obj)
-        @event_queue.delete_message_async(e)
+        chunk = @chunk_router.route(msg)
+        @chunk_buffer.save(chunk)
+        @event_queue.delete_message_async(msg)
       end
 
-      def handle_dispatch(e)
+      def handle_dispatch(msg)
         # Dispatching tasks may takes 10 minutes or more, it can exceeds visibility timeout.
         # To avoid this, delay dispatching until all events of current message batch are processed.
-        if @dispatch_message_id == e.message_id
+        if @dispatch_message_id == msg.message_id
           @dispatch_requested = true
         end
-        @event_queue.delete_message_async(e)
+        @event_queue.delete_message_async(msg)
       end
 
-      def dispatch_tasks
-        tasks = @object_buffer.flush_tasks
-        send_tasks tasks
+      def do_handle_dispatch
+        tasks = @chunk_buffer.flush_partial
+        dispatch_tasks tasks
         set_dispatch_timer
       end
 
       def set_dispatch_timer
-        res = @event_queue.send_message(DispatchEvent.create(delay_seconds: @dispatch_interval))
+        res = @event_queue.send_message(DispatchDispatcherMessage.create(delay_seconds: @dispatch_interval))
         @dispatch_message_id = res.message_id
       end
 
-      def handle_flushtable(e)
-        logger.info "*** flushtable requested: table=#{e.table_name}"
-        tasks = @object_buffer.flush_table_force(e.table_name)
-        send_tasks tasks
+      def handle_flushtable(msg)
+        # FIXME: badly named attribute. table_name is really stream_name, which is called as data_source_id, too.
+        stream_name = msg.table_name
+
+        logger.info "*** flushtable requested: stream_name=#{stream_name}"
+        tasks = @chunk_buffer.flush_stream(stream_name)
+        dispatch_tasks tasks
         # Delete this event immediately
-        @event_queue.delete_message(e)
+        @event_queue.delete_message(msg)
       end
 
-      def send_tasks(tasks)
+      def dispatch_tasks(tasks)
         tasks.each do |task|
-          @task_queue.put task
+          msg = StreamingLoadV3LoaderMessage.for_load_task(task)
+          @task_queue.put msg
         end
       end
 
@@ -207,9 +214,6 @@ module Bricolage
         @rest_arguments = nil
 
         @opts = opts = OptionParser.new("Usage: #{$0} CONFIG_PATH")
-        opts.on('--task-id=id', 'Execute oneshot load task (implicitly disables daemon mode).') {|task_id|
-          @task_id = task_id
-        }
         opts.on('-e', '--environment=NAME', "Sets execution environment [default: #{Context::DEFAULT_ENV}]") {|env|
           @environment = env
         }

data/lib/bricolage/streamingload/{event.rb → dispatchermessage.rb}

@@ -1,18 +1,19 @@
+require 'bricolage/streamingload/chunk'
 require 'bricolage/sqsdatasource'
 
 module Bricolage
 
   module StreamingLoad
 
-    class Event < SQSMessage
+    class DispatcherMessage < SQSMessage
 
-      def Event.get_concrete_class(msg, rec)
+      def DispatcherMessage.get_concrete_class(msg, rec)
         case
-        when rec['eventName'] == 'shutdown' then ShutdownEvent
-        when rec['eventName'] == 'dispatch' then DispatchEvent
-        when rec['eventName'] == 'flushtable' then FlushTableEvent
-        when rec['eventName'] == 'checkpoint' then CheckPointEvent
-        when !!rec['s3'] then S3ObjectEvent
+        when rec['eventName'] == 'shutdown' then ShutdownDispatcherMessage
+        when rec['eventName'] == 'dispatch' then DispatchDispatcherMessage
+        when rec['eventName'] == 'flushtable' then FlushTableDispatcherMessage
+        when rec['eventName'] == 'checkpoint' then CheckPointDispatcherMessage
+        when !!rec['s3'] then S3ObjectDispatcherMessage
         else UnknownSQSMessage
         end
       end
@@ -28,13 +29,13 @@ module Bricolage
     end
 
 
-    class ShutdownEvent < Event
+    class ShutdownDispatcherMessage < DispatcherMessage
 
-      def ShutdownEvent.create
+      def ShutdownDispatcherMessage.create
         super name: 'shutdown'
       end
 
-      def ShutdownEvent.parse_sqs_record(msg, rec)
+      def ShutdownDispatcherMessage.parse_sqs_record(msg, rec)
         {}
       end
 
@@ -47,13 +48,13 @@ module Bricolage
 
 
     # Flushes all tables and shutdown
-    class CheckPointEvent < Event
+    class CheckPointDispatcherMessage < DispatcherMessage
 
-      def CheckPointEvent.create
+      def CheckPointDispatcherMessage.create
         super name: 'checkpoint'
       end
 
-      def CheckPointEvent.parse_sqs_record(msg, rec)
+      def CheckPointDispatcherMessage.parse_sqs_record(msg, rec)
         {}
       end
 
@@ -65,13 +66,13 @@ module Bricolage
     end
 
 
-    class FlushTableEvent < Event
+    class FlushTableDispatcherMessage < DispatcherMessage
 
-      def FlushTableEvent.create(table_name:)
+      def FlushTableDispatcherMessage.create(table_name:)
         super name: 'flushtable', table_name: table_name
       end
 
-      def FlushTableEvent.parse_sqs_record(msg, rec)
+      def FlushTableDispatcherMessage.parse_sqs_record(msg, rec)
         {
           table_name: rec['tableName']
         }
@@ -94,9 +95,9 @@ module Bricolage
     end
 
 
-    class DispatchEvent < Event
+    class DispatchDispatcherMessage < DispatcherMessage
 
-      def DispatchEvent.create(delay_seconds:)
+      def DispatchDispatcherMessage.create(delay_seconds:)
         super name: 'dispatch', delay_seconds: delay_seconds
       end
 
@@ -108,9 +109,9 @@ module Bricolage
     end
 
 
-    class S3ObjectEvent < Event
+    class S3ObjectDispatcherMessage < DispatcherMessage
 
-      def S3ObjectEvent.parse_sqs_record(msg, rec)
+      def S3ObjectDispatcherMessage.parse_sqs_record(msg, rec)
         {
           region: rec['awsRegion'],
           bucket: rec['s3']['bucket']['name'],
@@ -144,12 +145,12 @@ module Bricolage
         true
       end
 
-      def created?
+      def created_event?
         !!(/\AObjectCreated:(?!Copy)/ =~ @name)
       end
 
-      def loadable_object(url_patterns)
-        LoadableObject.new(self, url_patterns.match(url))
+      def chunk
+        Chunk.new(id: nil, url: url, size: size)
       end
 
     end

data/lib/bricolage/streamingload/incomingchunk.rb

@@ -0,0 +1,35 @@
+require 'forwardable'
+
+module Bricolage
+
+  module StreamingLoad
+
+    # a Chunk which is not saved yet (received from SQS)
+    class IncomingChunk
+
+      extend Forwardable
+
+      def initialize(message, stream_name)
+        @chunk = message.chunk
+        @message = message
+        @stream_name = stream_name
+      end
+
+      def_delegator '@chunk', :id
+      def_delegator '@chunk', :url
+      def_delegator '@chunk', :size
+
+      def_delegator '@message', :message_id
+      def_delegator '@message', :receipt_handle
+
+      def event_time
+        @message.time
+      end
+
+      attr_reader :stream_name
+
+    end
+
+  end
+
+end

data/lib/bricolage/streamingload/{task.rb → loadermessage.rb}

@@ -4,11 +4,11 @@ module Bricolage
 
   module StreamingLoad
 
-    class Task < SQSMessage
+    class LoaderMessage < SQSMessage
 
-      def Task.get_concrete_class(msg, rec)
+      def LoaderMessage.get_concrete_class(msg, rec)
         case
-        when rec['eventName'] == 'streaming_load_v3' then LoadTask
+        when rec['eventName'] == 'streaming_load_v3' then StreamingLoadV3LoaderMessage
         else UnknownSQSMessage
         end
       end
@@ -24,13 +24,17 @@ module Bricolage
     end
 
 
-    class LoadTask < Task
+    class StreamingLoadV3LoaderMessage < LoaderMessage
 
-      def LoadTask.create(task_id:, force: false)
+      def StreamingLoadV3LoaderMessage.for_load_task(load_task)
+        create(task_id: load_task.id)
+      end
+
+      def StreamingLoadV3LoaderMessage.create(task_id:, force: false)
         super name: 'streaming_load_v3', task_id: task_id, force: force
       end
 
-      def LoadTask.parse_sqs_record(msg, rec)
+      def StreamingLoadV3LoaderMessage.parse_sqs_record(msg, rec)
         {
           task_id: rec['taskId'],
           force: (rec['force'].to_s == 'true')

data/lib/bricolage/streamingload/loadtask.rb

@@ -0,0 +1,17 @@
+module Bricolage
+
+  module StreamingLoad
+
+    class LoadTask
+      def initialize(id:, chunks: [])
+        @id = id
+        @chunks = chunks
+      end
+
+      attr_reader :id
+      attr_reader :chunks
+    end
+
+  end
+
+end

data/lib/bricolage/streamingload/taskhandler.rb

@@ -1,6 +1,6 @@
 require 'bricolage/context'
 require 'bricolage/sqsdatasource'
-require 'bricolage/streamingload/task'
+require 'bricolage/streamingload/loadermessage'
 require 'bricolage/streamingload/job'
 require 'bricolage/streamingload/alertinglogger'
 require 'bricolage/logger'
@@ -113,7 +113,7 @@ module Bricolage
       end
 
       def event_loop
-        @task_queue.handle_messages(handler: self, message_class: Task)
+        @task_queue.handle_messages(handler: self, message_class: LoaderMessage)
       end
 
       # message handler

data/lib/bricolage/streamingload/version.rb

@@ -1,5 +1,5 @@
 module Bricolage
   module StreamingLoad
-    VERSION = '0.12.0'
+    VERSION = '0.13.0'
   end
 end

data/test/streamingload/test_dispatcher.rb

@@ -3,6 +3,8 @@ require 'bricolage/context'
 require 'bricolage/sqsdatasource'
 require 'bricolage/sqsmock'
 require 'bricolage/streamingload/dispatcher'
+require 'bricolage/streamingload/chunkrouter'
+require 'bricolage/streamingload/chunkbuffer'
 
 module Bricolage
   module StreamingLoad
@@ -35,12 +37,12 @@ module Bricolage
 
         task_queue = SQSDataSource.new_mock
 
-        object_buffer = ObjectBuffer.new(
+        chunk_buffer = ChunkBuffer.new(
           control_data_source: ctl_ds,
           logger: ctx.logger
         )
 
-        url_patterns = URLPatterns.for_config([
+        chunk_router = ChunkRouter.for_config([
           {
             "url" => %r<\As3://test-bucket/testschema\.desttable/datafile-\d{4}\.json\.gz>.source,
             "schema" => 'testschema',
@@ -51,8 +53,8 @@ module Bricolage
         dispatcher = Dispatcher.new(
           event_queue: event_queue,
           task_queue: task_queue,
-          object_buffer: object_buffer,
-          url_patterns: url_patterns,
+          chunk_buffer: chunk_buffer,
+          chunk_router: chunk_router,
           dispatch_interval: 600,
           logger: ctx.logger
         )
@@ -127,12 +129,12 @@ module Bricolage
 
         task_queue = SQSDataSource.new_mock
 
-        object_buffer = ObjectBuffer.new(
+        chunk_buffer = ChunkBuffer.new(
           control_data_source: ctl_ds,
           logger: ctx.logger
         )
 
-        url_patterns = URLPatterns.for_config([
+        chunk_router = ChunkRouter.for_config([
           {
             "url" => %r<\As3://test-bucket/testschema\.(?<table>\w+)/datafile-\d{4}\.json\.gz>.source,
             "schema" => 'testschema',
@@ -143,8 +145,8 @@ module Bricolage
         dispatcher = Dispatcher.new(
           event_queue: event_queue,
           task_queue: task_queue,
-          object_buffer: object_buffer,
-          url_patterns: url_patterns,
+          chunk_buffer: chunk_buffer,
+          chunk_router: chunk_router,
           dispatch_interval: 600,
           logger: ctx.logger
         )

data/test/streamingload/{test_event.rb → test_dispatchermessage.rb}

@@ -1,12 +1,12 @@
 require 'test/unit'
-require 'bricolage/streamingload/event'
+require 'bricolage/streamingload/dispatchermessage'
 
 module Bricolage::StreamingLoad
 
-  class TestEvent < Test::Unit::TestCase
+  class TestDispatcherMessage < Test::Unit::TestCase
 
     def new_s3event(message_id: nil, receipt_handle: nil, name: nil, time: nil, source: nil, region: nil, bucket: nil, key: nil, size: nil)
-      S3ObjectEvent.new(
+      S3ObjectDispatcherMessage.new(
         message_id: message_id,
         receipt_handle: receipt_handle,
         name: name,
@@ -21,9 +21,9 @@ module Bricolage::StreamingLoad
 
     test "#created?" do
       e = new_s3event(name: "ObjectCreated:Put")
-      assert_true e.created?
+      assert_true e.created_event?
       e = new_s3event(name: "ObjectCreated:Copy")
-      assert_false e.created?
+      assert_false e.created_event?
     end
 
   end

data/test/test_sqsdatasource.rb

@@ -1,5 +1,4 @@
 require 'test/unit'
-require 'bricolage/streamingload/event'
 require 'bricolage/sqsmock'
 require 'bricolage/logger'
 

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: bricolage-streamingload
 version: !ruby/object:Gem::Version
-  version: 0.12.0
+  version: 0.13.0
 platform: ruby
 authors:
 - Minero Aoki
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2017-12-25 00:00:00.000000000 Z
+date: 2018-01-05 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bricolage
@@ -120,19 +120,22 @@ files:
 - lib/bricolage/sqsdatasource.rb
 - lib/bricolage/sqsmock.rb
 - lib/bricolage/streamingload/alertinglogger.rb
+- lib/bricolage/streamingload/chunk.rb
+- lib/bricolage/streamingload/chunkbuffer.rb
+- lib/bricolage/streamingload/chunkrouter.rb
 - lib/bricolage/streamingload/dispatcher.rb
-- lib/bricolage/streamingload/event.rb
+- lib/bricolage/streamingload/dispatchermessage.rb
+- lib/bricolage/streamingload/incomingchunk.rb
 - lib/bricolage/streamingload/job.rb
 - lib/bricolage/streamingload/jobparams.rb
+- lib/bricolage/streamingload/loadermessage.rb
+- lib/bricolage/streamingload/loadtask.rb
 - lib/bricolage/streamingload/manifest.rb
-- lib/bricolage/streamingload/objectbuffer.rb
-- lib/bricolage/streamingload/task.rb
 - lib/bricolage/streamingload/taskhandler.rb
-- lib/bricolage/streamingload/urlpatterns.rb
 - lib/bricolage/streamingload/version.rb
 - test/all.rb
 - test/streamingload/test_dispatcher.rb
-- test/streamingload/test_event.rb
+- test/streamingload/test_dispatchermessage.rb
 - test/streamingload/test_job.rb
 - test/test_sqsdatasource.rb
 homepage: https://github.com/aamine/bricolage-streamingload

data/lib/bricolage/streamingload/urlpatterns.rb

@@ -1,59 +0,0 @@
-module Bricolage
-
-  module StreamingLoad
-
-    class URLPatternNotMatched < StandardError; end
-
-
-    class URLPatterns
-
-      def URLPatterns.for_config(configs)
-        new(configs.map {|c|
-          Pattern.new(url: c.fetch('url'), schema: c.fetch('schema'), table: c.fetch('table'))
-        })
-      end
-
-      def initialize(patterns)
-        @patterns = patterns
-      end
-
-      def match(url)
-        @patterns.each do |pat|
-          components = pat.match(url)
-          return components if components
-        end
-        raise URLPatternNotMatched, "no URL pattern matches the object url: #{url.inspect}"
-      end
-
-      class Pattern
-        def initialize(url:, schema:, table:)
-          @url_pattern = /\A#{url}\z/
-          @schema = schema
-          @table = table
-        end
-
-        attr_reader :url_pattern
-        attr_reader :schema
-        attr_reader :table
-
-        def match(url)
-          m = @url_pattern.match(url) or return nil
-          Components.new(get_component(m, @schema), get_component(m, @table))
-        end
-
-        def get_component(m, label)
-          if /\A%/ =~ label
-            m[label[1..-1]]
-          else
-            label
-          end
-        end
-      end
-
-      Components = Struct.new(:schema_name, :table_name)
-
-    end
-
-  end   # module StreamingLoad
-
-end   # module Bricolage