bricolage-streamingload 0.5.1 → 0.6.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 6fcd9200426dc3c2389566e0b6c5618d7ee2ae2f
-  data.tar.gz: 6880ed950da15b2550c706bb0525364286889e23
+  metadata.gz: 17aa54eda4d063cb571f3a7671a4e6413ea079e1
+  data.tar.gz: 1e879a10e505c01a9f66393a079e18de997a3478
 SHA512:
-  metadata.gz: e278f2dbf3e9cc26b3c1420bea5a4e580af7eaf006b01e2bfecd641efa553214583b3291160d80911c9b7ed06ee00164eb76a64b13a4ef7ab586dcfaae790de7
-  data.tar.gz: ea493c3e3f2ee8644c3615dc1667fe2902120c8af5c86bfe71b882020aa967801c70b12eff69c0032f6ee486781fcde475a7c0e18e9d73bdb84ec2a32c62c535
+  metadata.gz: 5f5778b2ecff8e2bf8d06e1ab00e8e1ee394772a24e4812b74b921c8303566638a212632c896b9b46c7ed5918357db89c65116595664628dba1c2e3cfbef3375
+  data.tar.gz: ebb53db3b87334f5c725e28665f9716f2246d1aef4389d635627e31b07c356e0f9a9edea451162ca39505da82dba22f5fd710d8bbdcb24f8e0fd583c2c734955

data/lib/bricolage/streamingload/alertinglogger.rb

@@ -1,19 +1,37 @@
+require 'bricolage/logger'
+require 'logger'
+require 'forwardable'
+
 module Bricolage
   module StreamingLoad
     class AlertingLogger
       extend Forwardable
 
-      def initialize(logger: , sns_datasource: , alert_level: 'warn')
+      def initialize(logger:, sns_datasource:, alert_level: 'warn')
         @logger = logger
-        @sns_logger = Bricolage::Logger.new(device: sns_datasource)
-        @sns_logger.level = Kernel.const_get("Logger").const_get(alert_level.upcase)
+        @alerter = Bricolage::Logger.new(device: sns_datasource)
+        @alerter.level = ::Logger.const_get(alert_level.upcase)
       end
 
       def_delegators '@logger', :level, :level=, :debug?, :info?, :warn?, :error?, :fatal?, :unknown?
 
-      %w(log debug info warn error fatal unknown).each do |m|
+      %w[log debug info warn error fatal unknown].each do |m|
         define_method(m) do |*args|
-          [@logger, @sns_logger].map {|t| t.send(m, *args) }
+          @logger.__send__(m, *args)
+          begin
+            @alerter.__send__(m, *args)
+          rescue Exception => err
+            @logger.error "could not send alert: #{err.message}"
+          end
+        end
+      end
+
+      def exception(ex)
+        @logger.exception(ex)
+        begin
+          @alerter.error(ex.message)
+        rescue Exception => err
+          @logger.error "could not send alert: #{err.message}"
         end
       end
 

data/lib/bricolage/streamingload/dispatcher.rb

@@ -29,16 +29,16 @@ module Bricolage
         config = YAML.load(File.read(config_path))
         logger = opts.log_file_path ? new_logger(opts.log_file_path, config) : nil
         ctx = Context.for_application('.', environment: opts.environment, logger: logger)
-        event_queue = ctx.get_data_source('sqs', config.fetch('event-queue-ds'))
-        task_queue = ctx.get_data_source('sqs', config.fetch('task-queue-ds'))
+        event_queue = ctx.get_data_source('sqs', config.fetch('event-queue-ds', 'sqs_event'))
+        task_queue = ctx.get_data_source('sqs', config.fetch('task-queue-ds', 'sqs_task'))
         alert_logger = AlertingLogger.new(
           logger: ctx.logger,
-          sns_datasource: ctx.get_data_source('sns', config.fetch('sns-ds')),
+          sns_datasource: ctx.get_data_source('sns', config.fetch('sns-ds', 'sns')),
           alert_level: config.fetch('alert-level', 'warn')
         )
 
         object_buffer = ObjectBuffer.new(
-          control_data_source: ctx.get_data_source('sql', config.fetch('ctl-postgres-ds')),
+          control_data_source: ctx.get_data_source('sql', config.fetch('ctl-postgres-ds', 'db_data')),
           logger: alert_logger
         )
 
@@ -85,12 +85,14 @@ module Bricolage
         @dispatch_interval = dispatch_interval
         @dispatch_message_id = nil
         @logger = logger
+        @dispatch_requested = false
         @checkpoint_requested = false
       end
 
       attr_reader :logger
 
       def event_loop
+        logger.info "dispatcher started"
         set_dispatch_timer
         @event_queue.handle_messages(handler: self, message_class: Event)
         @event_queue.process_async_delete_force
@@ -99,9 +101,17 @@ module Bricolage
 
       # override
       def after_message_batch
+        # must be processed first
         @event_queue.process_async_delete
+
+        if @dispatch_requested
+          dispatch_tasks
+          @dispatch_requested = false
+        end
+
         if @checkpoint_requested
           create_checkpoint
+          @checkpoint_requested = false   # is needless, but reset it just in case
         end
       end
 
@@ -139,13 +149,19 @@ module Bricolage
       end
 
       def handle_dispatch(e)
+        logger.info "dispatching tasks requested"
+        # Dispatching tasks may takes 10 minutes or more, it can exceeds visibility timeout.
+        # To avoid this, delay dispatching until all events of current message batch are processed.
         if @dispatch_message_id == e.message_id
-          tasks = @object_buffer.flush_tasks
-          send_tasks tasks
-          set_dispatch_timer
+          @dispatch_requested = true
         end
-        # Delete this event immediately
-        @event_queue.delete_message(e)
+        @event_queue.delete_message_async(e)
+      end
+
+      def dispatch_tasks
+        tasks = @object_buffer.flush_tasks
+        send_tasks tasks
+        set_dispatch_timer
       end
 
       def set_dispatch_timer

data/lib/bricolage/streamingload/loader.rb

@@ -51,7 +51,7 @@ module Bricolage
                 strload_tasks
             where
                 task_id = #{@params.task_id}
-                and (task_id not in (select task_id from strload_jobs) or #{@params.force})
+                and (#{@params.force?} or task_id not in (select task_id from strload_jobs))
             returning job_id
             ;
           EndSQL
@@ -60,26 +60,20 @@ module Bricolage
       end
 
       def do_load
-        ManifestFile.create(
-          @params.ctl_bucket,
-          job_id: @job_id,
-          object_urls: @params.object_urls,
-          logger: @logger
-        ) {|manifest|
-          if @params.enable_work_table?
-            prepare_work_table @params.work_table
+        manifest = ManifestFile.create(@params.ctl_bucket, job_id: @job_id, object_urls: @params.object_urls, logger: @logger)
+        if @params.enable_work_table?
+          @connection.transaction {|txn|
+            # NOTE: This transaction ends with truncation, this DELETE does nothing
+            # from the second time.  So don't worry about DELETE cost here.
+            @connection.execute("delete from #{@params.work_table}")
             load_objects @params.work_table, manifest, @params.load_options_string
-            @connection.transaction {
-              commit_work_table @params
-              commit_job_result
-            }
-          else
-            @connection.transaction {
-              load_objects @params.dest_table, manifest, @params.load_options_string
-              commit_job_result
-            }
-          end
-        }
+            commit_work_table txn, @params
+          }
+          commit_job_result
+        else
+          load_objects @params.dest_table, manifest, @params.load_options_string
+          commit_job_result
+        end
       rescue JobFailure => ex
         write_job_error 'failure', ex.message
         raise
@@ -88,10 +82,6 @@ module Bricolage
         raise
       end
 
-      def prepare_work_table(work_table)
-        @connection.execute("truncate #{work_table}")
-      end
-
       def load_objects(dest_table, manifest, options)
         @connection.execute(<<-EndSQL.strip.gsub(/\s+/, ' '))
             copy #{dest_table}
@@ -106,14 +96,37 @@ module Bricolage
         @logger.info "load succeeded: #{manifest.url}"
       end
 
-      def commit_work_table(params)
+      def commit_work_table(txn, params)
         @connection.execute(params.sql_source)
-        # keep work table records for later tracking
+        txn.truncate_and_commit(params.work_table)
       end
 
       def commit_job_result
         @end_time = Time.now
-        write_job_result 'success', ''
+        @ctl_ds.open {|conn|
+          conn.transaction {
+            write_job_result conn, 'success', ''
+            update_loaded_flag conn
+          }
+        }
+      end
+
+      def update_loaded_flag(connection)
+        connection.execute(<<-EndSQL)
+          update
+              strload_objects
+          set
+              loaded = true
+          where
+              object_id in (
+                select
+                    object_id
+                from
+                    strload_task_objects
+                where task_id = (select task_id from strload_jobs where job_id = #{@job_id})
+              )
+          ;
+        EndSQL
       end
 
       MAX_MESSAGE_LENGTH = 1000
@@ -121,23 +134,23 @@ module Bricolage
       def write_job_error(status, message)
         @end_time = Time.now
         @logger.warn message.lines.first
-        write_job_result status, message.lines.first.strip[0, MAX_MESSAGE_LENGTH]
-      end
-
-      def write_job_result(status, message)
         @ctl_ds.open {|conn|
-          conn.execute(<<-EndSQL)
-            update
-                strload_jobs
-            set
-                (status, finish_time, message) = (#{s status}, current_timestamp, #{s message})
-            where
-                job_id = #{@job_id}
-            ;
-          EndSQL
+          write_job_result conn, status, message.lines.first.strip[0, MAX_MESSAGE_LENGTH]
         }
       end
 
+      def write_job_result(connection, status, message)
+        connection.execute(<<-EndSQL)
+          update
+              strload_jobs
+          set
+              (status, finish_time, message) = (#{s status}, current_timestamp, #{s message})
+          where
+              job_id = #{@job_id}
+          ;
+        EndSQL
+      end
+
     end
 
   end

data/lib/bricolage/streamingload/loaderparams.rb

@@ -1,3 +1,4 @@
+require 'bricolage/job'
 require 'bricolage/rubyjobclass'
 require 'bricolage/psqldatasource'
 
@@ -60,8 +61,8 @@ module Bricolage
         @task.table
       end
 
-      def force
-        @task.force
+      def force?
+        @task.force?
       end
 
       def object_urls

data/lib/bricolage/streamingload/loaderservice.rb

@@ -1,3 +1,4 @@
+require 'bricolage/context'
 require 'bricolage/sqsdatasource'
 require 'bricolage/streamingload/task'
 require 'bricolage/streamingload/loader'
@@ -5,6 +6,7 @@ require 'bricolage/streamingload/alertinglogger'
 require 'bricolage/logger'
 require 'bricolage/exception'
 require 'bricolage/version'
+require 'yaml'
 require 'optparse'
 
 module Bricolage
@@ -23,21 +25,25 @@ module Bricolage
         config_path, * = opts.rest_arguments
         config = YAML.load(File.read(config_path))
         logger = opts.log_file_path ? new_logger(opts.log_file_path, config) : nil
-        ctx = Context.for_application('.', environment: opts.environment, logger: logger)
-        redshift_ds = ctx.get_data_source('sql', config.fetch('redshift-ds'))
-        task_queue = ctx.get_data_source('sqs', config.fetch('task-queue-ds'))
-        alert_logger = AlertingLogger.new(
-          logger: ctx.logger,
-          sns_datasource: ctx.get_data_source('sns', config.fetch('sns-ds')),
-          alert_level: config.fetch('alert-level', 'warn')
-        )
+        ctx = Context.for_application(opts.working_dir, environment: opts.environment, logger: logger)
+        redshift_ds = ctx.get_data_source('sql', config.fetch('redshift-ds', 'db_data'))
+        task_queue = ctx.get_data_source('sqs', config.fetch('task-queue-ds', 'sqs_task'))
+        raw_logger = logger = ctx.logger
+        if config.key?('alert-level')
+          logger = AlertingLogger.new(
+            logger: raw_logger,
+            sns_datasource: ctx.get_data_source('sns', config.fetch('sns-ds', 'sns')),
+            alert_level: config.fetch('alert-level', 'warn')
+          )
+        end
 
         service = new(
           context: ctx,
-          control_data_source: ctx.get_data_source('sql', config.fetch('ctl-postgres-ds')),
+          control_data_source: ctx.get_data_source('sql', config.fetch('ctl-postgres-ds', 'db_ctl')),
           data_source: redshift_ds,
           task_queue: task_queue,
-          logger: alert_logger
+          working_dir: opts.working_dir,
+          logger: logger
         )
 
         if opts.task_id
@@ -46,12 +52,18 @@ module Bricolage
         else
           # Server mode
           Process.daemon(true) if opts.daemon?
+          Dir.chdir '/'
           create_pid_file opts.pid_file_path if opts.pid_file_path
-          service.event_loop
+          begin
+            logger.info "*** bricolage-streaming-loader started: pid=#{$$}"
+            service.event_loop
+            logger.info "*** bricolage-streaming-loader shutdown gracefully: pid=#{$$}"
+          rescue Exception => ex
+            logger.exception(ex)
+            logger.error "*** bricolage-streaming-loader abort: pid=#{$$}"
+            raise
+          end
         end
-      rescue Exception => e
-        alert_logger.error e.message
-        raise
       end
 
       def LoaderService.new_logger(path, config)
@@ -70,11 +82,12 @@ module Bricolage
         # ignore
       end
 
-      def initialize(context:, control_data_source:, data_source:, task_queue:, logger:)
+      def initialize(context:, control_data_source:, data_source:, task_queue:, working_dir:, logger:)
         @ctx = context
         @ctl_ds = control_data_source
         @ds = data_source
         @task_queue = task_queue
+        @working_dir = working_dir
         @logger = logger
       end
 
@@ -82,7 +95,6 @@ module Bricolage
 
       def event_loop
         @task_queue.handle_messages(handler: self, message_class: Task)
-        @logger.info "shutdown gracefully"
       end
 
       def execute_task_by_id(task_id)
@@ -95,19 +107,23 @@ module Bricolage
 
       # message handler
       def handle_streaming_load_v3(task)
-        # 1. Load task detail from table
-        # 2. Skip disabled (sqs message should not have disabled state since it will never be exectuted)
-        # 3. Try execute
-        #   - Skip if the task has already been executed AND force = false
-        loadtask = load_task(task.id, force: task.force)
-        return if loadtask.disabled # skip if disabled, but don't delete sqs msg
-        execute_task(loadtask)
-        # Delete load task immediately (do not use async delete)
-        @task_queue.delete_message(task)
+        Dir.chdir(@working_dir) {
+          loadtask = load_task(task.id, force: task.force?)
+          if loadtask.disabled
+            # Skip if disabled, and don't delete SQS message.
+            @logger.info "skip disabled task: task_id=#{task.id}"
+            return
+          end
+          execute_task(loadtask)
+          # Do not use async delete
+          @task_queue.delete_message(task)
+        }
+      rescue => ex
+        @logger.exception ex
       end
 
       def execute_task(task)
-        @logger.info "handling load task: table=#{task.qualified_name} task_id=#{task.id}"
+        @logger.info "execute task: task_id=#{task.id} table=#{task.qualified_name}"
         loader = Loader.load_from_file(@ctx, @ctl_ds, task, logger: @logger)
         loader.execute
       end
@@ -119,16 +135,18 @@ module Bricolage
       def initialize(argv)
         @argv = argv
         @task_id = nil
+        @environment = Context::DEFAULT_ENV
         @daemon = false
         @log_file_path = nil
         @pid_file_path = nil
+        @working_dir = Dir.getwd
         @rest_arguments = nil
 
         @opts = opts = OptionParser.new("Usage: #{$0} CONFIG_PATH")
         opts.on('--task-id=ID', 'Execute oneshot load task (implicitly disables daemon mode).') {|task_id|
           @task_id = task_id
         }
-        opts.on('-e', '--environment=NAME', "Sets execution environment [default: #{Context::DEFAULT_ENV}]") {|env|
+        opts.on('-e', '--environment=NAME', "Sets execution environment [default: #{@environment}]") {|env|
           @environment = env
         }
         opts.on('--daemon', 'Becomes daemon in server mode.') {
@@ -140,6 +158,9 @@ module Bricolage
         opts.on('--pid-file=PATH', 'Creates PID file.') {|path|
           @pid_file_path = path
         }
+        opts.on('--working-dir=PATH', "Loader working directory. [default: #{@working_dir}]") {|path|
+          @working_dir = path
+        }
         opts.on('--help', 'Prints this message and quit.') {
           puts opts.help
           exit 0
@@ -161,14 +182,18 @@ module Bricolage
         raise OptionError, err.message
       end
 
-      attr_reader :rest_arguments, :environment, :log_file_path
+      attr_reader :rest_arguments
+
       attr_reader :task_id
+      attr_reader :environment
 
       def daemon?
         @daemon
       end
 
+      attr_reader :log_file_path
       attr_reader :pid_file_path
+      attr_reader :working_dir
 
     end
 

data/lib/bricolage/streamingload/manifest.rb

@@ -6,7 +6,12 @@ module Bricolage
 
       def ManifestFile.create(ds, job_id:, object_urls:, logger:, noop: false, &block)
         manifest = new(ds, job_id, object_urls, logger: logger, noop: noop)
-        manifest.create_temporary(&block)
+        if block
+          manifest.create_temporary(&block)
+        else
+          manifest.put
+          return manifest
+        end
       end
 
       def initialize(ds, job_id, object_urls, logger:, noop: false)
@@ -22,7 +27,9 @@ module Bricolage
       end
 
       def name
-        @name ||= "manifest-#{@job_id}.json"
+        return @name if @name
+        now =Time.now
+        "#{now.strftime('%Y/%m/%d')}/manifest-#{now.strftime('%H%M%S')}-#{@job_id}.json"
       end
 
       def url

data/lib/bricolage/streamingload/objectbuffer.rb

@@ -42,26 +42,43 @@ module Bricolage
 
     class ObjectBuffer
 
+      TASK_GENERATION_TIME_LIMIT = 30 #sec
+
       include SQLUtils
 
       def initialize(control_data_source:, logger:)
         @ctl_ds = control_data_source
         @logger = logger
+        @task_generation_time_limit = TASK_GENERATION_TIME_LIMIT
       end
 
       def put(obj)
         @ctl_ds.open {|conn|
-          insert_object(conn, obj)
+          suppress_sql_logging {
+            conn.transaction {
+              object_id = insert_object(conn, obj)
+              if object_id
+                insert_task_objects(conn, object_id)
+              else
+                insert_dup_object(conn, obj)
+              end
+            }
+          }
         }
       end
 
       # Flushes multiple tables periodically
       def flush_tasks
-        task_ids = nil
-        @ctl_ds.open {|conn|
-          conn.transaction {|txn|
-            task_ids = insert_tasks(conn)
-            insert_task_object_mappings(conn) unless task_ids.empty?
+        task_ids = []
+        warn_slow_task_generation {
+          @ctl_ds.open {|conn|
+            conn.transaction {|txn|
+              task_ids = insert_tasks(conn)
+              unless task_ids.empty?
+                update_task_object_mappings(conn, task_ids)
+                log_mapped_object_num(conn, task_ids)
+              end
+            }
           }
         }
         return task_ids.map {|id| LoadTask.create(task_id: id) }
@@ -73,11 +90,12 @@ module Bricolage
         task_ids  = []
         @ctl_ds.open {|conn|
           conn.transaction {|txn|
-            # insert_task_object_mappings may not consume all saved objects
+            # update_task_object_mappings may not consume all saved objects
             # (e.g. there are too many objects for one table), we must create
             # tasks repeatedly until there are no unassigned objects.
             until (ids = insert_tasks_force(conn)).empty?
-              insert_task_object_mappings(conn)
+              update_task_object_mappings(conn, ids)
+              log_mapped_object_num(conn, ids)
               task_ids.concat ids
             end
           }
@@ -91,11 +109,12 @@ module Bricolage
         task_ids  = []
         @ctl_ds.open {|conn|
           conn.transaction {|txn|
-            # insert_task_object_mappings may not consume all saved objects
+            # update_task_object_mappings may not consume all saved objects
             # (e.g. there are too many objects for one table), we must create
             # tasks repeatedly until there are no unassigned objects.
             until (ids = insert_table_task_force(conn, table_name)).empty?
-              insert_task_object_mappings(conn)
+              update_task_object_mappings(conn, ids)
+              log_mapped_object_num(conn, ids)
               task_ids.concat ids
             end
           }
@@ -106,30 +125,66 @@ module Bricolage
       private
 
       def insert_object(conn, obj)
-        suppress_sql_logging {
-          conn.update(<<-EndSQL)
-              insert into strload_objects
-                  ( object_url
-                  , object_size
-                  , data_source_id
-                  , message_id
-                  , event_time
-                  , submit_time
-                  )
-              select
-                  #{s obj.url}
-                  , #{obj.size}
-                  , #{s obj.data_source_id}
-                  , #{s obj.message_id}
-                  , '#{obj.event_time}' AT TIME ZONE 'JST'
-                  , current_timestamp
-              from
-                  strload_tables
-              where
-                  data_source_id = #{s obj.data_source_id}
-              ;
-          EndSQL
-        }
+        object_ids = conn.query_values(<<-EndSQL)
+            insert into strload_objects
+                ( object_url
+                , object_size
+                , data_source_id
+                , message_id
+                , event_time
+                , submit_time
+                )
+            values
+                ( #{s obj.url}
+                , #{obj.size}
+                , #{s obj.data_source_id}
+                , #{s obj.message_id}
+                , '#{obj.event_time}' AT TIME ZONE 'JST'
+                , current_timestamp
+                )
+            on conflict on constraint strload_objects_object_url
+            do nothing
+            returning object_id
+            ;
+        EndSQL
+        return object_ids.first
+      end
+
+      def insert_dup_object(conn, obj)
+        @logger.info "Duplicated object recieved: object_url=#{obj.url}"
+        conn.update(<<-EndSQL)
+            insert into strload_dup_objects
+                ( object_url
+                , object_size
+                , data_source_id
+                , message_id
+                , event_time
+                , submit_time
+                )
+            values
+                ( #{s obj.url}
+                , #{obj.size}
+                , #{s obj.data_source_id}
+                , #{s obj.message_id}
+                , '#{obj.event_time}' AT TIME ZONE 'JST'
+                , current_timestamp
+                )
+            ;
+        EndSQL
+      end
+
+      def insert_task_objects(conn, object_id)
+        conn.update(<<-EndSQL)
+            insert into strload_task_objects
+                ( task_id
+                , object_id
+                )
+            values
+                ( -1
+                , #{object_id}
+                )
+            ;
+        EndSQL
       end
 
       def insert_tasks_force(conn)
@@ -140,14 +195,12 @@ module Bricolage
         task_ids = conn.query_values(<<-EndSQL)
             insert into strload_tasks
                 ( task_class
-                , schema_name
-                , table_name
+                , table_id
                 , submit_time
                 )
             select
                 'streaming_load_v3'
-                , tbl.schema_name
-                , tbl.table_name
+                , tbl.table_id
                 , current_timestamp
             from
                 strload_tables tbl
@@ -158,19 +211,9 @@ module Bricolage
                         data_source_id
                         , count(*) as object_count
                     from
-                        (
-                            select
-                                min(object_id) as object_id
-                                , object_url
-                            from
-                                strload_objects
-                            group by
-                                object_url
-                        ) uniq_objects
-                        inner join strload_objects using (object_id)
-                        left outer join strload_task_objects using (object_id)
+                        strload_objects
                     where
-                        task_id is null -- not assigned to a task
+                        object_id in (select object_id from strload_task_objects where task_id = -1)
                     group by
                         data_source_id
                 ) obj
@@ -179,28 +222,27 @@ module Bricolage
                 -- preceeding task's submit time
                 left outer join (
                     select
-                        schema_name
-                        , table_name
+                        table_id
                         , max(submit_time) as latest_submit_time
                     from
                         strload_tasks
                     group by
-                        schema_name, table_name
+                        table_id
                 ) task
-                using (schema_name, table_name)
+                using (table_id)
             where
                 not tbl.disabled -- not disabled
                 and (
                     #{force ? "true or" : ""}                                                      -- Creates tasks with no conditions if forced
                     obj.object_count > tbl.load_batch_size                                         -- batch_size exceeded?
-                    or extract(epoch from current_timestamp - latest_submit_time) > load_interval  -- load_interval exceeded?
-                    or latest_submit_time is null                                                  -- no previous tasks?
+                    or extract(epoch from current_timestamp - task.latest_submit_time) > tbl.load_interval  -- load_interval exceeded?
+                    or task.latest_submit_time is null                                             -- no previous tasks?
                 )
             returning task_id
             ;
         EndSQL
 
-        @logger.info "Number of task created: #{task_ids.size}"
+        log_created_tasks task_ids
         task_ids
       end
 
@@ -208,14 +250,12 @@ module Bricolage
         task_ids = conn.query_values(<<-EndSQL)
             insert into strload_tasks
                 ( task_class
-                , schema_name
-                , table_name
+                , table_id
                 , submit_time
                 )
             select
                 'streaming_load_v3'
-                , tbl.schema_name
-                , tbl.table_name
+                , tbl.table_id
                 , current_timestamp
             from
                 strload_tables tbl
@@ -227,21 +267,9 @@ module Bricolage
                         data_source_id
                         , count(*) as object_count
                     from
-                        (
-                            select
-                                min(object_id) as object_id
-                                , object_url
-                            from
-                                strload_objects
-                            where
-                                data_source_id = #{s table_name}
-                            group by
-                                object_url
-                        ) uniq_objects
-                        inner join strload_objects using (object_id)
-                        left outer join strload_task_objects using (object_id)
+                        strload_objects
                     where
-                        task_id is null -- not assigned to a task
+                        object_id in (select object_id from strload_task_objects where task_id = -1)
                     group by
                         data_source_id
                 ) obj
@@ -254,70 +282,55 @@ module Bricolage
         EndSQL
 
         # It must be 1
-        @logger.info "Number of task created: #{task_ids.size}"
+        log_created_tasks(task_ids)
         task_ids
       end
 
-      def insert_task_object_mappings(conn)
+      def update_task_object_mappings(conn, task_ids)
         conn.update(<<-EndSQL)
-            insert into strload_task_objects
-                ( task_id
-                , object_id
-                )
+            update strload_task_objects dst
+            set
+                task_id = tasks.task_id
+            from
+                strload_tasks tasks
+                inner join strload_tables tables using (table_id)
+                inner join (
+                    select
+                        object_id
+                        , data_source_id
+                        , row_number() over (partition by data_source_id order by object_id) as object_seq
+                    from
+                        strload_objects
+                    where
+                        object_id in (select object_id from strload_task_objects where task_id = -1)
+                ) tsk_obj
+                using (data_source_id)
+            where
+                dst.task_id = -1
+                and tasks.task_id in (#{task_ids.join(",")})
+                and dst.object_id = tsk_obj.object_id
+                and tsk_obj.object_seq <= tables.load_batch_size
+            ;
+        EndSQL
+      end
+
+      def log_mapped_object_num(conn, task_ids)
+        # This method is required since UPDATE does not "returning" multiple values
+        rows = conn.query_values(<<-EndSQL)
             select
                 task_id
-                , object_id
-            from (
-                select
-                    row_number() over (partition by task.task_id order by obj.object_id) as object_count
-                    , task.task_id
-                    , obj.object_id
-                    , load_batch_size
-                from
-                    -- unassigned objects
-                    (
-                        select
-                            data_source_id
-                            , uniq_objects.object_url
-                            , object_id
-                        from
-                            (
-                                select
-                                    min(object_id) as object_id
-                                    , object_url
-                                from
-                                    strload_objects
-                                group by
-                                    object_url
-                            ) uniq_objects
-                            inner join strload_objects using(object_id)
-                            left outer join strload_task_objects using(object_id)
-                        where
-                            task_id is null
-                     ) obj
-
-                    -- tasks without objects
-                    inner join (
-                        select
-                            tbl.data_source_id
-                            , min(task_id) as task_id   -- pick up oldest task
-                            , max(load_batch_size) as load_batch_size
-                        from
-                            strload_tasks
-                            inner join strload_tables tbl
-                            using (schema_name, table_name)
-                        where
-                            -- unassigned objects
-                            task_id not in (select distinct task_id from strload_task_objects)
-                        group by
-                            1
-                    ) task
-                    using (data_source_id)
-                ) as t
+                , count(*)
+            from
+                strload_task_objects
             where
-                object_count <= load_batch_size   -- limit number of objects assigned to single task
+                task_id in (#{task_ids.join(',')})
+            group by
+                task_id
             ;
         EndSQL
+        rows.each_slice(2) do |task_id, object_count|
+          @logger.info "Number of objects assigned to task: task_id=#{task_id} object_count=#{object_count}"
+        end
       end
 
       def suppress_sql_logging
@@ -331,6 +344,22 @@ module Bricolage
         end
       end
 
+      def log_created_tasks(task_ids)
+        created_task_num = task_ids.size
+        @logger.info "Number of task created: #{created_task_num}"
+        @logger.info "Created task ids: #{task_ids}" if created_task_num > 0
+      end
+
+      def warn_slow_task_generation(&block)
+        start_time = Time.now
+        yield
+        exec_time = (Time.now - start_time)
+        if exec_time > @task_generation_time_limit
+          @logger.warn "Long task generation time:  #{exec_time}"
+          @task_generation_time_limit = @task_generation_time_limit * 1.1
+        end
+      end
+
     end
 
   end

data/lib/bricolage/streamingload/task.rb

@@ -35,7 +35,7 @@ module Bricolage
       def LoadTask.parse_sqs_record(msg, rec)
         {
           task_id: rec['taskId'],
-          force: rec['force'],
+          force: (rec['force'].to_s == 'true')
         }
       end
 
@@ -49,7 +49,7 @@ module Bricolage
           from
               strload_tasks tsk
               inner join strload_tables tbl
-                  using(schema_name, table_name)
+                  using(table_id)
           where
               task_id = #{task_id}
           ;
@@ -94,7 +94,11 @@ module Bricolage
         @disabled = disabled
       end
 
-      attr_reader :id, :force
+      attr_reader :id
+
+      def force?
+        !!@force
+      end
 
       #
       # For writer only

data/lib/bricolage/streamingload/version.rb

@@ -1,5 +1,5 @@
 module Bricolage
   module StreamingLoad
-    VERSION = '0.5.1'
+    VERSION = '0.6.0'
   end
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: bricolage-streamingload
 version: !ruby/object:Gem::Version
-  version: 0.5.1
+  version: 0.6.0
 platform: ruby
 authors:
 - Minero Aoki
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2016-08-24 00:00:00.000000000 Z
+date: 2016-09-05 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bricolage
@@ -17,42 +17,42 @@ dependencies:
     requirements:
     - - '='
       - !ruby/object:Gem::Version
-        version: 5.16.8
+        version: 5.16.9
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - '='
       - !ruby/object:Gem::Version
-        version: 5.16.8
+        version: 5.16.9
 - !ruby/object:Gem::Dependency
   name: pg
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - '='
       - !ruby/object:Gem::Version
-        version: '0'
+        version: 0.18.4
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - '='
       - !ruby/object:Gem::Version
-        version: '0'
+        version: 0.18.4
 - !ruby/object:Gem::Dependency
   name: aws-sdk
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - '='
       - !ruby/object:Gem::Version
-        version: '2'
+        version: 2.5.6
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - '='
       - !ruby/object:Gem::Version
-        version: '2'
+        version: 2.5.6
 - !ruby/object:Gem::Dependency
   name: rake
   requirement: !ruby/object:Gem::Requirement
@@ -138,7 +138,7 @@ required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
-      version: 2.0.0
+      version: 2.1.0
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="