bricolage-streamingload 0.7.1 → 0.8.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 14a3e79cc5b39bd90b848b1d3b030bf244ddee15
-  data.tar.gz: b82cce95ef1573cb2dd0fd74214777afd3f972ae
+  metadata.gz: 8a33b6a5561c4bf69b725a96bca4abf75f06fe8a
+  data.tar.gz: 9c6357983ba1fea216fd3e0931d08f549f610320
 SHA512:
-  metadata.gz: 6daab7f6d72fd79261d047ac8f5a0ab0f8dcdd0fb392b1ee013dd6c0616fe90b83ab3637274dc4e7fdab23c81b3a6d3888ce614083ff1ca07b9778017aab6e76
-  data.tar.gz: 1c0041319e7091f869f51c182dad162e81c37f4683d990b5cd224ff5ec93ec004936cc22d44eb961e9628e8962eaaf0c253c30a8036f44331ad5970bae5a3fc6
+  metadata.gz: e701e7aa67d9a9b6503b436342a5d2ef3b1edb49513fa6881bf4c9b758e5d44f50a7b444b950527518187f3c8d8da896ee7fc597d9d0878611c8c34c2dbb3a19
+  data.tar.gz: 77b535d15e8e0724c054655e8f351b7620454f08e67a86a5c61f01b8bb623361b22344b93d590608538ee59e9809886e64ae48ec3b80f8cf61570897e3814e41

data/bin/bricolage-streaming-loader

@@ -1,6 +1,6 @@
 #!/usr/bin/env ruby
 
 Bundler.require(:default) if defined?(Bundler)
-require 'bricolage/streamingload/loaderservice'
+require 'bricolage/streamingload/taskhandler'
 
-Bricolage::StreamingLoad::LoaderService.main
+Bricolage::StreamingLoad::TaskHandler.main

data/lib/bricolage/sqsmock.rb

@@ -1,5 +1,4 @@
 require 'bricolage/sqsdatasource'
-require 'bricolage/nulllogger'
 require 'json'
 
 module Bricolage

data/lib/bricolage/streamingload/dispatcher.rb

@@ -27,7 +27,7 @@ module Bricolage
         end
         config_path, * = opts.rest_arguments
         config = YAML.load(File.read(config_path))
-        log = opts.log_file_path ? new_logger(opts.log_file_path, config) : nil
+        log = opts.log_file_path ? new_logger(File.expand_path(opts.log_file_path), config) : nil
         ctx = Context.for_application('.', environment: opts.environment, logger: log)
         logger = raw_logger = ctx.logger
         event_queue = ctx.get_data_source('sqs', config.fetch('event-queue-ds', 'sqs_event'))
@@ -58,6 +58,7 @@ module Bricolage
 
         Process.daemon(true) if opts.daemon?
         create_pid_file opts.pid_file_path if opts.pid_file_path
+        Dir.chdir '/'
         dispatcher.event_loop
       rescue Exception => e
         logger.exception e

data/lib/bricolage/streamingload/job.rb

@@ -0,0 +1,387 @@
+require 'bricolage/streamingload/jobparams'
+require 'bricolage/streamingload/manifest'
+require 'bricolage/sqlutils'
+require 'socket'
+require 'json'
+
+module Bricolage
+
+  module StreamingLoad
+
+    class JobCancelled < ApplicationError; end
+    class JobDefered < ApplicationError; end
+    class JobDuplicated < ApplicationError; end
+
+    class ControlConnectionFailed < JobFailure; end
+    class DataConnectionFailed < JobFailure; end
+
+
+    class Job
+
+      def initialize(context:, ctl_ds:, task_id:, force: false, logger:)
+        @context = context
+        @ctl_ds = ctl_ds
+        @task_id = task_id
+        @force = force
+        @logger = logger
+
+        @task = nil
+        @job_id = nil
+        @data_ds = nil
+        @manifest = nil
+      end
+
+      # For tests
+      attr_reader :job_id
+      attr_reader :process_id
+      attr_reader :task
+      attr_reader :data_ds
+      attr_reader :manifest
+
+      # Returns true -> Deletes a SQS message
+      # Returns false -> Keeps a SQS message
+      def execute(fail_fast: false)
+        execute_task
+        return true
+      rescue JobCancelled
+        return true
+      rescue JobDuplicated
+        return true
+      rescue JobDefered
+        return false
+      rescue ControlConnectionFailed => ex
+        @logger.error ex.message
+        wait_for_connection('ctl', @ctl_ds) unless fail_fast
+        return false
+      rescue DataConnectionFailed
+        wait_for_connection('data', @data_ds) unless fail_fast
+        return false
+      rescue JobFailure
+        return false
+      rescue JobError
+        return true
+      rescue Exception => ex
+        @logger.exception ex
+        return true
+      end
+
+      MAX_RETRY = 5
+
+      def execute_task
+        @process_id = "#{Socket.gethostname}-#{$$}"
+        @logger.info "execute task: task_id=#{@task_id} force=#{@force} process_id=#{@process_id}"
+        ctl = ControlConnection.new(@ctl_ds, @logger)
+
+        ctl.open {
+          @task = ctl.load_task(@task_id)
+          @logger.info "task details: task_id=#{@task_id} table=#{@task.schema_name}.#{@task.table_name}"
+          if @task.disabled
+            # We do not record disabled job in the DB.
+            @logger.info "task is disabled; defer task: task_id=#{@task_id}"
+            raise JobDefered, "defered: task_id=#{@task_id}"
+          end
+
+          @job_id = ctl.begin_job(@task_id, @process_id, @force)
+          unless @job_id
+            @logger.warn "task is already succeeded and not forced; discard task: task_id=#{@task_id}"
+            ctl.commit_duplicated_job @task_id, @process_id
+            raise JobDuplicated, "duplicated: task_id=#{@task_id}"
+          end
+        }
+
+        begin
+          do_load @task, @job_id
+          ctl.open {
+            ctl.commit_job @job_id, (@force ? 'forced' : nil)
+          }
+        rescue ControlConnectionFailed
+          raise
+        rescue JobFailure => ex
+          @logger.error ex.message
+          ctl.open {
+            fail_count = ctl.fail_count(@task_id)
+            final_retry = (fail_count >= MAX_RETRY)
+            retry_msg = (fail_count > 0) ? "(retry\##{fail_count}#{final_retry ? ' FINAL' : ''}) " : ''
+            ctl.abort_job job_id, 'failure', retry_msg + ex.message.lines.first.strip
+            raise JobCancelled, "retry count exceeds limit: task_id=#{@task_id}" if final_retry
+          }
+          raise
+        rescue JobError => ex
+          @logger.error ex.message
+          ctl.open {
+            ctl.abort_job job_id, 'error', ex.message.lines.first.strip
+          }
+          raise
+        rescue Exception => ex
+          @logger.exception ex
+          ctl.open {
+            ctl.abort_job job_id, 'error', ex.message.lines.first.strip
+          }
+          raise JobError, "#{ex.class}: #{ex.message}"
+        end
+      end
+
+      def do_load(task, job_id)
+        params = JobParams.load(@context, task.task_class, task.schema_name, task.table_name)
+        @data_ds = params.ds
+        @manifest = ManifestFile.create(ds: params.ctl_bucket, job_id: job_id, object_urls: task.object_urls, logger: @logger)
+        DataConnection.open(params.ds, @logger) {|data|
+          if params.enable_work_table?
+            data.load_with_work_table params.work_table, @manifest, params.load_options_string, params.sql_source
+          else
+            data.load_objects params.dest_table, @manifest, params.load_options_string
+          end
+        }
+      end
+
+      def wait_for_connection(type, ds)
+        @logger.warn "loader: #{type} DB connection lost; polling..."
+        start_time = Time.now
+        n = 0
+        while true
+          begin
+            ds.open {}
+            @logger.warn "loader: #{type} DB connection recovered; return to normal state"
+            return true
+          rescue ConnectionError
+          end
+          sleep 15
+          n += 1
+          if n == 120  # 30 min
+            # Could not get a connection in 30 minutes, now we return to the queue loop.
+            # Next job may fail too, but we should not stop to receive the task queue too long,
+            # because it contains control tasks.
+            @logger.warn "loader: #{type} DB connection still failing (since #{start_time}); give up."
+            return false
+          end
+        end
+      end
+
+
+      class DataConnection
+
+        include SQLUtils
+
+        def DataConnection.open(ds, logger = ds.logger, &block)
+          new(ds, logger).open(&block)
+        end
+
+        def initialize(ds, logger = ds.logger)
+          @ds = ds
+          @connection = nil
+          @logger = logger
+        end
+
+        def open(&block)
+          @ds.open {|conn|
+            @connection = conn
+            yield self
+          }
+        rescue ConnectionError => ex
+          raise DataConnectionFailed, "data connection failed: #{ex.message}"
+        end
+
+        def load_with_work_table(work_table, manifest, options, sql_source)
+          @connection.transaction {|txn|
+            # NOTE: This transaction ends with truncation, this DELETE does nothing
+            # from the second time.  So don't worry about DELETE cost here.
+            @connection.execute("delete from #{work_table}")
+            load_objects work_table, manifest, options
+            @connection.execute sql_source
+            txn.truncate_and_commit work_table
+          }
+        end
+
+        def load_objects(dest_table, manifest, options)
+          @connection.execute(<<-EndSQL.strip.gsub(/\s+/, ' '))
+              copy #{dest_table}
+              from #{s manifest.url}
+              credentials #{s manifest.credential_string}
+              manifest
+              statupdate false
+              compupdate false
+              #{options}
+              ;
+          EndSQL
+          @logger.info "load succeeded: #{manifest.url}"
+        end
+
+      end   # class DataConnection
+
+
+      class ControlConnection
+
+        include SQLUtils
+
+        def ControlConnection.open(ds, logger = ds.logger, &block)
+          new(ds, logger).open(&block)
+        end
+
+        def initialize(ds, logger = ds.logger)
+          @ds = ds
+          @connection = nil
+        end
+
+        def open(&block)
+          @ds.open {|conn|
+            @connection = conn
+            yield self
+          }
+        rescue ConnectionError => ex
+          raise ControlConnectionFailed, "control connection failed: #{ex.message}"
+        end
+
+        TaskInfo = Struct.new(:task_id, :task_class, :schema_name, :table_name, :disabled, :object_urls)
+
+        def load_task(task_id)
+          rec = @connection.query_row(<<-EndSQL) or raise JobError, "no such task: #{task_id}"
+            select
+                tsk.task_class
+                , tbl.schema_name
+                , tbl.table_name
+                , tbl.disabled
+            from
+                strload_tasks tsk
+                inner join strload_tables tbl using (table_id)
+            where
+                tsk.task_id = #{task_id}
+            ;
+          EndSQL
+          TaskInfo.new(
+            task_id,
+            rec['task_class'],
+            rec['schema_name'],
+            rec['table_name'],
+            (rec['disabled'] != 'f'),
+            load_object_urls(task_id)
+          )
+        end
+
+        def load_object_urls(task_id)
+          urls = @connection.query_values(<<-EndSQL)
+            select
+                o.object_url
+            from
+                strload_tasks t
+                inner join strload_task_objects tob using (task_id)
+                inner join strload_objects o using (object_id)
+            where
+                t.task_id = #{task_id}
+            ;
+          EndSQL
+          urls
+        end
+
+        def begin_job(task_id, process_id, force)
+          job_id = @connection.query_value(<<-EndSQL)
+            insert into strload_jobs
+                ( task_id
+                , process_id
+                , status
+                , start_time
+                )
+            select
+                task_id
+                , #{s process_id}
+                , 'running'
+                , current_timestamp
+            from
+                strload_tasks
+            where
+                task_id = #{task_id}
+                and (#{force ? 'true' : 'false'} or task_id not in (select task_id from strload_jobs where status = 'success'))
+            returning job_id
+            ;
+          EndSQL
+          return job_id ? job_id.to_i : nil
+        end
+
+        def fail_count(task_id)
+          statuses = @connection.query_values(<<-EndSQL)
+            select
+                j.status
+            from
+                strload_tasks t
+                inner join strload_jobs j using (task_id)
+            where
+                t.task_id = #{task_id}
+            order by
+                j.job_id desc
+          EndSQL
+          statuses.shift if statuses.first == 'running'   # current job
+          statuses.take_while {|st| %w[failure error].include?(st) }.size
+        end
+
+        def commit_job(job_id, message = nil)
+          @connection.transaction {|txn|
+            write_job_result job_id, 'success', (message || '')
+            update_loaded_flag job_id
+          }
+        end
+
+        def abort_job(job_id, status, message)
+          write_job_result(job_id, status, message)
+        end
+
+        MAX_MESSAGE_LENGTH = 1000
+
+        def write_job_result(job_id, status, message)
+          @connection.execute(<<-EndSQL)
+            update
+                strload_jobs
+            set
+                (status, finish_time, message) = (#{s status}, current_timestamp, #{s message[0, MAX_MESSAGE_LENGTH]})
+            where
+                job_id = #{job_id}
+            ;
+          EndSQL
+        end
+
+        def update_loaded_flag(job_id)
+          @connection.execute(<<-EndSQL)
+            update
+                strload_objects
+            set
+                loaded = true
+            where
+                object_id in (
+                  select
+                      object_id
+                  from
+                      strload_task_objects
+                  where task_id = (select task_id from strload_jobs where job_id = #{job_id})
+                )
+            ;
+          EndSQL
+        end
+
+        def commit_duplicated_job(task_id, process_id)
+          job_id = @connection.query_value(<<-EndSQL)
+            insert into strload_jobs
+                ( task_id
+                , process_id
+                , status
+                , start_time
+                , finish_time
+                , message
+                )
+            select
+                #{task_id}
+                , #{s process_id}
+                , 'duplicated'
+                , current_timestamp
+                , current_timestamp
+                , ''
+            returning job_id
+            ;
+          EndSQL
+          return job_id
+        end
+
+      end   # class ControlConnection
+
+    end   # class Job
+
+  end   # module StreamingLoad
+
+end   # module Bricolage

data/lib/bricolage/streamingload/{loaderparams.rb → jobparams.rb}

@@ -6,69 +6,44 @@ module Bricolage
 
   module StreamingLoad
 
-    class LoaderParams
+    class JobParams
 
-      def LoaderParams.load(ctx, task)
-        job = load_job(ctx, task)
-        schema = resolve_schema(ctx, task.schema)
-        job.provide_default 'dest-table', "#{schema}.#{task.table}"
+      def JobParams.load(ctx, job_class, schema, table)
+        job = load_bricolage_job(ctx, job_class, schema, table)
+        schema = resolve_schema(ctx, schema)
+        job.provide_default 'dest-table', "#{schema}.#{table}"
         #job.provide_sql_file_by_job_id   # FIXME: provide only when exist
         job.compile
-        new(task, job)
+        new(job)
       end
 
-      def LoaderParams.load_job(ctx, task)
-        if job_file = find_job_file(ctx, task.schema, task.table)
+      def JobParams.load_bricolage_job(ctx, job_class, schema, table)
+        if job_file = find_job_file(ctx, schema, table)
           ctx.logger.debug "using .job file: #{job_file}"
-          Job.load_file(job_file, ctx.subsystem(task.schema))
+          Bricolage::Job.load_file(job_file, ctx.subsystem(schema))
         else
           ctx.logger.debug "using default job parameters (no .job file)"
-          Job.instantiate(task.table, 'streaming_load_v3', ctx).tap {|job|
+          Bricolage::Job.instantiate(table, job_class, ctx).tap {|job|
             job.bind_parameters({})
           }
         end
       end
 
-      def LoaderParams.find_job_file(ctx, schema, table)
+      def JobParams.find_job_file(ctx, schema, table)
         paths = Dir.glob("#{ctx.home_path}/#{schema}/#{table}.*")
         paths.select {|path| File.extname(path) == '.job' }.sort.first
       end
 
-      def LoaderParams.resolve_schema(ctx, schema)
+      def JobParams.resolve_schema(ctx, schema)
         ctx.global_variables["#{schema}_schema"] || schema
       end
       private_class_method :resolve_schema
 
-      def initialize(task, job)
-        @task = task
+      def initialize(job)
         @job = job
         @params = job.params
       end
 
-      def task_id
-        @task.id
-      end
-
-      def task_id
-        @task.id
-      end
-
-      def schema
-        @task.schema
-      end
-
-      def table
-        @task.table
-      end
-
-      def force?
-        @task.force?
-      end
-
-      def object_urls
-        @task.object_urls
-      end
-
       def ds
         @params['redshift-ds']
       end
@@ -101,7 +76,7 @@ module Bricolage
     end
 
 
-    class LoaderJob < RubyJobClass
+    class StreamingLoadV3Job < RubyJobClass
 
       job_class_id 'streaming_load_v3'
 

data/lib/bricolage/streamingload/manifest.rb

@@ -4,7 +4,7 @@ module Bricolage
 
     class ManifestFile
 
-      def ManifestFile.create(ds, job_id:, object_urls:, logger:, noop: false, &block)
+      def ManifestFile.create(ds:, job_id:, object_urls:, logger:, noop: false, &block)
         manifest = new(ds, job_id, object_urls, logger: logger, noop: noop)
         if block
           manifest.create_temporary(&block)
@@ -49,11 +49,17 @@ module Bricolage
       def put
         @logger.info "s3: put: #{url}"
         @ds.object(name).put(body: content) unless @noop
+      rescue Aws::S3::Errors::ServiceError => ex
+        @logger.exception ex
+        raise S3Exception.wrap(ex)
       end
 
       def delete
         @logger.info "s3: delete: #{url}"
         @ds.object(name).delete unless @noop
+      rescue Aws::S3::Errors::ServiceError => ex
+        @logger.exception ex
+        raise S3Exception.wrap(ex)
       end
 
       def create_temporary

data/lib/bricolage/streamingload/objectbuffer.rb

@@ -1,8 +1,5 @@
 require 'bricolage/streamingload/task'
-require 'bricolage/streamingload/loaderparams'
 require 'bricolage/sqlutils'
-require 'json'
-require 'securerandom'
 require 'forwardable'
 
 module Bricolage

data/lib/bricolage/streamingload/task.rb

@@ -1,5 +1,4 @@
 require 'bricolage/sqsdatasource'
-require 'json'
 
 module Bricolage
 
@@ -38,85 +37,23 @@ module Bricolage
         }
       end
 
-      def LoadTask.load(conn, task_id, force: false)
-        rec = conn.query_row(<<-EndSQL)
-          select
-              task_class
-              , tbl.schema_name
-              , tbl.table_name
-              , disabled
-          from
-              strload_tasks tsk
-              inner join strload_tables tbl
-                  using(table_id)
-          where
-              task_id = #{task_id}
-          ;
-        EndSQL
-        object_urls = conn.query_values(<<-EndSQL)
-          select
-              object_url
-          from
-              strload_task_objects
-              inner join strload_objects
-              using (object_id)
-              inner join strload_tasks
-              using (task_id)
-          where
-              task_id = #{task_id}
-          ;
-        EndSQL
-        return nil unless rec
-        new(
-          name: rec['task_class'],
-          time: nil,
-          source: nil,
-          task_id: task_id,
-          schema: rec['schema_name'],
-          table: rec['table_name'],
-          object_urls: object_urls,
-          disabled: rec['disabled'] == 'f' ? false : true,
-          force: force
-        )
-      end
-
       alias message_type name
 
-      def init_message(task_id:, schema: nil, table: nil, object_urls: nil, disabled: false, force: false)
-        @id = task_id
+      def init_message(task_id:, force: false)
+        @task_id = task_id
         @force = force
-
-        # Effective only for queue reader process
-        @schema = schema
-        @table = table
-        @object_urls = object_urls
-        @disabled = disabled
       end
 
-      attr_reader :id
+      attr_reader :task_id
 
       def force?
         !!@force
       end
 
-      #
-      # For writer only
-      #
-
-      attr_reader :schema, :table, :object_urls, :disabled
-
-      def qualified_name
-        "#{@schema}.#{@table}"
-      end
-
       def body
         obj = super
-        obj['taskId'] = @id
-        obj['schemaName'] = @schema
-        obj['tableName'] = @table
-        obj['objectUrls'] = @object_urls
-        obj['disabled'] = @disabled
-        obj['force'] = @force
+        obj['taskId'] = @task_id
+        obj['force'] = true if @force
         obj
       end