bricolage-streamingload 0.7.1 → 0.8.0

data/lib/bricolage/streamingload/{loaderservice.rb → taskhandler.rb}

@@ -1,7 +1,7 @@
 require 'bricolage/context'
 require 'bricolage/sqsdatasource'
 require 'bricolage/streamingload/task'
-require 'bricolage/streamingload/loader'
+require 'bricolage/streamingload/job'
 require 'bricolage/streamingload/alertinglogger'
 require 'bricolage/logger'
 require 'bricolage/exception'
@@ -13,60 +13,60 @@ module Bricolage
 
   module StreamingLoad
 
-    class LoaderService < SQSDataSource::MessageHandler
+    class TaskHandler < SQSDataSource::MessageHandler
 
-      def LoaderService.main
-        opts = LoaderServiceOptions.new(ARGV)
+      def TaskHandler.main
+        opts = TaskHandlerOptions.new(ARGV)
         opts.parse
-        unless opts.rest_arguments.size == 1
+        unless opts.rest_arguments.size <= 1
           $stderr.puts opts.usage
           exit 1
         end
-        config_path, * = opts.rest_arguments
+        config_path = opts.rest_arguments[0] || "#{opts.working_dir}/config/#{opts.environment}/streamingload.yml"
         config = YAML.load(File.read(config_path))
+
         logger = opts.log_file_path ? new_logger(opts.log_file_path, config) : nil
         ctx = Context.for_application(opts.working_dir, environment: opts.environment, logger: logger)
-        redshift_ds = ctx.get_data_source('sql', config.fetch('redshift-ds', 'db_data'))
+
+        ctl_ds = ctx.get_data_source('sql', config.fetch('ctl-postgres-ds', 'db_ctl'))
         task_queue = ctx.get_data_source('sqs', config.fetch('task-queue-ds', 'sqs_task'))
-        raw_logger = logger = ctx.logger
-        if config.key?('alert-level')
-          logger = AlertingLogger.new(
-            logger: raw_logger,
-            sns_datasource: ctx.get_data_source('sns', config.fetch('sns-ds', 'sns')),
-            alert_level: config.fetch('alert-level', 'warn')
-          )
-        end
+        service_logger =
+          if config.key?('alert-level')
+            new_alerting_logger(ctx, config)
+          else
+            ctx.logger
+          end
 
-        service = new(
+        task_handler = new(
           context: ctx,
-          control_data_source: ctx.get_data_source('sql', config.fetch('ctl-postgres-ds', 'db_ctl')),
-          data_source: redshift_ds,
+          ctl_ds: ctl_ds,
           task_queue: task_queue,
           working_dir: opts.working_dir,
-          logger: logger
+          logger: service_logger,
+          job_class: opts.noop? ? NoopJob : Job
         )
 
         if opts.task_id
           # Single task mode
-          service.execute_task_by_id opts.task_id
+          task_handler.execute_task_by_id opts.task_id, force: opts.force?
         else
           # Server mode
           Process.daemon(true) if opts.daemon?
           Dir.chdir '/'
           create_pid_file opts.pid_file_path if opts.pid_file_path
           begin
-            logger.info "*** bricolage-streaming-loader started: pid=#{$$}"
-            service.event_loop
-            logger.info "*** bricolage-streaming-loader shutdown gracefully: pid=#{$$}"
+            service_logger.info "*** bricolage-streaming-loader started: pid=#{$$}"
+            task_handler.event_loop
+            service_logger.info "*** bricolage-streaming-loader shutdown gracefully: pid=#{$$}"
           rescue Exception => ex
-            logger.exception(ex)
-            logger.error "*** bricolage-streaming-loader abort: pid=#{$$}"
+            service_logger.exception(ex)
+            service_logger.error "*** bricolage-streaming-loader abort: pid=#{$$}"
             raise
           end
         end
       end
 
-      def LoaderService.new_logger(path, config)
+      def TaskHandler.new_logger(path, config)
         Logger.new(
           device: path,
           rotation_period: config.fetch('log-rotation-period', 'daily'),
@@ -74,7 +74,15 @@ module Bricolage
         )
       end
 
-      def LoaderService.create_pid_file(path)
+      def TaskHandler.new_alerting_logger(ctx, config)
+        AlertingLogger.new(
+          logger: ctx.logger,
+          sns_datasource: ctx.get_data_source('sns', config.fetch('sns-ds', 'sns')),
+          alert_level: config.fetch('alert-level', 'warn')
+        )
+      end
+
+      def TaskHandler.create_pid_file(path)
         File.open(path, 'w') {|f|
           f.puts $$
         }
@@ -82,76 +90,91 @@ module Bricolage
         # ignore
       end
 
-      def initialize(context:, control_data_source:, data_source:, task_queue:, working_dir:, logger:)
+      def initialize(context:, ctl_ds:, task_queue:, working_dir:, logger:, job_class: Job)
         @ctx = context
-        @ctl_ds = control_data_source
-        @ds = data_source
+        @ctl_ds = ctl_ds
         @task_queue = task_queue
         @working_dir = working_dir
         @logger = logger
+        @job_class = job_class
       end
 
       attr_reader :logger
 
-      def event_loop
-        @task_queue.handle_messages(handler: self, message_class: Task)
-      end
-
-      def execute_task_by_id(task_id)
-        execute_task load_task(task_id)
+      def execute_task_by_id(task_id, force: false)
+        job = @job_class.new(context: @ctx, ctl_ds: @ctl_ds, task_id: task_id, force: force, logger: @logger)
+        job.execute(fail_fast: true)
       end
 
-      def load_task(task_id, force: true)
-        @ctl_ds.open {|conn| LoadTask.load(conn, task_id, force: force) }
+      def event_loop
+        @task_queue.handle_messages(handler: self, message_class: Task)
       end
 
       # message handler
-      def handle_unknown(task)
-        @logger.warn "unknown task: #{task.message_body}"
-        @task_queue.delete_message task
+      def handle_unknown(t)
+        @logger.warn "unknown task: #{t.message_body}"
+        @task_queue.delete_message t
       end
 
       # message handler
-      def handle_streaming_load_v3(task)
+      def handle_streaming_load_v3(t)
         Dir.chdir(@working_dir) {
-          loadtask = load_task(task.id, force: task.force?)
-          if loadtask.disabled
-            # Skip if disabled, and don't delete SQS message.
-            @logger.info "skip disabled task: task_id=#{task.id}"
-            return
+          job = @job_class.new(context: @ctx, ctl_ds: @ctl_ds, task_id: t.task_id, force: t.force?, logger: @logger)
+          if job.execute
+            @task_queue.delete_message(t)
           end
-          execute_task(loadtask)
-          # Do not use async delete
-          @task_queue.delete_message(task)
         }
-      rescue => ex
+      rescue Exception => ex
         @logger.exception ex
       end
 
-      def execute_task(task)
-        @logger.info "execute task: task_id=#{task.id} table=#{task.qualified_name}"
-        loader = Loader.load_from_file(@ctx, @ctl_ds, task, logger: @logger)
-        loader.execute
+      def job_class
+        @job_class ||= Job
+      end
+
+      attr_writer :job_class
+
+    end
+
+
+    class NoopJob
+
+      def initialize(context:, ctl_ds:, task_id:, force: false, logger:)
+        @ctx = context
+        @ctl_ds = ctl_ds
+        @task_id = task_id
+        @force = force
+        @logger = logger
+      end
+
+      def execute(fail_fast: false)
+        @logger.info "execute: fail_fast=#{fail_fast}"
+        execute_task
+        true
+      end
+
+      def execute_task
+        @logger.info "execute_task: task_id=#{@task_id} force=#{@force} ctx=#{@ctx.home_path} ctl_ds=#{@ctl_ds.name} dir=#{@working_dir}"
       end
 
     end
 
-    class LoaderServiceOptions
+
+    class TaskHandlerOptions
 
       def initialize(argv)
         @argv = argv
-        @task_id = nil
         @environment = Context::DEFAULT_ENV
         @daemon = false
         @log_file_path = nil
         @pid_file_path = nil
         @working_dir = Dir.getwd
+        @task_id = nil
+        @force = false
+        @noop = false
         @rest_arguments = nil
 
         @opts = opts = OptionParser.new("Usage: #{$0} CONFIG_PATH")
-        opts.on('--task-id=ID', 'Execute oneshot load task (implicitly disables daemon mode).') {|task_id|
-          @task_id = task_id
-        }
         opts.on('-e', '--environment=NAME', "Sets execution environment [default: #{@environment}]") {|env|
           @environment = env
         }
@@ -167,6 +190,15 @@ module Bricolage
         opts.on('--working-dir=PATH', "Loader working directory. [default: #{@working_dir}]") {|path|
           @working_dir = path
         }
+        opts.on('--task-id=ID', 'Execute oneshot load task (implicitly disables daemon mode).') {|task_id|
+          @task_id = task_id
+        }
+        opts.on('--force', 'Disables loaded check.') {|path|
+          @force = true
+        }
+        opts.on('--noop', 'Does not execute tasks.') {
+          @noop = true
+        }
         opts.on('--help', 'Prints this message and quit.') {
           puts opts.help
           exit 0
@@ -190,7 +222,6 @@ module Bricolage
 
       attr_reader :rest_arguments
 
-      attr_reader :task_id
       attr_reader :environment
 
       def daemon?
@@ -201,6 +232,16 @@ module Bricolage
       attr_reader :pid_file_path
       attr_reader :working_dir
 
+      attr_reader :task_id
+
+      def force?
+        @force
+      end
+
+      def noop?
+        @noop
+      end
+
     end
 
   end

data/lib/bricolage/streamingload/version.rb

@@ -1,5 +1,5 @@
 module Bricolage
   module StreamingLoad
-    VERSION = '0.7.1'
+    VERSION = '0.8.0'
   end
 end

data/test/streamingload/test_dispatcher.rb

@@ -87,9 +87,9 @@ module Bricolage
 
         # Object Buffer
         assert_equal [], unassigned_objects(ctl_ds)
-        task = ctl_ds.open {|conn| LoadTask.load(conn, task_id) }
-        assert_equal 'testschema', task.schema
-        assert_equal 'desttable', task.table
+        task = Job::ControlConnection.open(ctl_ds) {|ctl| ctl.load_task(task_id) }
+        assert_equal 'testschema', task.schema_name
+        assert_equal 'desttable', task.table_name
         assert_equal 10, task.object_urls.size
       end
 
@@ -180,9 +180,9 @@ module Bricolage
 
         # Object Buffer
         assert_equal [], unassigned_table_objects(ctl_ds, 'testschema.bbb')
-        task = ctl_ds.open {|conn| LoadTask.load(conn, task_id) }
-        assert_equal 'testschema', task.schema
-        assert_equal 'bbb', task.table
+        task = Job::ControlConnection.open(ctl_ds) {|ctl| ctl.load_task(task_id) }
+        assert_equal 'testschema', task.schema_name
+        assert_equal 'bbb', task.table_name
         assert_equal 2, task.object_urls.size
       end