bricolage-streamingload 0.8.1 → 0.9.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 8bc418b99ed85932c9f2483d643b34d6a76788f0
-  data.tar.gz: 5a1310ac8711de6c9d009ea1405ab9ba7e50918b
+  metadata.gz: 8bddfd0337158c01e5acb0fac9ff9d4b3e152029
+  data.tar.gz: 7243c54d80c5bcd4c8e157fd8d5a14e66e629abe
 SHA512:
-  metadata.gz: c205fe5bf75dfee2fb4abe845afb0c8ac91118a461c6b54e4f8534b69262f3db37cff1ddac9d874f8370b94ad89cfc2772b96cb95539cb8e9a453e919f7def62
-  data.tar.gz: 6d4319ce64cf4a115606075d1edb16df725b781ab7c8a6c362df608d4ed6c095e50e80682cdaaf36285781bac34a8117d99688e832862e0b688b79e47f20c0ca
+  metadata.gz: 3e85208c4fcbf1a199169e75b40490181127fbaf7398e6ebb78daaead3f36704e76b7177ce43edfe404e5509e95f5a592ed8db4d4f7fa379efcca0d382fdedd6
+  data.tar.gz: eb8187f0b900731bdf21c4e3b3ffbb0ac24ad133bd6d7819b20e4b7eb24420e649bf690d43f0b0e3aa5caf496146e208a0d13fa33c6be72183b8469eca96bb1c

data/lib/bricolage/streamingload/job.rb

@@ -18,9 +18,10 @@ module Bricolage
 
     class Job
 
-      def initialize(context:, ctl_ds:, task_id:, force: false, logger:)
+      def initialize(context:, ctl_ds:, log_table: 'strload_load_logs', task_id:, force: false, logger:)
         @context = context
         @ctl_ds = ctl_ds
+        @log_table = log_table
         @task_id = task_id
         @force = force
         @logger = logger
@@ -53,14 +54,17 @@ module Bricolage
         @logger.error ex.message
         wait_for_connection('ctl', @ctl_ds) unless fail_fast
         return false
-      rescue DataConnectionFailed
+      rescue DataConnectionFailed => ex
+        @logger.error ex.message
         wait_for_connection('data', @data_ds) unless fail_fast
         # FIXME: tmp: We don't know the transaction was succeeded or not in the Redshift, auto-retry is too dangerous.
         #return false
         return true
-      rescue JobFailure
+      rescue JobFailure => ex
+        @logger.error ex.message
         return false
-      rescue JobError
+      rescue JobError => ex
+        @logger.error ex.message
         return true
       rescue Exception => ex
         @logger.exception ex
@@ -101,14 +105,12 @@ module Bricolage
 
         # FIXME: tmp: should be a failure, not an error.
         rescue DataConnectionFailed => ex
-          @logger.error ex.message
           ctl.open {
             ctl.abort_job job_id, 'error', ex.message.lines.first.strip
           }
           raise
 
         rescue JobFailure => ex
-          @logger.error ex.message
           ctl.open {
             fail_count = ctl.fail_count(@task_id)
             final_retry = (fail_count >= MAX_RETRY)
@@ -118,7 +120,6 @@ module Bricolage
           }
           raise
         rescue JobError => ex
-          @logger.error ex.message
           ctl.open {
             ctl.abort_job job_id, 'error', ex.message.lines.first.strip
           }
@@ -138,9 +139,9 @@ module Bricolage
         @manifest = ManifestFile.create(ds: params.ctl_bucket, job_id: job_id, object_urls: task.object_urls, logger: @logger)
         DataConnection.open(params.ds, @logger) {|data|
           if params.enable_work_table?
-            data.load_with_work_table params.work_table, @manifest, params.load_options_string, params.sql_source
+            data.load_with_work_table params.work_table, @manifest, params.load_options_string, params.sql_source, @log_table, job_id
           else
-            data.load_objects params.dest_table, @manifest, params.load_options_string
+            data.load_objects params.dest_table, @manifest, params.load_options_string, @log_table, job_id
           end
         }
       end
@@ -192,18 +193,26 @@ module Bricolage
           raise DataConnectionFailed, "data connection failed: #{ex.message}"
         end
 
-        def load_with_work_table(work_table, manifest, options, sql_source)
+        def load_with_work_table(work_table, manifest, options, sql_source, log_table, job_id)
           @connection.transaction {|txn|
             # NOTE: This transaction ends with truncation, this DELETE does nothing
             # from the second time.  So don't worry about DELETE cost here.
             @connection.execute("delete from #{work_table}")
-            load_objects work_table, manifest, options
+            execute_copy work_table, manifest, options
             @connection.execute sql_source
+            write_load_log log_table, job_id
             txn.truncate_and_commit work_table
           }
         end
 
-        def load_objects(dest_table, manifest, options)
+        def load_objects(dest_table, manifest, options, log_table, job_id)
+          @connection.transaction {|txn|
+            execute_copy dest_table, manifest, options
+            write_load_log log_table, job_id
+          }
+        end
+
+        def execute_copy(dest_table, manifest, options)
           @connection.execute(<<-EndSQL.strip.gsub(/\s+/, ' '))
               copy #{dest_table}
               from #{s manifest.url}
@@ -217,6 +226,10 @@ module Bricolage
           @logger.info "load succeeded: #{manifest.url}"
         end
 
+        def write_load_log(log_table, job_id)
+          @connection.execute("insert into #{log_table} (job_id, finish_time) values (#{job_id}, current_timestamp)")
+        end
+
       end   # class DataConnection
 
 

data/lib/bricolage/streamingload/taskhandler.rb

@@ -30,6 +30,7 @@ module Bricolage
 
         ctl_ds = ctx.get_data_source('sql', config.fetch('ctl-postgres-ds', 'db_ctl'))
         task_queue = ctx.get_data_source('sqs', config.fetch('task-queue-ds', 'sqs_task'))
+        log_table = config.fetch('log-table', 'strload_load_logs')
         service_logger =
           if config.key?('alert-level')
             new_alerting_logger(ctx, config)
@@ -40,6 +41,7 @@ module Bricolage
         task_handler = new(
           context: ctx,
           ctl_ds: ctl_ds,
+          log_table: log_table,
           task_queue: task_queue,
           working_dir: opts.working_dir,
           logger: service_logger,
@@ -90,9 +92,10 @@ module Bricolage
         # ignore
       end
 
-      def initialize(context:, ctl_ds:, task_queue:, working_dir:, logger:, job_class: Job)
+      def initialize(context:, ctl_ds:, log_table:, task_queue:, working_dir:, logger:, job_class: Job)
         @ctx = context
         @ctl_ds = ctl_ds
+        @log_table = log_table
         @task_queue = task_queue
         @working_dir = working_dir
         @logger = logger
@@ -102,7 +105,7 @@ module Bricolage
       attr_reader :logger
 
       def execute_task_by_id(task_id, force: false)
-        job = @job_class.new(context: @ctx, ctl_ds: @ctl_ds, task_id: task_id, force: force, logger: @logger)
+        job = new_job(task_id, force)
         job.execute(fail_fast: true)
       end
 
@@ -119,7 +122,7 @@ module Bricolage
       # message handler
       def handle_streaming_load_v3(t)
         Dir.chdir(@working_dir) {
-          job = @job_class.new(context: @ctx, ctl_ds: @ctl_ds, task_id: t.task_id, force: t.force?, logger: @logger)
+          job = new_job(t.task_id, t.force?)
           if job.execute
             @task_queue.delete_message(t)
           end
@@ -128,6 +131,10 @@ module Bricolage
         @logger.exception ex
       end
 
+      def new_job(task_id, force)
+        @job_class.new(context: @ctx, ctl_ds: @ctl_ds, log_table: @log_table, task_id: task_id, force: force, logger: @logger)
+      end
+
       def job_class
         @job_class ||= Job
       end

data/lib/bricolage/streamingload/version.rb

@@ -1,5 +1,5 @@
 module Bricolage
   module StreamingLoad
-    VERSION = '0.8.1'
+    VERSION = '0.9.0'
   end
 end

data/test/streamingload/test_job.rb

@@ -27,8 +27,13 @@ module Bricolage
           job = Job.new(context: ctx, ctl_ds: ctl_ds, task_id: 1, force: false, logger: ctx.logger)
           job.execute_task
 
-          copy_stmt = "copy testschema.desttable from '#{job.manifest.url}' credentials 'cccc' manifest statupdate false compupdate false json 'auto' gzip timeformat 'auto' dateformat 'auto' acceptanydate acceptinvchars ' ' truncatecolumns trimblanks ;"
-          assert_equal [copy_stmt], job.data_ds.sql_list
+          assert_equal [
+            "begin transaction;",
+            "copy testschema.desttable from '#{job.manifest.url}' credentials 'cccc' manifest statupdate false compupdate false json 'auto' gzip timeformat 'auto' dateformat 'auto' acceptanydate acceptinvchars ' ' truncatecolumns trimblanks ;",
+            "insert into strload_load_logs (job_id, finish_time) values (#{job.job_id}, current_timestamp)",
+            "commit;"
+          ], job.data_ds.sql_list
+
           job_row = db.query_row("select * from strload_jobs where job_id = #{job.job_id}")
           assert_equal 1, job_row['task_id'].to_i
           assert_equal job.process_id, job_row['process_id']
@@ -48,11 +53,14 @@ module Bricolage
           job = Job.new(context: ctx, ctl_ds: ctl_ds, task_id: 11, force: false, logger: ctx.logger)
           job.execute_task
 
-          assert_equal 'begin transaction;', job.data_ds.sql_list[0]
-          assert_equal 'delete from testschema.with_work_table_wk', job.data_ds.sql_list[1]
-          assert_equal "copy testschema.with_work_table_wk from '#{job.manifest.url}' credentials 'cccc' manifest statupdate false compupdate false json 'auto' gzip timeformat 'auto' dateformat 'auto' acceptanydate acceptinvchars ' ' truncatecolumns trimblanks ;", job.data_ds.sql_list[2]
-          assert_equal "insert into testschema.with_work_table select * from testschema.with_work_table_wk;\n", job.data_ds.sql_list[3]
-          assert_equal 'truncate testschema.with_work_table_wk;', job.data_ds.sql_list[4]
+          assert_equal [
+            "begin transaction;",
+            "delete from testschema.with_work_table_wk",
+            "copy testschema.with_work_table_wk from '#{job.manifest.url}' credentials 'cccc' manifest statupdate false compupdate false json 'auto' gzip timeformat 'auto' dateformat 'auto' acceptanydate acceptinvchars ' ' truncatecolumns trimblanks ;",
+            "insert into testschema.with_work_table select * from testschema.with_work_table_wk;\n",
+            "insert into strload_load_logs (job_id, finish_time) values (#{job.job_id}, current_timestamp)",
+            "truncate testschema.with_work_table_wk;"
+          ], job.data_ds.sql_list
 
           job_row = db.query_row("select * from strload_jobs where job_id = #{job.job_id}")
           assert_equal 11, job_row['task_id'].to_i
@@ -103,8 +111,12 @@ module Bricolage
           job = Job.new(context: ctx, ctl_ds: ctl_ds, task_id: 11, force: true, logger: ctx.logger)
           job.execute_task
 
-          copy_stmt = "copy testschema.desttable from '#{job.manifest.url}' credentials 'cccc' manifest statupdate false compupdate false json 'auto' gzip timeformat 'auto' dateformat 'auto' acceptanydate acceptinvchars ' ' truncatecolumns trimblanks ;"
-          assert_equal [copy_stmt], job.data_ds.sql_list
+          assert_equal [
+            "begin transaction;",
+            "copy testschema.desttable from '#{job.manifest.url}' credentials 'cccc' manifest statupdate false compupdate false json 'auto' gzip timeformat 'auto' dateformat 'auto' acceptanydate acceptinvchars ' ' truncatecolumns trimblanks ;",
+            "insert into strload_load_logs (job_id, finish_time) values (#{job.job_id}, current_timestamp)",
+            "commit;"
+          ], job.data_ds.sql_list
 
           job_row = db.query_row("select * from strload_jobs where job_id = #{job.job_id}")
           assert_equal 11, job_row['task_id'].to_i
@@ -127,8 +139,11 @@ module Bricolage
           assert_raise(JobFailure) {
             job.execute_task
           }
-          copy_stmt = "copy testschema.sql_fails from '#{job.manifest.url}' credentials 'cccc' manifest statupdate false compupdate false json 'auto' gzip timeformat 'auto' dateformat 'auto' acceptanydate acceptinvchars ' ' truncatecolumns trimblanks ;"
-          assert_equal [copy_stmt], job.data_ds.sql_list
+          assert_equal [
+            "begin transaction;",
+            "copy testschema.sql_fails from '#{job.manifest.url}' credentials 'cccc' manifest statupdate false compupdate false json 'auto' gzip timeformat 'auto' dateformat 'auto' acceptanydate acceptinvchars ' ' truncatecolumns trimblanks ;",
+            "abort;"
+          ], job.data_ds.sql_list
 
           job_row = db.query_row("select * from strload_jobs where job_id = #{job.job_id}")
           assert_equal 11, job_row['task_id'].to_i
@@ -153,8 +168,11 @@ module Bricolage
           assert_raise(JobFailure) {
             job.execute_task
           }
-          copy_stmt = "copy testschema.sql_fails from '#{job.manifest.url}' credentials 'cccc' manifest statupdate false compupdate false json 'auto' gzip timeformat 'auto' dateformat 'auto' acceptanydate acceptinvchars ' ' truncatecolumns trimblanks ;"
-          assert_equal [copy_stmt], job.data_ds.sql_list
+          assert_equal [
+            "begin transaction;",
+            "copy testschema.sql_fails from '#{job.manifest.url}' credentials 'cccc' manifest statupdate false compupdate false json 'auto' gzip timeformat 'auto' dateformat 'auto' acceptanydate acceptinvchars ' ' truncatecolumns trimblanks ;",
+            "abort;"
+          ], job.data_ds.sql_list
 
           job_row = db.query_row("select * from strload_jobs where job_id = #{job.job_id}")
           assert_equal 11, job_row['task_id'].to_i
@@ -183,8 +201,12 @@ module Bricolage
           assert_raise(JobCancelled) {
             job.execute_task
           }
-          copy_stmt = "copy testschema.sql_fails from '#{job.manifest.url}' credentials 'cccc' manifest statupdate false compupdate false json 'auto' gzip timeformat 'auto' dateformat 'auto' acceptanydate acceptinvchars ' ' truncatecolumns trimblanks ;"
-          assert_equal [copy_stmt], job.data_ds.sql_list
+          assert_equal [
+            "begin transaction;",
+            "copy testschema.sql_fails from '#{job.manifest.url}' credentials 'cccc' manifest statupdate false compupdate false json 'auto' gzip timeformat 'auto' dateformat 'auto' acceptanydate acceptinvchars ' ' truncatecolumns trimblanks ;",
+            "abort;"
+          ], job.data_ds.sql_list
+
           job_row = db.query_row("select * from strload_jobs where job_id = #{job.job_id}")
           assert_equal 11, job_row['task_id'].to_i
           assert_equal job.process_id, job_row['process_id']
@@ -206,7 +228,12 @@ module Bricolage
           assert_raise(JobError) {
             job.execute_task
           }
-          assert_equal 1, job.data_ds.sql_list.size
+          assert_equal [
+            "begin transaction;",
+            "copy testschema.job_error from '#{job.manifest.url}' credentials 'cccc' manifest statupdate false compupdate false json 'auto' gzip timeformat 'auto' dateformat 'auto' acceptanydate acceptinvchars ' ' truncatecolumns trimblanks ;",
+            "abort;"
+          ], job.data_ds.sql_list
+
           job_row = db.query_row("select * from strload_jobs where job_id = #{job.job_id}")
           assert_equal 11, job_row['task_id'].to_i
           assert_equal job.process_id, job_row['process_id']
@@ -227,7 +254,12 @@ module Bricolage
           assert_raise(JobError) {
             job.execute_task
           }
-          assert_equal 1, job.data_ds.sql_list.size
+          assert_equal [
+            "begin transaction;",
+            "copy testschema.unexpected_error from '#{job.manifest.url}' credentials 'cccc' manifest statupdate false compupdate false json 'auto' gzip timeformat 'auto' dateformat 'auto' acceptanydate acceptinvchars ' ' truncatecolumns trimblanks ;",
+            "abort;"
+          ], job.data_ds.sql_list
+
           job_row = db.query_row("select * from strload_jobs where job_id = #{job.job_id}")
           assert_equal 11, job_row['task_id'].to_i
           assert_equal job.process_id, job_row['process_id']
@@ -379,21 +411,39 @@ module Bricolage
 
           def transaction
             @ds.issue_sql "begin transaction;"
-            yield Transaction.new(@ds)
+            txn = Transaction.new(@ds)
+            yield txn
+          rescue
+            txn.abort unless txn.committed?
+            raise
+          ensure
+            txn.commit unless txn.committed?
           end
         end
 
         class Transaction
           def initialize(ds)
             @ds = ds
+            @commit = false
+          end
+
+          def committed?
+            @commit
           end
 
           def commit
             @ds.issue_sql "commit;"
+            @commit = true
+          end
+
+          def abort
+            @ds.issue_sql "abort;"
+            @commit = true
           end
 
           def truncate_and_commit(table)
             @ds.issue_sql "truncate #{table};"
+            @commit = true
           end
         end
       end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: bricolage-streamingload
 version: !ruby/object:Gem::Version
-  version: 0.8.1
+  version: 0.9.0
 platform: ruby
 authors:
 - Minero Aoki
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2016-10-19 00:00:00.000000000 Z
+date: 2016-10-20 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bricolage