bricolage-streamingload 0.11.0 → 0.12.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 338d44b3bf3b51f784bb6a2984c60ec8ef9338d9
-  data.tar.gz: 46fb68569a99af0f28243871d36949e78cc89f63
+  metadata.gz: c8c9c822c1c2d207827cc5d0b6573140254b1638
+  data.tar.gz: 166188b17014074c82a963aa9aecaff796ced70d
 SHA512:
-  metadata.gz: 5d512c32008682db824b92b7907da884a356f65e86bdc1abdce55345196d1eb1116528a65c7b27cb12e76a8dc9961ce9d46f0121ab2310a01152b540d75b2567
-  data.tar.gz: bd06304e4a2602d13fa30b912ed1bfda941718165b8f2776c14a64b0d4ff616bf19348d017c767f35fa6d48758fab8c378b7b5ca3b86d0e774f7df84600bb55a
+  metadata.gz: 17e35b2d5d49312d92a4f9f7e232cd6ff9daeb7975d7a1383936a935d12dd22c89550e1fece87920c36cac46fea6aa5517431c7af56e741cb5fe774a21cd7825
+  data.tar.gz: 7556f6eb45b74d041891f3d9254aee4bf32c30d12e9aa61093351f5a08af4c7f6a04926b75a37d8cfbde4c359a50e5efc610cd6ceda605f63ca147e3d96d3285

data/lib/bricolage/streamingload/job.rb

@@ -86,8 +86,8 @@ module Bricolage
           end
 
           if @task.unknown_state?
-            true_status = DataConnection.open(@data_ds, @logger) {|data|
-              data.get_job_status(@log_table, @task.last_job_id)
+            true_status = DataConnection.open(@data_ds, log_table: @log_table, logger: @logger) {|data|
+              data.get_job_status(@task.last_job_id)
             }
             @logger.info "fixiating unknown job status: job_id=#{@task.last_job_id}, status=(unknown->#{true_status})"
             @task.fix_last_job_status true_status
@@ -142,11 +142,14 @@ module Bricolage
         params = JobParams.load(@context, task.task_class, task.schema_name, task.table_name)
         @data_ds = params.ds
         @manifest = ManifestFile.create(ds: params.ctl_bucket, job_id: job_id, object_urls: task.object_urls, logger: @logger)
-        DataConnection.open(params.ds, @logger) {|data|
+        log = LoadLog.new
+        log.task_id = @task_id
+        log.job_id = job_id
+        DataConnection.open(params.ds, log_table: @log_table, logger: @logger) {|data|
           if params.enable_work_table?
-            data.load_with_work_table params.work_table, @manifest, params.load_options_string, params.sql_source, @log_table, job_id
+            data.load_with_work_table params.work_table, @manifest, params.load_options_string, params.sql_source, log
           else
-            data.load_objects params.dest_table, @manifest, params.load_options_string, @log_table, job_id
+            data.load_objects params.dest_table, @manifest, params.load_options_string, log
           end
         }
       end
@@ -175,18 +178,22 @@ module Bricolage
       end
 
 
+      LoadLog = Struct.new(:task_id, :job_id)
+
+
       class DataConnection
 
         include SQLUtils
 
-        def DataConnection.open(ds, logger = ds.logger, &block)
-          new(ds, logger).open(&block)
+        def DataConnection.open(ds, log_table:, logger: ds.logger, &block)
+          new(ds, log_table: log_table, logger: logger).open(&block)
         end
 
-        def initialize(ds, logger = ds.logger)
+        def initialize(ds, log_table:, logger: ds.logger)
           @ds = ds
-          @connection = nil
+          @log_table = log_table
           @logger = logger
+          @connection = nil
         end
 
         def open(&block)
@@ -198,27 +205,27 @@ module Bricolage
           raise DataConnectionFailed, "data connection failed: #{ex.message}"
         end
 
-        def get_job_status(log_table, job_id)
-          count = @connection.query_value("select count(*) from #{log_table} where job_id = #{job_id}")
+        def get_job_status(job_id)
+          count = @connection.query_value("select count(*) from #{@log_table} where job_id = #{job_id}")
           count.to_i > 0 ? 'success' : 'failure'
         end
 
-        def load_with_work_table(work_table, manifest, options, sql_source, log_table, job_id)
+        def load_with_work_table(work_table, manifest, options, sql_source, log)
           @connection.transaction {|txn|
             # NOTE: This transaction ends with truncation, this DELETE does nothing
             # from the second time.  So don't worry about DELETE cost here.
             @connection.execute("delete from #{work_table}")
             execute_copy work_table, manifest, options
             @connection.execute sql_source
-            write_load_log log_table, job_id
+            write_load_log log
             txn.truncate_and_commit work_table
           }
         end
 
-        def load_objects(dest_table, manifest, options, log_table, job_id)
+        def load_objects(dest_table, manifest, options, log)
           @connection.transaction {|txn|
             execute_copy dest_table, manifest, options
-            write_load_log log_table, job_id
+            write_load_log log
           }
         end
 
@@ -236,8 +243,8 @@ module Bricolage
           @logger.info "load succeeded: #{manifest.url}"
         end
 
-        def write_load_log(log_table, job_id)
-          @connection.execute("insert into #{log_table} (job_id, finish_time) values (#{job_id}, current_timestamp)")
+        def write_load_log(log)
+          @connection.execute("insert into #{@log_table} (task_id, job_id, finish_time) values (#{log.task_id}, #{log.job_id}, current_timestamp)")
         end
 
       end   # class DataConnection

data/lib/bricolage/streamingload/version.rb

@@ -1,5 +1,5 @@
 module Bricolage
   module StreamingLoad
-    VERSION = '0.11.0'
+    VERSION = '0.12.0'
   end
 end

data/test/streamingload/test_job.rb

@@ -30,7 +30,7 @@ module Bricolage
           assert_equal [
             "begin transaction;",
             "copy testschema.desttable from '#{job.manifest.url}' credentials 'cccc' manifest statupdate false compupdate false json 'auto' gzip timeformat 'auto' dateformat 'auto' acceptanydate acceptinvchars ' ' truncatecolumns trimblanks ;",
-            "insert into strload_load_logs (job_id, finish_time) values (#{job.job_id}, current_timestamp)",
+            "insert into strload_load_logs (task_id, job_id, finish_time) values (1, #{job.job_id}, current_timestamp)",
             "commit;"
           ], job.data_ds.sql_list
 
@@ -58,7 +58,7 @@ module Bricolage
             "delete from testschema.with_work_table_wk",
             "copy testschema.with_work_table_wk from '#{job.manifest.url}' credentials 'cccc' manifest statupdate false compupdate false json 'auto' gzip timeformat 'auto' dateformat 'auto' acceptanydate acceptinvchars ' ' truncatecolumns trimblanks ;",
             "insert into testschema.with_work_table select * from testschema.with_work_table_wk;\n",
-            "insert into strload_load_logs (job_id, finish_time) values (#{job.job_id}, current_timestamp)",
+            "insert into strload_load_logs (task_id, job_id, finish_time) values (11, #{job.job_id}, current_timestamp)",
             "truncate testschema.with_work_table_wk;"
           ], job.data_ds.sql_list
 
@@ -114,7 +114,7 @@ module Bricolage
           assert_equal [
             "begin transaction;",
             "copy testschema.desttable from '#{job.manifest.url}' credentials 'cccc' manifest statupdate false compupdate false json 'auto' gzip timeformat 'auto' dateformat 'auto' acceptanydate acceptinvchars ' ' truncatecolumns trimblanks ;",
-            "insert into strload_load_logs (job_id, finish_time) values (#{job.job_id}, current_timestamp)",
+            "insert into strload_load_logs (task_id, job_id, finish_time) values (11, #{job.job_id}, current_timestamp)",
             "commit;"
           ], job.data_ds.sql_list
 
@@ -303,7 +303,7 @@ module Bricolage
           assert_equal [
             "begin transaction;",
             "copy testschema.desttable from '#{job.manifest.url}' credentials 'cccc' manifest statupdate false compupdate false json 'auto' gzip timeformat 'auto' dateformat 'auto' acceptanydate acceptinvchars ' ' truncatecolumns trimblanks ;",
-            "insert into strload_load_logs (job_id, finish_time) values (#{job.job_id}, current_timestamp)",
+            "insert into strload_load_logs (task_id, job_id, finish_time) values (11, #{job.job_id}, current_timestamp)",
             "commit;"
           ], job.data_ds.sql_list
 

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: bricolage-streamingload
 version: !ruby/object:Gem::Version
-  version: 0.11.0
+  version: 0.12.0
 platform: ruby
 authors:
 - Minero Aoki
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2017-02-06 00:00:00.000000000 Z
+date: 2017-12-25 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bricolage
@@ -155,7 +155,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.5.1
+rubygems_version: 2.6.11
 signing_key: 
 specification_version: 4
 summary: Bricolage Streaming Load Daemon