bricolage 5.19.1 → 5.20.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 08ee4bdc2cc0df34636a9d370bd8dcc658f1fa5a
-  data.tar.gz: a26fcb9a3150e12572bacc4bd98168ebab68888f
+  metadata.gz: bb66877b6bc66492a6091a11eafbd9bb953f95e5
+  data.tar.gz: a31a4383dad5d6b8b1a319fa3a2d68900f57db55
 SHA512:
-  metadata.gz: 5915c56686b6757f8a059aa3d7bd05ae14ff0d2b8d806cca59392698fcd1ffcdf8e574a3b1204208c9b4f7dbefe92a8984165920b4a400b1c282950e3cc7c649
-  data.tar.gz: 30c0701ff39c2e15c2723a0fb4aac188a655c0ceea04c73df7c3097594367e316a563b51a62958cd3e3747f7b0b2ac38caaa7dba0081baac7e3751f4337944c5
+  metadata.gz: c2b402f7f2392a0dec1b4ed72753abed5a9683cf3860b3c6a906d86a1901abbccd9f88d5a4c7d6e52bd6e4ba235e4de0e8ce355971c3d91e064946224f8c91b7
+  data.tar.gz: c3cf0419ce2a8dc470ed15b9d9837bf8bf32cbfbc440d655813f167d101d2b9ebcb56d21099bd0653e79e44eba85b4062e189179cf4b4ff00e8b98ee1481c2fe

data/jobclass/streaming_load.rb

@@ -139,19 +139,19 @@ class StreamingLoadJobClass < RubyJobClass
       end
       create_load_log_file(objects) {|log_url|
         @ds.open {|conn|
-          create_tmp_log_table(conn, log_url) {|tmp_log_table|
-            loaded, not_loaded = partition_loaded_objects(conn, objects, tmp_log_table)
-            loaded.each do |obj|
-              obj.dequeue(force: true, noop: @noop)
-            end
-          }
+          execute_update conn, "delete #{log_table_wk};"
+          execute_update conn, load_log_copy_stmt(log_table_wk, log_url, @src.credential_string)
+          loaded, not_loaded = partition_loaded_objects(conn, objects, log_table_wk)
+          loaded.each do |obj|
+            obj.dequeue(force: true, noop: @noop)
+          end
         }
       }
     end
 
     def load
       log_basic_info
-      @logger.info 'load with manifest'
+      @logger.info 'streaming load start'
       objects = @src.queued_objects
       if objects.empty?
         @logger.info 'no target data files; exit'
@@ -159,19 +159,28 @@ class StreamingLoadJobClass < RubyJobClass
       end
       create_load_log_file(objects) {|log_url|
         @ds.open {|conn|
-          create_tmp_log_table(conn, log_url) {|tmp_log_table|
-            loaded, not_loaded = partition_loaded_objects(conn, objects, tmp_log_table)
+          execute_update conn, "truncate #{work_table}"
+          conn.transaction {|txn|
+            execute_update conn, "delete #{log_table_wk}"
+            execute_update conn, load_log_copy_stmt(log_table_wk, log_url, @src.credential_string)
+            loaded, not_loaded = partition_loaded_objects(conn, objects, log_table_wk)
             unless @load_only
               loaded.each do |obj|
                 obj.dequeue(force: true, noop: @noop)
               end
             end
-            unless not_loaded.empty?
+            if not_loaded.empty?
+              @logger.info "no data files to load" unless @noop
+              txn.truncate_and_commit log_table_wk
+            else
               create_manifest_file(not_loaded) {|manifest_url|
-                init_work_table conn
                 execute_update conn, manifest_copy_stmt(work_table, manifest_url)
                 @logger.info "load succeeded: #{manifest_url}" unless @noop
-                commit conn, work_table, tmp_log_table unless @load_only
+                unless @load_only
+                  commit_work_table conn, work_table
+                  commit_load_log conn, log_table_wk
+                end
+                txn.truncate_and_commit log_table_wk
               }
               unless @load_only
                 not_loaded.each do |obj|
@@ -184,24 +193,11 @@ class StreamingLoadJobClass < RubyJobClass
       }
     end
 
-    def commit(conn, work_table, tmp_log_table)
-      @end_time = Time.now   # commit_load_log writes this, generate before that
-      transaction(conn) {
-        commit_work_table conn, work_table
-        commit_load_log conn, tmp_log_table
-      }
-    end
-
     private
 
-    def init_work_table(conn)
-      execute_update conn, "truncate #{work_table};"
-    end
-
     def commit_work_table(conn, work_table)
       insert_stmt = @sql ? @sql.source : "insert into #{@table} select * from #{work_table};"
       execute_update conn, insert_stmt
-      # keep work table records for tracing
     end
 
     def create_manifest_file(objects)
@@ -243,8 +239,11 @@ class StreamingLoadJobClass < RubyJobClass
       csv = make_load_log_csv(objects)
       @logger.info "load_log:\n" + csv
       url = @src.put_control_file(log_name, csv, noop: @noop)
-      yield url
-      @src.remove_control_file(File.basename(url), noop: @noop)
+      begin
+        yield url
+      ensure
+        @src.remove_control_file(File.basename(url), noop: @noop)
+      end
     end
 
     def make_load_log_csv(objects)
@@ -270,10 +269,6 @@ class StreamingLoadJobClass < RubyJobClass
     LoadLogRecord = Struct.new(:job_process_id, :start_time, :end_time, :target_table, :data_file)
 
     def create_tmp_log_table(conn, log_url)
-      target_table = log_table_wk
-      execute_update conn, "truncate #{target_table};"
-      execute_update conn, load_log_copy_stmt(target_table, log_url, @src.credential_string)
-      yield target_table
     end
 
     def log_table_wk
@@ -313,6 +308,7 @@ class StreamingLoadJobClass < RubyJobClass
     end
 
     def commit_load_log(conn, tmp_table_name)
+      @end_time = Time.now
       conn.execute(<<-EndSQL)
         insert into #{@log_table}
         select
@@ -342,12 +338,6 @@ class StreamingLoadJobClass < RubyJobClass
       %Q('#{escaped}')
     end
 
-    def transaction(conn)
-      execute_update conn, 'begin transaction'
-      yield
-      execute_update conn, 'commit'
-    end
-
     def execute_update(conn, sql)
       if @noop
         log_query(sql)

data/lib/bricolage/version.rb

@@ -1,4 +1,4 @@
 module Bricolage
   APPLICATION_NAME = 'Bricolage'
-  VERSION = '5.19.1'
+  VERSION = '5.20.0'
 end

data/test/home/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: ../..
   specs:
-    bricolage (5.19.1)
+    bricolage (5.20.0)
       aws-sdk (~> 2)
       mysql2
       pg (~> 0.18.0)

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: bricolage
 version: !ruby/object:Gem::Version
-  version: 5.19.1
+  version: 5.20.0
 platform: ruby
 authors:
 - Minero Aoki