bricolage 5.23.0 → 5.23.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 68aff8b010487891f214258a2c0b3e4c175e9195
-  data.tar.gz: c9cf9a4ee6d938e7b05456a0643a687094cc379a
+  metadata.gz: 8a460416e7ef2fb18814d325a89d9107916a2741
+  data.tar.gz: b2f0be30fee48a26dc1b6064b47b4039829a9e95
 SHA512:
-  metadata.gz: 3c923cd19ed97f203a12e090efafd55edfbe79f2807cd739e92704d1788caa45bebe074c56ce156276ee0af69fbf9cbfcbb7b0a25bdcb73068c93d9df8b5aed3
-  data.tar.gz: 25d863da9ce8fb1f0d380f4fe9ce80cbe8f9201fc82ca1ad3fe49259453f6a12325723254884ac87b77b0a22da5c8856c0cb10332539e7832fa5ebffc1b01c92
+  metadata.gz: 15018c00f2be9725375fc7f97ac9f5ef42f3dd6af39a4afc3cdb740bb2ef287ce4541bad920d8a78b13d275b3cede650dab737c14c09f12d577ca937c157504d
+  data.tar.gz: 80dae05f2fcfebab99363c2c42e533e732454e5585ce45b8944da403ab2e497a014e41bd6199aeb2207baf57bbb94b8e8924ad93b386947dfdf68b345e419429

data/jobclass/streaming_load.rb

@@ -27,6 +27,7 @@ class StreamingLoadJobClass < RubyJobClass
     params.add Bricolage::OptionalBoolParam.new('noop', 'Does not change any data.')
     params.add Bricolage::OptionalBoolParam.new('load-only', 'Just issues COPY statement to work table and quit. No INSERT, no dequeue, no load log.')
     params.add Bricolage::OptionalBoolParam.new('dequeue-only', 'Dequeues already loaded files.')
+    params.add Bricolage::OptionalBoolParam.new('skip-work', 'Skip work table and load directly into dest-table')
   end
 
   def StreamingLoadJobClass.declarations(params)
@@ -71,7 +72,8 @@ class StreamingLoadJobClass < RubyJobClass
       sql: params['sql-file'],
       logger: ds.logger,
       noop: params['noop'],
-      load_only: params['load-only']
+      load_only: params['load-only'],
+      skip_work: params['skip-work']
     )
   end
 
@@ -95,7 +97,7 @@ class StreamingLoadJobClass < RubyJobClass
     def initialize(data_source:, queue:, keep_ctl:,
         table:, work_table: nil, log_table: nil, load_options: nil,
         sql: nil,
-        logger:, noop: false, load_only: false)
+        logger:, noop: false, load_only: false, skip_work: false)
       @ds = data_source
       @src = queue
       @keep_ctl = keep_ctl
@@ -107,6 +109,7 @@ class StreamingLoadJobClass < RubyJobClass
       @logger = logger
       @noop = noop
       @load_only = load_only
+      @skip_work = skip_work
 
       @start_time = Time.now
       @end_time = nil
@@ -159,7 +162,7 @@ class StreamingLoadJobClass < RubyJobClass
       end
       create_load_log_file(objects) {|log_url|
         @ds.open {|conn|
-          execute_update conn, "truncate #{work_table}"
+          execute_update conn, "truncate #{work_table}" unless @skip_work
           conn.transaction {|txn|
             execute_update conn, "delete #{log_table_wk}"
             execute_update conn, load_log_copy_stmt(log_table_wk, log_url, @src.credential_string)
@@ -174,10 +177,11 @@ class StreamingLoadJobClass < RubyJobClass
               txn.truncate_and_commit log_table_wk
             else
               create_manifest_file(not_loaded) {|manifest_url|
-                execute_update conn, manifest_copy_stmt(work_table, manifest_url)
+                dest_table = @skip_work ? @table : work_table
+                execute_update conn, manifest_copy_stmt(dest_table, manifest_url)
                 @logger.info "load succeeded: #{manifest_url}" unless @noop
                 unless @load_only
-                  commit_work_table conn, work_table
+                  commit_work_table conn, work_table unless @skip_work
                   commit_load_log conn, log_table_wk
                 end
                 txn.truncate_and_commit log_table_wk

data/lib/bricolage/version.rb

@@ -1,4 +1,4 @@
 module Bricolage
   APPLICATION_NAME = 'Bricolage'
-  VERSION = '5.23.0'
+  VERSION = '5.23.1'
 end

data/test/home/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: ../..
   specs:
-    bricolage (5.23.0)
+    bricolage (5.23.1)
       aws-sdk (~> 2)
       mysql2
       pg (~> 0.18.0)

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: bricolage
 version: !ruby/object:Gem::Version
-  version: 5.23.0
+  version: 5.23.1
 platform: ruby
 authors:
 - Minero Aoki
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2017-02-10 00:00:00.000000000 Z
+date: 2017-02-14 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: pg