embulk-output-vertica 0.5.8 → 0.5.9

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: a5c852962bbea4e096164da3d4e1686a2c4dddda
-  data.tar.gz: db0801b7d7645f8a93442339e91427ea80c8fac9
+  metadata.gz: 43ce7f3d95fb038f1ed88f0d6439971ca7fcaae3
+  data.tar.gz: ba0bbaf232f3e38694ef9671b677998e2773c6a1
 SHA512:
-  metadata.gz: 74423ccdc71170c85ead4e7b5181d4308f6c7ab391de32fcd533547cc4729aee8aaefb57792954d9680179ec0a7b29e7728fc54d9745139b9fa680aa8b2ff1fd
-  data.tar.gz: fbdb7aa14a69cb04fbbce927ed8da214daf099b1043c9f42faba65574058e1b19fee0c97625d31f6272220944cad470bebfb1001774a5bf6fa36d6b0b9ec331b
+  metadata.gz: 25060fb55332281e7f54a88cdfb6d91d90287dbc3250722e4ca9b56c935bb728e1269e6d954d91e82822dcbfb5ecb4e5ed29833d9ae9deb04f1449a5e0ae4c2d
+  data.tar.gz: 6826c93e0abd8cbb0664145fbbb19899d754e8e67948b1ac9e151bca324235f66d5ab995c8ed209ca138e3e78d78be9c08574bbac5c03fe32f87d74715f22ec6

data/CHANGELOG.md

@@ -1,3 +1,15 @@
+# 0.5.9 (2016/01/26)
+
+Enhancements:
+
+* log speed (rows/sec) of progress
+
+Trivial changes:
+
+* Use ::Jvertica.quote to quote resource_pool
+* change log_level of push / pop finish
+* change log_level of select result from debug to trace
+
 # 0.5.8 (2016/01/24)
 
 Enhancements:

data/embulk-output-vertica.gemspec

@@ -1,6 +1,6 @@
 Gem::Specification.new do |spec|
   spec.name          = "embulk-output-vertica"
-  spec.version       = "0.5.8"
+  spec.version       = "0.5.9"
   spec.authors       = ["eiji.sekiya", "Naotoshi Seo"]
   spec.email         = ["eiji.sekiya.0326@gmail.com", "sonots@gmail.com"]
   spec.summary       = "Vertica output plugin for Embulk"

data/lib/embulk/output/vertica.rb

@@ -109,7 +109,7 @@ module Embulk
           connect(task) do |jv|
             # clean up the temp table
             query(jv, %[DROP TABLE IF EXISTS #{quoted_schema}.#{quoted_temp_table}])
-            Embulk.logger.debug { "embulk-output-vertica: select result\n#{query(jv, %[SELECT * FROM #{quoted_schema}.#{quoted_table} LIMIT 10]).map {|row| row.to_h }.join("\n") rescue nil}" }
+            Embulk.logger.trace { "embulk-output-vertica: select result\n#{query(jv, %[SELECT * FROM #{quoted_schema}.#{quoted_table} LIMIT 10]).map {|row| row.to_h }.join("\n") rescue nil}" }
           end
         end
         # this is for -o next_config option, add some paramters for next time execution if wants
@@ -154,8 +154,8 @@ module Embulk
           database: task['database'],
         })
 
-        if task['resource_pool']
-          query(jv, "SET SESSION RESOURCE_POOL = '#{task['resource_pool']}'")
+        if resource_pool = task['resource_pool']
+          query(jv, "SET SESSION RESOURCE_POOL = #{::Jvertica.quote(resource_pool)}")
         end
 
         if block_given?

data/lib/embulk/output/vertica/output_thread.rb

@@ -54,6 +54,7 @@ module Embulk
           @outer_thread = Thread.current
           @thread_active = false
           @progress_log_timer = Time.now
+          @previous_num_input_rows = 0
 
           case task['compress']
           when 'GZIP'
@@ -92,13 +93,20 @@ module Embulk
             buf << record << "\n"
             @num_input_rows += 1
           end
+          sleep 10
           now = Time.now
           if @progress_log_timer < now - 10 # once in 10 seconds
+            speed = ((@num_input_rows - @previous_num_input_rows) / (now - @progress_log_timer).to_f).round(1)
             @progress_log_timer = now
-            Embulk.logger.info { "embulk-output-vertica: num_input_rows #{@num_input_rows}" }
+            @previous_num_input_rows = @num_input_rows
+            Embulk.logger.info { "embulk-output-vertica: num_input_rows #{num_format(@num_input_rows)} (#{num_format(speed)} rows/sec)" }
           end
         end
 
+        def num_format(number)
+          number.to_s.gsub(/(\d)(?=(\d{3})+(?!\d))/, '\1,')
+        end
+
         def run
           Embulk.logger.debug { "embulk-output-vertica: thread started" }
           Vertica.connect(@task) do |jv|
@@ -107,7 +115,7 @@ module Embulk
               num_output_rows, rejects = copy(jv, copy_sql) do |stdin|
                 while json_page = @queue.pop
                   if json_page == 'finish'
-                    Embulk.logger.trace { "embulk-output-vertica: popped finish" }
+                    Embulk.logger.debug { "embulk-output-vertica: popped finish" }
                     break
                   end
                   Embulk.logger.trace { "embulk-output-vertica: dequeued" }
@@ -155,8 +163,8 @@ module Embulk
         def commit
           @thread_active = false
           if @thread.alive?
+            Embulk.logger.debug { "embulk-output-vertica: push finish" }
             @queue.push('finish')
-            Embulk.logger.trace { "embulk-output-vertica: pushed finish" }
             Thread.pass
             @thread.join
           else

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: embulk-output-vertica
 version: !ruby/object:Gem::Version
-  version: 0.5.8
+  version: 0.5.9
 platform: ruby
 authors:
 - eiji.sekiya
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2016-01-23 00:00:00.000000000 Z
+date: 2016-01-27 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: jvertica