embulk-output-vertica 0.5.5 → 0.5.6

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 6d81c4848d4fad7d85e7faeff05cf759c49488d1
-  data.tar.gz: 21937e37ae064520bd43baaa0af03f9d593509a9
+  metadata.gz: 4fb8733ef4ad44731139ff6f8cb63aa7dfb6aa1c
+  data.tar.gz: b3d80ae9599a03762ac7e65cc2bcfb033f51ecae
 SHA512:
-  metadata.gz: ef5e6d9dee4d26a7ddd13d67abdc59135e06c72a40151741941828fc23dfa8fc3552791f89e25d8e05027cf15035028323e9cb1e6fd29339a7242e92ddd443ce
-  data.tar.gz: c8068505624bc2f868dc3b402bc6a4af15af219c99696d5fbd91c13efa438adf1236fb1e27d12985febaa1d8c4f3362839f5a99ba6ce621c2b108bd8e26f2ea1
+  metadata.gz: c84bb2d342b70fbbfbbd5656e41e3942115ed25fa2e786bdf7c9d10bf4540cabd0370cfc5d339c107657f71bb8b4b5365eac390d7d104f9f2e905ba3411867e9
+  data.tar.gz: 92df6faa9d4b0b2be4e6085cf6b2f87986c9f6636ce622f4c901d7e691133d691f5f286578ed48a87a40bfffce49653477abeb26849d272f9e48dea4cebfb148

data/CHANGELOG.md

@@ -1,3 +1,9 @@
+# 0.5.6 (2016/01/22)
+
+Enhancements:
+
+* Perform to_json in embulk threads. This achieves better performance when pool < num_threads
+
 # 0.5.5 (2016/01/22)
 
 Changes:

data/embulk-output-vertica.gemspec

@@ -1,6 +1,6 @@
 Gem::Specification.new do |spec|
   spec.name          = "embulk-output-vertica"
-  spec.version       = "0.5.5"
+  spec.version       = "0.5.6"
   spec.authors       = ["eiji.sekiya", "Naotoshi Seo"]
   spec.email         = ["eiji.sekiya.0326@gmail.com", "sonots@gmail.com"]
   spec.summary       = "Vertica output plugin for Embulk"

data/lib/embulk/output/vertica/output_thread.rb

@@ -6,14 +6,19 @@ module Embulk
       class OutputThreadPool
         def initialize(task, schema, size)
           @size = size
-          converters = ValueConverterFactory.create_converters(schema, task['default_timezone'], task['column_options'])
-          @output_threads = size.times.map { OutputThread.new(task, schema, converters) }
+          @schema = schema
+          @converters = ValueConverterFactory.create_converters(schema, task['default_timezone'], task['column_options'])
+          @output_threads = size.times.map { OutputThread.new(task) }
           @current_index = 0
         end
 
         def enqueue(page)
+          json_page = []
+          page.each do |record|
+            json_page << to_json(record)
+          end
           @mutex.synchronize do
-            @output_threads[@current_index].enqueue(page)
+            @output_threads[@current_index].enqueue(json_page)
             @current_index = (@current_index + 1) % @size
           end
         end
@@ -26,14 +31,18 @@ module Embulk
         def commit
           task_reports = @size.times.map {|i| @output_threads[i].commit }
         end
+
+        def to_json(record)
+          Hash[*(@schema.names.zip(record).map do |column_name, value|
+            [column_name, @converters[column_name].call(value)]
+          end.flatten!(1))].to_json
+        end
       end
 
       class OutputThread
-        def initialize(task, schema, converters)
+        def initialize(task)
           @task = task
-          @schema = schema
           @queue = SizedQueue.new(1)
-          @converters = converters
           @num_input_rows = 0
           @num_output_rows = 0
           @num_rejected_rows = 0
@@ -49,10 +58,10 @@ module Embulk
           end
         end
 
-        def enqueue(page)
+        def enqueue(json_page)
           if @thread_active and @thread.alive?
             Embulk.logger.trace { "embulk-output-vertica: enqueue" }
-            @queue.push(page)
+            @queue.push(json_page)
           else
             Embulk.logger.info { "embulk-output-vertica: thread is dead, but still trying to enqueue" }
             raise RuntimeError, "embulk-output-vertica: thread is died, but still trying to enqueue"
@@ -71,13 +80,11 @@ module Embulk
           io << buf
         end
 
-        def write_buf(buf, page, &block)
-          page.each do |record|
+        def write_buf(buf, json_page, &block)
+          json_page.each do |record|
             yield(record) if block_given?
             Embulk.logger.trace { "embulk-output-vertica: record #{record}" }
-            json = to_json(record)
-            Embulk.logger.trace { "embulk-output-vertica: to_json #{json}" }
-            buf << json << "\n"
+            buf << record << "\n"
             @num_input_rows += 1
           end
           now = Time.now
@@ -90,18 +97,17 @@ module Embulk
         def run
           Embulk.logger.debug { "embulk-output-vertica: thread started" }
           Vertica.connect(@task) do |jv|
-            json = nil # for log
             begin
               last_record = nil
               num_output_rows, rejects = copy(jv, copy_sql) do |stdin|
-                while page = @queue.pop
-                  if page == 'finish'
+                while json_page = @queue.pop
+                  if json_page == 'finish'
                     Embulk.logger.trace { "embulk-output-vertica: popped finish" }
                     break
                   end
                   Embulk.logger.trace { "embulk-output-vertica: dequeued" }
 
-                  @write_proc.call(stdin, page) do |record|
+                  @write_proc.call(stdin, json_page) do |record|
                     last_record = record
                   end
                 end
@@ -115,11 +121,11 @@ module Embulk
               Embulk.logger.debug { "embulk-output-vertica: COMMITTED!" }
             rescue java.sql.SQLDataException => e
               if @task['reject_on_materialized_type_error'] and e.message =~ /Rejected by user-defined parser/
-                Embulk.logger.warn "embulk-output-vertica: ROLLBACK! some of column types and values types do not fit #{json}"
+                Embulk.logger.warn "embulk-output-vertica: ROLLBACK! some of column types and values types do not fit #{last_record}"
               else
                 Embulk.logger.warn "embulk-output-vertica: ROLLBACK!"
               end
-              Embulk.logger.info { "embulk-output-vertica: last_record: #{to_json(last_record)}" }
+              Embulk.logger.info { "embulk-output-vertica: last_record: #{last_record}" }
               jv.rollback
               raise e # die transaction
             rescue => e
@@ -170,12 +176,6 @@ module Embulk
           @copy_sql ||= "COPY #{quoted_schema}.#{quoted_temp_table} FROM STDIN#{compress}#{fjsonparser}#{copy_mode}#{abort_on_error} NO COMMIT"
         end
 
-        def to_json(record)
-          Hash[*(@schema.names.zip(record).map do |column_name, value|
-            [column_name, @converters[column_name].call(value)]
-          end.flatten!(1))].to_json
-        end
-
         def quoted_schema
           ::Jvertica.quote_identifier(@task['schema'])
         end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: embulk-output-vertica
 version: !ruby/object:Gem::Version
-  version: 0.5.5
+  version: 0.5.6
 platform: ruby
 authors:
 - eiji.sekiya