embulk-output-vertica 0.7.1 → 0.7.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 5378b73846425007a9de7f30869bfd2206b06bed
-  data.tar.gz: cc430ff8ec752696eec4933fdc982832bf179133
+  metadata.gz: e3a0449ef97c75dd1e28f722b344935ba2552952
+  data.tar.gz: 71983eb9e264edc18535f9e9b75ebd3ef4e3921b
 SHA512:
-  metadata.gz: 7c599fac58e80276b4f8a312bb962b6281f2e95752197a429a3f5b05d9b71e131037ad653f5556a5939bb6cfa7f008b91a700e89d9345d54d8c185fa98e3b5dc
-  data.tar.gz: 25a0159744f47d3e4e0343673ba940f816335b1fae2a9bbbd84afec0d7d45daf06cc8d19edb7b453a3a5c477ecdd3cb92b136f6d7ad6aed6b9ffc93d98d8e935
+  metadata.gz: ab6d594e5037f6c12f14a65b7aa0a0e78c1ae1ca18fa79ff5e506cb7aaa86c5e532f2c5eceb76345e6b2d8c99924d1f26a498decd2f2b221875f3d8ba2d97f39
+  data.tar.gz: ee3105565d94a409ea6c22aa17fbaf3be326c76958cfe1d01e0d657b6d1dcd34e4f934d4ecc94e081eb8048a016006a6e36730c610fcd86fccd368db4deca936

data/CHANGELOG.md

@@ -1,3 +1,10 @@
+# 0.7.2 (2016/02/10)
+
+Fixes:
+
+* Fix to get stuck if no input is comming
+* Fix to get stcuk if one record is longer thant IO pipe maximum size (64k bytes for Linux, and Mac OSX)
+
 # 0.7.1 (2016/01/30)
 
 Enhancements:

data/embulk-output-vertica.gemspec

@@ -1,6 +1,6 @@
 Gem::Specification.new do |spec|
   spec.name          = "embulk-output-vertica"
-  spec.version       = "0.7.1"
+  spec.version       = "0.7.2"
   spec.authors       = ["eiji.sekiya", "Naotoshi Seo"]
   spec.email         = ["eiji.sekiya.0326@gmail.com", "sonots@gmail.com"]
   spec.summary       = "Vertica output plugin for Embulk"

data/lib/embulk/output/vertica/output_thread.rb

@@ -77,13 +77,25 @@ module Embulk
         def write_gzip(io, page, &block)
           buf = Zlib::Deflate.new
           write_buf(buf, page, &block)
-          io << buf.finish
+          write_io(io, buf.finish)
         end
 
         def write_uncompressed(io, page, &block)
           buf = ''
           write_buf(buf, page, &block)
-          io << buf
+          write_io(io, buf)
+        end
+
+        PIPE_BUF = 4096
+
+        def write_io(io, str)
+          str = str.force_encoding('ASCII-8BIT')
+          i = 0
+          # split str not to be blocked (max size of pipe buf is 64k bytes on Linux, Mac at default)
+          while substr = str[i, PIPE_BUF]
+            io.write(substr)
+            i += PIPE_BUF
+          end
         end
 
         def write_buf(buf, json_page, &block)
@@ -106,42 +118,62 @@ module Embulk
           number.to_s.gsub(/(\d)(?=(\d{3})+(?!\d))/, '\1,')
         end
 
+        # @return [Array] dequeued json_page
+        # @return [String] 'finish' is dequeued to finish
+        def dequeue
+          json_page = @queue.pop
+          Embulk.logger.trace { "embulk-output-vertica: dequeued" }
+          Embulk.logger.debug { "embulk-output-vertica: dequeued finish" } if json_page == 'finish'
+          json_page
+        end
+
+        def copy(jv, sql, &block)
+          Embulk.logger.debug "embulk-output-vertica: #{sql}"
+          num_output_rows = 0; rejected_row_nums = []; last_record = nil
+
+          json_page = dequeue
+          return [num_output_rows, rejected_row_nums, last_record] if json_page == 'finish'
+
+          num_output_rows, rejected_row_nums = jv.copy(sql) do |stdin, stream|
+            @write_proc.call(stdin, json_page) {|record| last_record = record }
+
+            while true
+              json_page = dequeue
+              break if json_page == 'finish'
+              @write_proc.call(stdin, json_page) {|record| last_record = record }
+            end
+          end
+
+          @num_output_rows += num_output_rows
+          @num_rejected_rows += rejected_row_nums.size
+          Embulk.logger.info { "embulk-output-vertica: COMMIT!" }
+          jv.commit
+          Embulk.logger.debug { "embulk-output-vertica: COMMITTED!" }
+
+          if rejected_row_nums.size > 0
+            Embulk.logger.debug { "embulk-output-vertica: rejected_row_nums: #{rejected_row_nums}" }
+          end
+
+          [num_output_rows, rejected_row_nums, last_record]
+        end
+
         def run
           Embulk.logger.debug { "embulk-output-vertica: thread started" }
           Vertica.connect(@task) do |jv|
             begin
-              last_record = nil
-              num_output_rows, rejects = copy(jv, copy_sql) do |stdin|
-                while json_page = @queue.pop
-                  if json_page == 'finish'
-                    Embulk.logger.debug { "embulk-output-vertica: popped finish" }
-                    break
-                  end
-                  Embulk.logger.trace { "embulk-output-vertica: dequeued" }
-
-                  @write_proc.call(stdin, json_page) do |record|
-                    last_record = record
-                  end
-                end
-              end
+              num_output_rows, rejected_row_nums, last_record = copy(jv, copy_sql)
               Embulk.logger.debug { "embulk-output-vertica: thread finished" }
-              num_rejected_rows = rejects.size
-              @num_output_rows += num_output_rows
-              @num_rejected_rows += num_rejected_rows
-              Embulk.logger.info { "embulk-output-vertica: COMMIT!" }
-              jv.commit
-              Embulk.logger.debug { "embulk-output-vertica: COMMITTED!" }
             rescue java.sql.SQLDataException => e
               if @task['reject_on_materialized_type_error'] and e.message =~ /Rejected by user-defined parser/
-                Embulk.logger.warn "embulk-output-vertica: ROLLBACK! some of column types and values types do not fit #{last_record}"
+                Embulk.logger.warn "embulk-output-vertica: ROLLBACK! some of column types and values types do not fit #{rejected_row_nums}"
               else
-                Embulk.logger.warn "embulk-output-vertica: ROLLBACK!"
+                Embulk.logger.warn "embulk-output-vertica: ROLLBACK! #{rejected_row_nums}"
               end
               Embulk.logger.info { "embulk-output-vertica: last_record: #{last_record}" }
               jv.rollback
               raise e # die transaction
             rescue => e
-              Embulk.logger.warn "embulk-output-vertica: ROLLBACK!"
+              Embulk.logger.warn "embulk-output-vertica: ROLLBACK! #{e.class} #{e.message}"
               jv.rollback
               raise e
             end
@@ -179,11 +211,6 @@ module Embulk
 
         # private
 
-        def copy(conn, sql, &block)
-          Embulk.logger.debug "embulk-output-vertica: #{sql}"
-          results, rejects = conn.copy(sql, &block)
-        end
-
         def copy_sql
           @copy_sql ||= "COPY #{quoted_schema}.#{quoted_temp_table} FROM STDIN#{compress}#{fjsonparser}#{copy_mode}#{abort_on_error} NO COMMIT"
         end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: embulk-output-vertica
 version: !ruby/object:Gem::Version
-  version: 0.7.1
+  version: 0.7.2
 platform: ruby
 authors:
 - eiji.sekiya
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2016-01-30 00:00:00.000000000 Z
+date: 2016-02-10 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: jvertica