embulk-output-vertica 0.7.2 → 0.7.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: e3a0449ef97c75dd1e28f722b344935ba2552952
-  data.tar.gz: 71983eb9e264edc18535f9e9b75ebd3ef4e3921b
+  metadata.gz: 7e89fddffd4579262fb1d91d92a3288c0e8a8ba2
+  data.tar.gz: e5708526eb702e7008e2758126251a4e306e75f1
 SHA512:
-  metadata.gz: ab6d594e5037f6c12f14a65b7aa0a0e78c1ae1ca18fa79ff5e506cb7aaa86c5e532f2c5eceb76345e6b2d8c99924d1f26a498decd2f2b221875f3d8ba2d97f39
-  data.tar.gz: ee3105565d94a409ea6c22aa17fbaf3be326c76958cfe1d01e0d657b6d1dcd34e4f934d4ecc94e081eb8048a016006a6e36730c610fcd86fccd368db4deca936
+  metadata.gz: efa9d97f5d999a983f4d55ffc5cb5413bf1b9da63f825276682e01c14fbdc9541da1277a108b2b639f9960f7e6af21f4b1c7b99894ca51ae1954814e777e5e27
+  data.tar.gz: c4c1b97985f3211d187192b8581f1602109cb2eb9b2d1f58dc05e30e232a7b823e00fd7b76831fea95883b80232e759d632d621892f3c69429612cf3b5e1fe12

data/CHANGELOG.md

@@ -1,3 +1,15 @@
+# 0.7.3 (2016/03/27)
+
+Enhancements:
+
+* Add write_timeout option
+* Add finish_timeout option
+* Add dequeuen_timeout option
+
+Changes:
+
+* Transfer repository from eratostennis to sonots
+
 # 0.7.2 (2016/02/10)
 
 Fixes:

data/README.md

@@ -44,7 +44,7 @@
 ### Modes
 
 * **insert**:
-  * Behavior: This mode copies rows to some intermediate tables first. If all those tasks run correctly, runs INSERT INTO <target_table> SELECT * FROM <intermediate_table>
+  * Behavior: This mode copies rows to an intermediate table first. If all those tasks run correctly, runs `INSERT INTO <target_table> SELECT * FROM <intermediate_table>`
 * **replace**:
   * Behavior: This mode copies rows to an intermediate table first. If all those tasks run correctly, swaps the target table and the intermediate table, and drops the old target table
 * **drop_insert**:

data/embulk-output-vertica.gemspec

@@ -1,11 +1,11 @@
 Gem::Specification.new do |spec|
   spec.name          = "embulk-output-vertica"
-  spec.version       = "0.7.2"
-  spec.authors       = ["eiji.sekiya", "Naotoshi Seo"]
-  spec.email         = ["eiji.sekiya.0326@gmail.com", "sonots@gmail.com"]
+  spec.version       = "0.7.3"
+  spec.authors       = ["Naotoshi Seo", "Eiji Sekiya"]
+  spec.email         = ["sonots@gmail.com", "eiji.sekiya.0326@gmail.com"]
   spec.summary       = "Vertica output plugin for Embulk"
   spec.description   = "Dump records to vertica"
-  spec.homepage      = "https://github.com/eratostennis/embulk-output-vertica"
+  spec.homepage      = "https://github.com/sonots/embulk-output-vertica"
   spec.license       = "MIT"
 
   spec.files         = `git ls-files -z`.split("\x0")

data/example/timeout.yml

@@ -0,0 +1,41 @@
+# in:
+#   type: random
+#   rows: 100
+#   schema:
+#     id: primary_key
+#     name: string
+#     score: integer
+in:
+  type: file
+  path_prefix: example/example.csv
+  parser:
+    type: csv
+    charset: UTF-8
+    newline: CRLF
+    null_string: 'NULL'
+    skip_header_lines: 1
+    comment_line_marker: '#'
+    columns:
+      - {name: timestamp_date, type: timestamp, format: "%Y-%m-%d", timezone: "+09:00"}
+      - {name: string_date,    type: string}
+      - {name: foo,   type: string}
+      - {name: bar,   type: string}
+      - {name: id,    type: long}
+      - {name: name,  type: string}
+      - {name: score, type: double} 
+out:
+  type: vertica 
+  host: 127.0.0.1
+  user: dbadmin
+  password: xxxxxxx
+  database: vdb
+  schema: sandbox
+  table: embulk_test
+  mode: drop_insert
+  copy_mode: DIRECT
+  abort_on_error: true
+  reject_on_materialized_type_error: true
+  default_timezone: 'Asia/Tokyo'
+  write_timeout: 660
+  dequeue_timeout: 780
+  finish_timeout: 180

data/lib/embulk/output/vertica.rb

@@ -14,7 +14,7 @@ module Embulk
         @thread_pool ||= @thread_pool_proc.call
       end
 
-      def self.transaction(config, schema, processor_count, &control)
+      def self.transaction(config, schema, task_count, &control)
         task = {
           'host'             => config.param('host',             :string,  :default => 'localhost'),
           'port'             => config.param('port',             :integer, :default => 5433),
@@ -33,7 +33,10 @@ module Embulk
           'json_payload'     => config.param('json_payload',     :bool,    :default => false),
           'resource_pool'    => config.param('resource_pool',    :string,  :default => nil),
           'reject_on_materialized_type_error' => config.param('reject_on_materialized_type_error', :bool, :default => false),
-          'pool'             => config.param('pool',             :integer, :default => processor_count),
+          'pool'             => config.param('pool',             :integer, :default => task_count),
+          'write_timeout'    => config.param('write_timeout',    :integer, :default => nil), # like 11 * 60 sec
+          'dequeue_timeout'  => config.param('dequeue_timeout',  :integer, :default => nil), # like 13 * 60 sec
+          'finish_timeout'   => config.param('finish_timeout',   :integer, :default => nil), # like 3 * 60 sec
         }
 
         @thread_pool_proc = Proc.new do

data/lib/embulk/output/vertica/output_thread.rb

@@ -3,6 +3,14 @@ require 'zlib'
 module Embulk
   module Output
     class Vertica < OutputPlugin
+      class CommitError < ::StandardError; end
+      class TimeoutError < ::Timeout::Error; end
+      class DequeueTimeoutError < TimeoutError; end
+      class FinishTimeoutError < TimeoutError; end
+      class WriteTimeoutError < TimeoutError; end
+
+      $embulk_output_vertica_thread_dumped = false
+
       class OutputThreadPool
         def initialize(task, schema, size)
           @task = task
@@ -30,7 +38,14 @@ module Embulk
         end
 
         def commit
-          task_reports = @size.times.map {|i| @output_threads[i].commit }
+          Embulk.logger.debug "embulk-output-vertica: commit"
+          task_reports = @mutex.synchronize do
+            @size.times.map {|i| @output_threads[i].commit }
+          end
+          unless task_reports.all? {|task_report| task_report['success'] }
+            raise CommitError, "some of output_threads failed to commit"
+          end
+          task_reports
         end
 
         def to_json(record)
@@ -64,12 +79,26 @@ module Embulk
           end
         end
 
+        def thread_dump
+          unless $embulk_output_vertica_thread_dumped
+            $embulk_output_vertica_thread_dumped = true
+            Embulk.logger.debug "embulk-output-vertica: kill -3 #{$$} (Thread dump)"
+            begin
+              Process.kill :QUIT, $$
+            rescue SignalException
+            ensure
+              sleep 1
+            end
+          end
+        end
+
         def enqueue(json_page)
           if @thread_active and @thread.alive?
             Embulk.logger.trace { "embulk-output-vertica: enqueue" }
             @queue.push(json_page)
           else
             Embulk.logger.info { "embulk-output-vertica: thread is dead, but still trying to enqueue" }
+            thread_dump
             raise RuntimeError, "embulk-output-vertica: thread is died, but still trying to enqueue"
           end
         end
@@ -93,7 +122,8 @@ module Embulk
           i = 0
           # split str not to be blocked (max size of pipe buf is 64k bytes on Linux, Mac at default)
           while substr = str[i, PIPE_BUF]
-            io.write(substr)
+            Embulk.logger.trace { "embulk-output-vertica: io.write with write_timeout:#{@task['write_timeout']}" }
+            Timeout.timeout(@task['write_timeout'], WriteTimeoutError) { io.write(substr) }
             i += PIPE_BUF
           end
         end
@@ -121,19 +151,24 @@ module Embulk
         # @return [Array] dequeued json_page
         # @return [String] 'finish' is dequeued to finish
         def dequeue
-          json_page = @queue.pop
+          json_page = nil
+          Embulk.logger.trace { "embulk-output-vertica: @queue.pop with dequeue_timeout:#{@task['dequeue_timeout']}" }
+          Timeout.timeout(@task['dequeue_timeout'], DequeueTimeoutError) { json_page = @queue.pop }
           Embulk.logger.trace { "embulk-output-vertica: dequeued" }
           Embulk.logger.debug { "embulk-output-vertica: dequeued finish" } if json_page == 'finish'
           json_page
         end
 
         def copy(jv, sql, &block)
-          Embulk.logger.debug "embulk-output-vertica: #{sql}"
+          Embulk.logger.debug "embulk-output-vertica: copy, waiting a first message"
+
           num_output_rows = 0; rejected_row_nums = []; last_record = nil
 
           json_page = dequeue
           return [num_output_rows, rejected_row_nums, last_record] if json_page == 'finish'
 
+          Embulk.logger.debug "embulk-output-vertica: #{sql}"
+
           num_output_rows, rejected_row_nums = jv.copy(sql) do |stdin, stream|
             @write_proc.call(stdin, json_page) {|record| last_record = record }
 
@@ -159,7 +194,8 @@ module Embulk
 
         def run
           Embulk.logger.debug { "embulk-output-vertica: thread started" }
-          Vertica.connect(@task) do |jv|
+          begin
+            jv = Vertica.connect(@task)
             begin
               num_output_rows, rejected_row_nums, last_record = copy(jv, copy_sql)
               Embulk.logger.debug { "embulk-output-vertica: thread finished" }
@@ -170,20 +206,50 @@ module Embulk
                 Embulk.logger.warn "embulk-output-vertica: ROLLBACK! #{rejected_row_nums}"
               end
               Embulk.logger.info { "embulk-output-vertica: last_record: #{last_record}" }
-              jv.rollback
-              raise e # die transaction
+              rollback(jv)
+              raise e
             rescue => e
-              Embulk.logger.warn "embulk-output-vertica: ROLLBACK! #{e.class} #{e.message}"
-              jv.rollback
+              Embulk.logger.warn "embulk-output-vertica: ROLLBACK! #{e.class} #{e.message} #{e.backtrace.join("\n  ")}"
+              rollback(jv)
               raise e
             end
+          ensure
+            close(jv)
           end
-        rescue => e
+        rescue TimeoutError => e
+          Embulk.logger.error "embulk-output-vertica: UNKNOWN TIMEOUT!! #{e.class}"
+          @thread_active = false # not to be enqueued any more
+          dequeue_all
+          thread_dump
+          exit(1)
+        rescue Exception => e
+          Embulk.logger.error "embulk-output-vertica: UNKNOWN ERROR! #{e.class} #{e.message} #{e.backtrace.join("\n  ")}"
           @thread_active = false # not to be enqueued any more
+          dequeue_all
+          @outer_thread.raise e
+        end
+
+        def dequeue_all
+          Embulk.logger.debug "embulk-output-vertica: dequeue all"
           while @queue.size > 0
             @queue.pop # dequeue all because some might be still trying @queue.push and get blocked, need to release
           end
-          @outer_thread.raise e.class.new("#{e.message}\n  #{e.backtrace.join("\n  ")}")
+        end
+
+        def close(jv)
+          begin
+            jv.close
+          rescue java.sql.SQLException => e # The connection is closed
+            Embulk.logger.debug "embulk-output-vertica: #{e.class} #{e.message}"
+          end
+        end
+
+        def rollback(jv)
+          begin
+            jv.rollback
+          rescue java.sql.SQLException => e # The connection is closed
+            Embulk.logger.debug "embulk-output-vertica: #{e.class} #{e.message}"
+          end
         end
 
         def start
@@ -192,20 +258,29 @@ module Embulk
         end
 
         def commit
+          Embulk.logger.debug "embulk-output-vertica: output_thread commit"
           @thread_active = false
+          success = true
           if @thread.alive?
-            Embulk.logger.debug { "embulk-output-vertica: push finish" }
+            Embulk.logger.debug { "embulk-output-vertica: push finish with finish_timeout:#{@task['finish_timeout']}" }
             @queue.push('finish')
             Thread.pass
-            @thread.join
+            @thread.join(@task['finish_timeout'])
+            if @thread.alive?
+              @thread.kill
+              Embulk.logger.error "embulk-output-vertica: finish_timeout #{@task['finish_timeout']}sec exceeded, thread is killed forcely"
+              success = false
+            end
           else
-            raise RuntimeError, "embulk-output-vertica: thread died accidently"
+            Embulk.logger.error "embulk-output-vertica: thread died accidently"
+            success = false
           end
 
           task_report = {
             'num_input_rows' => @num_input_rows,
             'num_output_rows' => @num_output_rows,
             'num_rejected_rows' => @num_rejected_rows,
+            'success' => success
           }
         end
 

metadata

@@ -1,15 +1,15 @@
 --- !ruby/object:Gem::Specification
 name: embulk-output-vertica
 version: !ruby/object:Gem::Version
-  version: 0.7.2
+  version: 0.7.3
 platform: ruby
 authors:
-- eiji.sekiya
 - Naotoshi Seo
+- Eiji Sekiya
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2016-02-10 00:00:00.000000000 Z
+date: 2016-03-27 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: jvertica
@@ -69,8 +69,8 @@ dependencies:
         version: '10.0'
 description: Dump records to vertica
 email:
-- eiji.sekiya.0326@gmail.com
 - sonots@gmail.com
+- eiji.sekiya.0326@gmail.com
 executables: []
 extensions: []
 extra_rdoc_files: []
@@ -89,10 +89,11 @@ files:
 - example/json_payload.yml
 - example/replace.yml
 - example/resource_pool.yml
+- example/timeout.yml
 - lib/embulk/output/vertica.rb
 - lib/embulk/output/vertica/output_thread.rb
 - lib/embulk/output/vertica/value_converter_factory.rb
-homepage: https://github.com/eratostennis/embulk-output-vertica
+homepage: https://github.com/sonots/embulk-output-vertica
 licenses:
 - MIT
 metadata: {}