embulk-output-vertica 0.4.1 → 0.5.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: dd4013fbcdb34ce19df2fe94eb19b226d92d6b83
-  data.tar.gz: 5e99ffef341da9c253a6f5b6420239c7e2000999
+  metadata.gz: 14be9147838a9b3e7e7c7ace08b5d26b491538cf
+  data.tar.gz: 1f53ce38472f8c015e38e2b502cb652dfcaeef6e
 SHA512:
-  metadata.gz: dabf1e7a0a5b6767dbd7352c42e777b37012e3172ef46024cb515298f8b734a8e64aa6d0b685b7fa920e41821a9351183ab9a7764ecb0a0790cf0190055ff563
-  data.tar.gz: bc761f55280145dde00983580b960af45b03e50f4ce32f4a6338cc8154a7b76d7f78280657828f5780b60f2370987415d6d3ba678535814f6e75df875df18a67
+  metadata.gz: f265d51ec0ffc498cbeaf66f6b86622d2cce48f2d9d5cac846250930bec95376b3dde1069f5aff12fd3c89a16d50172c3faed476652fd1cae9ac9d7eb582a04c
+  data.tar.gz: 9eebc8e2346c03d908c9ca2c735623a6890d2e6ae852c3083efef543694b905e6d9d3babd24e1c7a9ced34433e13ee3e42a1bd0e28babfab6ab070e8009e8ead

data/CHANGELOG.md

@@ -1,3 +1,9 @@
+# 0.5.0 (2015/12/04)
+
+Changes:
+
+* Use thread pool instead of connection pool #13
+
 # 0.4.1 (2015/12/04)
 
 Fixes:

data/README.md

@@ -18,8 +18,7 @@
 - **table**:    table name (string, required)
 - **mode**:     "insert", or "replace". See bellow. (string, default: insert)
 - **copy_mode**: specifies how data is loaded into the database. See vertica documents for details. (`AUTO`, `DIRECT`, or `TRICKLE`. default: `AUTO`)
-- **pool**: number of connection pools, this number controls number of concurrency to issue COPY statements (integer, default: processor_count, that is, number of threads in input plugin)
-- **pool_timeout**: timeout to checkout a connection from connection pools (seconds, default: 600)
+- **pool**: number of output threads, this number controls number of concurrency to issue COPY statements (integer, default: processor_count, that is, number of threads in input plugin)
 - **abort_on_error**: stops the COPY command if a row is rejected and rolls back the command. No data is loaded. (bool, default: false)
 - **reject_on_materialized_type_error**: uses `reject_on_materialized_type_error` option for fjsonparser(). This rejects rows if any of column types and value types do not fit, ex) double value into INT column fails. See vertica documents for details. (bool, default: false)
 - **default_timezone**: the default timezone for column_options (string, default is "UTC")

data/embulk-output-vertica.gemspec

@@ -1,6 +1,6 @@
 Gem::Specification.new do |spec|
   spec.name          = "embulk-output-vertica"
-  spec.version       = "0.4.1"
+  spec.version       = "0.5.0"
   spec.authors       = ["eiji.sekiya", "Naotoshi Seo"]
   spec.email         = ["eiji.sekiya.0326@gmail.com", "sonots@gmail.com"]
   spec.summary       = "Vertica output plugin for Embulk"
@@ -15,7 +15,6 @@ Gem::Specification.new do |spec|
 
   spec.add_dependency "jvertica", "~> 0.2"
   spec.add_dependency "tzinfo"
-  spec.add_dependency "connection_pool"
   spec.add_development_dependency "bundler", "~> 1.7"
   spec.add_development_dependency "rake", "~> 10.0"
 end

data/lib/embulk/output/vertica/output_thread.rb

@@ -0,0 +1,148 @@
+module Embulk
+  module Output
+    class Vertica < OutputPlugin
+      class OutputThreadPool
+        def initialize(task, schema, size)
+          @size = size
+          converters = ValueConverterFactory.create_converters(schema, task['default_timezone'], task['column_options'])
+          @output_threads = size.times.map { OutputThread.new(task, schema, converters) }
+          @current_index = 0
+        end
+
+        def enqueue(page)
+          @output_threads[@current_index].enqueue(page)
+          @current_index = (@current_index + 1) % @size
+        end
+
+        def start
+          @size.times.map {|i| @output_threads[i].start }
+        end
+
+        def commit
+          task_reports = @size.times.map {|i| @output_threads[i].commit }
+        end
+      end
+
+      class OutputThread
+        def initialize(task, schema, converters)
+          @task = task
+          @schema = schema
+          @queue = SizedQueue.new(1)
+          @converters = converters
+          @num_input_rows = 0
+          @num_output_rows = 0
+          @num_rejected_rows = 0
+        end
+
+        def enqueue(page)
+          if @thread.status.nil? # thread died by an error
+            @thread.join # raise the same error raised inside thread
+          end
+          if @thread.alive?
+            Embulk.logger.trace { "embulk-output-vertica: enqueued" }
+            @queue.push(page)
+          end
+        end
+
+        def run
+          Embulk.logger.debug { "embulk-output-vertica: thread started" }
+          Vertica.connect(@task) do |jv|
+            json = nil # for log
+            begin
+              num_output_rows, rejects = copy(jv, copy_sql) do |stdin|
+                while page = @queue.pop
+                  if page == 'finish'
+                    Embulk.logger.debug { "embulk-output-vertica: thread finished" }
+                    break
+                  end
+                  Embulk.logger.trace { "embulk-output-vertica: dequeued" }
+
+                  page.each do |record|
+                    json = to_json(record)
+                    Embulk.logger.trace { "embulk-output-vertica: to_json #{json}" }
+                    stdin << json << "\n"
+                    @num_input_rows += 1
+                  end
+                end
+              end
+              num_rejected_rows = rejects.size
+              @num_output_rows += num_output_rows
+              @num_rejected_rows += num_rejected_rows
+              jv.commit
+              Embulk.logger.info { "embulk-output-vertica: COMMIT!" }
+            rescue java.sql.SQLDataException => e
+              jv.rollback
+              if @task['reject_on_materialized_type_error'] and e.message =~ /Rejected by user-defined parser/
+                Embulk.logger.warn "embulk-output-vertica: ROLLBACK! some of column types and values types do not fit #{json}"
+              else
+                Embulk.logger.warn "embulk-output-vertica: ROLLBACK!"
+              end
+              raise e # die transaction
+            end
+          end
+        end
+
+        def start
+          @thread = Thread.new(&method(:run))
+        end
+
+        def commit
+          @queue.push('finish') if @thread.alive?
+          Thread.pass
+          @thread.join # the same error with run would be raised at here
+
+          task_report = {
+            'num_input_rows' => @num_input_rows,
+            'num_output_rows' => @num_output_rows,
+            'num_rejected_rows' => @num_rejected_rows,
+          }
+        end
+
+        # private
+
+        def copy(conn, sql, &block)
+          Embulk.logger.debug "embulk-output-vertica: #{sql}"
+          results, rejects = conn.copy(sql, &block)
+        end
+
+        def copy_sql
+          @copy_sql ||= "COPY #{quoted_schema}.#{quoted_temp_table} FROM STDIN#{fjsonparser}#{copy_mode}#{abort_on_error} NO COMMIT"
+        end
+
+        def to_json(record)
+          Hash[*(@schema.names.zip(record).map do |column_name, value|
+            [column_name, @converters[column_name].call(value)]
+          end.flatten!(1))].to_json
+        end
+
+        def quoted_schema
+          ::Jvertica.quote_identifier(@task['schema'])
+        end
+
+        def quoted_table
+          ::Jvertica.quote_identifier(@task['table'])
+        end
+
+        def quoted_temp_table
+          ::Jvertica.quote_identifier(@task['temp_table'])
+        end
+
+        def copy_mode
+          " #{@task['copy_mode']}"
+        end
+
+        def abort_on_error
+          @task['abort_on_error'] ? ' ABORT ON ERROR' : ''
+        end
+
+        def fjsonparser
+          " PARSER fjsonparser(#{reject_on_materialized_type_error})"
+        end
+
+        def reject_on_materialized_type_error
+          @task['reject_on_materialized_type_error'] ? 'reject_on_materialized_type_error=true' : ''
+        end
+      end
+    end
+  end
+end

data/lib/embulk/output/vertica.rb

@@ -1,6 +1,6 @@
 require 'jvertica'
-require 'connection_pool'
 require_relative 'vertica/value_converter_factory'
+require_relative 'vertica/output_thread'
 
 module Embulk
   module Output
@@ -10,8 +10,8 @@ module Embulk
       class Error < StandardError; end
       class NotSupportedType < Error; end
 
-      def self.connection_pool
-        @connection_pool ||= @connection_pool_proc.call
+      def self.thread_pool
+        @thread_pool ||= @thread_pool_proc.call
       end
 
       def self.transaction(config, schema, processor_count, &control)
@@ -27,24 +27,14 @@ module Embulk
           'mode'             => config.param('mode',             :string,  :default => 'insert'),
           'copy_mode'        => config.param('copy_mode',        :string,  :default => 'AUTO'),
           'abort_on_error'   => config.param('abort_on_error',   :bool,    :default => false),
-          'default_timezone' => config.param('default_timezone', :string,  :default => 'UTC'),
+          'default_timezone' => config.param('default_timezone', :string, :default => 'UTC'),
           'column_options'   => config.param('column_options',   :hash,    :default => {}),
           'reject_on_materialized_type_error' => config.param('reject_on_materialized_type_error', :bool, :default => false),
           'pool'             => config.param('pool',             :integer, :default => processor_count),
-          'pool_timeout'     => config.param('pool_timeout',     :integer, :default => 600),
         }
-        task['user'] ||= task['username']
 
-        @connection_pool_proc = Proc.new do
-          ConnectionPool.new(size: task['pool'], timeout: task['pool_timeout']) do
-            ::Jvertica.connect({
-              host: task['host'],
-              port: task['port'],
-              user: task['user'],
-              password: task['password'],
-              database: task['database'],
-            })
-          end
+        @thread_pool_proc = Proc.new do
+          OutputThreadPool.new(task, schema, task['pool'])
         end
 
         task['user'] ||= task['username']
@@ -73,7 +63,7 @@ module Embulk
         sql_schema_table = self.sql_schema_from_embulk_schema(schema, task['column_options'])
 
         # create the target table
-        connection_pool.with do |jv|
+        connect(task) do |jv|
           query(jv, %[DROP TABLE IF EXISTS #{quoted_schema}.#{quoted_table}]) if task['mode'] == 'REPLACE'
           query(jv, %[CREATE TABLE IF NOT EXISTS #{quoted_schema}.#{quoted_table} (#{sql_schema_table})])
         end
@@ -81,7 +71,7 @@ module Embulk
         sql_schema_temp_table = self.sql_schema_from_table(task)
 
         # create a temp table
-        connection_pool.with do |jv|
+        connect(task) do |jv|
           query(jv, %[DROP TABLE IF EXISTS #{quoted_schema}.#{quoted_temp_table}])
           query(jv, %[CREATE TABLE #{quoted_schema}.#{quoted_temp_table} (#{sql_schema_temp_table})])
           # Create internal vertica projection beforehand, otherwirse, parallel copies lock table to create a projection and we get S Lock error sometimes
@@ -96,30 +86,21 @@ module Embulk
 
         begin
           # insert data into the temp table
-          task_reports = yield(task) # obtain an array of task_reports where one report is of a task
-          connection_pool.shutdown do |jv| # just don't know how to loop all connections
-            jv.commit
-            Embulk.logger.info { "embulk-output-vertica: COMMIT!" }
-            jv.close rescue nil
-          end
-          @connection_pool = nil
+          thread_pool.start
+          yield(task)
+          task_reports = thread_pool.commit
           Embulk.logger.info { "embulk-output-vertica: task_reports: #{task_reports.to_json}" }
 
           # insert select from the temp table
-          connection_pool.with do |jv|
+          connect(task) do |jv|
             query(jv, %[INSERT INTO #{quoted_schema}.#{quoted_table} SELECT * FROM #{quoted_schema}.#{quoted_temp_table}])
             jv.commit
           end
         ensure
-          connection_pool.with do |jv|
+          connect(task) do |jv|
             # clean up the temp table
-            Embulk.logger.debug { "embulk-output-vertica: select count #{query(jv, %[SELECT count(*) FROM #{quoted_schema}.#{quoted_temp_table}]).map {|row| row.to_h }.join("\n") rescue nil}" }
-            Embulk.logger.trace { "embulk-output-vertica: select limit 10\n#{query(jv, %[SELECT * FROM #{quoted_schema}.#{quoted_temp_table} LIMIT 10]).map {|row| row.to_h }.join("\n") rescue nil}" }
             query(jv, %[DROP TABLE IF EXISTS #{quoted_schema}.#{quoted_temp_table}])
-          end
-
-          connection_pool.shutdown do |jv|
-            jv.close rescue nil
+            Embulk.logger.debug { "embulk-output-vertica: select result\n#{query(jv, %[SELECT * FROM #{quoted_schema}.#{quoted_table} LIMIT 10]).map {|row| row.to_h }.join("\n") rescue nil}" }
           end
         end
         # this is for -o next_config option, add some paramters for next time execution if wants
@@ -130,46 +111,15 @@ module Embulk
       # instance is created on each thread
       def initialize(task, schema, index)
         super
-        @converters = ValueConverterFactory.create_converters(schema, task['default_timezone'], task['column_options'])
-        Embulk.logger.trace { @converters.to_s }
-        @num_input_rows = 0
-        @num_output_rows = 0
-        @num_rejected_rows = 0
-      end
-
-      def connection_pool
-        self.class.connection_pool
       end
 
+      # called for each page in each thread
       def close
-        # do not close connection_pool on each thread / page
       end
 
+      # called for each page in each thread
       def add(page)
-        connection_pool.with do |jv| # block if no available connection left
-          json = nil # for log
-          begin
-            num_output_rows, rejects = copy(jv, copy_sql) do |stdin|
-              page.each do |record|
-                json = to_json(record)
-                Embulk.logger.debug { "embulk-output-vertica: to_json #{json}" }
-                stdin << json << "\n"
-                @num_input_rows += 1
-              end
-            end
-            num_rejected_rows = rejects.size
-            @num_output_rows += num_output_rows
-            @num_rejected_rows += num_rejected_rows
-          rescue java.sql.SQLDataException => e
-            jv.rollback
-            if @task['reject_on_materialized_type_error'] and e.message =~ /Rejected by user-defined parser/
-              Embulk.logger.warn "embulk-output-vertica: ROLLBACK! some of column types and values types do not fit #{json}"
-            else
-              Embulk.logger.warn "embulk-output-vertica: ROLLBACK!"
-            end
-            raise e # die transaction
-          end
-        end
+        self.class.thread_pool.enqueue(page)
       end
 
       def finish
@@ -178,19 +128,33 @@ module Embulk
       def abort
       end
 
-      # this is called after processing all pages in a thread
-      # we do commit on #transaction for all connection pools, not at here
+      # called after processing all pages in each thread
+      # we do commit on #transaction for all pools, not at here
       def commit
-        Embulk.logger.debug { "embulk-output-vertica: #{@num_output_rows} rows" }
-        task_report = {
-          "num_input_rows" => @num_input_rows,
-          "num_output_rows" => @num_output_rows,
-          "num_rejected_rows" => @num_rejected_rows,
-        }
+        {}
       end
 
       private
 
+      def self.connect(task)
+        jv = ::Jvertica.connect({
+          host: task['host'],
+          port: task['port'],
+          user: task['user'],
+          password: task['password'],
+          database: task['database'],
+        })
+
+        if block_given?
+          begin
+            yield jv
+          ensure
+            jv.close
+          end
+        end
+        jv
+      end
+
       # @param [Schema] schema embulk defined column types
       # @param [Hash]   column_options user defined column types
       # @return [String] sql schema used to CREATE TABLE
@@ -224,7 +188,7 @@ module Embulk
           "WHERE table_schema = #{quoted_schema} AND table_name = #{quoted_table}"
 
         sql_schema = {}
-        connection_pool.with do |jv|
+        connect(task) do |jv|
           result = query(jv, sql)
           sql_schema = result.map {|row| [row[0], row[1]] }
         end
@@ -239,49 +203,6 @@ module Embulk
       def query(conn, sql)
         self.class.query(conn, sql)
       end
-
-      def copy(conn, sql, &block)
-        Embulk.logger.debug "embulk-output-vertica: #{sql}"
-        results, rejects = conn.copy(sql, &block)
-      end
-
-      def copy_sql
-        @copy_sql ||= "COPY #{quoted_schema}.#{quoted_temp_table} FROM STDIN#{fjsonparser}#{copy_mode}#{abort_on_error} NO COMMIT"
-      end
-
-      def to_json(record)
-        Hash[*(schema.names.zip(record).map do |column_name, value|
-          [column_name, @converters[column_name].call(value)]
-        end.flatten!(1))].to_json
-      end
-
-      def quoted_schema
-        ::Jvertica.quote_identifier(@task['schema'])
-      end
-
-      def quoted_table
-        ::Jvertica.quote_identifier(@task['table'])
-      end
-
-      def quoted_temp_table
-        ::Jvertica.quote_identifier(@task['temp_table'])
-      end
-
-      def copy_mode
-        " #{@task['copy_mode']}"
-      end
-
-      def abort_on_error
-        @task['abort_on_error'] ? ' ABORT ON ERROR' : ''
-      end
-
-      def fjsonparser
-        " PARSER fjsonparser(#{reject_on_materialized_type_error})"
-      end
-
-      def reject_on_materialized_type_error
-        @task['reject_on_materialized_type_error'] ? 'reject_on_materialized_type_error=true' : ''
-      end
     end
   end
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: embulk-output-vertica
 version: !ruby/object:Gem::Version
-  version: 0.4.1
+  version: 0.5.0
 platform: ruby
 authors:
 - eiji.sekiya
@@ -39,20 +39,6 @@ dependencies:
         version: '0'
   prerelease: false
   type: :runtime
-- !ruby/object:Gem::Dependency
-  name: connection_pool
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
-  prerelease: false
-  type: :runtime
 - !ruby/object:Gem::Dependency
   name: bundler
   version_requirements: !ruby/object:Gem::Requirement
@@ -99,6 +85,7 @@ files:
 - example.csv
 - example.yml
 - lib/embulk/output/vertica.rb
+- lib/embulk/output/vertica/output_thread.rb
 - lib/embulk/output/vertica/value_converter_factory.rb
 homepage: https://github.com/eratostennis/embulk-output-vertica
 licenses: