embulk-output-vertica 0.3.1 → 0.4.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 2db4b28e6259edb2d4cacc084586ec82c876ccac
-  data.tar.gz: bbfe10034e08509f04e259293bde211543cdd1f1
+  metadata.gz: b178cb9ffe687570117d6aa8edfb55e7c77c446d
+  data.tar.gz: 72195fdd08f4929162602381bcd79432545067c0
 SHA512:
-  metadata.gz: 8ad723dfc4972dcc125026d10098b572baeac9f33e384a5f560674f4c322d34185e168723584e287e40461975d7f0fcd7856b2c7fd4f7d03ef1e506126d74281
-  data.tar.gz: f5b89412688b7609d2a54e2878d469ec946204df22f423d3cf0108b26324559f2791a9509192981f6c937cff8221e948bbb883253a2da396e2ec8f145125a06e
+  metadata.gz: 3f132dba10f45571d856cf33478b80f73c154413dd78268a43510461f24f223b04def25cada97e204b3053c6d2ebd09054a6d08816bfe4d26cf102b908ff5bc4
+  data.tar.gz: 10fcde36ab57437e724b4e0eb63586fcb218b300d799875777b15ba4769e9e3402dc85cc3b77f882897fdc73515e258b2ce3e57261d79eb98d921b8bab226b7f

data/CHANGELOG.md

@@ -1,3 +1,9 @@
+# 0.4.0 (2015/11/24)
+
+Enhancements:
+
+* Support connection pool
+
 # 0.3.1 (2015/11/20)
 
 Fixes:

data/README.md

@@ -18,6 +18,8 @@
 - **table**:    table name (string, required)
 - **mode**:     "insert", or "replace". See bellow. (string, default: insert)
 - **copy_mode**: specifies how data is loaded into the database. See vertica documents for details. (`AUTO`, `DIRECT`, or `TRICKLE`. default: `AUTO`)
+- **pool**: number of connection pools, this number controls number of concurrency to issue COPY statements (integer, default: processor_count, that is, number of threads in input plugin)
+- **pool_timeout**: timeout to checkout a connection from connection pools (seconds, default: 600)
 - **abort_on_error**: stops the COPY command if a row is rejected and rolls back the command. No data is loaded. (bool, default: false)
 - **reject_on_materialized_type_error**: uses `reject_on_materialized_type_error` option for fjsonparser(). This rejects rows if any of column types and value types do not fit, ex) double value into INT column fails. See vertica documents for details. (bool, default: false)
 - **default_timezone**: the default timezone for column_options (string, default is "UTC")

data/embulk-output-vertica.gemspec

@@ -1,6 +1,6 @@
 Gem::Specification.new do |spec|
   spec.name          = "embulk-output-vertica"
-  spec.version       = "0.3.1"
+  spec.version       = "0.4.0"
   spec.authors       = ["eiji.sekiya", "Naotoshi Seo"]
   spec.email         = ["eiji.sekiya.0326@gmail.com", "sonots@gmail.com"]
   spec.summary       = "Vertica output plugin for Embulk"
@@ -15,6 +15,7 @@ Gem::Specification.new do |spec|
 
   spec.add_dependency "jvertica", "~> 0.2"
   spec.add_dependency "tzinfo"
+  spec.add_dependency "connection_pool"
   spec.add_development_dependency "bundler", "~> 1.7"
   spec.add_development_dependency "rake", "~> 10.0"
 end

data/lib/embulk/output/vertica.rb

@@ -1,4 +1,5 @@
 require 'jvertica'
+require 'connection_pool'
 require_relative 'vertica/value_converter_factory'
 
 module Embulk
@@ -9,23 +10,42 @@ module Embulk
       class Error < StandardError; end
       class NotSupportedType < Error; end
 
+      def self.connection_pool
+        @connection_pool ||= @connection_pool_proc.call
+      end
+
       def self.transaction(config, schema, processor_count, &control)
         task = {
-          'host'             => config.param('host',           :string,  :default => 'localhost'),
-          'port'             => config.param('port',           :integer, :default => 5433),
-          'user'             => config.param('user',           :string,  :default => nil),
-          'username'         => config.param('username',       :string,  :default => nil), # alias to :user for backward compatibility
-          'password'         => config.param('password',       :string,  :default => ''),
-          'database'         => config.param('database',       :string,  :default => 'vdb'),
-          'schema'           => config.param('schema',         :string,  :default => 'public'),
-          'table'            => config.param('table',          :string),
-          'mode'             => config.param('mode',           :string,  :default => 'insert'),
-          'copy_mode'        => config.param('copy_mode',      :string,  :default => 'AUTO'),
-          'abort_on_error'   => config.param('abort_on_error', :bool,    :default => false),
-          'default_timezone' => config.param('default_timezone', :string, :default => 'UTC'),
-          'column_options'   => config.param('column_options', :hash,    :default => {}),
+          'host'             => config.param('host',             :string,  :default => 'localhost'),
+          'port'             => config.param('port',             :integer, :default => 5433),
+          'user'             => config.param('user',             :string,  :default => nil),
+          'username'         => config.param('username',         :string,  :default => nil), # alias to :user for backward compatibility
+          'password'         => config.param('password',         :string,  :default => ''),
+          'database'         => config.param('database',         :string,  :default => 'vdb'),
+          'schema'           => config.param('schema',           :string,  :default => 'public'),
+          'table'            => config.param('table',            :string),
+          'mode'             => config.param('mode',             :string,  :default => 'insert'),
+          'copy_mode'        => config.param('copy_mode',        :string,  :default => 'AUTO'),
+          'abort_on_error'   => config.param('abort_on_error',   :bool,    :default => false),
+          'default_timezone' => config.param('default_timezone', :string,  :default => 'UTC'),
+          'column_options'   => config.param('column_options',   :hash,    :default => {}),
           'reject_on_materialized_type_error' => config.param('reject_on_materialized_type_error', :bool, :default => false),
+          'pool'             => config.param('pool',             :integer, :default => processor_count),
+          'pool_timeout'     => config.param('pool_timeout',     :integer, :default => 600),
         }
+        task['user'] ||= task['username']
+
+        @connection_pool_proc = Proc.new do
+          ConnectionPool.new(size: task['pool'], timeout: task['pool_timeout']) do
+            ::Jvertica.connect({
+              host: task['host'],
+              port: task['port'],
+              user: task['user'],
+              password: task['password'],
+              database: task['database'],
+            })
+          end
+        end
 
         task['user'] ||= task['username']
         unless task['user']
@@ -53,7 +73,7 @@ module Embulk
         sql_schema_table = self.sql_schema_from_embulk_schema(schema, task['column_options'])
 
         # create the target table
-        connect(task) do |jv|
+        connection_pool.with do |jv|
           query(jv, %[DROP TABLE IF EXISTS #{quoted_schema}.#{quoted_table}]) if task['mode'] == 'REPLACE'
           query(jv, %[CREATE TABLE IF NOT EXISTS #{quoted_schema}.#{quoted_table} (#{sql_schema_table})])
         end
@@ -61,7 +81,7 @@ module Embulk
         sql_schema_temp_table = self.sql_schema_from_table(task)
 
         # create a temp table
-        connect(task) do |jv|
+        connection_pool.with do |jv|
           query(jv, %[DROP TABLE IF EXISTS #{quoted_schema}.#{quoted_temp_table}])
           query(jv, %[CREATE TABLE #{quoted_schema}.#{quoted_temp_table} (#{sql_schema_temp_table})])
         end
@@ -69,18 +89,29 @@ module Embulk
         begin
           # insert data into the temp table
           task_reports = yield(task) # obtain an array of task_reports where one report is of a task
+          connection_pool.shutdown do |jv| # just don't know how to loop all connections
+            jv.commit
+            Embulk.logger.info { "embulk-output-vertica: COMMIT!" }
+            jv.close rescue nil
+          end
+          @connection_pool = nil
           Embulk.logger.info { "embulk-output-vertica: task_reports: #{task_reports.to_json}" }
 
           # insert select from the temp table
-          connect(task) do |jv|
+          connection_pool.with do |jv|
             query(jv, %[INSERT INTO #{quoted_schema}.#{quoted_table} SELECT * FROM #{quoted_schema}.#{quoted_temp_table}])
             jv.commit
           end
         ensure
-          connect(task) do |jv|
+          connection_pool.with do |jv|
             # clean up the temp table
+            Embulk.logger.debug { "embulk-output-vertica: select count #{query(jv, %[SELECT count(*) FROM #{quoted_schema}.#{quoted_temp_table}]).map {|row| row.to_h }.join("\n") rescue nil}" }
+            Embulk.logger.trace { "embulk-output-vertica: select limit 10\n#{query(jv, %[SELECT * FROM #{quoted_schema}.#{quoted_temp_table} LIMIT 10]).map {|row| row.to_h }.join("\n") rescue nil}" }
             query(jv, %[DROP TABLE IF EXISTS #{quoted_schema}.#{quoted_temp_table}])
-            Embulk.logger.debug { "embulk-output-vertica: select result #{query(jv, %[SELECT * FROM #{quoted_schema}.#{quoted_table} LIMIT 10]).map {|row| row.to_h }.join("\n") rescue nil}" }
+          end
+
+          connection_pool.shutdown do |jv|
+            jv.close rescue nil
           end
         end
         # this is for -o next_config option, add some paramters for next time execution if wants
@@ -91,39 +122,44 @@ module Embulk
       def initialize(task, schema, index)
         super
         @converters = ValueConverterFactory.create_converters(schema, task['default_timezone'], task['column_options'])
-        Embulk.logger.debug { @converters.to_s }
-        @jv = self.class.connect(task)
+        Embulk.logger.trace { @converters.to_s }
         @num_input_rows = 0
         @num_output_rows = 0
         @num_rejected_rows = 0
       end
 
+      def connection_pool
+        self.class.connection_pool
+      end
+
       def close
-        @jv.close
+        # do not close connection_pool on each thread / page
       end
 
       def add(page)
-        json = nil # for log
-        begin
-          num_output_rows, rejects = copy(@jv, copy_sql) do |stdin|
-            page.each do |record|
-              json = to_json(record)
-              Embulk.logger.debug { "embulk-output-vertica: to_json #{json}" }
-              stdin << json << "\n"
-              @num_input_rows += 1
+        connection_pool.with do |jv| # block if no available connection left
+          json = nil # for log
+          begin
+            num_output_rows, rejects = copy(jv, copy_sql) do |stdin|
+              page.each do |record|
+                json = to_json(record)
+                Embulk.logger.debug { "embulk-output-vertica: to_json #{json}" }
+                stdin << json << "\n"
+                @num_input_rows += 1
+              end
             end
+            num_rejected_rows = rejects.size
+            @num_output_rows += num_output_rows
+            @num_rejected_rows += num_rejected_rows
+          rescue java.sql.SQLDataException => e
+            jv.rollback
+            if @task['reject_on_materialized_type_error'] and e.message =~ /Rejected by user-defined parser/
+              Embulk.logger.warn "embulk-output-vertica: ROLLBACK! some of column types and values types do not fit #{json}"
+            else
+              Embulk.logger.warn "embulk-output-vertica: ROLLBACK!"
+            end
+            raise e # die transaction
           end
-          num_rejected_rows = rejects.size
-          @num_output_rows += num_output_rows
-          @num_rejected_rows += num_rejected_rows
-        rescue java.sql.SQLDataException => e
-          @jv.rollback
-          if @task['reject_on_materialized_type_error'] and e.message =~ /Rejected by user-defined parser/
-            Embulk.logger.warn "embulk-output-vertica: ROLLBACK! some of column types and values types do not fit #{json}"
-          else
-            Embulk.logger.warn "embulk-output-vertica: ROLLBACK!"
-          end
-          raise e # die transaction
         end
       end
 
@@ -133,9 +169,10 @@ module Embulk
       def abort
       end
 
+      # this is called after processing all pages in a thread
+      # we do commit on #transaction for all connection pools, not at here
       def commit
-        @jv.commit
-        Embulk.logger.debug { "embulk-output-vertica: COMMIT! #{@num_output_rows} rows" }
+        Embulk.logger.debug { "embulk-output-vertica: #{@num_output_rows} rows" }
         task_report = {
           "num_input_rows" => @num_input_rows,
           "num_output_rows" => @num_output_rows,
@@ -145,25 +182,6 @@ module Embulk
 
       private
 
-      def self.connect(task)
-        jv = ::Jvertica.connect({
-          host: task['host'],
-          port: task['port'],
-          user: task['user'],
-          password: task['password'],
-          database: task['database'],
-        })
-
-        if block_given?
-          begin
-            yield jv
-          ensure
-            jv.close
-          end
-        end
-        jv
-      end
-
       # @param [Schema] schema embulk defined column types
       # @param [Hash]   column_options user defined column types
       # @return [String] sql schema used to CREATE TABLE
@@ -197,7 +215,7 @@ module Embulk
           "WHERE table_schema = #{quoted_schema} AND table_name = #{quoted_table}"
 
         sql_schema = {}
-        connect(task) do |jv|
+        connection_pool.with do |jv|
           result = query(jv, sql)
           sql_schema = result.map {|row| [row[0], row[1]] }
         end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: embulk-output-vertica
 version: !ruby/object:Gem::Version
-  version: 0.3.1
+  version: 0.4.0
 platform: ruby
 authors:
 - eiji.sekiya
@@ -9,7 +9,7 @@ authors:
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2015-11-20 00:00:00.000000000 Z
+date: 2015-11-24 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: jvertica
@@ -39,6 +39,20 @@ dependencies:
         version: '0'
   prerelease: false
   type: :runtime
+- !ruby/object:Gem::Dependency
+  name: connection_pool
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  prerelease: false
+  type: :runtime
 - !ruby/object:Gem::Dependency
   name: bundler
   version_requirements: !ruby/object:Gem::Requirement