embulk-output-vertica 0.2.2 → 0.2.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 1fadc45702f906302a1ce19933883397ca681a33
-  data.tar.gz: cab2e2a81546ba3f42001b1577cdcf573f665b58
+  metadata.gz: 2ab7bcf92a11a6a2899ca833c6a8c07677d5b1e5
+  data.tar.gz: d05503ff66756df167a83e70da5ca7cd6f7ba2f0
 SHA512:
-  metadata.gz: 8b3dddf8c4b834c31d44e777795fe81946182449d58261a6b089842b22548b976ee0fae95e7a095bbdea059cac78441c470e2d987bdd1a59c477bf5d323f07f0
-  data.tar.gz: fd2b18e00e2b03b6e8df5f710e93301f313f146a8a6f1183b97005a62a30ce0001d657add51d815f2d5b46ae0fc438fec5df44d058ce27b3acbc21f002b61185
+  metadata.gz: 2c2dd21ae409cbcfcf40428bdcc6e2ad67f9911935145d69f05a7b534546523ae4618e879f60b6ca7a60ada7496ee9754da5f909b7b50773a8474c52a5f08787
+  data.tar.gz: e3e962118458e4412370e5efa25227f801d5986a4bced0f29a67ab490608d2e83f15d3a9e3c1ea769530cabc7006019b4f16b935819dbf560d32ac899322b9ca

data/CHANGELOG.md

@@ -1,3 +1,13 @@
+# 0.2.3 (2015/09/16)
+
+Changes:
+
+* Commit all pages at burst (in each task)
+
+Enhancements:
+
+* Return task_reports
+
 # 0.2.2 (2015/07/24)
 
 Changes:
@@ -20,19 +30,19 @@ Enhancements:
 
 Enhancements:
 
-* Support timezone for string converter
+* Support `timezone` for string converter
 
 # 0.1.8 (2015/07/24)
 
 Enhancements:
 
-* Support value_type, timezone_format, timezone option for column_options[
+* Support `value_type`, `timezone_format`, `timezone` option for column_options
 
 # 0.1.7 (2015/07/24)
 
 Enhancements:
 
-* Add reject_on_materialized_type_error option
+* Add `reject_on_materialized_type_error` option
 
 # 0.1.6 (2015/07/23)
 

data/README.md

@@ -4,7 +4,7 @@
 
 * **Plugin type**: output
 * **Resume supported**: no
-* **Cleanup supported**: no
+* **Cleanup supported**: yes
 * **Dynamic table creating**: yes
 
 ## Configuration
@@ -17,9 +17,9 @@
 - **schema**:   schema name (string, default: public)
 - **table**:    table name (string, required)
 - **mode**:     "insert", or "replace". See bellow. (string, default: insert)
-- **copy_mode**: specifies how data is loaded into the database. (`AUTO`, `DIRECT`, or `TRICKLE`. default: AUTO) See vertica documents for details.
-- **abort_on_error**: Stops the COPY command if a row is rejected and rolls back the command. No data is loaded. (bool, default: false)
-- **reject_on_materialized_type_error**: Use `reject_on_materialized_type_error` option for fjsonparser(). This rejects rows if any of olumn types and value types do not fit. ex) double value into INT column fails. See vertica documents for details. (bool, default: false)
+- **copy_mode**: specifies how data is loaded into the database. See vertica documents for details. (`AUTO`, `DIRECT`, or `TRICKLE`. default: `AUTO`)
+- **abort_on_error**: stops the COPY command if a row is rejected and rolls back the command. No data is loaded. (bool, default: false)
+- **reject_on_materialized_type_error**: uses `reject_on_materialized_type_error` option for fjsonparser(). This rejects rows if any of column types and value types do not fit, ex) double value into INT column fails. See vertica documents for details. (bool, default: false)
 - **default_timezone**: the default timezone for column_options (string, default is "UTC")
 - **column_options**: advanced: a key-value pairs where key is a column name and value is options for the column.
   - **type**: type of a column when this plugin creates new tables such as `VARCHAR(255)`, `INTEGER NOT NULL UNIQUE`. This is used on creating intermediate tables (insert and truncate_insert modes) and on creating a new target table. (string, default: depends on input column type, see below)
@@ -29,11 +29,11 @@
     - string:    `VARCHAR`
     - timestamp: `TIMESTAMP`
   - **value_type**:  The types (embulk types) of values to convert (string, default: no conversion. See below for available types)
-    - boolean:   `boolean`, `string` (to\_s)
-    - long:      `boolean` (true), `long`, `double` (to\_f), `string` (to\_s), `timestamp` (Time.at)
-    - double:    `boolean` (true), `long` (to\_i), `double`, `string` (to\_s), `timestamp` (Time.at)
-    - string:    `boolean` (true), `long` (to\_i), `double` (to\_f), `string`, `timestamp` (Time.strptime)
-    - timestamp: `boolean` (true), `long` (to\_i), `double` (to\_f), `string` (strftime), `timestamp`
+    - boolean:   `boolean`, `string`
+    - long:      `boolean`, `long`, `double`, `string`, `timestamp`
+    - double:    `boolean`, `long`, `double`, `string`, `timestamp`
+    - string:    `boolean`, `long`, `double`, `string`, `timestamp`
+    - timestamp: `boolean`, `long`, `double`, `string`, `timestamp`
   - **timestamp_format**: timestamp format to convert into/from `timestamp` (string, default is "%Y-%m-%d %H:%M:%S %z")
   - **timezone**: timezone to convert into/from `timestamp` (string, default is `default_timezone`).
 
@@ -43,7 +43,7 @@
   * Behavior: This mode copys rows to some intermediate tables first. If all those tasks run correctly, runs INSERT INTO <target_table> SELECT * FROM <intermediate_table>
   * Transactional: Yes if `abort_on_error` option is used
 * **replace**:
-  * Behavior: Same with insert mode excepting that it drop the target table first.
+  * Behavior: Same with insert mode excepting that it drops the target table first.
   * Transactional: Yes if `abort_on_error` option is used
 
 ## Example
@@ -62,7 +62,7 @@ out:
   column_options:
     id:   {type: INT}
     name: {type: VARCHAR(255)}
-    date: {type: DATE, value_type: timezone, timezone: "+09:00"}
+    date: {type: DATE, value_type: timestamp, timezone: "+09:00"}
 ```
 
 ## Development

data/embulk-output-vertica.gemspec

@@ -1,6 +1,6 @@
 Gem::Specification.new do |spec|
   spec.name          = "embulk-output-vertica"
-  spec.version       = "0.2.2"
+  spec.version       = "0.2.3"
   spec.authors       = ["eiji.sekiya", "Naotoshi Seo"]
   spec.email         = ["eiji.sekiya.0326@gmail.com", "sonots@gmail.com"]
   spec.summary       = "Vertica output plugin for Embulk"

data/lib/embulk/output/vertica.rb

@@ -53,7 +53,9 @@ module Embulk
         end
 
         begin
-          yield(task)
+          # obtain an array of task_reports where one report is of a task
+          task_reports = yield(task)
+          Embulk.logger.info { "embulk-output-vertica: task_reports: #{task_reports}" }
           connect(task) do |jv|
             query(jv, %[CREATE TABLE IF NOT EXISTS #{quoted_schema}.#{quoted_table} (#{sql_schema})])
             query(jv, %[INSERT INTO #{quoted_schema}.#{quoted_table} SELECT * FROM #{quoted_schema}.#{quoted_temp_table}])
@@ -65,7 +67,9 @@ module Embulk
             Embulk.logger.debug { "embulk-output-vertica: #{query(jv, %[SELECT * FROM #{quoted_schema}.#{quoted_table} LIMIT 10]).map {|row| row.to_h }.join("\n") rescue nil}" }
           end
         end
-        return {}
+        # this is for -o next_config option, add some paramters for next time execution if wants
+        next_config_diff = {}
+        return next_config_diff
       end
 
       def initialize(task, schema, index)
@@ -73,6 +77,9 @@ module Embulk
         @converters = ValueConverterFactory.create_converters(schema, task['default_timezone'], task['column_options'])
         Embulk.logger.debug { @converters.to_s }
         @jv = self.class.connect(task)
+        @num_input_rows = 0
+        @num_output_rows = 0
+        @num_rejected_rows = 0
       end
 
       def close
@@ -82,15 +89,17 @@ module Embulk
       def add(page)
         json = nil # for log
         begin
-          copy(@jv, copy_sql) do |stdin|
+          num_output_rows, rejects = copy(@jv, copy_sql) do |stdin|
             page.each do |record|
               json = to_json(record)
               Embulk.logger.debug { "embulk-output-vertica: #{json}" }
               stdin << json << "\n"
+              @num_input_rows += 1
             end
           end
-          Embulk.logger.info "embulk-output-vertica: COMMIT!"
-          @jv.commit
+          num_rejected_rows = rejects.size
+          @num_output_rows += num_output_rows
+          @num_rejected_rows += num_rejected_rows
         rescue java.sql.SQLDataException => e
           @jv.rollback
           if @task['reject_on_materialized_type_error'] and e.message =~ /Rejected by user-defined parser/
@@ -98,7 +107,7 @@ module Embulk
           else
             Embulk.logger.warn "embulk-output-vertica: ROLLBACK!"
           end
-          raise e
+          raise e # die transaction
         end
       end
 
@@ -109,7 +118,13 @@ module Embulk
       end
 
       def commit
-        {}
+        @jv.commit
+        Embulk.logger.info { "embulk-output-vertica: COMMIT! #{@num_output_rows} rows" }
+        task_report = {
+          "num_input_rows" => @num_input_rows,
+          "num_output_rows" => @num_output_rows,
+          "num_rejected_rows" => @num_rejected_rows,
+        }
       end
 
       private
@@ -169,7 +184,7 @@ module Embulk
 
       def copy(conn, sql, &block)
         Embulk.logger.info "embulk-output-vertica: #{sql}"
-        conn.copy(sql, &block)
+        results, rejects = conn.copy(sql, &block)
       end
 
       def copy_sql

data/lib/embulk/output/vertica/value_converter_factory.rb

@@ -9,11 +9,12 @@ module Embulk
 
         DEFAULT_TIMESTAMP_FORMAT = "%Y-%m-%d %H:%M:%S %z"
 
+        # @param [Schema] schema embulk defined column types
+        # @param [String] default_timezone
+        # @param [Hash]   column_options user defined column types
+        # @return [Hash] hash whose key is column_name, and value is its converter (Proc)
         def self.create_converters(schema, default_timezone, column_options)
-          # @param [Schema] schema embulk defined column types
-          # @param [Hash]   column_options user defined column types
-          # @return [Array] value converters (array of Proc)
-          Hash[*(schema.names.zip(schema.types).map do |column_name, schema_type|
+          Hash[schema.names.zip(schema.types).map do |column_name, schema_type|
             if column_options[column_name]
               value_type       = column_options[column_name]['value_type']
               timestamp_format = column_options[column_name]['timestamp_format'] || DEFAULT_TIMESTAMP_FORMAT
@@ -22,7 +23,7 @@ module Embulk
             else
               [column_name, Proc.new {|val| val }]
             end
-          end.flatten!(1))]
+          end]
         end
 
         def initialize(schema_type, value_type = nil, timestamp_format = nil, timezone = nil)

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: embulk-output-vertica
 version: !ruby/object:Gem::Version
-  version: 0.2.2
+  version: 0.2.3
 platform: ruby
 authors:
 - eiji.sekiya
@@ -9,64 +9,64 @@ authors:
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2015-07-24 00:00:00.000000000 Z
+date: 2015-09-16 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
+  name: jvertica
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.2'
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
         version: '0.2'
-  name: jvertica
   prerelease: false
   type: :runtime
+- !ruby/object:Gem::Dependency
+  name: tzinfo
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - ">="
       - !ruby/object:Gem::Version
-        version: '0.2'
-- !ruby/object:Gem::Dependency
+        version: '0'
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
-  name: tzinfo
   prerelease: false
   type: :runtime
+- !ruby/object:Gem::Dependency
+  name: bundler
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '0'
-- !ruby/object:Gem::Dependency
+        version: '1.7'
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
         version: '1.7'
-  name: bundler
   prerelease: false
   type: :development
+- !ruby/object:Gem::Dependency
+  name: rake
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.7'
-- !ruby/object:Gem::Dependency
+        version: '10.0'
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
         version: '10.0'
-  name: rake
   prerelease: false
   type: :development
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - ~>
-      - !ruby/object:Gem::Version
-        version: '10.0'
 description: Dump records to vertica
 email:
 - eiji.sekiya.0326@gmail.com
@@ -75,7 +75,7 @@ executables: []
 extensions: []
 extra_rdoc_files: []
 files:
-- .gitignore
+- ".gitignore"
 - CHANGELOG.md
 - Gemfile
 - LICENSE.txt
@@ -96,17 +96,17 @@ require_paths:
 - lib
 required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
-  - - '>='
+  - - ">="
     - !ruby/object:Gem::Version
       version: '0'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
-  - - '>='
+  - - ">="
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
 rubyforge_project:
-rubygems_version: 2.4.5
+rubygems_version: 2.4.8
 signing_key:
 specification_version: 4
 summary: Vertica output plugin for Embulk