embulk-output-vertica 0.1.6 → 0.1.7

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 3f78066cc105d0aa31d8fe9a70f9166facdef222
-  data.tar.gz: 9f60547ac77c5f54461722b907f8fafd5c32000c
+  metadata.gz: daadcaba0204e6e4c1103f4edd8e5b3c6b70924f
+  data.tar.gz: 339ab9e6350031e1325ac69d2773f45bc6f7e0b0
 SHA512:
-  metadata.gz: 9fee2ba24167506cfe0ef7fbae612361ecc15bc42d35bbee245d0221f573742434768e8b49f11a62332ee8c8e8475e26d44fcce4259b963e459bb36c9f2c39d8
-  data.tar.gz: 257182e4efed2b0ef43b955ab989bcd658a9d14b5efb09dc56106c1dd1a7f68e6080414c91e534c5c4db480a29a54accff892bd63d6522fe7e72c6f5940aa3d0
+  metadata.gz: ecec7a142322fdff6596c3359fb67949235d8082ae626e491df8ed5cce55961777eb7715cb69a7d12243f3305ec97595a97c3e820188f6198f3a68e43595e01f
+  data.tar.gz: 7d17307369af315a30ab296aad4bb94da6ff08806db4f73fef541c1aa710326a7e251804e730004c854be97a68a194059306867b484f607af7864e02210e9bd5

data/CHANGELOG.md

@@ -1,3 +1,9 @@
+# 0.1.7 (2015/07/25)
+
+Enhancements:
+
+* Add reject_on_materialized_type_error option
+
 # 0.1.6 (2015/07/24)
 
 Enhancements:

data/README.md

@@ -19,13 +19,14 @@
 - **mode**:     "insert", or "replace". See bellow. (string, default: insert)
 - **copy_mode**: specifies how data is loaded into the database. (`AUTO`, `DIRECT`, or `TRICKLE`. default: AUTO) See vertica documents for details.
 - **abort_on_error**: Stops the COPY command if a row is rejected and rolls back the command. No data is loaded. (bool, default: false)
+- **reject_on_materialized_type_error**: Use `reject_on_materialized_type_error` option for fjsonparser(). This rejects rows if any of olumn types and value types do not fit. ex) double value into INT column fails. See vertica documents for details. (bool, default: false)
 - **column_options**: advanced: a key-value pairs where key is a column name and value is options for the column.
   - **type**: type of a column when this plugin creates new tables such as `VARCHAR(255)`, `INTEGER NOT NULL UNIQUE`. This is used on creating intermediate tables (insert and truncate_insert modes) and on creating a new target table. (string, default: depends on input column type, see below)
-    - `INT` (same with `BIGINT` in vertica) for `long`
-    - `BOOLEAN` for `boolean`
-    - `FLOAT` (same with `DOUBLE PRECISION` in vertica) for `double`
-    - `VARCHAR` for `string`
-    - `TIMESTAMP` for `timestamp`
+    - long: `INT` (same with `BIGINT` in vertica)
+    - boolean: `BOOLEAN`
+    - double: `FLOAT` (same with `DOUBLE PRECISION` in vertica)
+    - string: `VARCHAR`
+    - timestamp: `TIMESTAMP`
 
 ### Modes
 

data/embulk-output-vertica.gemspec

@@ -1,6 +1,6 @@
 Gem::Specification.new do |spec|
   spec.name          = "embulk-output-vertica"
-  spec.version       = "0.1.6"
+  spec.version       = "0.1.7"
   spec.authors       = ["eiji.sekiya", "Naotoshi Seo"]
   spec.email         = ["eiji.sekiya.0326@gmail.com", "sonots@gmail.com"]
   spec.summary       = "Vertica output plugin for Embulk"

data/example.yml

@@ -30,7 +30,10 @@ out:
   database: vdb
   schema: sandbox
   table: embulk_test
+  mode: replace
   copy_mode: direct
+  abort_on_error: true
+  reject_on_materialized_type_error: true
   column_options:
     date: {type: DATE}
     id: {type: INT}

data/lib/embulk/output/vertica.rb

@@ -20,6 +20,7 @@ module Embulk
           'mode'           => config.param('mode',           :string,  :default => 'insert'),
           'copy_mode'      => config.param('copy_mode',      :string,  :default => 'AUTO'),
           'abort_on_error' => config.param('abort_on_error', :bool,    :default => false),
+          'reject_on_materialized_type_error' => config.param('reject_on_materialized_type_error', :bool, :default => false),
           'column_options' => config.param('column_options', :hash,    :default => {}),
         }
 
@@ -59,7 +60,7 @@ module Embulk
         ensure
           connect(task) do |jv|
             query(jv, %[DROP TABLE IF EXISTS #{quoted_schema}.#{quoted_temp_table}])
-            Embulk.logger.debug { query(jv, %[SELECT * FROM #{quoted_schema}.#{quoted_table} LIMIT 10]).map {|row| row.to_h }.join("\n") rescue nil }
+            Embulk.logger.debug { "embulk-output-vertica: #{query(jv, %[SELECT * FROM #{quoted_schema}.#{quoted_table} LIMIT 10]).map {|row| row.to_h }.join("\n") rescue nil}" }
           end
         end
         return {}
@@ -75,12 +76,25 @@ module Embulk
       end
 
       def add(page)
-        copy(@jv, copy_sql) do |stdin|
-          page.each do |record|
-            stdin << to_json(record) << "\n"
+        json = nil # for log
+        begin
+          copy(@jv, copy_sql) do |stdin|
+            page.each do |record|
+              json = to_json(record)
+              stdin << json << "\n"
+            end
+          end
+          Embulk.logger.debug "embulk-output-vertica: COMMIT!"
+          @jv.commit
+        rescue java.sql.SQLDataException => e
+          @jv.rollback
+          if @task['reject_on_materialized_type_error'] and e.message =~ /Rejected by user-defined parser/
+            Embulk.logger.warn "embulk-output-vertica: ROLLBACK! some of column types and values types do not fit #{json}"
+          else
+            Embulk.logger.warn "embulk-output-vertica: ROLLBACK!"
           end
+          raise e
         end
-        @jv.commit
       end
 
       def finish
@@ -154,7 +168,7 @@ module Embulk
       end
 
       def copy_sql
-        @copy_sql ||= "COPY #{quoted_schema}.#{quoted_temp_table} FROM STDIN PARSER fjsonparser() #{copy_mode}#{abort_on_error} NO COMMIT"
+        @copy_sql ||= "COPY #{quoted_schema}.#{quoted_temp_table} FROM STDIN#{fjsonparser}#{copy_mode}#{abort_on_error} NO COMMIT"
       end
 
       def to_json(record)
@@ -174,13 +188,20 @@ module Embulk
       end
 
       def copy_mode
-        @task['copy_mode']
+        " #{@task['copy_mode']}"
       end
 
       def abort_on_error
         @task['abort_on_error'] ? ' ABORT ON ERROR' : ''
       end
 
+      def fjsonparser
+        " PARSER fjsonparser(#{reject_on_materialized_type_error})"
+      end
+
+      def reject_on_materialized_type_error
+        @task['reject_on_materialized_type_error'] ? 'reject_on_materialized_type_error=true' : ''
+      end
     end
   end
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: embulk-output-vertica
 version: !ruby/object:Gem::Version
-  version: 0.1.6
+  version: 0.1.7
 platform: ruby
 authors:
 - eiji.sekiya
@@ -9,7 +9,7 @@ authors:
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2015-07-23 00:00:00.000000000 Z
+date: 2015-07-24 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement