embulk-output-vertica 0.6.2 → 0.7.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: eb34c1b088bf2c2f7320d0c6bb1e076b84495683
-  data.tar.gz: 300769b3f384905879b654fce8e783b0b85d2262
+  metadata.gz: e728473aef9d73c5ff031ba16f354129fce903ac
+  data.tar.gz: 701b369758f736ca57455e4ba15d395ece62669f
 SHA512:
-  metadata.gz: e94f3b6f86bb97b2dc01862db20113111cb030605996cc10f02c1dacada7bc227a17be8e9589312cb49a916a60a58a15acb42ad40a93a49fa1d5dff7c99d03ec
-  data.tar.gz: 98547b91c62437e1a732f72f0dcc35b378ed68d41423c1f6681c3e86acf4cd63be8bd80d24536e35a53de6299c5c2ddc414eba62dfd92f18bde665442a19dc56
+  metadata.gz: 5e7ca636569766779460699fbec289c9f6c246c8a35d85ed3764a8f0e7af6c3453c4a32771fcf81313217ab2867d06326026aa86ff61e721f60fc0d224717891
+  data.tar.gz: 873396d69807a19426a8ec5d088d86c8a081b3e381e283298d4952615963598fd1d33acd96605f4775d2fa4c028dc586d535838915db6df5493eaabc32c9365f

data/CHANGELOG.md

@@ -1,3 +1,13 @@
+# 0.7.0 (2016/01/29)
+
+Breaking Changes:
+
+* rename mode `REPLACE` to `DROP_INSERT`
+
+Enhancements:
+
+* Add mode `REPLACE` which copies rows to an intermediate table first. If all those tasks run correctly, swaps the target table and the intermediate table, and drops the old target table
+
 # 0.6.2 (2016/01/28)
 
 Enhancements:

data/README.md

@@ -44,11 +44,11 @@
 ### Modes
 
 * **insert**:
-  * Behavior: This mode copys rows to some intermediate tables first. If all those tasks run correctly, runs INSERT INTO <target_table> SELECT * FROM <intermediate_table>
-  * Transactional: Yes if `abort_on_error` option is used
+  * Behavior: This mode copies rows to some intermediate tables first. If all those tasks run correctly, runs INSERT INTO <target_table> SELECT * FROM <intermediate_table>
 * **replace**:
-  * Behavior: Same with insert mode excepting that it drops the target table first.
-  * Transactional: Yes if `abort_on_error` option is used
+  * Behavior: This mode copies rows to an intermediate table first. If all those tasks run correctly, swaps the target table and the intermediate table, and drops the old target table
+* **drop_insert**:
+  * Behavior: Same with insert mode excepting that it drops the target table first. This is aimed to use for development.
 
 ## Example
 

data/embulk-output-vertica.gemspec

@@ -1,6 +1,6 @@
 Gem::Specification.new do |spec|
   spec.name          = "embulk-output-vertica"
-  spec.version       = "0.6.2"
+  spec.version       = "0.7.0"
   spec.authors       = ["eiji.sekiya", "Naotoshi Seo"]
   spec.email         = ["eiji.sekiya.0326@gmail.com", "sonots@gmail.com"]
   spec.summary       = "Vertica output plugin for Embulk"
@@ -13,7 +13,7 @@ Gem::Specification.new do |spec|
   spec.test_files    = spec.files.grep(%r{^(test|spec|features)/})
   spec.require_paths = ["lib"]
 
-  spec.add_dependency "jvertica", "~> 0.2"
+  spec.add_dependency "jvertica", ">= 0.2.2"
   spec.add_dependency "tzinfo"
   spec.add_development_dependency "bundler", "~> 1.7"
   spec.add_development_dependency "rake", "~> 10.0"

data/example/abort.yml

@@ -24,7 +24,7 @@ out:
   database: vdb
   schema: sandbox
   table: embulk_test
-  mode: replace
+  mode: drop_insert
   copy_mode: DIRECT
   abort_on_error: true
   reject_on_materialized_type_error: true

data/example/example.yml

@@ -31,7 +31,7 @@ out:
   database: vdb
   schema: sandbox
   table: embulk_test
-  mode: replace
+  mode: drop_insert
   copy_mode: DIRECT
   abort_on_error: true
   reject_on_materialized_type_error: true

data/example/replace.yml

@@ -0,0 +1,33 @@
+in:
+  type: file
+  path_prefix: example/example.csv
+  parser:
+    type: csv
+    charset: UTF-8
+    newline: CRLF
+    null_string: 'NULL'
+    skip_header_lines: 1
+    comment_line_marker: '#'
+    columns:
+      - {name: timestamp_date, type: timestamp, format: "%Y-%m-%d", timezone: "+09:00"}
+      - {name: string_date,    type: string}
+      - {name: foo,   type: string}
+      - {name: bar,   type: string}
+      - {name: id,    type: long}
+      - {name: name,  type: string}
+      - {name: score, type: double} 
+out:
+  type: vertica 
+  host: 127.0.0.1
+  user: dbadmin
+  password: xxxxxxx
+  database: vdb
+  schema: sandbox
+  table: jvertica_test
+  mode: replace
+  copy_mode: DIRECT
+  abort_on_error: true
+  reject_on_materialized_type_error: true
+  default_timezone: 'Asia/Tokyo'
+  pool: 3
+  compress: GZIP

data/example/resource_pool.yml

@@ -0,0 +1,34 @@
+in:
+  type: file
+  path_prefix: example/example.csv
+  parser:
+    type: csv
+    charset: UTF-8
+    newline: CRLF
+    null_string: 'NULL'
+    skip_header_lines: 1
+    comment_line_marker: '#'
+    columns:
+      - {name: timestamp_date, type: timestamp, format: "%Y-%m-%d", timezone: "+09:00"}
+      - {name: string_date,    type: string}
+      - {name: foo,   type: string}
+      - {name: bar,   type: string}
+      - {name: id,    type: long}
+      - {name: name,  type: string}
+      - {name: score, type: double} 
+out:
+  type: vertica 
+  host: 127.0.0.1
+  user: dbadmin
+  password: xxxxxxx
+  database: vdb
+  schema: sandbox
+  table: jvertica_test
+  mode: drop_inesrt
+  copy_mode: DIRECT
+  abort_on_error: true
+  reject_on_materialized_type_error: true
+  default_timezone: 'Asia/Tokyo'
+  pool: 3
+  compress: GZIP
+  resource_pool: default

data/lib/embulk/output/vertica.rb

@@ -46,8 +46,8 @@ module Embulk
         end
 
         task['mode'] = task['mode'].upcase
-        unless %w[INSERT REPLACE].include?(task['mode'])
-          raise ConfigError.new "`mode` must be one of INSERT, REPLACE"
+        unless %w[INSERT REPLACE DROP_INSERT].include?(task['mode'])
+          raise ConfigError.new "`mode` must be one of INSERT, REPLACE, DROP_INSERT"
         end
 
         task['copy_mode'] = task['copy_mode'].upcase
@@ -74,7 +74,7 @@ module Embulk
             sql_schema_table = self.sql_schema_from_embulk_schema(schema, task['column_options'])
 
             # create the target table
-            query(jv, %[DROP TABLE IF EXISTS #{quoted_schema}.#{quoted_table}]) if task['mode'] == 'REPLACE'
+            query(jv, %[DROP TABLE IF EXISTS #{quoted_schema}.#{quoted_table}]) if task['mode'] == 'DROP_INSERT'
             query(jv, %[CREATE TABLE IF NOT EXISTS #{quoted_schema}.#{quoted_table} (#{sql_schema_table})])
           end
 
@@ -98,11 +98,20 @@ module Embulk
           task_reports = thread_pool.commit
           Embulk.logger.info { "embulk-output-vertica: task_reports: #{task_reports.to_json}" }
 
-          # insert select from the temp table
           connect(task) do |jv|
-            hint = '/*+ direct */ ' if task['copy_mode'] == 'DIRECT' # I did not prepare a specific option, does anyone want?
-            query(jv, %[INSERT #{hint}INTO #{quoted_schema}.#{quoted_table} SELECT * FROM #{quoted_schema}.#{quoted_temp_table}])
-            jv.commit
+            if task['mode'] == 'REPLACE'
+              # swap table and drop the old table
+              quoted_old_table = ::Jvertica.quote_identifier("#{task['table']}_LOAD_OLD_#{unique_name}")
+              from = "#{quoted_schema}.#{quoted_table},#{quoted_schema}.#{quoted_temp_table}"
+              to   = "#{quoted_old_table},#{quoted_table}"
+              query(jv, %[ALTER TABLE #{from} RENAME TO #{to}])
+              query(jv, %[DROP TABLE #{quoted_schema}.#{quoted_old_table}])
+            else
+              # insert select from the temp table
+              hint = '/*+ direct */ ' if task['copy_mode'] == 'DIRECT' # I did not prepare a specific option, does anyone want?
+              query(jv, %[INSERT #{hint}INTO #{quoted_schema}.#{quoted_table} SELECT * FROM #{quoted_schema}.#{quoted_temp_table}])
+              jv.commit
+            end
           end
         ensure
           connect(task) do |jv|

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: embulk-output-vertica
 version: !ruby/object:Gem::Version
-  version: 0.6.2
+  version: 0.7.0
 platform: ruby
 authors:
 - eiji.sekiya
@@ -9,22 +9,22 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2016-01-28 00:00:00.000000000 Z
+date: 2016-01-29 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: jvertica
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
-        version: '0.2'
+        version: 0.2.2
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
-        version: '0.2'
+        version: 0.2.2
 - !ruby/object:Gem::Dependency
   name: tzinfo
   requirement: !ruby/object:Gem::Requirement
@@ -87,6 +87,8 @@ files:
 - example/example.jsonl
 - example/example.yml
 - example/json_payload.yml
+- example/replace.yml
+- example/resource_pool.yml
 - lib/embulk/output/vertica.rb
 - lib/embulk/output/vertica/output_thread.rb
 - lib/embulk/output/vertica/value_converter_factory.rb