embulk-output-vertica 0.2.6 → 0.2.7

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 7ee9565adfe070ea8678d99eb5d94d328f5fcbe0
-  data.tar.gz: e801c980bea0733372909d7f1672d03910f0d0f9
+  metadata.gz: 758fc8b31ffadedaf028b5775fd8820867e2998b
+  data.tar.gz: 8be72ce20cfe83be5f9d7ef827f81a0c9691cfec
 SHA512:
-  metadata.gz: 24db1540c0ceaab5bb18af2112df702d8843fcc4c2c0ef9a43ad449e6d69bcb098e35aacf047a1c0bfeec82347f9808793d55b2ee9adadfdfeb6d5aeb09b46c4
-  data.tar.gz: 39eed0dda265c8da99969ad8e059ee42d90e158609e31bacecefbd2e3f7235cdac0967d83301d09994c9a7cb564f9e9c92faf6da29c6ee5528ed963f6bfeb8a4
+  metadata.gz: 08188e9dcc7209b7317e8b4256eca1e262dcd5759d44a43a4223ea5ebb6dac8a7f205140d789816de2b1032dc95e0074a074c6e85f107e43beb7450be855614e
+  data.tar.gz: c7184dd4edae3f1a4c988e1ae62442ff634b72765e805e590fa365bb698594fd00e6ef5bf4ea98b39ffc2cec756726c3c2034e171f3f33d65e76446005499736

data/CHANGELOG.md

@@ -1,3 +1,9 @@
+# 0.2.7 (2015/11/06)
+
+Enhancements:
+
+* Get sql schema information from existing table unless REPLACE mode
+
 # 0.2.6 (2015/11/06)
 
 Fixes:

data/embulk-output-vertica.gemspec

@@ -1,6 +1,6 @@
 Gem::Specification.new do |spec|
   spec.name          = "embulk-output-vertica"
-  spec.version       = "0.2.6"
+  spec.version       = "0.2.7"
   spec.authors       = ["eiji.sekiya", "Naotoshi Seo"]
   spec.email         = ["eiji.sekiya.0326@gmail.com", "sonots@gmail.com"]
   spec.summary       = "Vertica output plugin for Embulk"

data/lib/embulk/output/vertica.rb

@@ -46,31 +46,38 @@ module Embulk
         unique_name = "%08x%08x" % [now.tv_sec, now.tv_nsec]
         task['temp_table'] = "#{task['table']}_LOAD_TEMP_#{unique_name}"
 
-        sql_schema = self.to_sql_schema(schema, task['column_options'])
-
         quoted_schema     = ::Jvertica.quote_identifier(task['schema'])
         quoted_table      = ::Jvertica.quote_identifier(task['table'])
         quoted_temp_table = ::Jvertica.quote_identifier(task['temp_table'])
 
+        sql_schema ||= self.existing_sql_schema(task) unless task['mode'] == 'REPLACE'
+        sql_schema ||= self.to_sql_schema(schema, task['column_options'])
+        sql_schema_expression = sql_schema.map {|name, type| "#{::Jvertica.quote_identifier(name)} #{type}" }.join(',')
+
         connect(task) do |jv|
-          if task['mode'] == 'REPLACE'
-            query(jv, %[DROP TABLE IF EXISTS #{quoted_schema}.#{quoted_table}])
-          end
+          # create a temp table
           query(jv, %[DROP TABLE IF EXISTS #{quoted_schema}.#{quoted_temp_table}])
-          query(jv, %[CREATE TABLE #{quoted_schema}.#{quoted_temp_table} (#{sql_schema})])
+          query(jv, %[CREATE TABLE #{quoted_schema}.#{quoted_temp_table} (#{sql_schema_expression})])
         end
 
         begin
-          # obtain an array of task_reports where one report is of a task
-          task_reports = yield(task)
+          # insert data into a temp table
+          task_reports = yield(task) # obtain an array of task_reports where one report is of a task
           Embulk.logger.info { "embulk-output-vertica: task_reports: #{task_reports.to_json}" }
+
           connect(task) do |jv|
-            query(jv, %[CREATE TABLE IF NOT EXISTS #{quoted_schema}.#{quoted_table} (#{sql_schema})])
+            # create the target table if not exists or mode == replace
+            if task['mode'] == 'REPLACE'
+              query(jv, %[DROP TABLE IF EXISTS #{quoted_schema}.#{quoted_table}])
+            end
+            query(jv, %[CREATE TABLE IF NOT EXISTS #{quoted_schema}.#{quoted_table} (#{sql_schema_expression})])
+            # insert select from the temp table
             query(jv, %[INSERT INTO #{quoted_schema}.#{quoted_table} SELECT * FROM #{quoted_schema}.#{quoted_temp_table}])
             jv.commit
           end
         ensure
           connect(task) do |jv|
+            # clean up the temp table
             query(jv, %[DROP TABLE IF EXISTS #{quoted_schema}.#{quoted_temp_table}])
             Embulk.logger.debug { "embulk-output-vertica: #{query(jv, %[SELECT * FROM #{quoted_schema}.#{quoted_table} LIMIT 10]).map {|row| row.to_h }.join("\n") rescue nil}" }
           end
@@ -166,8 +173,8 @@ module Embulk
           else
             sql_type = to_sql_type(type)
           end
-          "#{::Jvertica.quote_identifier(column_name)} #{sql_type}"
-        end.join(',')
+          [column_name, sql_type]
+        end.to_h
       end
 
       def self.to_sql_type(type)
@@ -181,6 +188,22 @@ module Embulk
         end
       end
 
+      # Get sql schema if table exists
+      def self.existing_sql_schema(task)
+        quoted_schema = Jvertica.quote(task['schema'])
+        quoted_table  = Jvertica.quote(task['table'])
+        sql = "SELECT column_name, data_type FROM v_catalog.columns " \
+          "WHERE table_schema = #{quoted_schema} AND table_name = #{quoted_table}"
+
+        sql_schema = {}
+        connect(task) do |jv|
+          result = query(jv, sql)
+          sql_schema = result.map {|row| [row[0], row[1]] }.to_h
+        end
+        Embulk.logger.info "embulk-output-vertica: existing_sql_schema: #{sql_schema}"
+        sql_schema.empty? ? nil : sql_schema
+      end
+
       def self.query(conn, sql)
         Embulk.logger.info "embulk-output-vertica: #{sql}"
         conn.query(sql)

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: embulk-output-vertica
 version: !ruby/object:Gem::Version
-  version: 0.2.6
+  version: 0.2.7
 platform: ruby
 authors:
 - eiji.sekiya