embulk-output-vertica 0.2.7 → 0.2.8

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 758fc8b31ffadedaf028b5775fd8820867e2998b
-  data.tar.gz: 8be72ce20cfe83be5f9d7ef827f81a0c9691cfec
+  metadata.gz: ff28392e54434eac04115cb7bd4f8822befe9dfc
+  data.tar.gz: 7f072f39b7046ab26347bb940a0974565836ae46
 SHA512:
-  metadata.gz: 08188e9dcc7209b7317e8b4256eca1e262dcd5759d44a43a4223ea5ebb6dac8a7f205140d789816de2b1032dc95e0074a074c6e85f107e43beb7450be855614e
-  data.tar.gz: c7184dd4edae3f1a4c988e1ae62442ff634b72765e805e590fa365bb698594fd00e6ef5bf4ea98b39ffc2cec756726c3c2034e171f3f33d65e76446005499736
+  metadata.gz: d8ef85915db3658ff923913a6d2d1c78ee02c7752192f94f8518e63af50644865c4f5aa159b7edc369cf3c568fa08cfbdb305f864a0a5705930c0566faccc3e1
+  data.tar.gz: b929a4e1d49d336272b8659c44d6fa199ef802ff85829ba554b184a3efa4593378a1d1b2e8b1f1f5c1515e9153ff338b2a6fd857fecae51bfd58f4ea3f0ae584

data/CHANGELOG.md

@@ -1,8 +1,12 @@
-# 0.2.7 (2015/11/06)
+# 0.2.8 (2015/11/06)
 
 Enhancements:
 
-* Get sql schema information from existing table unless REPLACE mode
+* Get sql schema from the existing target table to create internal temporary tables to avoid schema conflicts
+
+# 0.2.7 (2015/11/06)
+
+Skipped
 
 # 0.2.6 (2015/11/06)
 

data/embulk-output-vertica.gemspec

@@ -1,6 +1,6 @@
 Gem::Specification.new do |spec|
   spec.name          = "embulk-output-vertica"
-  spec.version       = "0.2.7"
+  spec.version       = "0.2.8"
   spec.authors       = ["eiji.sekiya", "Naotoshi Seo"]
   spec.email         = ["eiji.sekiya.0326@gmail.com", "sonots@gmail.com"]
   spec.summary       = "Vertica output plugin for Embulk"

data/lib/embulk/output/vertica.rb

@@ -50,28 +50,29 @@ module Embulk
         quoted_table      = ::Jvertica.quote_identifier(task['table'])
         quoted_temp_table = ::Jvertica.quote_identifier(task['temp_table'])
 
-        sql_schema ||= self.existing_sql_schema(task) unless task['mode'] == 'REPLACE'
-        sql_schema ||= self.to_sql_schema(schema, task['column_options'])
-        sql_schema_expression = sql_schema.map {|name, type| "#{::Jvertica.quote_identifier(name)} #{type}" }.join(',')
+        sql_schema_table = self.sql_schema_from_embulk_schema(schema, task['column_options'])
 
+        # create the target table
+        connect(task) do |jv|
+          query(jv, %[DROP TABLE IF EXISTS #{quoted_schema}.#{quoted_table}]) if task['mode'] == 'REPLACE'
+          query(jv, %[CREATE TABLE IF NOT EXISTS #{quoted_schema}.#{quoted_table} (#{sql_schema_table})])
+        end
+
+        sql_schema_temp_table = self.sql_schema_from_table(task)
+
+        # create a temp table
         connect(task) do |jv|
-          # create a temp table
           query(jv, %[DROP TABLE IF EXISTS #{quoted_schema}.#{quoted_temp_table}])
-          query(jv, %[CREATE TABLE #{quoted_schema}.#{quoted_temp_table} (#{sql_schema_expression})])
+          query(jv, %[CREATE TABLE #{quoted_schema}.#{quoted_temp_table} (#{sql_schema_temp_table})])
         end
 
         begin
-          # insert data into a temp table
+          # insert data into the temp table
           task_reports = yield(task) # obtain an array of task_reports where one report is of a task
           Embulk.logger.info { "embulk-output-vertica: task_reports: #{task_reports.to_json}" }
 
+          # insert select from the temp table
           connect(task) do |jv|
-            # create the target table if not exists or mode == replace
-            if task['mode'] == 'REPLACE'
-              query(jv, %[DROP TABLE IF EXISTS #{quoted_schema}.#{quoted_table}])
-            end
-            query(jv, %[CREATE TABLE IF NOT EXISTS #{quoted_schema}.#{quoted_table} (#{sql_schema_expression})])
-            # insert select from the temp table
             query(jv, %[INSERT INTO #{quoted_schema}.#{quoted_table} SELECT * FROM #{quoted_schema}.#{quoted_temp_table}])
             jv.commit
           end
@@ -166,18 +167,19 @@ module Embulk
       # @param [Schema] schema embulk defined column types
       # @param [Hash]   column_options user defined column types
       # @return [String] sql schema used to CREATE TABLE
-      def self.to_sql_schema(schema, column_options)
-        schema.names.zip(schema.types).map do |column_name, type|
+      def self.sql_schema_from_embulk_schema(schema, column_options)
+        sql_schema = schema.names.zip(schema.types).map do |column_name, type|
           if column_options[column_name] and column_options[column_name]['type']
             sql_type = column_options[column_name]['type']
           else
-            sql_type = to_sql_type(type)
+            sql_type = sql_type_from_embulk_type(type)
           end
           [column_name, sql_type]
-        end.to_h
+        end
+        sql_schema.map {|name, type| "#{::Jvertica.quote_identifier(name)} #{type}" }.join(',')
       end
 
-      def self.to_sql_type(type)
+      def self.sql_type_from_embulk_type(type)
         case type
         when :boolean then 'BOOLEAN'
         when :long then 'INT' # BIGINT is a synonym for INT in vertica
@@ -188,8 +190,7 @@ module Embulk
         end
       end
 
-      # Get sql schema if table exists
-      def self.existing_sql_schema(task)
+      def self.sql_schema_from_table(task)
         quoted_schema = Jvertica.quote(task['schema'])
         quoted_table  = Jvertica.quote(task['table'])
         sql = "SELECT column_name, data_type FROM v_catalog.columns " \
@@ -198,10 +199,9 @@ module Embulk
         sql_schema = {}
         connect(task) do |jv|
           result = query(jv, sql)
-          sql_schema = result.map {|row| [row[0], row[1]] }.to_h
+          sql_schema = result.map {|row| [row[0], row[1]] }
         end
-        Embulk.logger.info "embulk-output-vertica: existing_sql_schema: #{sql_schema}"
-        sql_schema.empty? ? nil : sql_schema
+        sql_schema.map {|name, type| "#{::Jvertica.quote_identifier(name)} #{type}" }.join(',')
       end
 
       def self.query(conn, sql)

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: embulk-output-vertica
 version: !ruby/object:Gem::Version
-  version: 0.2.7
+  version: 0.2.8
 platform: ruby
 authors:
 - eiji.sekiya