embulk-output-vertica 0.1.2 → 0.1.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 452d0c45a1e7f46e91cce223618aeffc0141c902
-  data.tar.gz: 16e2bc4294f270f3df7c92243ce28b2ac56f2801
+  metadata.gz: 1316243ed313380058a701d08ef37a034d54fa57
+  data.tar.gz: 1e88fcd58409f47a8114759f3f25e23702b7b303
 SHA512:
-  metadata.gz: 2538b4999f2ea7c7c8ab64c5f1aa5fa90f8dbd7bfc258643c59b00713fdd675bc29654b8c37a07d22cc29e7f9f08db03536de6e4236b14426963c0769f834607
-  data.tar.gz: 0b9a30e215cf0836160e252cf2b77d4e4dc5223b8a5ed1c492b2fc0e16d02bcff26696a61b5c8e23e379a822f416828ea3c29d8700dfe5c62b2f6f4f13af5e0a
+  metadata.gz: d705ccab43c8394f6933e463acb8b05010849853eaba47978ef6f1eb86e79d0b685cd6099b1cc2b3735acce143dd1cfe90fb2e76bf7e4fd8074b8f5ce7cabf3b
+  data.tar.gz: 9bbf3719c8e2b1b915536cb7a5527613351bca648652f59ba1586279cb276784e106dab5d45baf4cee261f675fa468107516c4cbb4885239da2d697aeff66391

data/CHANGELOG.md

@@ -1,3 +1,10 @@
+# 0.1.3 (2015/07/05)
+
+Enhancements:
+
+* Escape schama, table names
+* Add `abort_on_error` option
+
 # 0.1.2 (2015/06/26)
 
 Enhancements:

data/README.md

@@ -17,6 +17,7 @@
 - **schema**:   schema name (string, default: public)
 - **table**:    table name (string, required)
 - **copy_mode**: specifies how data is loaded into the database. (`AUTO`, `DIRECT`, or `TRICKLE`. default: AUTO)
+- **abort_on_error**: Stops the COPY command if a row is rejected and rolls back the command. No data is loaded. (bool, default: false)
 - **column_options**: advanced: a key-value pairs where key is a column name and value is options for the column.
   - **type**: type of a column when this plugin creates new tables (e.g. VARCHAR(255), INTEGER NOT NULL UNIQUE). This used when this plugin creates intermediate tables (insert and truncate_insert modes), and when it creates nonexistent target table automatically. (string, default: depends on input column type. INT (same with BIGINT in vertica) if input column type is long, BOOLEAN if boolean, FLOAT (same with DOUBLE PRECISION in vertica) if double, VARCHAR if string, TIMESTAMP if timestamp)
 
@@ -32,6 +33,7 @@ out:
   schema: sandbox
   table: embulk_test
   copy_mode: DIRECT
+  abort_on_error: true
   column_options:
     id: {type: INT}
     name: {type: VARCHAR(255)}

data/embulk-output-vertica.gemspec

@@ -1,6 +1,6 @@
 Gem::Specification.new do |spec|
   spec.name          = "embulk-output-vertica"
-  spec.version       = "0.1.2"
+  spec.version       = "0.1.3"
   spec.authors       = ["eiji.sekiya", "Naotoshi Seo"]
   spec.email         = ["eiji.sekiya.0326@gmail.com", "sonots@gmail.com"]
   spec.summary       = "Vertica output plugin for Embulk"

data/lib/embulk/output/vertica.rb

@@ -18,6 +18,7 @@ module Embulk
           'schema'         => config.param('schema',         :string,  :default => 'public'),
           'table'          => config.param('table',          :string),
           'copy_mode'      => config.param('copy_mode',      :string,  :default => 'AUTO'),
+          'abort_on_error' => config.param('abort_on_error', :bool,    :default => false),
           'column_options' => config.param('column_options', :hash,    :default => {}),
         }
 
@@ -29,13 +30,17 @@ module Embulk
         unique_name = "%08x%08x" % [now.tv_sec, now.tv_nsec]
         task['temp_table'] = "#{task['table']}_LOAD_TEMP_#{unique_name}"
 
-        sql_schema = self.to_vertica_schema(schema, task['column_options'])
+        sql_schema = self.to_sql_schema(schema, task['column_options'])
+
+        quoted_schema     = ::Jvertica.quote_identifier(task['schema'])
+        quoted_table      = ::Jvertica.quote_identifier(task['table'])
+        quoted_temp_table = ::Jvertica.quote_identifier(task['temp_table'])
 
         connect(task) do |jv|
           # drop table if exists "DEST"
           # 'create table if exists "TEMP" ("COL" json)'
-          jv.query %[drop table if exists #{task['schema']}.#{task['temp_table']}]
-          jv.query %[create table #{task['schema']}.#{task['temp_table']} (#{sql_schema})]
+          jv.query %[drop table if exists #{quoted_schema}.#{quoted_temp_table}]
+          jv.query %[create table #{quoted_schema}.#{quoted_temp_table} (#{sql_schema})]
         end
 
         begin
@@ -43,14 +48,14 @@ module Embulk
           connect(task) do |jv|
             # create table if not exists "DEST" ("COL" json)
             # 'insert into "DEST" ("COL") select "COL" from "TEMP"'
-            jv.query %[create table if not exists #{task['schema']}.#{task['table']} (#{sql_schema})]
-            jv.query %[insert into #{task['schema']}.#{task['table']} select * from #{task['schema']}.#{task['temp_table']}]
+            jv.query %[create table if not exists #{quoted_schema}.#{quoted_table} (#{sql_schema})]
+            jv.query %[insert into #{quoted_schema}.#{quoted_table} select * from #{quoted_schema}.#{quoted_temp_table}]
             jv.commit
           end
         ensure
           connect(task) do |jv|
             # 'drop table if exists TEMP'
-            jv.query %[drop table if exists #{task['schema']}.#{task['temp_table']}]
+            jv.query %[drop table if exists #{quoted_schema}.#{quoted_temp_table}]
           end
         end
         return {}
@@ -75,11 +80,14 @@ module Embulk
         jv
       end
 
-      def self.to_vertica_schema(schema, column_options)
-        schema.names.zip(schema.types).map do |name, type|
-          sql_type = (column_options[name] and column_options[name]['type']) ?
-            column_options[name]['type'] : to_sql_type(type)
-          "#{name} #{sql_type}"
+      # @param [Schema] schema embulk defined column types
+      # @param [Hash]   column_options user defined column types
+      # @return [String] sql schema used to CREATE TABLE
+      def self.to_sql_schema(schema, column_options)
+        schema.names.zip(schema.types).map do |column_name, type|
+          sql_type = (column_options[column_name] and column_options[column_name]['type']) ?
+            column_options[column_name]['type'] : to_sql_type(type)
+          "#{::Jvertica.quote_identifier(column_name)} #{sql_type}"
         end.join(',')
       end
 
@@ -104,9 +112,7 @@ module Embulk
       end
 
       def add(page)
-        sql = "COPY #{@task['schema']}.#{@task['temp_table']} FROM STDIN DELIMITER ',' #{@task['copy_mode']} NO COMMIT"
-        Embulk.logger.debug sql
-        @jv.copy(sql) do |stdin|
+        @jv.copy(copy_sql) do |stdin|
           page.each_with_index do |record, idx|
             stdin << record.map {|v| ::Jvertica.quote(v) }.join(",") << "\n"
           end
@@ -123,6 +129,18 @@ module Embulk
       def commit
         {}
       end
+
+      private
+
+      def copy_sql
+        quoted_schema     = ::Jvertica.quote_identifier(@task['schema'])
+        quoted_temp_table = ::Jvertica.quote_identifier(@task['temp_table'])
+        copy_mode         = @task['copy_mode']
+        abort_on_error    = @task['abort_on_error'] ? ' ABORT ON ERROR' : ''
+        sql = "COPY #{quoted_schema}.#{quoted_temp_table} FROM STDIN DELIMITER ',' #{copy_mode}#{abort_on_error} NO COMMIT"
+        Embulk.logger.debug sql
+        sql
+      end
     end
   end
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: embulk-output-vertica
 version: !ruby/object:Gem::Version
-  version: 0.1.2
+  version: 0.1.3
 platform: ruby
 authors:
 - eiji.sekiya
@@ -9,7 +9,7 @@ authors:
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2015-06-26 00:00:00.000000000 Z
+date: 2015-07-05 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement