embulk-output-vertica 0.1.4 → 0.1.5

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 30466810c9c7e57dec6627ad52ae406ecd1b4dd1
-  data.tar.gz: fbf6ca1e55f6f739cf4afbcafd5e440ecd1e72b1
+  metadata.gz: 81d94a9a3a707573b6eaed08fcd3b64f2890963d
+  data.tar.gz: 56035fc8679597c91ac57eab7ddba7cab8938710
 SHA512:
-  metadata.gz: 859c52e8b2cf51c6eb9b5357d51c575c7df38d9583cbd5ffa88f203e511ecb1973bdf8cf73254e41d808bffa6d0c08381da025d8709788fd3ccadd2acaac5d40
-  data.tar.gz: 2746be9c7f2d6d63f0c2adde70c52a0272acae0c6f2bdbc29cd6a52fd4f268762edcab93a06dde532003b46949f0e4e0eda5d254e445632ed3deca01d84cbf3c
+  metadata.gz: f2a54c45a4407fa37a2e1fb2fbb7326d3091c3ceb8d6dd30d6bea701be34c6cb75743e48bae9ad6ef6247a85f0f31f3fc736fe12be7444c29a4eb181e0f066ae
+  data.tar.gz: 1a192bde5bdfd73e3988500cfb198462a86cd2979b4df688fe189f7f1fc36ab1c158e95f5c67067545524bff1edb0bd85fcf1d43734686430fcd86f618722e1c

data/CHANGELOG.md

@@ -1,3 +1,9 @@
+# 0.1.5 (2015/07/24)
+
+Fixes:
+
+* Use PARSER fjsonparser() instead of DELIMITER ',', otherwise escape is too difficult to do
+
 # 0.1.4 (2015/07/10)
 
 Fixes:

data/README.md

@@ -16,10 +16,25 @@
 - **database**: database name (string, default: vdb)
 - **schema**:   schema name (string, default: public)
 - **table**:    table name (string, required)
-- **copy_mode**: specifies how data is loaded into the database. (`AUTO`, `DIRECT`, or `TRICKLE`. default: AUTO)
+- **mode**:     "insert", or "replace". See bellow. (string, default: insert)
+- **copy_mode**: specifies how data is loaded into the database. (`AUTO`, `DIRECT`, or `TRICKLE`. default: AUTO) See vertica documents for details.
 - **abort_on_error**: Stops the COPY command if a row is rejected and rolls back the command. No data is loaded. (bool, default: false)
 - **column_options**: advanced: a key-value pairs where key is a column name and value is options for the column.
-  - **type**: type of a column when this plugin creates new tables (e.g. VARCHAR(255), INTEGER NOT NULL UNIQUE). This used when this plugin creates intermediate tables (insert and truncate_insert modes), and when it creates nonexistent target table automatically. (string, default: depends on input column type. INT (same with BIGINT in vertica) if input column type is long, BOOLEAN if boolean, FLOAT (same with DOUBLE PRECISION in vertica) if double, VARCHAR if string, TIMESTAMP if timestamp)
+  - **type**: type of a column when this plugin creates new tables such as `VARCHAR(255)`, `INTEGER NOT NULL UNIQUE`. This is used on creating intermediate tables (insert and truncate_insert modes) and on creating a new target table. (string, default: depends on input column type, see below)
+    - `INT` (same with `BIGINT` in vertica) for `long`
+    - `BOOLEAN` for `boolean`
+    - `FLOAT` (same with `DOUBLE PRECISION` in vertica) for `double`
+    - `VARCHAR` for `string`
+    - `TIMESTAMP` for `timestamp`
+
+### Modes
+
+* **insert**:
+  * Behavior: This mode copys rows to some intermediate tables first. If all those tasks run correctly, runs INSERT INTO <target_table> SELECT * FROM <intermediate_table>
+  * Transactional: Yes if `abort_on_error` option is used
+* **replace**:
+  * Behavior: Same with insert mode excepting that it drop the target table first.
+  * Transactional: Yes if `abort_on_error` option is used
 
 ## Example
 

data/embulk-output-vertica.gemspec

@@ -1,6 +1,6 @@
 Gem::Specification.new do |spec|
   spec.name          = "embulk-output-vertica"
-  spec.version       = "0.1.4"
+  spec.version       = "0.1.5"
   spec.authors       = ["eiji.sekiya", "Naotoshi Seo"]
   spec.email         = ["eiji.sekiya.0326@gmail.com", "sonots@gmail.com"]
   spec.summary       = "Vertica output plugin for Embulk"

data/example.csv

@@ -1,2 +1,9 @@
-1,foo
-2,bar
+date,foo,bar,id,name,score
+2015-07-13,,bar,90,l6lTsvxdlcTfcqx2c0lQSd9HejVQg40W25f0wGNQViY,903.4
+2015-07-13,,bar,91,XoALSEQg9ycuGqrEWHOb8vdrLbheZSgFO53Wr3mciXY,394.5
+2015-07-13,,bar,92,0hgDRI_mijs5w7rkiLIe__LEayOOLxL0qVT1IHa5QBw,810.9
+2015-07-13,,bar,93,KjCRAc-AVcS-R13toBUR6pK_7d9Y8Gl4TRdYYMaSirc,477.4
+2015-07-13,,bar,94,fyQVGlT8Bqmu_LiajPlgfbmavoNyAqXaBsBP_e4OnN8,725.3
+2015-07-13,,bar,95,FpBYRPWKu6DmLpx5tsB25URWfj3sNCbcydNAXULaiD8,316.6
+2015-07-13,,bar,96,9ikvnUqp1Rf2yVwLvs5bBvxQP-KyqxGi4gZRSZ8c1d4,369.5
+2015-07-13,,bar,97,RRNYDAzKaq4Trtt96Bxgk3N0fXLIV8hXoK0qQ7uw_Wc,506.5

data/example.yml

@@ -1,10 +1,27 @@
+# in:
+#   type: random
+#   rows: 100
+#   schema:
+#     id: primary_key
+#     name: string
+#     score: integer
 in:
-  type: random
-  rows: 100
-  schema:
-    id: primary_key
-    name: string
-    score: integer
+  type: file
+  path_prefix: example.csv
+  parser:
+    type: csv
+    charset: UTF-8
+    newline: CRLF
+    null_string: 'NULL'
+    skip_header_lines: 1
+    comment_line_marker: '#'
+    columns:
+      - {name: date,  type: timestamp, format: "%Y-%m-%d"}
+      - {name: foo,   type: string}
+      - {name: bar,   type: string}
+      - {name: id,    type: long}
+      - {name: name,  type: string}
+      - {name: score, type: double} 
 out:
   type: vertica 
   host: 127.0.0.1
@@ -15,7 +32,6 @@ out:
   table: embulk_test
   copy_mode: direct
   column_options:
+    date: {type: DATE}
     id: {type: INT}
     name: {type: VARCHAR}
-    score: {type: INT}
-

data/lib/embulk/output/vertica.rb

@@ -17,12 +17,17 @@ module Embulk
           'database'       => config.param('database',       :string,  :default => 'vdb'),
           'schema'         => config.param('schema',         :string,  :default => 'public'),
           'table'          => config.param('table',          :string),
+          'mode'           => config.param('mode',           :string,  :default => 'insert'),
           'copy_mode'      => config.param('copy_mode',      :string,  :default => 'AUTO'),
           'abort_on_error' => config.param('abort_on_error', :bool,    :default => false),
           'column_options' => config.param('column_options', :hash,    :default => {}),
         }
 
-        unless %w[AUTO DIRECT TRICKLE].include?(task['copy_mode'].upcase)
+        unless %w[INSERT REPLACE].include?(task['mode'].upcase!)
+          raise ConfigError, "`mode` must be one of INSERT, REPLACE"
+        end
+
+        unless %w[AUTO DIRECT TRICKLE].include?(task['copy_mode'].upcase!)
           raise ConfigError, "`copy_mode` must be one of AUTO, DIRECT, TRICKLE"
         end
 
@@ -37,30 +42,59 @@ module Embulk
         quoted_temp_table = ::Jvertica.quote_identifier(task['temp_table'])
 
         connect(task) do |jv|
-          # drop table if exists "DEST"
-          # 'create table if exists "TEMP" ("COL" json)'
-          jv.query %[drop table if exists #{quoted_schema}.#{quoted_temp_table}]
-          jv.query %[create table #{quoted_schema}.#{quoted_temp_table} (#{sql_schema})]
+          if task['mode'] == 'REPLACE'
+            query(jv, %[DROP TABLE IF EXISTS #{quoted_schema}.#{quoted_table}])
+          end
+          query(jv, %[DROP TABLE IF EXISTS #{quoted_schema}.#{quoted_temp_table}])
+          query(jv, %[CREATE TABLE #{quoted_schema}.#{quoted_temp_table} (#{sql_schema})])
         end
 
         begin
           yield(task)
           connect(task) do |jv|
-            # create table if not exists "DEST" ("COL" json)
-            # 'insert into "DEST" ("COL") select "COL" from "TEMP"'
-            jv.query %[create table if not exists #{quoted_schema}.#{quoted_table} (#{sql_schema})]
-            jv.query %[insert into #{quoted_schema}.#{quoted_table} select * from #{quoted_schema}.#{quoted_temp_table}]
+            query(jv, %[CREATE TABLE IF NOT EXISTS #{quoted_schema}.#{quoted_table} (#{sql_schema})])
+            query(jv, %[INSERT INTO #{quoted_schema}.#{quoted_table} SELECT * FROM #{quoted_schema}.#{quoted_temp_table}])
             jv.commit
           end
         ensure
           connect(task) do |jv|
-            # 'drop table if exists TEMP'
-            jv.query %[drop table if exists #{quoted_schema}.#{quoted_temp_table}]
+            query(jv, %[DROP TABLE IF EXISTS #{quoted_schema}.#{quoted_temp_table}])
+            Embulk.logger.debug { query(jv, %[SELECT * FROM #{quoted_schema}.#{quoted_table} LIMIT 10]).map {|row| row.to_h }.join("\n") }
           end
         end
         return {}
       end
 
+      def initialize(task, schema, index)
+        super
+        @jv = self.class.connect(task)
+      end
+
+      def close
+        @jv.close
+      end
+
+      def add(page)
+        copy(@jv, copy_sql) do |stdin|
+          page.each do |record|
+            stdin << to_json(record) << "\n"
+          end
+        end
+        @jv.commit
+      end
+
+      def finish
+      end
+
+      def abort
+      end
+
+      def commit
+        {}
+      end
+
+      private
+
       def self.connect(task)
         jv = ::Jvertica.connect({
           host: task['host'],
@@ -85,8 +119,11 @@ module Embulk
       # @return [String] sql schema used to CREATE TABLE
       def self.to_sql_schema(schema, column_options)
         schema.names.zip(schema.types).map do |column_name, type|
-          sql_type = (column_options[column_name] and column_options[column_name]['type']) ?
-            column_options[column_name]['type'] : to_sql_type(type)
+          if column_options[column_name] and column_options[column_name]['type']
+            sql_type = column_options[column_name]['type']
+          else
+            sql_type = to_sql_type(type)
+          end
           "#{::Jvertica.quote_identifier(column_name)} #{sql_type}"
         end.join(',')
       end
@@ -102,45 +139,48 @@ module Embulk
         end
       end
 
-      def initialize(task, schema, index)
-        super
-        @jv = self.class.connect(task)
+      def self.query(conn, sql)
+        Embulk.logger.debug sql
+        conn.query(sql)
       end
 
-      def close
-        @jv.close
+      def query(conn, sql)
+        self.class.query(conn, sql)
       end
 
-      def add(page)
-        @jv.copy(copy_sql) do |stdin|
-          page.each_with_index do |record, idx|
-            stdin << record.map {|v| ::Jvertica.quote(v) }.join(",") << "\n"
-          end
-        end
-        @jv.commit
+      def copy(conn, sql, &block)
+        Embulk.logger.debug sql
+        conn.copy(sql, &block)
       end
 
-      def finish
+      def copy_sql
+        @copy_sql ||= "COPY #{quoted_schema}.#{quoted_temp_table} FROM STDIN PARSER fjsonparser() #{copy_mode}#{abort_on_error} NO COMMIT"
       end
 
-      def abort
+      def to_json(record)
+        Hash[*(schema.names.zip(record).flatten!(1))].to_json
       end
 
-      def commit
-        {}
+      def quoted_schema
+        ::Jvertica.quote_identifier(@task['schema'])
       end
 
-      private
+      def quoted_table
+        ::Jvertica.quote_identifier(@task['table'])
+      end
 
-      def copy_sql
-        quoted_schema     = ::Jvertica.quote_identifier(@task['schema'])
-        quoted_temp_table = ::Jvertica.quote_identifier(@task['temp_table'])
-        copy_mode         = @task['copy_mode']
-        abort_on_error    = @task['abort_on_error'] ? ' ABORT ON ERROR' : ''
-        sql = "COPY #{quoted_schema}.#{quoted_temp_table} FROM STDIN DELIMITER ',' #{copy_mode}#{abort_on_error} NO COMMIT"
-        Embulk.logger.debug sql
-        sql
+      def quoted_temp_table
+        ::Jvertica.quote_identifier(@task['temp_table'])
+      end
+
+      def copy_mode
+        @task['copy_mode']
       end
+
+      def abort_on_error
+        @task['abort_on_error'] ? ' ABORT ON ERROR' : ''
+      end
+
     end
   end
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: embulk-output-vertica
 version: !ruby/object:Gem::Version
-  version: 0.1.4
+  version: 0.1.5
 platform: ruby
 authors:
 - eiji.sekiya
@@ -9,7 +9,7 @@ authors:
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2015-07-10 00:00:00.000000000 Z
+date: 2015-07-23 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement