embulk-output-vertica 0.5.3 → 0.5.4

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 64254a208edbd6f0c254ed8cabdbee4fac53dbdf
-  data.tar.gz: 41b73b19cdbc334be205a45a57c2dded37ff3a53
+  metadata.gz: fce85ac19d1f8d081c0c35217be4a624a281e242
+  data.tar.gz: 35c8a1b4e33cf62653e84e69fb4e611db79f83f1
 SHA512:
-  metadata.gz: 3a97e62ed025bfb4b81423db3027a38709946a6900d68976f96fb20772c513852b80315722bbea0e0014c8425312e5b3340170319540e42a3945de511e653498
-  data.tar.gz: ba2307d15902dafcb306c478f6ec993a411a4eef5a975b14125ae8ca9a73f1c0abe4064bc662124194017ac104172fb960101f3f5182937aefcb550601bd4f48
+  metadata.gz: 5d2126b126a0ffb47be5ed8218823096395e23cc9afc3fc67bcc1b0cd5e2ed3a90c17daeedddb4ebf1020972f005dec9288c363b8e2cae88b576845774c8c3e4
+  data.tar.gz: c912765ff8a69422e607fe0b6d52ec16ee09c02434277c1be45f5d511470ce89f62a8b28fb80d03b804b2e22995fc91916fc45fddafbe80f97412faab4cffbaf

data/.gitignore

@@ -14,4 +14,4 @@
 mkmf.log
 vendor
 .ruby-version
-example2.yml
+example2/

data/CHANGELOG.md

@@ -1,3 +1,9 @@
+# 0.5.4 (2016/01/21)
+
+Enhancements:
+
+* Log rejected record
+
 # 0.5.3 (2016/01/09)
 
 Enhancements:

data/README.md

@@ -20,6 +20,7 @@
 - **copy_mode**: specifies how data is loaded into the database. See vertica documents for details. (`AUTO`, `DIRECT`, or `TRICKLE`. default: `AUTO`)
 - **pool**: number of output threads, this number controls number of concurrency to issue COPY statements (integer, default: processor_count, that is, number of threads in input plugin)
 - **abort_on_error**: stops the COPY command if a row is rejected and rolls back the command. No data is loaded. (bool, default: false)
+- **compress**: compress input (`GZIP`, or `UNCOMPRESSED`, default: `UNCOMPRESSED`)
 - **reject_on_materialized_type_error**: uses `reject_on_materialized_type_error` option for fjsonparser(). This rejects rows if any of column types and value types do not fit, ex) double value into INT column fails. See vertica documents for details. (bool, default: false)
 - **default_timezone**: the default timezone for column_options (string, default is "UTC")
 - **column_options**: advanced: a key-value pairs where key is a column name and value is options for the column.
@@ -72,7 +73,7 @@ Run example:
 
 ```
 $ embulk bundle install --path vendor/bundle
-$ embulk -J-O -R--dev run -b . run -l debug example.yml
+$ embulk -J-O -R--dev run -b . -l debug example.yml
 ```
 
 Release gem:

data/embulk-output-vertica.gemspec

@@ -1,6 +1,6 @@
 Gem::Specification.new do |spec|
   spec.name          = "embulk-output-vertica"
-  spec.version       = "0.5.3"
+  spec.version       = "0.5.4"
   spec.authors       = ["eiji.sekiya", "Naotoshi Seo"]
   spec.email         = ["eiji.sekiya.0326@gmail.com", "sonots@gmail.com"]
   spec.summary       = "Vertica output plugin for Embulk"

data/example/abort.yml

@@ -0,0 +1,33 @@
+in:
+  type: file
+  path_prefix: example/example.csv
+  parser:
+    type: csv
+    charset: UTF-8
+    newline: CRLF
+    null_string: 'NULL'
+    skip_header_lines: 1
+    comment_line_marker: '#'
+    columns:
+      - {name: timestamp_date, type: timestamp, format: "%Y-%m-%d", timezone: "+09:00"}
+      - {name: string_date,    type: string}
+      - {name: foo,   type: string}
+      - {name: bar,   type: string}
+      - {name: id,    type: long}
+      - {name: name,  type: string}
+      - {name: score, type: double} 
+out:
+  type: vertica 
+  host: 127.0.0.1
+  user: dbadmin
+  password: xxxxxxx
+  database: vdb
+  schema: sandbox
+  table: embulk_test
+  mode: replace
+  copy_mode: DIRECT
+  abort_on_error: true
+  reject_on_materialized_type_error: true
+  default_timezone: 'Asia/Tokyo'
+  column_options:
+    bar: {type: INT}

data/{example.yml → example/example.yml}

@@ -7,7 +7,7 @@
 #     score: integer
 in:
   type: file
-  path_prefix: example.csv
+  path_prefix: example/example.csv
   parser:
     type: csv
     charset: UTF-8

data/lib/embulk/output/vertica/output_thread.rb

@@ -1,3 +1,5 @@
+require 'zlib'
+
 module Embulk
   module Output
     class Vertica < OutputPlugin
@@ -37,6 +39,13 @@ module Embulk
           @num_rejected_rows = 0
           @outer_thread = Thread.current
           @thread_active = false
+
+          case task['compress']
+          when 'GZIP'
+            @write_proc = self.method(:write_gzip)
+          else
+            @write_proc = self.method(:write_uncompressed)
+          end
         end
 
         def enqueue(page)
@@ -49,28 +58,49 @@ module Embulk
           end
         end
 
+        def write_gzip(io, page, &block)
+          buf = Zlib::Deflate.new
+          write_buf(buf, page, &block)
+          io << buf.finish
+        end
+
+        def write_uncompressed(io, page, &block)
+          buf = ''
+          write_buf(buf, page, &block)
+          io << buf
+        end
+
+        def write_buf(buf, page, &block)
+          page.each do |record|
+            yield(record) if block_given?
+            Embulk.logger.trace { "embulk-output-vertica: record #{record}" }
+            json = to_json(record)
+            Embulk.logger.trace { "embulk-output-vertica: to_json #{json}" }
+            buf << json << "\n"
+            @num_input_rows += 1
+          end
+        end
+
         def run
           Embulk.logger.debug { "embulk-output-vertica: thread started" }
           Vertica.connect(@task) do |jv|
             json = nil # for log
             begin
+              last_record = nil
               num_output_rows, rejects = copy(jv, copy_sql) do |stdin|
                 while page = @queue.pop
                   if page == 'finish'
-                    Embulk.logger.debug { "embulk-output-vertica: thread finished" }
+                    Embulk.logger.trace { "embulk-output-vertica: popped finish" }
                     break
                   end
                   Embulk.logger.trace { "embulk-output-vertica: dequeued" }
 
-                  page.each do |record|
-                    Embulk.logger.trace { "embulk-output-vertica: record #{record}" }
-                    json = to_json(record)
-                    Embulk.logger.trace { "embulk-output-vertica: to_json #{json}" }
-                    stdin << json << "\n"
-                    @num_input_rows += 1
+                  @write_proc.call(stdin, page) do |record|
+                    last_record = record
                   end
                 end
               end
+              Embulk.logger.debug { "embulk-output-vertica: thread finished" }
               num_rejected_rows = rejects.size
               @num_output_rows += num_output_rows
               @num_rejected_rows += num_rejected_rows
@@ -83,6 +113,7 @@ module Embulk
               else
                 Embulk.logger.warn "embulk-output-vertica: ROLLBACK!"
               end
+              Embulk.logger.info { "embulk-output-vertica: last_record: #{to_json(last_record)}" }
               jv.rollback
               raise e # die transaction
             rescue => e
@@ -108,6 +139,7 @@ module Embulk
           @thread_active = false
           if @thread.alive?
             @queue.push('finish')
+            Embulk.logger.trace { "embulk-output-vertica: pushed finish" }
             Thread.pass
             @thread.join
           else
@@ -129,7 +161,7 @@ module Embulk
         end
 
         def copy_sql
-          @copy_sql ||= "COPY #{quoted_schema}.#{quoted_temp_table} FROM STDIN#{fjsonparser}#{copy_mode}#{abort_on_error} NO COMMIT"
+          @copy_sql ||= "COPY #{quoted_schema}.#{quoted_temp_table} FROM STDIN#{compress}#{fjsonparser}#{copy_mode}#{abort_on_error} NO COMMIT"
         end
 
         def to_json(record)
@@ -150,6 +182,10 @@ module Embulk
           ::Jvertica.quote_identifier(@task['temp_table'])
         end
 
+        def compress
+          " #{@task['compress']}"
+        end
+
         def copy_mode
           " #{@task['copy_mode']}"
         end

data/lib/embulk/output/vertica.rb

@@ -27,6 +27,7 @@ module Embulk
           'mode'             => config.param('mode',             :string,  :default => 'insert'),
           'copy_mode'        => config.param('copy_mode',        :string,  :default => 'AUTO'),
           'abort_on_error'   => config.param('abort_on_error',   :bool,    :default => false),
+          'compress'         => config.param('compress',         :string,  :default => 'UNCOMPRESSED'),
           'default_timezone' => config.param('default_timezone', :string, :default => 'UTC'),
           'column_options'   => config.param('column_options',   :hash,    :default => {}),
           'reject_on_materialized_type_error' => config.param('reject_on_materialized_type_error', :bool, :default => false),
@@ -52,6 +53,12 @@ module Embulk
           raise ConfigError.new "`copy_mode` must be one of AUTO, DIRECT, TRICKLE"
         end
 
+        # ToDo: Support BZIP, LZO
+        task['compress'] = task['compress'].upcase
+        unless %w[GZIP UNCOMPRESSED].include?(task['compress'])
+          raise ConfigError.new "`compress` must be one of GZIP, UNCOMPRESSED"
+        end
+
         now = Time.now
         unique_name = "%08x%08x" % [now.tv_sec, now.tv_nsec]
         task['temp_table'] = "#{task['table']}_LOAD_TEMP_#{unique_name}"

metadata

@@ -1,72 +1,72 @@
 --- !ruby/object:Gem::Specification
 name: embulk-output-vertica
 version: !ruby/object:Gem::Version
-  version: 0.5.3
+  version: 0.5.4
 platform: ruby
 authors:
 - eiji.sekiya
 - Naotoshi Seo
-autorequire:
+autorequire: 
 bindir: bin
 cert_chain: []
-date: 2016-01-09 00:00:00.000000000 Z
+date: 2016-01-21 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: jvertica
-  version_requirements: !ruby/object:Gem::Requirement
+  requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '0.2'
-  requirement: !ruby/object:Gem::Requirement
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '0.2'
-  prerelease: false
-  type: :runtime
 - !ruby/object:Gem::Dependency
   name: tzinfo
-  version_requirements: !ruby/object:Gem::Requirement
+  requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
-  requirement: !ruby/object:Gem::Requirement
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
-  prerelease: false
-  type: :runtime
 - !ruby/object:Gem::Dependency
   name: bundler
-  version_requirements: !ruby/object:Gem::Requirement
+  requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '1.7'
-  requirement: !ruby/object:Gem::Requirement
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '1.7'
-  prerelease: false
-  type: :development
 - !ruby/object:Gem::Dependency
   name: rake
-  version_requirements: !ruby/object:Gem::Requirement
+  requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '10.0'
-  requirement: !ruby/object:Gem::Requirement
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '10.0'
-  prerelease: false
-  type: :development
 description: Dump records to vertica
 email:
 - eiji.sekiya.0326@gmail.com
@@ -82,8 +82,9 @@ files:
 - README.md
 - Rakefile
 - embulk-output-vertica.gemspec
-- example.csv
-- example.yml
+- example/abort.yml
+- example/example.csv
+- example/example.yml
 - lib/embulk/output/vertica.rb
 - lib/embulk/output/vertica/output_thread.rb
 - lib/embulk/output/vertica/value_converter_factory.rb
@@ -91,7 +92,7 @@ homepage: https://github.com/eratostennis/embulk-output-vertica
 licenses:
 - MIT
 metadata: {}
-post_install_message:
+post_install_message: 
 rdoc_options: []
 require_paths:
 - lib
@@ -106,9 +107,9 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubyforge_project:
-rubygems_version: 2.4.8
-signing_key:
+rubyforge_project: 
+rubygems_version: 2.5.1
+signing_key: 
 specification_version: 4
 summary: Vertica output plugin for Embulk
 test_files: []