embulk-output-vertica 0.1.1 → 0.1.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: fe435fb50a129cc904e31d2e87d9119804acf854
-  data.tar.gz: 23458272c05c2b660033b4ec893f18913080fb6e
+  metadata.gz: 452d0c45a1e7f46e91cce223618aeffc0141c902
+  data.tar.gz: 16e2bc4294f270f3df7c92243ce28b2ac56f2801
 SHA512:
-  metadata.gz: 92258f1615b0bb2157a67840e782d141c37752ae2dbb6bf8ed272b958375727a740c5ca902162b66c691df42e4356797499430ce02fb3bbe4d1b5ee7e95e56b2
-  data.tar.gz: 170a6dd3a87987a25d9977fad7674330be58a20bcf89002a0060a16db96c587ce7c978cdb81c3063226c35778bf12ab7fc14834df52d60d926bfaa97175bc529
+  metadata.gz: 2538b4999f2ea7c7c8ab64c5f1aa5fa90f8dbd7bfc258643c59b00713fdd675bc29654b8c37a07d22cc29e7f9f08db03536de6e4236b14426963c0769f834607
+  data.tar.gz: 0b9a30e215cf0836160e252cf2b77d4e4dc5223b8a5ed1c492b2fc0e16d02bcff26696a61b5c8e23e379a822f416828ea3c29d8700dfe5c62b2f6f4f13af5e0a

data/CHANGELOG.md

@@ -0,0 +1,17 @@
+# 0.1.2 (2015/06/26)
+
+Enhancements:
+
+* Add `column_options` option
+
+# 0.1.1 (2015/06/25)
+
+Enhancements:
+
+* Add `copy_mode` option
+* Use `jvertica` gem instead of `vertica` gem
+
+# 0.1.0
+
+first version
+

data/Gemfile

@@ -1,4 +1,7 @@
 source 'https://rubygems.org'
 
-# Specify your gem's dependencies in embulk-plugin-vertica.gemspec
 gemspec
+gem 'embulk'
+gem 'embulk-input-random'
+gem 'pry'
+gem 'pry-nav'

data/README.md

@@ -3,9 +3,9 @@
 ## Overview
 
 * **Plugin type**: output
-* **Load all or nothing**: no
 * **Resume supported**: no
-* **Cleanup supported**: yes
+* **Cleanup supported**: no
+* **Dynamic table creating**: yes
 
 ## Configuration
 
@@ -17,6 +17,8 @@
 - **schema**:   schema name (string, default: public)
 - **table**:    table name (string, required)
 - **copy_mode**: specifies how data is loaded into the database. (`AUTO`, `DIRECT`, or `TRICKLE`. default: AUTO)
+- **column_options**: advanced: a key-value pairs where key is a column name and value is options for the column.
+  - **type**: type of a column when this plugin creates new tables (e.g. VARCHAR(255), INTEGER NOT NULL UNIQUE). This used when this plugin creates intermediate tables (insert and truncate_insert modes), and when it creates nonexistent target table automatically. (string, default: depends on input column type. INT (same with BIGINT in vertica) if input column type is long, BOOLEAN if boolean, FLOAT (same with DOUBLE PRECISION in vertica) if double, VARCHAR if string, TIMESTAMP if timestamp)
 
 ## Example
 
@@ -30,6 +32,9 @@ out:
   schema: sandbox
   table: embulk_test
   copy_mode: DIRECT
+  column_options:
+    id: {type: INT}
+    name: {type: VARCHAR(255)}
 ```
 
 
@@ -38,9 +43,8 @@ out:
 Run example:
 
 ```
-$ embulk gem install embulk-input-random
-$ embulk gem install jvertica
-$ embulk run -I lib -l debug example.yml
+$ bundle install
+$ bundle exec embulk run -l debug example.yml
 ```
 
 Release gem:
@@ -48,3 +52,7 @@ Release gem:
 ```
 $ bundle exec rake release
 ```
+
+## ChangeLog
+
+[CHANGELOG.md](CHANGELOG.md)

data/embulk-output-vertica.gemspec

@@ -1,6 +1,6 @@
 Gem::Specification.new do |spec|
   spec.name          = "embulk-output-vertica"
-  spec.version       = "0.1.1"
+  spec.version       = "0.1.2"
   spec.authors       = ["eiji.sekiya", "Naotoshi Seo"]
   spec.email         = ["eiji.sekiya.0326@gmail.com", "sonots@gmail.com"]
   spec.summary       = "Vertica output plugin for Embulk"

data/example.yml

@@ -13,3 +13,9 @@ out:
   database: vdb
   schema: sandbox
   table: embulk_test
+  copy_mode: direct
+  column_options:
+    id: {type: INT}
+    name: {type: VARCHAR}
+    score: {type: INT}
+

data/lib/embulk/output/vertica.rb

@@ -10,14 +10,15 @@ module Embulk
 
       def self.transaction(config, schema, processor_count, &control)
         task = {
-          'host'      => config.param('host',      :string,  :default => 'localhost'),
-          'port'      => config.param('port',      :integer, :default => 5433),
-          'username'  => config.param('username',  :string),
-          'password'  => config.param('password',  :string,  :default => ''),
-          'database'  => config.param('database',  :string,  :default => 'vdb'),
-          'schema'    => config.param('schema',    :string,  :default => 'public'),
-          'table'     => config.param('table',     :string),
-          'copy_mode' => config.param('copy_mode', :string, :default => 'AUTO'),
+          'host'           => config.param('host',           :string,  :default => 'localhost'),
+          'port'           => config.param('port',           :integer, :default => 5433),
+          'username'       => config.param('username',       :string),
+          'password'       => config.param('password',       :string,  :default => ''),
+          'database'       => config.param('database',       :string,  :default => 'vdb'),
+          'schema'         => config.param('schema',         :string,  :default => 'public'),
+          'table'          => config.param('table',          :string),
+          'copy_mode'      => config.param('copy_mode',      :string,  :default => 'AUTO'),
+          'column_options' => config.param('column_options', :hash,    :default => {}),
         }
 
         unless %w[AUTO DIRECT TRICKLE].include?(task['copy_mode'].upcase)
@@ -28,7 +29,7 @@ module Embulk
         unique_name = "%08x%08x" % [now.tv_sec, now.tv_nsec]
         task['temp_table'] = "#{task['table']}_LOAD_TEMP_#{unique_name}"
 
-        sql_schema = self.to_vertica_schema schema
+        sql_schema = self.to_vertica_schema(schema, task['column_options'])
 
         connect(task) do |jv|
           # drop table if exists "DEST"
@@ -74,18 +75,20 @@ module Embulk
         jv
       end
 
-      def self.to_vertica_schema(schema)
-        schema.names.zip(schema.types)
-          .map { |name, type| "#{name} #{to_sql_type(type)}" }
-          .join(',')
+      def self.to_vertica_schema(schema, column_options)
+        schema.names.zip(schema.types).map do |name, type|
+          sql_type = (column_options[name] and column_options[name]['type']) ?
+            column_options[name]['type'] : to_sql_type(type)
+          "#{name} #{sql_type}"
+        end.join(',')
       end
 
       def self.to_sql_type(type)
         case type
         when :boolean then 'BOOLEAN'
-        when :long then 'INT'
-        when :double then 'FLOAT'
-        when :string then 'VARCHAR'
+        when :long then 'INT' # BIGINT is a synonym for INT in vertica
+        when :double then 'FLOAT' # DOUBLE PRECISION is a synonym for FLOAT in vertica
+        when :string then 'VARCHAR' # LONG VARCHAR is not recommended
         when :timestamp then 'TIMESTAMP'
         else raise NotSupportedType, "embulk-output-vertica cannot take column type #{type}"
         end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: embulk-output-vertica
 version: !ruby/object:Gem::Version
-  version: 0.1.1
+  version: 0.1.2
 platform: ruby
 authors:
 - eiji.sekiya
@@ -9,7 +9,7 @@ authors:
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2015-06-25 00:00:00.000000000 Z
+date: 2015-06-26 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
@@ -62,6 +62,7 @@ extensions: []
 extra_rdoc_files: []
 files:
 - .gitignore
+- CHANGELOG.md
 - Gemfile
 - LICENSE.txt
 - README.md