embulk-output-vertica 0.1.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 7b5979942b9ed3ae9f640b1150f9a05679e2f4ce
+  data.tar.gz: 0af6db714dc9fa90258c6f3ff5587847037e43ce
+SHA512:
+  metadata.gz: eb62e8c19e59aa31f71659db45a5c4a6c840540b6b074483e79f3ff9f844ace4d283a351877c4cbfaadce82fda3dc07425704e2101fdb16ff9e6c3988bfa5d25
+  data.tar.gz: e559a967086ae51ccd02f0dd95c8acb4f362c410edae12ebd972f6e17c5bd22a7edc5ba278e54b48e073643f331df0d04b06d05bb4b57f1f4bb1bfef96027cef

data/.gitignore

@@ -0,0 +1,17 @@
+/.bundle/
+/.yardoc
+/Gemfile.lock
+/_yardoc/
+/coverage/
+/doc/
+/pkg/
+/spec/reports/
+/tmp/
+*.bundle
+*.so
+*.o
+*.a
+mkmf.log
+vendor
+.ruby-version
+example2.yml

data/Gemfile

@@ -0,0 +1,4 @@
+source 'https://rubygems.org'
+
+# Specify your gem's dependencies in embulk-plugin-vertica.gemspec
+gemspec

data/LICENSE.txt

@@ -0,0 +1,22 @@
+Copyright (c) 2015 eiji.sekiya
+
+MIT License
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.md

@@ -0,0 +1,48 @@
+# Vertica output plugin for Embulk
+
+## Overview
+
+* **Plugin type**: output
+* **Load all or nothing**: no
+* **Resume supported**: no
+* **Cleanup supported**: yes
+
+## Configuration
+
+- **host**: hostname (string, default: localhost)
+- **port**: port number (integer, default: 5433)
+- **username**: username (string, required)
+- **password**: password (string, default: '')
+- **database**: database name (string, default: vdb)
+- **schema**:   schema name (string, default: public)
+- **table**:    table name (string, required)
+
+## Example
+
+```yaml
+out:
+  type: vertica 
+  host: 127.0.0.1
+  username: dbadmin
+  password: xxxxxxx
+  database: vdb
+  schema: sandbox
+  table: embulk_test
+```
+
+
+## Development
+
+Run example:
+
+```
+$ embulk gem install embulk-input-random
+$ embulk gem install jvertica
+$ embulk run -I lib example.yml
+```
+
+Release gem:
+
+```
+$ bundle exec rake release
+```

data/Rakefile

@@ -0,0 +1,5 @@
+require "bundler/gem_tasks"
+Bundler::GemHelper.install_tasks
+
+task :default => [:build]
+

data/embulk-output-vertica.gemspec

@@ -0,0 +1,19 @@
+Gem::Specification.new do |spec|
+  spec.name          = "embulk-output-vertica"
+  spec.version       = "0.1.0"
+  spec.authors       = ["eiji.sekiya", "Naotoshi Seo"]
+  spec.email         = ["eiji.sekiya.0326@gmail.com", "sonots@gmail.com"]
+  spec.summary       = "Vertica output plugin for Embulk"
+  spec.description   = "Dump records to vertica"
+  spec.homepage      = "https://github.com/eratostennis/embulk-plugin-vertica"
+  spec.license       = "MIT"
+
+  spec.files         = `git ls-files -z`.split("\x0")
+  spec.executables   = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
+  spec.test_files    = spec.files.grep(%r{^(test|spec|features)/})
+  spec.require_paths = ["lib"]
+
+  spec.add_dependency "jvertica", "~> 0.2"
+  spec.add_development_dependency "bundler", "~> 1.7"
+  spec.add_development_dependency "rake", "~> 10.0"
+end

data/example.csv

@@ -0,0 +1,2 @@
+1,foo
+2,bar

data/example.yml

@@ -0,0 +1,15 @@
+in:
+  type: random
+  rows: 100
+  schema:
+    id: primary_key
+    name: string
+    score: integer
+out:
+  type: vertica 
+  host: 127.0.0.1
+  username: dbadmin
+  password: xxxxxxx
+  database: vdb
+  schema: sandbox
+  table: embulk_test

data/lib/embulk/output/vertica.rb

@@ -0,0 +1,119 @@
+require 'jvertica'
+
+module Embulk
+  module Output
+    class Vertica < OutputPlugin
+      Plugin.register_output('vertica', self)
+
+      class Error < StandardError; end
+      class NotSupportedType < Error; end
+
+      def self.transaction(config, schema, processor_count, &control)
+        task = {
+          'host'     => config.param('host',     :string,  :default => 'localhost'),
+          'port'     => config.param('port',     :integer, :default => 5433),
+          'username' => config.param('username', :string),
+          'password' => config.param('password', :string,  :default => ''),
+          'database' => config.param('database', :string,  :default => 'vdb'),
+          'schema'   => config.param('schema',   :string,  :default => 'public'),
+          'table'    => config.param('table',    :string),
+        }
+
+        now = Time.now
+        unique_name = "%08x%08x" % [now.tv_sec, now.tv_nsec]
+        task['temp_table'] = "#{task['table']}_LOAD_TEMP_#{unique_name}"
+
+        sql_schema = self.to_vertica_schema schema
+
+        connect(task) do |jv|
+          # drop table if exists "DEST"
+          # 'create table if exists "TEMP" ("COL" json)'
+          jv.query %[drop table if exists #{task['schema']}.#{task['temp_table']}]
+          jv.query %[create table #{task['schema']}.#{task['temp_table']} (#{sql_schema})]
+        end
+
+        begin
+          yield(task)
+          connect(task) do |jv|
+            # create table if not exists "DEST" ("COL" json)
+            # 'insert into "DEST" ("COL") select "COL" from "TEMP"'
+            jv.query %[create table if not exists #{task['schema']}.#{task['table']} (#{sql_schema})]
+            jv.query %[insert into #{task['schema']}.#{task['table']} select * from #{task['schema']}.#{task['temp_table']}]
+            jv.commit
+          end
+        ensure
+          connect(task) do |jv|
+            # 'drop table if exists TEMP'
+            jv.query %[drop table if exists #{task['schema']}.#{task['temp_table']}]
+          end
+        end
+        return {}
+      end
+
+      def self.connect(task)
+        jv = ::Jvertica.connect({
+          host: task['host'],
+          port: task['port'],
+          user: task['username'],
+          password: task['password'],
+          database: task['database'],
+        })
+
+        if block_given?
+          begin
+            yield jv
+          ensure
+            jv.close
+          end
+        end
+        jv
+      end
+
+      def self.to_vertica_schema(schema)
+        schema.names.zip(schema.types)
+          .map { |name, type| "#{name} #{to_sql_type(type)}" }
+          .join(',')
+      end
+
+      def self.to_sql_type(type)
+        case type
+        when :boolean then 'BOOLEAN'
+        when :long then 'INT'
+        when :double then 'FLOAT'
+        when :string then 'VARCHAR'
+        when :timestamp then 'TIMESTAMP'
+        else raise NotSupportedType, "embulk-output-vertica cannot take column type #{type}"
+        end
+      end
+
+      def initialize(task, schema, index)
+        super
+        @jv = self.class.connect(task)
+      end
+
+      def close
+        @jv.close
+      end
+
+      def add(page)
+        sql = "COPY #{@task['schema']}.#{@task['temp_table']} FROM STDIN DELIMITER ',' NO COMMIT"
+        @jv.copy(sql) do |stdin|
+          page.each_with_index do |record, idx|
+            stdin << record.map {|v| ::Jvertica.quote(v) }.join(",") << "\n"
+          end
+        end
+        @jv.commit
+      end
+
+      def finish
+      end
+
+      def abort
+      end
+
+      def commit
+        {}
+      end
+    end
+  end
+end

metadata

@@ -0,0 +1,97 @@
+--- !ruby/object:Gem::Specification
+name: embulk-output-vertica
+version: !ruby/object:Gem::Version
+  version: 0.1.0
+platform: ruby
+authors:
+- eiji.sekiya
+- Naotoshi Seo
+autorequire:
+bindir: bin
+cert_chain: []
+date: 2015-06-25 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '0.2'
+  name: jvertica
+  prerelease: false
+  type: :runtime
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '0.2'
+- !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '1.7'
+  name: bundler
+  prerelease: false
+  type: :development
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '1.7'
+- !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '10.0'
+  name: rake
+  prerelease: false
+  type: :development
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '10.0'
+description: Dump records to vertica
+email:
+- eiji.sekiya.0326@gmail.com
+- sonots@gmail.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- .gitignore
+- Gemfile
+- LICENSE.txt
+- README.md
+- Rakefile
+- embulk-output-vertica.gemspec
+- example.csv
+- example.yml
+- lib/embulk/output/vertica.rb
+homepage: https://github.com/eratostennis/embulk-plugin-vertica
+licenses:
+- MIT
+metadata: {}
+post_install_message:
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project:
+rubygems_version: 2.4.5
+signing_key:
+specification_version: 4
+summary: Vertica output plugin for Embulk
+test_files: []