embulk-plugin-vertica 0.0.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
checksums.yaml ADDED
@@ -0,0 +1,7 @@
1
+ ---
2
+ SHA1:
3
+ metadata.gz: 931ea8d51b9e89dfba76d026dd96f3831f5245d5
4
+ data.tar.gz: bd50c0038dcb84e8bfb1d9766c3a4d0c2e618ecf
5
+ SHA512:
6
+ metadata.gz: a68c80d915b9aa92d358684cd15bccd9c50338e96dc76e78b705dae460b6c65c0e47503d85706b73fc41589b10b32a9fabff0953ffb8ed5a583ce4cf94b216e5
7
+ data.tar.gz: d96fefcc7a28b17c2c9519938a4aa754f7eb69cbae91f8e1a4b6ee106400ff8158ac1f802857063062d6477298133b64a2c24ac9a329fd4494671c3a71abd825
data/.gitignore ADDED
@@ -0,0 +1,15 @@
1
+ /.bundle/
2
+ /.yardoc
3
+ /Gemfile.lock
4
+ /_yardoc/
5
+ /coverage/
6
+ /doc/
7
+ /pkg/
8
+ /spec/reports/
9
+ /tmp/
10
+ *.bundle
11
+ *.so
12
+ *.o
13
+ *.a
14
+ mkmf.log
15
+ vendor
data/Gemfile ADDED
@@ -0,0 +1,4 @@
1
+ source 'https://rubygems.org'
2
+
3
+ # Specify your gem's dependencies in embulk-plugin-vertica.gemspec
4
+ gemspec
data/LICENSE.txt ADDED
@@ -0,0 +1,22 @@
1
+ Copyright (c) 2015 eiji.sekiya
2
+
3
+ MIT License
4
+
5
+ Permission is hereby granted, free of charge, to any person obtaining
6
+ a copy of this software and associated documentation files (the
7
+ "Software"), to deal in the Software without restriction, including
8
+ without limitation the rights to use, copy, modify, merge, publish,
9
+ distribute, sublicense, and/or sell copies of the Software, and to
10
+ permit persons to whom the Software is furnished to do so, subject to
11
+ the following conditions:
12
+
13
+ The above copyright notice and this permission notice shall be
14
+ included in all copies or substantial portions of the Software.
15
+
16
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
17
+ EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
18
+ MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
19
+ NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
20
+ LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
21
+ OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
22
+ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
data/README.md ADDED
@@ -0,0 +1,32 @@
1
+ # Embulk input/output plugin for Vertica
2
+
3
+ This Embulk output plugin writes record to a json column of a table.
4
+
5
+ ## Installation
6
+
7
+ Add this line to your application's Gemfile:
8
+
9
+ ```ruby
10
+ gem 'embulk'
11
+ gem 'embulk-plugin-vertica'
12
+ ```
13
+
14
+ And then execute:
15
+
16
+ $ bundle
17
+
18
+ Or install it yourself as:
19
+
20
+ $ gem install embulk-plugin-vertica
21
+
22
+ ## Usage
23
+
24
+ TODO: Write usage instructions here
25
+
26
+ ## Contributing
27
+
28
+ 1. Fork it ( https://github.com/[my-github-username]/embulk-plugin-vertica/fork )
29
+ 2. Create your feature branch (`git checkout -b my-new-feature`)
30
+ 3. Commit your changes (`git commit -am 'Add some feature'`)
31
+ 4. Push to the branch (`git push origin my-new-feature`)
32
+ 5. Create a new Pull Request
data/Rakefile ADDED
@@ -0,0 +1,5 @@
1
+ require "bundler/gem_tasks"
2
+ Bundler::GemHelper.install_tasks
3
+
4
+ task :default => [:build]
5
+
data/config.yml ADDED
@@ -0,0 +1,12 @@
1
+ exec: {}
2
+ in:
3
+ type: file
4
+ paths: [test.csv]
5
+ out:
6
+ type: vertica
7
+ host: 127.0.0.1
8
+ username: dbadmin
9
+ password: xxxxxxx
10
+ database: vdb
11
+ schema: sandbox
12
+ table: embulk_test
@@ -0,0 +1,25 @@
1
+ # coding: utf-8
2
+ lib = File.expand_path('../lib', __FILE__)
3
+ $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
4
+ require 'embulk/plugin/vertica/version'
5
+
6
+ Gem::Specification.new do |spec|
7
+ spec.name = "embulk-plugin-vertica"
8
+ spec.version = Embulk::Plugin::Vertica::VERSION
9
+ spec.authors = ["eiji.sekiya"]
10
+ spec.email = ["eiji.sekiya.0326@gmail.com"]
11
+ spec.summary = %q{Embulk plugin for vertica.}
12
+ spec.description = spec.summary
13
+ spec.homepage = "https://github.com/eratostennis/embulk-plugin-vertica"
14
+ spec.license = "MIT"
15
+
16
+ spec.files = `git ls-files -z`.split("\x0")
17
+ spec.executables = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
18
+ spec.test_files = spec.files.grep(%r{^(test|spec|features)/})
19
+ spec.require_paths = ["lib"]
20
+
21
+ spec.add_development_dependency "bundler", "~> 1.7"
22
+ spec.add_development_dependency "rake", "~> 10.0"
23
+
24
+ spec.add_dependency "vertica", "~> 0.11.2"
25
+ end
@@ -0,0 +1,120 @@
1
+ module Embulk
2
+ module Plugin
3
+ module Vertica
4
+
5
+ class NotSupportedSchema < Exception; end
6
+ class OutputVertica < OutputPlugin
7
+
8
+ require 'vertica'
9
+ Plugin.register_output('vertica', self)
10
+
11
+ def self.transaction(config, schema, processor_count, &control)
12
+ task = {
13
+ 'host' => config.param('host', :string, :default => 'localhost'),
14
+ 'port' => config.param('port', :integer, :default => 5433),
15
+ 'username' => config.param('username', :string),
16
+ 'password' => config.param('password', :string, :default => ''),
17
+ 'database' => config.param('database', :string, :default => 'vdb'),
18
+ 'schema' => config.param('schema', :string, :default => 'public'),
19
+ 'table' => config.param('table', :string),
20
+ }
21
+
22
+ now = Time.now
23
+ unique_name = "%08x%08x" % [now.tv_sec, now.tv_nsec]
24
+ task['temp_table'] = "#{task['table']}_LOAD_TEMP_#{unique_name}"
25
+
26
+ sql_schema = self.to_vertica_schema schema
27
+
28
+ connect(task) do |jv|
29
+ # drop table if exists "DEST"
30
+ # 'create table if exists "TEMP" ("COL" json)'
31
+ jv.query %[drop table if exists #{task['schema']}.#{task['temp_table']}]
32
+ jv.query %[create table #{task['schema']}.#{task['temp_table']} (#{sql_schema})]
33
+ end
34
+
35
+ begin
36
+ yield(task)
37
+ connect(task) do |jv|
38
+ # create table if not exists "DEST" ("COL" json)
39
+ # 'insert into "DEST" ("COL") select "COL" from "TEMP"'
40
+ jv.query %[create table if not exists #{task['schema']}.#{task['table']} (#{sql_schema})]
41
+ jv.query %[insert into #{task['schema']}.#{task['table']} select * from #{task['schema']}.#{task['temp_table']}]
42
+ jv.query %[COMMIT]
43
+ end
44
+ ensure
45
+ connect(task) do |jv|
46
+ # 'drop table if exists TEMP'
47
+ jv.query %[drop table if exists #{task['schema']}.#{task['temp_table']}]
48
+ end
49
+ end
50
+ return {}
51
+ end
52
+
53
+ def self.connect(task)
54
+ jv = ::Vertica.connect({
55
+ host: task['host'],
56
+ port: task['port'],
57
+ user: task['username'],
58
+ password: task['password'],
59
+ database: task['database'],
60
+ })
61
+
62
+ if block_given?
63
+ begin
64
+ yield jv
65
+ ensure
66
+ jv.close
67
+ end
68
+ end
69
+ jv
70
+ end
71
+
72
+ def self.to_vertica_schema(schema)
73
+ schema.names.zip(schema.types)
74
+ .map { |name, type| "#{name} #{to_sql_type(type)}" }
75
+ .join(',')
76
+ end
77
+
78
+ def self.to_sql_type(type)
79
+ case type
80
+ when :boolean then 'BOOLEAN'
81
+ when :long then 'INT'
82
+ when :double then 'FLOAT'
83
+ when :string then 'VARCHAR'
84
+ when :timestamp then 'TIMESTAMP'
85
+ else fail NotSupportedSchema, "embulk-plugin-output-vertica cannot take column type #{type}"
86
+ end
87
+ end
88
+
89
+ def initialize(task, schema, index)
90
+ super
91
+ @jv = self.class.connect(task)
92
+ end
93
+
94
+ def close
95
+ @jv.close
96
+ end
97
+
98
+ def add(page)
99
+ sql = "COPY #{@task['schema']}.#{@task['temp_table']} FROM STDIN DELIMITER ','"
100
+ @jv.copy(sql) do |stdin|
101
+ page.each_with_index do |record, idx|
102
+ stdin << record.map {|v| ::Vertica.quote(v)}.join(",")
103
+ stdin << "\n" unless record.size-1 == idx
104
+ end
105
+ end
106
+ end
107
+
108
+ def finish
109
+ end
110
+
111
+ def abort
112
+ end
113
+
114
+ def commit
115
+ {}
116
+ end
117
+ end
118
+ end
119
+ end
120
+ end
@@ -0,0 +1,7 @@
1
+ module Embulk
2
+ module Plugin
3
+ module Vertica
4
+ VERSION = "0.0.2"
5
+ end
6
+ end
7
+ end
metadata ADDED
@@ -0,0 +1,95 @@
1
+ --- !ruby/object:Gem::Specification
2
+ name: embulk-plugin-vertica
3
+ version: !ruby/object:Gem::Version
4
+ version: 0.0.2
5
+ platform: ruby
6
+ authors:
7
+ - eiji.sekiya
8
+ autorequire:
9
+ bindir: bin
10
+ cert_chain: []
11
+ date: 2015-02-14 00:00:00.000000000 Z
12
+ dependencies:
13
+ - !ruby/object:Gem::Dependency
14
+ name: bundler
15
+ version_requirements: !ruby/object:Gem::Requirement
16
+ requirements:
17
+ - - ~>
18
+ - !ruby/object:Gem::Version
19
+ version: '1.7'
20
+ requirement: !ruby/object:Gem::Requirement
21
+ requirements:
22
+ - - ~>
23
+ - !ruby/object:Gem::Version
24
+ version: '1.7'
25
+ prerelease: false
26
+ type: :development
27
+ - !ruby/object:Gem::Dependency
28
+ name: rake
29
+ version_requirements: !ruby/object:Gem::Requirement
30
+ requirements:
31
+ - - ~>
32
+ - !ruby/object:Gem::Version
33
+ version: '10.0'
34
+ requirement: !ruby/object:Gem::Requirement
35
+ requirements:
36
+ - - ~>
37
+ - !ruby/object:Gem::Version
38
+ version: '10.0'
39
+ prerelease: false
40
+ type: :development
41
+ - !ruby/object:Gem::Dependency
42
+ name: vertica
43
+ version_requirements: !ruby/object:Gem::Requirement
44
+ requirements:
45
+ - - ~>
46
+ - !ruby/object:Gem::Version
47
+ version: 0.11.2
48
+ requirement: !ruby/object:Gem::Requirement
49
+ requirements:
50
+ - - ~>
51
+ - !ruby/object:Gem::Version
52
+ version: 0.11.2
53
+ prerelease: false
54
+ type: :runtime
55
+ description: Embulk plugin for vertica.
56
+ email:
57
+ - eiji.sekiya.0326@gmail.com
58
+ executables: []
59
+ extensions: []
60
+ extra_rdoc_files: []
61
+ files:
62
+ - .gitignore
63
+ - Gemfile
64
+ - LICENSE.txt
65
+ - README.md
66
+ - Rakefile
67
+ - config.yml
68
+ - embulk-plugin-vertica.gemspec
69
+ - lib/embulk/output_vertica.rb
70
+ - lib/embulk/plugin/vertica/version.rb
71
+ homepage: https://github.com/eratostennis/embulk-plugin-vertica
72
+ licenses:
73
+ - MIT
74
+ metadata: {}
75
+ post_install_message:
76
+ rdoc_options: []
77
+ require_paths:
78
+ - lib
79
+ required_ruby_version: !ruby/object:Gem::Requirement
80
+ requirements:
81
+ - - '>='
82
+ - !ruby/object:Gem::Version
83
+ version: '0'
84
+ required_rubygems_version: !ruby/object:Gem::Requirement
85
+ requirements:
86
+ - - '>='
87
+ - !ruby/object:Gem::Version
88
+ version: '0'
89
+ requirements: []
90
+ rubyforge_project:
91
+ rubygems_version: 2.1.9
92
+ signing_key:
93
+ specification_version: 4
94
+ summary: Embulk plugin for vertica.
95
+ test_files: []