embulk-output-vertica 0.5.7 → 0.5.8
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +6 -0
- data/README.md +1 -0
- data/embulk-output-vertica.gemspec +1 -1
- data/lib/embulk/output/vertica.rb +14 -13
- metadata +2 -2
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: a5c852962bbea4e096164da3d4e1686a2c4dddda
|
4
|
+
data.tar.gz: db0801b7d7645f8a93442339e91427ea80c8fac9
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 74423ccdc71170c85ead4e7b5181d4308f6c7ab391de32fcd533547cc4729aee8aaefb57792954d9680179ec0a7b29e7728fc54d9745139b9fa680aa8b2ff1fd
|
7
|
+
data.tar.gz: fbdb7aa14a69cb04fbbce927ed8da214daf099b1043c9f42faba65574058e1b19fee0c97625d31f6272220944cad470bebfb1001774a5bf6fa36d6b0b9ec331b
|
data/CHANGELOG.md
CHANGED
data/README.md
CHANGED
@@ -39,6 +39,7 @@
|
|
39
39
|
- **timestamp_format**: timestamp format to convert into/from `timestamp` (string, default is "%Y-%m-%d %H:%M:%S %z")
|
40
40
|
- **timezone**: timezone to convert into/from `timestamp` (string, default is `default_timezone`).
|
41
41
|
- **json_payload**: Assuming first columns of records are json string, COPY each json into vertica directly using fjsonparser. This would improve performance by avoiding construction of json in this jruby plugin. ToDo: auto table creation is not supported for this mode yet (bool, default is false)
|
42
|
+
- **resource_pool**: Configure resource pool (string, default is nil)
|
42
43
|
|
43
44
|
### Modes
|
44
45
|
|
@@ -1,6 +1,6 @@
|
|
1
1
|
Gem::Specification.new do |spec|
|
2
2
|
spec.name = "embulk-output-vertica"
|
3
|
-
spec.version = "0.5.
|
3
|
+
spec.version = "0.5.8"
|
4
4
|
spec.authors = ["eiji.sekiya", "Naotoshi Seo"]
|
5
5
|
spec.email = ["eiji.sekiya.0326@gmail.com", "sonots@gmail.com"]
|
6
6
|
spec.summary = "Vertica output plugin for Embulk"
|
@@ -31,6 +31,7 @@ module Embulk
|
|
31
31
|
'default_timezone' => config.param('default_timezone', :string, :default => 'UTC'),
|
32
32
|
'column_options' => config.param('column_options', :hash, :default => {}),
|
33
33
|
'json_payload' => config.param('json_payload', :bool, :default => false),
|
34
|
+
'resource_pool' => config.param('resource_pool', :string, :default => nil),
|
34
35
|
'reject_on_materialized_type_error' => config.param('reject_on_materialized_type_error', :bool, :default => false),
|
35
36
|
'pool' => config.param('pool', :integer, :default => processor_count),
|
36
37
|
}
|
@@ -68,20 +69,18 @@ module Embulk
|
|
68
69
|
quoted_table = ::Jvertica.quote_identifier(task['table'])
|
69
70
|
quoted_temp_table = ::Jvertica.quote_identifier(task['temp_table'])
|
70
71
|
|
71
|
-
|
72
|
-
|
72
|
+
connect(task) do |jv|
|
73
|
+
unless task['json_payload'] # ToDo: auto table creation is not supported to json_payload mode yet
|
74
|
+
sql_schema_table = self.sql_schema_from_embulk_schema(schema, task['column_options'])
|
73
75
|
|
74
|
-
|
75
|
-
connect(task) do |jv|
|
76
|
+
# create the target table
|
76
77
|
query(jv, %[DROP TABLE IF EXISTS #{quoted_schema}.#{quoted_table}]) if task['mode'] == 'REPLACE'
|
77
78
|
query(jv, %[CREATE TABLE IF NOT EXISTS #{quoted_schema}.#{quoted_table} (#{sql_schema_table})])
|
78
79
|
end
|
79
|
-
end
|
80
80
|
|
81
|
-
|
81
|
+
sql_schema_temp_table = self.sql_schema_from_table(jv, task)
|
82
82
|
|
83
|
-
|
84
|
-
connect(task) do |jv|
|
83
|
+
# create a temp table
|
85
84
|
query(jv, %[DROP TABLE IF EXISTS #{quoted_schema}.#{quoted_temp_table}])
|
86
85
|
query(jv, %[CREATE TABLE #{quoted_schema}.#{quoted_temp_table} (#{sql_schema_temp_table})])
|
87
86
|
# Create internal vertica projection beforehand, otherwirse, parallel copies lock table to create a projection and we get S Lock error sometimes
|
@@ -155,6 +154,10 @@ module Embulk
|
|
155
154
|
database: task['database'],
|
156
155
|
})
|
157
156
|
|
157
|
+
if task['resource_pool']
|
158
|
+
query(jv, "SET SESSION RESOURCE_POOL = '#{task['resource_pool']}'")
|
159
|
+
end
|
160
|
+
|
158
161
|
if block_given?
|
159
162
|
begin
|
160
163
|
yield jv
|
@@ -191,17 +194,15 @@ module Embulk
|
|
191
194
|
end
|
192
195
|
end
|
193
196
|
|
194
|
-
def self.sql_schema_from_table(task)
|
197
|
+
def self.sql_schema_from_table(jv, task)
|
195
198
|
quoted_schema = Jvertica.quote(task['schema'])
|
196
199
|
quoted_table = Jvertica.quote(task['table'])
|
197
200
|
sql = "SELECT column_name, data_type FROM v_catalog.columns " \
|
198
201
|
"WHERE table_schema = #{quoted_schema} AND table_name = #{quoted_table}"
|
199
202
|
|
200
203
|
sql_schema = {}
|
201
|
-
|
202
|
-
|
203
|
-
sql_schema = result.map {|row| [row[0], row[1]] }
|
204
|
-
end
|
204
|
+
result = query(jv, sql)
|
205
|
+
sql_schema = result.map {|row| [row[0], row[1]] }
|
205
206
|
sql_schema.map {|name, type| "#{::Jvertica.quote_identifier(name)} #{type}" }.join(',')
|
206
207
|
end
|
207
208
|
|
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: embulk-output-vertica
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.5.
|
4
|
+
version: 0.5.8
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- eiji.sekiya
|
@@ -9,7 +9,7 @@ authors:
|
|
9
9
|
autorequire:
|
10
10
|
bindir: bin
|
11
11
|
cert_chain: []
|
12
|
-
date: 2016-01-
|
12
|
+
date: 2016-01-23 00:00:00.000000000 Z
|
13
13
|
dependencies:
|
14
14
|
- !ruby/object:Gem::Dependency
|
15
15
|
name: jvertica
|