embulk-output-bigobject 0.2.8 → 0.3.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/Gemfile +0 -0
- data/LICENSE.txt +0 -0
- data/README.md +8 -2
- data/Rakefile +0 -0
- data/embulk-output-bigobject.gemspec +8 -8
- data/lib/embulk/output/bigobject.rb +91 -38
- metadata +21 -35
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 43abe5dc1730fe8a446ddeed77a4de07f514c89c
|
4
|
+
data.tar.gz: df2bee768096c9abdfe944e6137e934aa2b66e96
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 4cfa4216e75df1bb68990700e529a3477d6a5f0c1eecdfbb91df1b70e9e4681c73525999019031269776b3c94679321ef0079952b879d27b2b0625d7dceefa5f
|
7
|
+
data.tar.gz: 8aa257ce70a52b46b3cf3f4cb6e2cda9d734678cf7636e03582a84ad918e2bbe08f17438c4504f89983669aab47d794b87b2808a47430fd0e59a288f721edef3
|
data/Gemfile
CHANGED
File without changes
|
data/LICENSE.txt
CHANGED
File without changes
|
data/README.md
CHANGED
@@ -17,8 +17,9 @@ embulk gem install embulk-output-bigobject
|
|
17
17
|
|
18
18
|
## Configuration
|
19
19
|
|
20
|
-
- **host**: database host name (string,
|
21
|
-
- **
|
20
|
+
- **host**: database host name (string, default: localhost)
|
21
|
+
- **restport**: database port number (integer, default: 9090)
|
22
|
+
- **ncport**: database port number (integer, default: 9090)
|
22
23
|
- **table**: database table name (string, required)
|
23
24
|
|
24
25
|
## Example
|
@@ -28,6 +29,11 @@ out:
|
|
28
29
|
type: bigobject
|
29
30
|
host: localhost
|
30
31
|
table: mytest
|
32
|
+
column_options:
|
33
|
+
- {name: "col1", type: 'INT64', is_key: true}
|
34
|
+
- {name: "col2", type: 'BYTE', is_key: true}
|
35
|
+
- {name: "col3", type: 'DATE32'}
|
36
|
+
- {name: "col4", type: 'STRING(16)'}
|
31
37
|
```
|
32
38
|
|
33
39
|
|
data/Rakefile
CHANGED
File without changes
|
@@ -1,11 +1,11 @@
|
|
1
1
|
|
2
2
|
Gem::Specification.new do |spec|
|
3
3
|
spec.name = "embulk-output-bigobject"
|
4
|
-
spec.version = "0.
|
5
|
-
spec.authors = ["
|
4
|
+
spec.version = "0.3.0"
|
5
|
+
spec.authors = ["cchuang"]
|
6
6
|
spec.summary = "Bigobject output plugin for Embulk"
|
7
7
|
spec.description = "Dumps records to Bigobject."
|
8
|
-
spec.email = ["
|
8
|
+
spec.email = ["cchuang.tw@gmail.com"]
|
9
9
|
spec.licenses = ["MIT"]
|
10
10
|
spec.homepage = "https://github.com/bigobject-inc/embulk-output-bigobject"
|
11
11
|
|
@@ -13,9 +13,9 @@ Gem::Specification.new do |spec|
|
|
13
13
|
spec.test_files = spec.files.grep(%r{^(test|spec)/})
|
14
14
|
spec.require_paths = ["lib"]
|
15
15
|
|
16
|
-
spec.add_dependency 'rest-client', ['
|
17
|
-
spec.add_development_dependency 'rest-client', ['
|
18
|
-
spec.add_development_dependency 'embulk', ['
|
19
|
-
spec.add_development_dependency 'bundler', ['
|
20
|
-
spec.add_development_dependency 'rake', ['>= 10.0']
|
16
|
+
spec.add_dependency 'rest-client', ['>= 1.8.0']
|
17
|
+
#spec.add_development_dependency 'rest-client', ['>= 1.8.0']
|
18
|
+
spec.add_development_dependency 'embulk', ['>= 0.8.9']
|
19
|
+
spec.add_development_dependency 'bundler', ['>= 1.10.6']
|
20
|
+
spec.add_development_dependency 'rake', ['>= 10.5.0']
|
21
21
|
end
|
@@ -1,37 +1,55 @@
|
|
1
1
|
module Embulk
|
2
|
+
|
2
3
|
module Output
|
3
4
|
|
4
5
|
class Bigobject < OutputPlugin
|
5
6
|
|
6
7
|
require 'rest-client'
|
7
8
|
require 'json'
|
9
|
+
require 'socket'
|
8
10
|
|
9
11
|
Plugin.register_output("bigobject", self)
|
10
12
|
|
11
|
-
def self.
|
13
|
+
def self.configure(config, schema, count)
|
12
14
|
# configuration code:
|
13
15
|
task = {
|
14
|
-
"host"
|
15
|
-
"
|
16
|
-
"
|
16
|
+
"host" => config.param("host", :string, :default => "localhost"), # string, optional
|
17
|
+
"restport" => config.param("restport", :integer, :default => 9090), # integer, optional
|
18
|
+
"ncport" => config.param("ncport", :integer, :default => 9091), # integer, optional
|
19
|
+
"table" => config.param("table", :string), # string, required
|
20
|
+
"column_options" => config.param("column_options", :array, :default => []),
|
17
21
|
}
|
18
22
|
|
19
|
-
|
20
|
-
|
23
|
+
task
|
24
|
+
end
|
25
|
+
|
26
|
+
def self.column_options_map(column_options)
|
27
|
+
(column_options || {}).map do |column_option|
|
28
|
+
[column_option['name'], column_option]
|
29
|
+
end.to_h
|
30
|
+
end
|
31
|
+
|
32
|
+
def self.transaction(config, schema, count, &control)
|
33
|
+
task = self.configure(config, schema, count)
|
34
|
+
task['co_map'] = self.column_options_map task["column_options"]
|
35
|
+
task['rest_uri'] = "http://#{task['host']}:#{task['restport']}/cmd".freeze
|
36
|
+
task['ttl_counter'] = 0
|
21
37
|
|
38
|
+
Embulk.logger.debug { "Transaction #{count}" }
|
22
39
|
# resumable output:
|
23
40
|
# resume(task, schema, count, &control)
|
24
41
|
|
25
|
-
# Create
|
26
|
-
response =
|
27
|
-
if response["Status"] == 0 then # table
|
28
|
-
|
29
|
-
elsif response["Status"] == -11 then # table not exist
|
30
|
-
response =
|
31
|
-
|
32
|
-
|
33
|
-
|
34
|
-
|
42
|
+
# Create-Table if it does not exist
|
43
|
+
response = rest_exec(task['rest_uri'], "desc #{task['table']}") # check table
|
44
|
+
if response["Status"] == 0 then # the table exists
|
45
|
+
Embulk.logger.debug { "#{response}" }
|
46
|
+
elsif response["Status"] == -11 then # the table does not exist
|
47
|
+
response = rest_exec(task['rest_uri'], "#{create_botable_stmt("#{task['table']}",schema, task['co_map'])}")
|
48
|
+
if response["Status"] != 0 then
|
49
|
+
Embulk.logger.error { "#{response}" }
|
50
|
+
raise "Create table #{task['table']} in BigObject Failed"
|
51
|
+
end
|
52
|
+
Embulk.logger.info { "embulk-output-bigobject: Create table #{task['table']}" }
|
35
53
|
else # should not be here
|
36
54
|
Embulk.logger.error { "#{response}" }
|
37
55
|
raise "Please check table #{task['table']} in BigObject First"
|
@@ -43,6 +61,22 @@ module Embulk
|
|
43
61
|
return next_config_diff
|
44
62
|
end
|
45
63
|
|
64
|
+
def safe_io_puts(buff)
|
65
|
+
@@io ||= create_shared_io
|
66
|
+
@@mutext ||= Mutex.new
|
67
|
+
@@mutext.synchronize do
|
68
|
+
@@io.puts buff
|
69
|
+
end
|
70
|
+
end
|
71
|
+
|
72
|
+
def create_shared_io
|
73
|
+
io = TCPSocket.new @task['host'], @task['ncport']
|
74
|
+
#@@io = File.new "embulkout.dump", "w"
|
75
|
+
io.write "csv\x01"
|
76
|
+
io.puts @task['table']
|
77
|
+
io
|
78
|
+
end
|
79
|
+
|
46
80
|
#def self.resume(task, schema, count, &control)
|
47
81
|
# task_reports = yield(task)
|
48
82
|
#
|
@@ -50,7 +84,10 @@ module Embulk
|
|
50
84
|
# return next_config_diff
|
51
85
|
#end
|
52
86
|
|
53
|
-
def
|
87
|
+
def initialize(task, schema, index)
|
88
|
+
super
|
89
|
+
|
90
|
+
# Embulk.logger.debug { "Initialize #{index}" }
|
54
91
|
# initialization code:
|
55
92
|
@table = task["table"]
|
56
93
|
@counter = 0
|
@@ -62,26 +99,20 @@ module Embulk
|
|
62
99
|
def add(page)
|
63
100
|
# output code:
|
64
101
|
|
65
|
-
data = Array.new
|
102
|
+
#data = Array.new
|
66
103
|
values = Array.new
|
104
|
+
count = 0
|
67
105
|
|
68
106
|
page.each do |records|
|
69
107
|
values = []
|
70
108
|
records.each do |row| values << "#{row}".to_json end
|
71
|
-
data.push("(#{values.join(",")})")
|
109
|
+
#data.push("(#{values.join(",")})")
|
110
|
+
safe_io_puts "#{values.join(",")}"
|
111
|
+
count += 1
|
72
112
|
end
|
73
113
|
|
74
|
-
@counter +=
|
75
|
-
|
76
|
-
|
77
|
-
Embulk.logger.trace { "INSERT INTO #@table VALUES #{data.join(",")}" }
|
78
|
-
rsp = self.class.send_stmt2borest("INSERT INTO #@table VALUES #{data.join(",")}")
|
79
|
-
if rsp["Status"] == 0 then
|
80
|
-
Embulk.logger.debug { "add #{data.length} to BigObject - #@counter | #@@ttlcounter" }
|
81
|
-
else
|
82
|
-
Embulk.logger.error { "add #{data.length} to BigObject failed!!" }
|
83
|
-
abort
|
84
|
-
end
|
114
|
+
@counter += count
|
115
|
+
@task['ttl_counter'] += count
|
85
116
|
|
86
117
|
end
|
87
118
|
|
@@ -99,21 +130,43 @@ module Embulk
|
|
99
130
|
return task_report
|
100
131
|
end
|
101
132
|
|
102
|
-
def self.
|
133
|
+
def self.rest_exec(uri, stmt)
|
103
134
|
begin
|
104
|
-
response = RestClient.post
|
135
|
+
response = RestClient.post uri, { "Stmt" => "#{stmt}" }.to_json, :content_type => :json, :accept => :json, :timeout => 2
|
105
136
|
JSON.parse(response.body)
|
106
|
-
rescue Exception => e
|
107
|
-
Embulk.logger.error { e }
|
137
|
+
rescue RestClient::Exception => e
|
138
|
+
#Embulk.logger.error { "RestClient: #{e.http_code}, #{e.message}, response: #{e.response}" }
|
139
|
+
Embulk.logger.warn { "Timeout: statement: #{stmt}" }
|
140
|
+
begin
|
141
|
+
response = RestClient.post uri, { "Stmt" => "#{stmt}" }.to_json, :content_type => :json, :accept => :json, :timeout => 4
|
142
|
+
JSON.parse(response.body)
|
143
|
+
rescue RestClient::Exception => e2
|
144
|
+
Embulk.logger.error { "RestClient: #{e2.http_code}, #{e2.message}, response: #{e2.response}" }
|
145
|
+
end
|
146
|
+
rescue JSON::Exception => e
|
147
|
+
Embulk.logger.error { "JSON: #{e.message}" }
|
108
148
|
end
|
109
149
|
end
|
110
150
|
|
111
|
-
def self.create_botable_stmt(tbl,schema)
|
112
|
-
|
113
|
-
|
151
|
+
def self.create_botable_stmt(tbl,schema, cos)
|
152
|
+
Embulk.logger.debug {"schema: #{schema}"}
|
153
|
+
bo_table_schema = schema.map {|column| "#{column.name} #{to_bigobject_column_type(column.type.to_s, column.format.to_s, cos[column.name])}" }.join(',')
|
154
|
+
keys = Array.new
|
155
|
+
cos.each do |k, co|
|
156
|
+
keys.push co["name"] if co["is_key"]
|
157
|
+
end
|
158
|
+
if keys.length == 0
|
159
|
+
"CREATE TABLE #{tbl} (#{bo_table_schema})"
|
160
|
+
else
|
161
|
+
"CREATE TABLE #{tbl} (#{bo_table_schema} KEY(#{keys.join(',')}))"
|
162
|
+
end
|
114
163
|
end
|
115
164
|
|
116
|
-
def self.to_bigobject_column_type(type, format)
|
165
|
+
def self.to_bigobject_column_type(type, format, co)
|
166
|
+
co = co || {}
|
167
|
+
Embulk.logger.debug {"type: #{type}, format #{format}, option #{co}"}
|
168
|
+
return co["type"] if co["type"]
|
169
|
+
|
117
170
|
case type
|
118
171
|
when 'long'
|
119
172
|
botype = :INT64
|
metadata
CHANGED
@@ -1,88 +1,74 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: embulk-output-bigobject
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.
|
4
|
+
version: 0.3.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
|
-
-
|
7
|
+
- cchuang
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2016-
|
11
|
+
date: 2016-07-06 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: rest-client
|
15
15
|
requirement: !ruby/object:Gem::Requirement
|
16
16
|
requirements:
|
17
|
-
- -
|
17
|
+
- - ">="
|
18
18
|
- !ruby/object:Gem::Version
|
19
19
|
version: 1.8.0
|
20
20
|
type: :runtime
|
21
21
|
prerelease: false
|
22
22
|
version_requirements: !ruby/object:Gem::Requirement
|
23
23
|
requirements:
|
24
|
-
- -
|
25
|
-
- !ruby/object:Gem::Version
|
26
|
-
version: 1.8.0
|
27
|
-
- !ruby/object:Gem::Dependency
|
28
|
-
name: rest-client
|
29
|
-
requirement: !ruby/object:Gem::Requirement
|
30
|
-
requirements:
|
31
|
-
- - ~>
|
32
|
-
- !ruby/object:Gem::Version
|
33
|
-
version: 1.8.0
|
34
|
-
type: :development
|
35
|
-
prerelease: false
|
36
|
-
version_requirements: !ruby/object:Gem::Requirement
|
37
|
-
requirements:
|
38
|
-
- - ~>
|
24
|
+
- - ">="
|
39
25
|
- !ruby/object:Gem::Version
|
40
26
|
version: 1.8.0
|
41
27
|
- !ruby/object:Gem::Dependency
|
42
28
|
name: embulk
|
43
29
|
requirement: !ruby/object:Gem::Requirement
|
44
30
|
requirements:
|
45
|
-
- -
|
31
|
+
- - ">="
|
46
32
|
- !ruby/object:Gem::Version
|
47
|
-
version: 0.
|
33
|
+
version: 0.8.9
|
48
34
|
type: :development
|
49
35
|
prerelease: false
|
50
36
|
version_requirements: !ruby/object:Gem::Requirement
|
51
37
|
requirements:
|
52
|
-
- -
|
38
|
+
- - ">="
|
53
39
|
- !ruby/object:Gem::Version
|
54
|
-
version: 0.
|
40
|
+
version: 0.8.9
|
55
41
|
- !ruby/object:Gem::Dependency
|
56
42
|
name: bundler
|
57
43
|
requirement: !ruby/object:Gem::Requirement
|
58
44
|
requirements:
|
59
|
-
- -
|
45
|
+
- - ">="
|
60
46
|
- !ruby/object:Gem::Version
|
61
|
-
version:
|
47
|
+
version: 1.10.6
|
62
48
|
type: :development
|
63
49
|
prerelease: false
|
64
50
|
version_requirements: !ruby/object:Gem::Requirement
|
65
51
|
requirements:
|
66
|
-
- -
|
52
|
+
- - ">="
|
67
53
|
- !ruby/object:Gem::Version
|
68
|
-
version:
|
54
|
+
version: 1.10.6
|
69
55
|
- !ruby/object:Gem::Dependency
|
70
56
|
name: rake
|
71
57
|
requirement: !ruby/object:Gem::Requirement
|
72
58
|
requirements:
|
73
|
-
- -
|
59
|
+
- - ">="
|
74
60
|
- !ruby/object:Gem::Version
|
75
|
-
version:
|
61
|
+
version: 10.5.0
|
76
62
|
type: :development
|
77
63
|
prerelease: false
|
78
64
|
version_requirements: !ruby/object:Gem::Requirement
|
79
65
|
requirements:
|
80
|
-
- -
|
66
|
+
- - ">="
|
81
67
|
- !ruby/object:Gem::Version
|
82
|
-
version:
|
68
|
+
version: 10.5.0
|
83
69
|
description: Dumps records to Bigobject.
|
84
70
|
email:
|
85
|
-
-
|
71
|
+
- cchuang.tw@gmail.com
|
86
72
|
executables: []
|
87
73
|
extensions: []
|
88
74
|
extra_rdoc_files: []
|
@@ -103,17 +89,17 @@ require_paths:
|
|
103
89
|
- lib
|
104
90
|
required_ruby_version: !ruby/object:Gem::Requirement
|
105
91
|
requirements:
|
106
|
-
- -
|
92
|
+
- - ">="
|
107
93
|
- !ruby/object:Gem::Version
|
108
94
|
version: '0'
|
109
95
|
required_rubygems_version: !ruby/object:Gem::Requirement
|
110
96
|
requirements:
|
111
|
-
- -
|
97
|
+
- - ">="
|
112
98
|
- !ruby/object:Gem::Version
|
113
99
|
version: '0'
|
114
100
|
requirements: []
|
115
101
|
rubyforge_project:
|
116
|
-
rubygems_version: 2.
|
102
|
+
rubygems_version: 2.5.1
|
117
103
|
signing_key:
|
118
104
|
specification_version: 4
|
119
105
|
summary: Bigobject output plugin for Embulk
|