embulk-output-bigobject 0.2.3 → 0.2.8
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/embulk-output-bigobject.gemspec +3 -2
- data/lib/embulk/output/bigobject.rb +60 -74
- metadata +30 -2
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 03d0eda74a099f6b37f90a96d23cfa4eea9da2b4
|
4
|
+
data.tar.gz: c54cab398511de5290c0a36cf46709cfe76967ee
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 29126af1038a32a65246b50679fe005d2c1dde141d5de96b5a7bffad1a12b3e36692fc38849e5499b8f0bbeec44f9c53e234b2d680ca104dad0d42bf55153d0e
|
7
|
+
data.tar.gz: eaf4267a3554f840ca863ba3fc4c9aae13ea573cc727b02e096b6fe875ce58c9207f8d72894b7d6d8701083cc7ff532be00af214f41fb36bf0b93760e4d492d5
|
@@ -1,7 +1,7 @@
|
|
1
1
|
|
2
2
|
Gem::Specification.new do |spec|
|
3
3
|
spec.name = "embulk-output-bigobject"
|
4
|
-
spec.version = "0.2.
|
4
|
+
spec.version = "0.2.8"
|
5
5
|
spec.authors = ["randyviola"]
|
6
6
|
spec.summary = "Bigobject output plugin for Embulk"
|
7
7
|
spec.description = "Dumps records to Bigobject."
|
@@ -13,7 +13,8 @@ Gem::Specification.new do |spec|
|
|
13
13
|
spec.test_files = spec.files.grep(%r{^(test|spec)/})
|
14
14
|
spec.require_paths = ["lib"]
|
15
15
|
|
16
|
-
|
16
|
+
spec.add_dependency 'rest-client', ['~> 1.8.0']
|
17
|
+
spec.add_development_dependency 'rest-client', ['~> 1.8.0']
|
17
18
|
spec.add_development_dependency 'embulk', ['~> 0.7.10']
|
18
19
|
spec.add_development_dependency 'bundler', ['~> 1.0']
|
19
20
|
spec.add_development_dependency 'rake', ['>= 10.0']
|
@@ -2,8 +2,8 @@ module Embulk
|
|
2
2
|
module Output
|
3
3
|
|
4
4
|
class Bigobject < OutputPlugin
|
5
|
-
|
6
|
-
require '
|
5
|
+
|
6
|
+
require 'rest-client'
|
7
7
|
require 'json'
|
8
8
|
|
9
9
|
Plugin.register_output("bigobject", self)
|
@@ -15,12 +15,28 @@ module Embulk
|
|
15
15
|
"port" => config.param("port", :integer, default: 9090), # integer, optional
|
16
16
|
"table" => config.param("table", :string), # string, required
|
17
17
|
}
|
18
|
-
|
19
|
-
@@
|
18
|
+
|
19
|
+
@@bo_url = "http://#{task['host']}:#{task['port']}/cmd".freeze
|
20
20
|
@@ttlcounter = 0
|
21
|
+
|
21
22
|
# resumable output:
|
22
23
|
# resume(task, schema, count, &control)
|
23
24
|
|
25
|
+
# Create Table if not exists
|
26
|
+
response = send_stmt2borest("desc #{task['table']}") # check table
|
27
|
+
if response["Status"] == 0 then # table exist
|
28
|
+
Embulk.logger.debug { "#{response}" }
|
29
|
+
elsif response["Status"] == -11 then # table not exist
|
30
|
+
response = send_stmt2borest("#{create_botable_stmt("#{task['table']}",schema)}")
|
31
|
+
if response["Status"] != 0 then
|
32
|
+
Embulk.logger.error { "#{response}" }
|
33
|
+
raise "Create table #{task['table']} in BigObject Failed"
|
34
|
+
end
|
35
|
+
else # should not be here
|
36
|
+
Embulk.logger.error { "#{response}" }
|
37
|
+
raise "Please check table #{task['table']} in BigObject First"
|
38
|
+
end
|
39
|
+
|
24
40
|
# non-resumable output:
|
25
41
|
task_reports = yield(task)
|
26
42
|
next_config_diff = {}
|
@@ -36,24 +52,11 @@ module Embulk
|
|
36
52
|
|
37
53
|
def init
|
38
54
|
# initialization code:
|
39
|
-
|
40
|
-
@host = task["host"]
|
41
|
-
@port = task["port"]
|
42
55
|
@table = task["table"]
|
43
|
-
|
44
|
-
@uri = URI.parse("http://#@host:#@port/cmd")
|
45
|
-
@http = Net::HTTP.new(@uri.host, @uri.port)
|
46
|
-
@counter = 0
|
47
|
-
|
48
|
-
if !istbexist(@table)
|
49
|
-
if !createtbl(@table)
|
50
|
-
abort
|
51
|
-
end
|
52
|
-
end
|
56
|
+
@counter = 0
|
53
57
|
end
|
54
58
|
|
55
59
|
def close
|
56
|
-
@http.finish
|
57
60
|
end
|
58
61
|
|
59
62
|
def add(page)
|
@@ -67,14 +70,16 @@ module Embulk
|
|
67
70
|
records.each do |row| values << "#{row}".to_json end
|
68
71
|
data.push("(#{values.join(",")})")
|
69
72
|
end
|
73
|
+
|
70
74
|
@counter += data.length
|
71
75
|
@@ttlcounter += data.length
|
72
|
-
|
73
|
-
|
76
|
+
|
77
|
+
Embulk.logger.trace { "INSERT INTO #@table VALUES #{data.join(",")}" }
|
78
|
+
rsp = self.class.send_stmt2borest("INSERT INTO #@table VALUES #{data.join(",")}")
|
74
79
|
if rsp["Status"] == 0 then
|
75
|
-
|
80
|
+
Embulk.logger.debug { "add #{data.length} to BigObject - #@counter | #@@ttlcounter" }
|
76
81
|
else
|
77
|
-
|
82
|
+
Embulk.logger.error { "add #{data.length} to BigObject failed!!" }
|
78
83
|
abort
|
79
84
|
end
|
80
85
|
|
@@ -88,64 +93,45 @@ module Embulk
|
|
88
93
|
end
|
89
94
|
|
90
95
|
def commit
|
91
|
-
task_report = {
|
96
|
+
task_report = {
|
97
|
+
"records" => @counter
|
98
|
+
}
|
92
99
|
return task_report
|
93
100
|
end
|
94
101
|
|
95
|
-
|
96
|
-
|
97
|
-
|
98
|
-
|
99
|
-
|
100
|
-
|
101
|
-
return JSON.parse(response.body)
|
102
|
-
rescue Exception => e
|
103
|
-
@@log.error e
|
104
|
-
abort
|
105
|
-
end
|
106
|
-
|
102
|
+
def self.send_stmt2borest(stmt)
|
103
|
+
begin
|
104
|
+
response = RestClient.post @@bo_url, { "Stmt" => "#{stmt}" }.to_json, :content_type => :json, :accept => :json
|
105
|
+
JSON.parse(response.body)
|
106
|
+
rescue Exception => e
|
107
|
+
Embulk.logger.error { e }
|
107
108
|
end
|
108
|
-
|
109
|
-
|
110
|
-
|
111
|
-
|
112
|
-
|
113
|
-
|
109
|
+
end
|
110
|
+
|
111
|
+
def self.create_botable_stmt(tbl,schema)
|
112
|
+
bo_table_schema = schema.map {|column| "#{column.name} #{to_bigobject_column_type(column.type.to_s, column.format.to_s)}" }.join(',')
|
113
|
+
"CREATE TABLE #{tbl} (#{bo_table_schema})"
|
114
|
+
end
|
115
|
+
|
116
|
+
def self.to_bigobject_column_type(type, format)
|
117
|
+
case type
|
118
|
+
when 'long'
|
119
|
+
botype = :INT64
|
120
|
+
when 'boolean'
|
121
|
+
botype = :INT8
|
122
|
+
when 'string'
|
123
|
+
botype = :STRING
|
124
|
+
when 'double'
|
125
|
+
botype = :DOUBLE
|
126
|
+
when 'timestamp'
|
127
|
+
if format.include? "%H"
|
128
|
+
botype = :DATETIME32
|
129
|
+
else
|
130
|
+
botype = :DATE32
|
114
131
|
end
|
115
|
-
return @@tblexist
|
116
|
-
end
|
117
|
-
|
118
|
-
def createtbl(tbl)
|
119
|
-
rsp = send_stmt("CREATE TABLE #{tbl} (#{change2botype(schema).join(",")})")
|
120
|
-
rsp["Status"] == 0 ? @@tblexist = true : false
|
121
|
-
return @@tblexist
|
122
132
|
end
|
123
|
-
|
124
|
-
|
125
|
-
colntype = Array.new
|
126
|
-
schema.each do |s|
|
127
|
-
botype = ""
|
128
|
-
case s[2]
|
129
|
-
when /long/
|
130
|
-
botype = :INT64
|
131
|
-
when /boolean/
|
132
|
-
botype = :INT8
|
133
|
-
when /string/
|
134
|
-
botype = :STRING
|
135
|
-
when /double/
|
136
|
-
botype = :DOUBLE
|
137
|
-
when /timestamp/
|
138
|
-
if s[3].include? "%H"
|
139
|
-
botype = :DATETIME32
|
140
|
-
else
|
141
|
-
botype = :DATE32
|
142
|
-
end
|
143
|
-
end
|
144
|
-
|
145
|
-
colntype << "'#{s[1]}' #{botype}"
|
146
|
-
end
|
147
|
-
return colntype
|
148
|
-
end
|
133
|
+
botype
|
134
|
+
end
|
149
135
|
|
150
136
|
end
|
151
137
|
|
metadata
CHANGED
@@ -1,15 +1,43 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: embulk-output-bigobject
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.2.
|
4
|
+
version: 0.2.8
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- randyviola
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2016-02-
|
11
|
+
date: 2016-02-18 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
|
+
- !ruby/object:Gem::Dependency
|
14
|
+
name: rest-client
|
15
|
+
requirement: !ruby/object:Gem::Requirement
|
16
|
+
requirements:
|
17
|
+
- - ~>
|
18
|
+
- !ruby/object:Gem::Version
|
19
|
+
version: 1.8.0
|
20
|
+
type: :runtime
|
21
|
+
prerelease: false
|
22
|
+
version_requirements: !ruby/object:Gem::Requirement
|
23
|
+
requirements:
|
24
|
+
- - ~>
|
25
|
+
- !ruby/object:Gem::Version
|
26
|
+
version: 1.8.0
|
27
|
+
- !ruby/object:Gem::Dependency
|
28
|
+
name: rest-client
|
29
|
+
requirement: !ruby/object:Gem::Requirement
|
30
|
+
requirements:
|
31
|
+
- - ~>
|
32
|
+
- !ruby/object:Gem::Version
|
33
|
+
version: 1.8.0
|
34
|
+
type: :development
|
35
|
+
prerelease: false
|
36
|
+
version_requirements: !ruby/object:Gem::Requirement
|
37
|
+
requirements:
|
38
|
+
- - ~>
|
39
|
+
- !ruby/object:Gem::Version
|
40
|
+
version: 1.8.0
|
13
41
|
- !ruby/object:Gem::Dependency
|
14
42
|
name: embulk
|
15
43
|
requirement: !ruby/object:Gem::Requirement
|