embulk-output-bigobject 0.3.4 → 0.4.4
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/embulk-output-bigobject.gemspec +1 -1
- data/lib/embulk/output/bigobject.rb +21 -34
- metadata +2 -2
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 4a1f4e9350156b99f2607618762c641c774cd573
|
4
|
+
data.tar.gz: cfeb2e35a4d6d3ead4d5c4cd910e578e65f57f7a
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 8423a4ac35e7ba3843cfd2da2ff7b0255390b78d3596e19e71565c496359ffdd4d814c734314a541b4550e4793e15ceba9468fc613e5fb6b04d96172c81811dd
|
7
|
+
data.tar.gz: 1374988497f7158f5c4f07aa159a91be58834c64c04521e74795f48acd8d57903b14a7ce53b27574d52b43a093e963bc8c705405f7445abdb256d05f0f50035d
|
@@ -1,7 +1,7 @@
|
|
1
1
|
|
2
2
|
Gem::Specification.new do |spec|
|
3
3
|
spec.name = "embulk-output-bigobject"
|
4
|
-
spec.version = "0.
|
4
|
+
spec.version = "0.4.4"
|
5
5
|
spec.authors = ["Cheng-Ching Huang"]
|
6
6
|
spec.summary = "Bigobject output plugin for Embulk"
|
7
7
|
spec.description = "Dumps records to Bigobject."
|
@@ -17,6 +17,7 @@ module Embulk
|
|
17
17
|
"restport" => config.param("restport", :integer, :default => 9090), # integer, optional
|
18
18
|
"ncport" => config.param("ncport", :integer, :default => 9091), # integer, optional
|
19
19
|
"table" => config.param("table", :string), # string, required
|
20
|
+
"rowcolumn" => config.param("rowcolumn", :string, :default => "ROW"), # string, optional
|
20
21
|
"column_options" => config.param("column_options", :array, :default => []),
|
21
22
|
"payload_column_index" => config.param("payload_column_index", :integer, :default => nil),
|
22
23
|
}
|
@@ -45,7 +46,7 @@ module Embulk
|
|
45
46
|
if response["Status"] == 0 then # the table exists
|
46
47
|
Embulk.logger.debug { "#{response}" }
|
47
48
|
elsif response["Status"] == -11 then # the table does not exist
|
48
|
-
response = rest_exec(task['rest_uri'], "#{create_botable_stmt("#{task['table']}",schema, task["column_options"])}")
|
49
|
+
response = rest_exec(task['rest_uri'], "#{create_botable_stmt("#{task['table']}", "#{task['rowcolumn']}", schema, task["column_options"])}")
|
49
50
|
if response["Status"] != 0 then
|
50
51
|
Embulk.logger.error { "#{response}" }
|
51
52
|
raise "Create table #{task['table']} in BigObject Failed"
|
@@ -97,29 +98,23 @@ module Embulk
|
|
97
98
|
def close
|
98
99
|
end
|
99
100
|
|
100
|
-
def
|
101
|
-
data = Array.new
|
102
|
-
|
103
|
-
page.each do |record|
|
104
|
-
values = []
|
105
|
-
record.each do |row| values << "\"#{row.to_s.gsub(/\"/,"\"\"")}\"" end
|
106
|
-
data.push "#{values.join(",")}\n"
|
107
|
-
end
|
108
|
-
|
109
|
-
safe_io_write "#{data.join}"
|
110
|
-
|
111
|
-
@counter += data.length
|
112
|
-
@task['ttl_counter'] += data.length
|
113
|
-
|
114
|
-
end
|
115
|
-
|
116
|
-
def add_payload(page)
|
101
|
+
def add(page)
|
117
102
|
data = Array.new
|
118
103
|
pindex = @task['payload_column_index']
|
104
|
+
Embulk.logger.debug "#{pindex}"
|
119
105
|
|
120
|
-
|
121
|
-
|
122
|
-
|
106
|
+
if pindex
|
107
|
+
page.each do |record|
|
108
|
+
data.push "#{record[pindex]}\n"
|
109
|
+
Embulk.logger.debug "#{record[pindex]}\n"
|
110
|
+
end
|
111
|
+
else
|
112
|
+
page.each do |record|
|
113
|
+
values = []
|
114
|
+
record.each do |row| values << "\"#{row.to_s.gsub(/\"/,"\"\"")}\"" end
|
115
|
+
data.push "#{values.join(",")}\n"
|
116
|
+
end
|
117
|
+
end
|
123
118
|
|
124
119
|
safe_io_write "#{data.join}"
|
125
120
|
|
@@ -128,14 +123,6 @@ module Embulk
|
|
128
123
|
|
129
124
|
end
|
130
125
|
|
131
|
-
def add(page)
|
132
|
-
if (@task['payload_column_index'])
|
133
|
-
add_payload(page)
|
134
|
-
else
|
135
|
-
add_csv(page)
|
136
|
-
end
|
137
|
-
end
|
138
|
-
|
139
126
|
def finish
|
140
127
|
end
|
141
128
|
|
@@ -168,7 +155,7 @@ module Embulk
|
|
168
155
|
end
|
169
156
|
end
|
170
157
|
|
171
|
-
def self.create_botable_stmt(tbl,schema, cos)
|
158
|
+
def self.create_botable_stmt(tbl,rowcol,schema, cos)
|
172
159
|
val_array = Array.new
|
173
160
|
schema.each do |c|
|
174
161
|
co = cos[c.index] || {}
|
@@ -176,16 +163,16 @@ module Embulk
|
|
176
163
|
val_array.push "#{co["name"] || c.name} #{to_bigobject_column_type(c.type.to_s, c.format.to_s, co)}"
|
177
164
|
end
|
178
165
|
bo_table_schema = val_array.join(',')
|
179
|
-
Embulk.logger.debug {"schema (#{schema.class}): #{schema}"}
|
180
|
-
Embulk.logger.debug {"schema: #{bo_table_schema}"}
|
166
|
+
#Embulk.logger.debug {"schema (#{schema.class}): #{schema}"}
|
167
|
+
#Embulk.logger.debug {"schema: #{bo_table_schema}"}
|
181
168
|
keys = Array.new
|
182
169
|
cos.each do |co|
|
183
170
|
keys.push co["name"] if co["is_key"]
|
184
171
|
end
|
185
172
|
if keys.length == 0
|
186
|
-
"CREATE TABLE #{tbl} (#{bo_table_schema})"
|
173
|
+
"CREATE TABLE #{tbl} #{rowcol} (#{bo_table_schema})"
|
187
174
|
else
|
188
|
-
"CREATE TABLE #{tbl} (#{bo_table_schema} KEY(#{keys.join(',')}))"
|
175
|
+
"CREATE TABLE #{tbl} #{rowcol} (#{bo_table_schema} KEY(#{keys.join(',')}))"
|
189
176
|
end
|
190
177
|
end
|
191
178
|
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: embulk-output-bigobject
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.
|
4
|
+
version: 0.4.4
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Cheng-Ching Huang
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2016-
|
11
|
+
date: 2016-11-18 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: rest-client
|