fluent-plugin-bigobject 0.0.4 → 0.0.7
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/README.md +32 -11
- data/fluent-plugin-bigobject.gemspec +2 -1
- data/lib/fluent/plugin/out_bigobject.rb +89 -30
- metadata +16 -2
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: a7bfafe5bf6153cafec9e23ce80ef2ab726dfe71
|
4
|
+
data.tar.gz: 4e45524019de554685fc87474f1cadc0f3bf7003
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: dccf9efcdd694916ba407cc7f84889297d43a5e17ed630a65a2cc33d1ec5bb78106a354e4c8b54f02da0f23b871af0b5ffe713d3d42d1b9084cfc3ea6e71e758
|
7
|
+
data.tar.gz: ce965346785948872b30ca6bf0b1e4a3deb08a91309428615c1b45cdb0c847f81c9285e73ddc6db0700b881a01ef31b997ae6400bd7a0552b8b3344c64ffe396
|
data/README.md
CHANGED
@@ -28,27 +28,48 @@ Configure BigObject URL and the table/column to be mapped in BigObject
|
|
28
28
|
|
29
29
|
log_level info
|
30
30
|
|
31
|
-
# specify the
|
32
|
-
|
31
|
+
# specify the bigobject to connect to
|
32
|
+
bigobject_hostname 192.168.59.103
|
33
|
+
bigobject_port 9091
|
33
34
|
|
34
35
|
remove_tag_prefix bo.insert.
|
35
36
|
flush_interval 5s
|
36
37
|
|
37
38
|
<table>
|
38
|
-
|
39
|
-
|
39
|
+
#example of sending data to BigObject using binary avro
|
40
|
+
#table name is specified in avsc file
|
40
41
|
pattern customer
|
41
|
-
|
42
|
-
|
42
|
+
schema_file /fluentd/input/avsc/Customer.avsc
|
43
|
+
|
44
|
+
#optional -
|
45
|
+
#column_mapping id1:id,name,language,state,company,gender,age
|
43
46
|
</table>
|
44
47
|
|
48
|
+
</match>
|
49
|
+
|
50
|
+
<match bo.insert_rest.*>
|
51
|
+
type bigobject
|
52
|
+
|
53
|
+
log_level info
|
54
|
+
|
55
|
+
# specify the bigobject_url to connect to
|
56
|
+
bigobject_hostname 192.168.59.103
|
57
|
+
bigobject_port 9090
|
58
|
+
|
59
|
+
remove_tag_prefix bo.insert_rest.
|
60
|
+
flush_interval 5s
|
61
|
+
|
45
62
|
<table>
|
46
|
-
|
47
|
-
|
48
|
-
|
49
|
-
pattern test3
|
50
|
-
</table>
|
63
|
+
#example of sending data to BigObject using restful API
|
64
|
+
table Customer2
|
65
|
+
pattern customer2
|
51
66
|
|
67
|
+
#optional --
|
68
|
+
#column_mapping id1:id,name,language,state,company,gender,age
|
69
|
+
#bo_workspace
|
70
|
+
#bo_opts
|
71
|
+
|
72
|
+
</table>
|
52
73
|
</match>
|
53
74
|
```
|
54
75
|
|
@@ -1,7 +1,7 @@
|
|
1
1
|
# -*- encoding: utf-8 -*-
|
2
2
|
Gem::Specification.new do |gem|
|
3
3
|
gem.name = "fluent-plugin-bigobject"
|
4
|
-
gem.version = "0.0.
|
4
|
+
gem.version = "0.0.7"
|
5
5
|
gem.authors = ["Andrea Sung"]
|
6
6
|
gem.email = ["andrea@bigobject.io"]
|
7
7
|
gem.description = %q{Fluentd output plugin to insert BIGOBJECT }
|
@@ -17,5 +17,6 @@ Gem::Specification.new do |gem|
|
|
17
17
|
gem.add_runtime_dependency "fluentd"
|
18
18
|
gem.add_runtime_dependency "rest-client"
|
19
19
|
gem.add_runtime_dependency "json"
|
20
|
+
gem.add_development_dependency "avro"
|
20
21
|
gem.add_development_dependency "rake"
|
21
22
|
end
|
@@ -1,10 +1,12 @@
|
|
1
1
|
class Fluent::BigObjectOutput < Fluent::BufferedOutput
|
2
|
+
|
2
3
|
Fluent::Plugin.register_output('bigobject', self)
|
3
4
|
|
4
5
|
include Fluent::SetTimeKeyMixin
|
5
6
|
include Fluent::SetTagKeyMixin
|
6
7
|
|
7
|
-
config_param :
|
8
|
+
config_param :bigobject_hostname, :string
|
9
|
+
config_param :bigobject_port, :integer
|
8
10
|
config_param :remove_tag_prefix, :string, :default => nil
|
9
11
|
config_param :send_unknown_chunks, :string, :default=>true
|
10
12
|
|
@@ -17,34 +19,58 @@ class Fluent::BigObjectOutput < Fluent::BufferedOutput
|
|
17
19
|
class TableElement
|
18
20
|
include Fluent::Configurable
|
19
21
|
|
20
|
-
config_param :table, :string
|
21
|
-
config_param :column_mapping, :string
|
22
|
+
config_param :table, :string, :default=>nil
|
23
|
+
config_param :column_mapping, :string, :default=>nil
|
22
24
|
config_param :pattern, :string, :default=>nil
|
23
25
|
config_param :bo_workspace, :string, :default=>nil
|
24
26
|
config_param :bo_opts, :string, :default=>nil
|
27
|
+
config_param :schema_file, :string, :default => nil
|
25
28
|
|
26
29
|
attr_reader :mpattern
|
27
30
|
|
28
|
-
def initialize(log)
|
31
|
+
def initialize(log, bo_hostname, bo_port)
|
29
32
|
super()
|
30
33
|
@log = log
|
34
|
+
@bo_hostname = bo_hostname
|
35
|
+
@bo_port = bo_port
|
36
|
+
@bo_url="http://#{@bo_hostname}:#{@bo_port}/cmd"
|
31
37
|
end
|
32
38
|
|
33
39
|
def configure(conf)
|
34
40
|
super
|
41
|
+
if (@table==nil)&&(@schema_file==nil)
|
42
|
+
raise "Table name and schema_file cannot be both nil. Please specify <schema_file> if using avro input or <table> is using restful api."
|
43
|
+
end
|
44
|
+
if (isBinary)
|
45
|
+
@avro_schema = Avro::Schema.parse(File.open(@schema_file, "rb").read)
|
46
|
+
@avro_writer = Avro::IO::DatumWriter.new(@avro_schema)
|
47
|
+
else
|
48
|
+
@avro_schema = nil
|
49
|
+
@avro_writer = nil
|
50
|
+
end
|
51
|
+
|
35
52
|
@mpattern = Fluent::MatchPattern.create(pattern)
|
36
|
-
@mapping = parse_column_mapping(@column_mapping)
|
53
|
+
@mapping = (@column_mapping==nil)? nil:parse_column_mapping(@column_mapping)
|
37
54
|
@log.info("column mapping for #{table} - #{@mapping}")
|
38
55
|
@format_proc = Proc.new { |record|
|
39
|
-
|
40
|
-
|
41
|
-
|
42
|
-
|
43
|
-
|
56
|
+
if (@mapping==nil)
|
57
|
+
record
|
58
|
+
else
|
59
|
+
new_record = {}
|
60
|
+
@mapping.each { |k, c|
|
61
|
+
new_record[c] = record[k]
|
62
|
+
}
|
63
|
+
new_record
|
64
|
+
end
|
44
65
|
}
|
45
66
|
end
|
67
|
+
|
68
|
+
def isBinary()
|
69
|
+
return !(@schema_file.to_s.empty?)
|
70
|
+
end
|
46
71
|
|
47
|
-
|
72
|
+
#Send Data to Bigobject using Restful API
|
73
|
+
def send_rest(chunk)
|
48
74
|
stmts = Array.new
|
49
75
|
i=0
|
50
76
|
columns = nil
|
@@ -59,25 +85,64 @@ class Fluent::BigObjectOutput < Fluent::BufferedOutput
|
|
59
85
|
if columns.to_s.empty?
|
60
86
|
columns = "(#{keys.join(",")})"
|
61
87
|
end
|
62
|
-
#single quote each column data
|
63
88
|
stmts.push("('#{values.join("','")}')")
|
64
89
|
i+=1
|
65
90
|
}
|
66
91
|
|
67
92
|
sendStmt = "INSERT INTO #{@table} #{columns} VALUES" + stmts.join(",")
|
68
|
-
|
69
|
-
|
70
|
-
resp = sendBO(bourl, sendStmt)
|
93
|
+
|
94
|
+
resp = sendBO(@bo_url, sendStmt)
|
71
95
|
parsed = JSON.parse(resp)
|
72
96
|
err = parsed['Err']
|
73
|
-
# puts "Content=#{parsed['Content']}, Err=#{err}"
|
74
97
|
if (err.to_s!='')
|
75
98
|
@log.error("[BigObject] #{err}")
|
76
99
|
end
|
77
|
-
@log.
|
100
|
+
@log.debug("bigobject insert #{i} rows")
|
78
101
|
|
79
102
|
end
|
80
103
|
|
104
|
+
#Send data to Bigobject using binary AVRO
|
105
|
+
def send_binary(chunk)
|
106
|
+
|
107
|
+
buffer = StringIO.new()
|
108
|
+
dw = Avro::DataFile::Writer.new(buffer, @avro_writer, @avro_schema)
|
109
|
+
i=0
|
110
|
+
chunk.msgpack_each { |tag, time, data|
|
111
|
+
data = @format_proc.call(data)
|
112
|
+
dw<<data
|
113
|
+
i+=1
|
114
|
+
}
|
115
|
+
dw.flush
|
116
|
+
|
117
|
+
begin
|
118
|
+
socket = TCPSocket.open(@bo_hostname, @bo_port)
|
119
|
+
begin
|
120
|
+
#timeout=60
|
121
|
+
opt = [1, 60].pack('I!I!') # { int l_onoff; int l_linger; }
|
122
|
+
socket.setsockopt(Socket::SOL_SOCKET, Socket::SO_LINGER, opt)
|
123
|
+
|
124
|
+
opt = [60, 0].pack('L!L!') # struct timeval
|
125
|
+
socket.setsockopt(Socket::SOL_SOCKET, Socket::SO_SNDTIMEO, opt)
|
126
|
+
socket.write(buffer.string)
|
127
|
+
ensure
|
128
|
+
socket.close
|
129
|
+
end
|
130
|
+
|
131
|
+
rescue Exception => e
|
132
|
+
@log.error(e.message)
|
133
|
+
raise "Failed to send_binary: #{e.message}"
|
134
|
+
end
|
135
|
+
@log.debug("bigobject send #{i} rows")
|
136
|
+
end
|
137
|
+
|
138
|
+
def send(chunk)
|
139
|
+
if (isBinary)
|
140
|
+
send_binary(chunk)
|
141
|
+
else
|
142
|
+
send_rest(chunk)
|
143
|
+
end
|
144
|
+
end
|
145
|
+
|
81
146
|
def to_s
|
82
147
|
"table:#{table}, column_mapping:#{column_mapping}, pattern:#{pattern}"
|
83
148
|
end
|
@@ -108,8 +173,6 @@ class Fluent::BigObjectOutput < Fluent::BufferedOutput
|
|
108
173
|
|
109
174
|
def sendBO(bourl, sendStmt)
|
110
175
|
params = formatRequest(sendStmt)
|
111
|
-
@log.debug("\nbourl=#{bourl} params=#{params.to_json}")
|
112
|
-
|
113
176
|
begin
|
114
177
|
resp = RestClient.post bourl, params.to_json, :content_type =>:json, :accept =>:json
|
115
178
|
@log.debug("resp= #{resp.body}")
|
@@ -117,6 +180,7 @@ class Fluent::BigObjectOutput < Fluent::BufferedOutput
|
|
117
180
|
@log.error(e.message)
|
118
181
|
raise "Failed to sendBO: #{e.message}"
|
119
182
|
end
|
183
|
+
|
120
184
|
return resp
|
121
185
|
end
|
122
186
|
|
@@ -126,6 +190,7 @@ class Fluent::BigObjectOutput < Fluent::BufferedOutput
|
|
126
190
|
super
|
127
191
|
require 'rest-client'
|
128
192
|
require 'json'
|
193
|
+
require 'avro'
|
129
194
|
log.info("bigobject initialize")
|
130
195
|
end
|
131
196
|
|
@@ -138,12 +203,12 @@ class Fluent::BigObjectOutput < Fluent::BufferedOutput
|
|
138
203
|
|
139
204
|
@tables = []
|
140
205
|
@default_table = nil
|
206
|
+
|
141
207
|
conf.elements.select { |e|
|
142
208
|
e.name == 'table'
|
143
209
|
}.each { |e|
|
144
|
-
te = TableElement.new(log)
|
210
|
+
te = TableElement.new(log, @bigobject_hostname, @bigobject_port)
|
145
211
|
te.configure(e)
|
146
|
-
# puts "conf.elements #{e}"
|
147
212
|
@tables << te
|
148
213
|
}
|
149
214
|
|
@@ -157,13 +222,10 @@ class Fluent::BigObjectOutput < Fluent::BufferedOutput
|
|
157
222
|
|
158
223
|
def shutdown
|
159
224
|
super
|
160
|
-
log.info("bigobject shutdown")
|
161
225
|
end
|
162
|
-
|
163
|
-
|
226
|
+
|
164
227
|
# This method is called when an event reaches to Fluentd.
|
165
228
|
def format(tag, time, record)
|
166
|
-
# puts "tag=#{tag}, time=#{time}, record=#{record}"
|
167
229
|
[tag, time, record].to_msgpack
|
168
230
|
end
|
169
231
|
|
@@ -173,11 +235,8 @@ class Fluent::BigObjectOutput < Fluent::BufferedOutput
|
|
173
235
|
def write(chunk)
|
174
236
|
unknownChunks = []
|
175
237
|
@tables.each { |table|
|
176
|
-
# puts "write table #{table}"
|
177
|
-
# puts "chunk.key= #{chunk.key}"
|
178
238
|
if table.mpattern.match(chunk.key)
|
179
|
-
|
180
|
-
return table.send(@bigobject_url, chunk)
|
239
|
+
return table.send(chunk)
|
181
240
|
end
|
182
241
|
}
|
183
242
|
|
@@ -196,4 +255,4 @@ class Fluent::BigObjectOutput < Fluent::BufferedOutput
|
|
196
255
|
def emit(tag, es, chain)
|
197
256
|
super(tag, es, chain, format_tag(tag))
|
198
257
|
end
|
199
|
-
end
|
258
|
+
end
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: fluent-plugin-bigobject
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.0.
|
4
|
+
version: 0.0.7
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Andrea Sung
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2015-
|
11
|
+
date: 2015-10-27 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: fluentd
|
@@ -52,6 +52,20 @@ dependencies:
|
|
52
52
|
- - ">="
|
53
53
|
- !ruby/object:Gem::Version
|
54
54
|
version: '0'
|
55
|
+
- !ruby/object:Gem::Dependency
|
56
|
+
name: avro
|
57
|
+
requirement: !ruby/object:Gem::Requirement
|
58
|
+
requirements:
|
59
|
+
- - ">="
|
60
|
+
- !ruby/object:Gem::Version
|
61
|
+
version: '0'
|
62
|
+
type: :development
|
63
|
+
prerelease: false
|
64
|
+
version_requirements: !ruby/object:Gem::Requirement
|
65
|
+
requirements:
|
66
|
+
- - ">="
|
67
|
+
- !ruby/object:Gem::Version
|
68
|
+
version: '0'
|
55
69
|
- !ruby/object:Gem::Dependency
|
56
70
|
name: rake
|
57
71
|
requirement: !ruby/object:Gem::Requirement
|