logstash-output-bcdb 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/CHANGELOG.md +4 -0
- data/CONTRIBUTORS +10 -0
- data/Gemfile +11 -0
- data/LICENSE +202 -0
- data/NOTICE.TXT +5 -0
- data/README.md +109 -0
- data/docs/index.asciidoc +391 -0
- data/lib/logstash/outputs/bcdb.rb +520 -0
- data/logstash-output-bcdb.gemspec +43 -0
- data/spec/outputs/bcdb_spec.rb +364 -0
- data/spec/supports/compressed_requests.rb +38 -0
- metadata +141 -0
@@ -0,0 +1,520 @@
|
|
1
|
+
# encoding: utf-8
|
2
|
+
require "logstash/outputs/base"
|
3
|
+
require "logstash/namespace"
|
4
|
+
require "logstash/json"
|
5
|
+
require "uri"
|
6
|
+
require "logstash/plugin_mixins/http_client"
|
7
|
+
require "zlib"
|
8
|
+
require 'json'
|
9
|
+
require 'net/http'
|
10
|
+
|
11
|
+
class LogStash::Outputs::Bcdb < LogStash::Outputs::Base
|
12
|
+
include LogStash::PluginMixins::HttpClient
|
13
|
+
|
14
|
+
concurrency :shared
|
15
|
+
|
16
|
+
attr_accessor :is_batch
|
17
|
+
|
18
|
+
VALID_METHODS = ["put", "post", "patch", "delete", "get", "head"]
|
19
|
+
|
20
|
+
RETRYABLE_MANTICORE_EXCEPTIONS = [
|
21
|
+
::Manticore::Timeout,
|
22
|
+
::Manticore::SocketException,
|
23
|
+
::Manticore::ClientProtocolException,
|
24
|
+
::Manticore::ResolutionFailure,
|
25
|
+
::Manticore::SocketTimeout
|
26
|
+
]
|
27
|
+
|
28
|
+
# This output lets you send events to a
|
29
|
+
# generic HTTP(S) endpoint
|
30
|
+
#
|
31
|
+
# This output will execute up to 'pool_max' requests in parallel for performance.
|
32
|
+
# Consider this when tuning this plugin for performance.
|
33
|
+
#
|
34
|
+
# Additionally, note that when parallel execution is used strict ordering of events is not
|
35
|
+
# guaranteed!
|
36
|
+
#
|
37
|
+
# Beware, this gem does not yet support codecs. Please use the 'format' option for now.
|
38
|
+
|
39
|
+
config_name "bcdb"
|
40
|
+
|
41
|
+
config :url, :validate => :string
|
42
|
+
# BCDB data endpoint
|
43
|
+
config :base_url, :validate => :string, :required => :true
|
44
|
+
|
45
|
+
# BCDB Auth endpoint
|
46
|
+
config :auth_url, :validate => :string, :required => :true
|
47
|
+
|
48
|
+
# BCDB database entity model name
|
49
|
+
config :bcdb_entity, :validate => :string, :default => "loglines"
|
50
|
+
|
51
|
+
config :username, :validate => :string, :required => :true
|
52
|
+
config :password, :validate => :string, :required => :true
|
53
|
+
config :client_id, :validate => :string, :required => :true
|
54
|
+
config :client_secret, :validate => :string, :required => :true
|
55
|
+
config :grant_type, :validate => ["password", "authorization_code"], :default => "password"
|
56
|
+
|
57
|
+
# The HTTP Verb. One of "put", "post", "patch", "delete", "get", "head"
|
58
|
+
config :http_method, :validate => VALID_METHODS, :default => "post"
|
59
|
+
|
60
|
+
# Custom headers to use
|
61
|
+
# format is `headers => ["X-My-Header", "%{host}"]`
|
62
|
+
config :headers, :validate => :hash, :default => {}
|
63
|
+
|
64
|
+
# Content type
|
65
|
+
#
|
66
|
+
# If not specified, this defaults to the following:
|
67
|
+
#
|
68
|
+
# * if format is "json", "application/json"
|
69
|
+
# * if format is "form", "application/x-www-form-urlencoded"
|
70
|
+
config :content_type, :validate => :string
|
71
|
+
|
72
|
+
# Set this to false if you don't want this output to retry failed requests
|
73
|
+
config :retry_failed, :validate => :boolean, :default => true
|
74
|
+
|
75
|
+
# If encountered as response codes this plugin will retry these requests
|
76
|
+
config :retryable_codes, :validate => :number, :list => true, :default => [429, 500, 502, 503, 504]
|
77
|
+
|
78
|
+
# If you would like to consider some non-2xx codes to be successes
|
79
|
+
# enumerate them here. Responses returning these codes will be considered successes
|
80
|
+
config :ignorable_codes, :validate => :number, :list => true
|
81
|
+
|
82
|
+
# This lets you choose the structure and parts of the event that are sent.
|
83
|
+
#
|
84
|
+
#
|
85
|
+
# For example:
|
86
|
+
# [source,ruby]
|
87
|
+
# mapping => {"foo" => "%{host}"
|
88
|
+
# "bar" => "%{type}"}
|
89
|
+
config :mapping, :validate => :hash
|
90
|
+
|
91
|
+
# Set the format of the http body.
|
92
|
+
#
|
93
|
+
# If form, then the body will be the mapping (or whole event) converted
|
94
|
+
# into a query parameter string, e.g. `foo=bar&baz=fizz...`
|
95
|
+
#
|
96
|
+
# If message, then the body will be the result of formatting the event according to message
|
97
|
+
#
|
98
|
+
# Otherwise, the event is sent as json.
|
99
|
+
config :format, :validate => ["json", "json_batch"], :default => "json"
|
100
|
+
|
101
|
+
# Set this to true if you want to enable gzip compression for your http requests
|
102
|
+
config :http_compression, :validate => :boolean, :default => false
|
103
|
+
|
104
|
+
config :message, :validate => :string
|
105
|
+
|
106
|
+
|
107
|
+
def bcdb_authorise()
|
108
|
+
auth_uri = URI.parse(@auth_url)
|
109
|
+
auth_data = {
|
110
|
+
:username => @username,
|
111
|
+
:password => @password,
|
112
|
+
:client_id => @client_id,
|
113
|
+
:client_secret => @client_secret,
|
114
|
+
:grant_type => @grant_type
|
115
|
+
}
|
116
|
+
status = true
|
117
|
+
begin
|
118
|
+
unless (@token_oauth && (@expires_token && Time.now.utc < @expires_token))
|
119
|
+
https= Net::HTTP.new(auth_uri.host,auth_uri.port)
|
120
|
+
https.use_ssl = auth_uri.scheme == 'https'
|
121
|
+
|
122
|
+
request = Net::HTTP::Post.new(auth_uri.path)
|
123
|
+
request.set_form_data(auth_data)
|
124
|
+
request['Content-Type'] = "application/x-www-form-urlencoded"
|
125
|
+
resp = https.request(request)
|
126
|
+
bcdb_response = {}
|
127
|
+
bcdb_response = JSON.parse(resp.body) rescue bcdb_response["code"] = 5000.to_s
|
128
|
+
if resp.code == 200.to_s && bcdb_response['access_token']
|
129
|
+
@token_oauth = bcdb_response['access_token']
|
130
|
+
@headers["Authorization"] = "Bearer #{@token_oauth}"
|
131
|
+
@expires_token = Time.now.utc + bcdb_response['expires_in'].to_i
|
132
|
+
else
|
133
|
+
status = false
|
134
|
+
@logger.error("Authentification failed please check your credentials")
|
135
|
+
end
|
136
|
+
end
|
137
|
+
rescue => e
|
138
|
+
# This should never happen unless there's a flat out bug in the code
|
139
|
+
@logger.error("Error Makeing Authorization Request to BCDB",
|
140
|
+
:class => e.class.name,
|
141
|
+
:message => e.message,
|
142
|
+
:backtrace => e.backtrace)
|
143
|
+
sleep(2)
|
144
|
+
bcdb_authorise()
|
145
|
+
end
|
146
|
+
return status
|
147
|
+
end
|
148
|
+
|
149
|
+
def bcdb_update_schema(data, cached_keys=false)
|
150
|
+
bcdb_authorise()
|
151
|
+
schema_uri = URI.parse(@create_schema_url)
|
152
|
+
schema_properties = {}
|
153
|
+
data.each do |key|
|
154
|
+
schema_properties["#{key}"] = {
|
155
|
+
:"$id" => "/properties/#{schema_properties["#{key}"]}",
|
156
|
+
:type => "string",
|
157
|
+
:title => "The #{schema_properties["#{key}"]} Schema"
|
158
|
+
}
|
159
|
+
end
|
160
|
+
schema_data = {
|
161
|
+
:type => "object",
|
162
|
+
:"$id" => @bcdb_entity,
|
163
|
+
:"$schema" => "http://json-schema.org/draft-07/schema#",
|
164
|
+
:title => "The Root Schema",
|
165
|
+
:properties => schema_properties,
|
166
|
+
:autoId => true
|
167
|
+
}
|
168
|
+
body = JSON(schema_data)
|
169
|
+
|
170
|
+
if cached_keys
|
171
|
+
request = bcdb_url(schema_uri,'put', body)
|
172
|
+
else
|
173
|
+
request = bcdb_url(schema_uri,'post',body)
|
174
|
+
resp = JSON.parse(request.body)["code"] rescue @logger.error("[BCDB SCHEMA] Unexpected error")
|
175
|
+
if request.code == 403
|
176
|
+
@logger.error("Authentification failed please check your credentials")
|
177
|
+
elsif resp == 4009 || resp ==4000
|
178
|
+
request = bcdb_url(schema_uri,'put', body)
|
179
|
+
end
|
180
|
+
end
|
181
|
+
return data, true
|
182
|
+
end
|
183
|
+
def bcdb_url(uri,type,body)
|
184
|
+
bcdb_request = Net::HTTP.new(uri.host,uri.port)
|
185
|
+
bcdb_request.use_ssl = uri.scheme == 'https'
|
186
|
+
case type
|
187
|
+
when 'post'
|
188
|
+
request = Net::HTTP::Post.new(uri.path)
|
189
|
+
when 'put'
|
190
|
+
request = Net::HTTP::Put.new(uri.path)
|
191
|
+
end
|
192
|
+
request.body = body
|
193
|
+
request['Content-Type'] = "application/json"
|
194
|
+
request['authorization'] = "Bearer #{@token_oauth}"
|
195
|
+
response = bcdb_request.request(request)
|
196
|
+
return response
|
197
|
+
end
|
198
|
+
|
199
|
+
def register
|
200
|
+
@http_method = @http_method.to_sym
|
201
|
+
|
202
|
+
# We count outstanding requests with this queue
|
203
|
+
# This queue tracks the requests to create backpressure
|
204
|
+
# When this queue is empty no new requests may be sent,
|
205
|
+
# tokens must be added back by the client on success
|
206
|
+
@request_tokens = SizedQueue.new(@pool_max)
|
207
|
+
@pool_max.times {|t| @request_tokens << true }
|
208
|
+
|
209
|
+
@requests = Array.new
|
210
|
+
|
211
|
+
if @content_type.nil?
|
212
|
+
case @format
|
213
|
+
when "form" ; @content_type = "application/x-www-form-urlencoded"
|
214
|
+
when "json" ; @content_type = "application/json"
|
215
|
+
when "json_batch" ; @content_type = "application/json"
|
216
|
+
when "message" ; @content_type = "text/plain"
|
217
|
+
end
|
218
|
+
end
|
219
|
+
|
220
|
+
@is_batch = @format == "json_batch"
|
221
|
+
|
222
|
+
@headers["Content-Type"] = @content_type
|
223
|
+
|
224
|
+
validate_format!
|
225
|
+
bcdb_authorise()
|
226
|
+
@create_schema_url = "#{@base_url}" + "/data/catalog/_JsonSchema/" + "#{@bcdb_entity}"
|
227
|
+
if @format == "json_batch"
|
228
|
+
@url = "#{@base_url}" + "/data/bulkAsync/" + "#{@bcdb_entity}"
|
229
|
+
else
|
230
|
+
@url = "#{@base_url}" + "/data/" + "#{@bcdb_entity}"
|
231
|
+
end
|
232
|
+
|
233
|
+
# Run named Timer as daemon thread
|
234
|
+
@timer = java.util.Timer.new("HTTP Output #{self.params['id']}", true)
|
235
|
+
end # def register
|
236
|
+
|
237
|
+
def multi_receive(events)
|
238
|
+
return if events.empty?
|
239
|
+
send_events(events)
|
240
|
+
end
|
241
|
+
|
242
|
+
class RetryTimerTask < java.util.TimerTask
|
243
|
+
def initialize(pending, event, attempt)
|
244
|
+
@pending = pending
|
245
|
+
@event = event
|
246
|
+
@attempt = attempt
|
247
|
+
super()
|
248
|
+
end
|
249
|
+
|
250
|
+
def run
|
251
|
+
@pending << [@event, @attempt]
|
252
|
+
end
|
253
|
+
end
|
254
|
+
|
255
|
+
def log_retryable_response(response)
|
256
|
+
if (response.code == 429)
|
257
|
+
@logger.debug? && @logger.debug("Encountered a 429 response, will retry. This is not serious, just flow control via HTTP")
|
258
|
+
else
|
259
|
+
@logger.warn("Encountered a retryable HTTP request in HTTP output, will retry", :code => response.code, :body => response.body)
|
260
|
+
end
|
261
|
+
end
|
262
|
+
|
263
|
+
def log_error_response(response, url, event)
|
264
|
+
log_failure(
|
265
|
+
"Encountered non-2xx HTTP code #{response.code}",
|
266
|
+
:response_code => response.code,
|
267
|
+
:url => url,
|
268
|
+
:event => event
|
269
|
+
)
|
270
|
+
end
|
271
|
+
|
272
|
+
def send_events(events)
|
273
|
+
successes = java.util.concurrent.atomic.AtomicInteger.new(0)
|
274
|
+
failures = java.util.concurrent.atomic.AtomicInteger.new(0)
|
275
|
+
retries = java.util.concurrent.atomic.AtomicInteger.new(0)
|
276
|
+
event_count = @is_batch ? 1 : events.size
|
277
|
+
|
278
|
+
pending = Queue.new
|
279
|
+
if @is_batch
|
280
|
+
pending << [events, 0]
|
281
|
+
else
|
282
|
+
events.each {|e| pending << [e, 0]}
|
283
|
+
end
|
284
|
+
|
285
|
+
while popped = pending.pop
|
286
|
+
break if popped == :done
|
287
|
+
|
288
|
+
event, attempt = popped
|
289
|
+
|
290
|
+
action, event, attempt = send_event(event, attempt)
|
291
|
+
begin
|
292
|
+
action = :failure if action == :retry && !@retry_failed
|
293
|
+
|
294
|
+
case action
|
295
|
+
when :success
|
296
|
+
successes.incrementAndGet
|
297
|
+
when :retry
|
298
|
+
retries.incrementAndGet
|
299
|
+
|
300
|
+
next_attempt = attempt+1
|
301
|
+
sleep_for = sleep_for_attempt(next_attempt)
|
302
|
+
@logger.info("Retrying http request, will sleep for #{sleep_for} seconds")
|
303
|
+
timer_task = RetryTimerTask.new(pending, event, next_attempt)
|
304
|
+
@timer.schedule(timer_task, sleep_for*1000)
|
305
|
+
when :failure
|
306
|
+
failures.incrementAndGet
|
307
|
+
else
|
308
|
+
raise "Unknown action #{action}"
|
309
|
+
end
|
310
|
+
|
311
|
+
if action == :success || action == :failure
|
312
|
+
if successes.get+failures.get == event_count
|
313
|
+
pending << :done
|
314
|
+
end
|
315
|
+
end
|
316
|
+
rescue => e
|
317
|
+
# This should never happen unless there's a flat out bug in the code
|
318
|
+
@logger.error("Error sending HTTP Request",
|
319
|
+
:class => e.class.name,
|
320
|
+
:message => e.message,
|
321
|
+
:backtrace => e.backtrace)
|
322
|
+
failures.incrementAndGet
|
323
|
+
raise e
|
324
|
+
end
|
325
|
+
end
|
326
|
+
rescue => e
|
327
|
+
@logger.error("Error in http output loop",
|
328
|
+
:class => e.class.name,
|
329
|
+
:message => e.message,
|
330
|
+
:backtrace => e.backtrace)
|
331
|
+
raise e
|
332
|
+
end
|
333
|
+
|
334
|
+
def sleep_for_attempt(attempt)
|
335
|
+
sleep_for = attempt**2
|
336
|
+
sleep_for = sleep_for <= 60 ? sleep_for : 60
|
337
|
+
(sleep_for/2) + (rand(0..sleep_for)/2)
|
338
|
+
end
|
339
|
+
|
340
|
+
def send_event(event, attempt)
|
341
|
+
bcdb_authorise()
|
342
|
+
|
343
|
+
body = event_body(event)
|
344
|
+
# Send the request
|
345
|
+
url = @is_batch ? @url : event.sprintf(@url)
|
346
|
+
headers = @is_batch ? @headers : event_headers(event)
|
347
|
+
|
348
|
+
# Compress the body and add appropriate header
|
349
|
+
if @http_compression == true
|
350
|
+
headers["Content-Encoding"] = "gzip"
|
351
|
+
body = gzip(body)
|
352
|
+
end
|
353
|
+
|
354
|
+
# Create an async request
|
355
|
+
response = client.send(@http_method, url, :body => body, :headers => headers).call
|
356
|
+
@logger.debug("[MAKEING REQUEST] Url: #{url}, response #{response.inspect}")
|
357
|
+
if !response_success?(response)
|
358
|
+
if retryable_response?(response)
|
359
|
+
log_retryable_response(response)
|
360
|
+
return :retry, event, attempt
|
361
|
+
else
|
362
|
+
log_error_response(response, url, event)
|
363
|
+
return :failure, event, attempt
|
364
|
+
end
|
365
|
+
else
|
366
|
+
return :success, event, attempt
|
367
|
+
end
|
368
|
+
|
369
|
+
rescue => exception
|
370
|
+
will_retry = retryable_exception?(exception)
|
371
|
+
log_failure("Could not fetch URL",
|
372
|
+
:url => url,
|
373
|
+
:method => @http_method,
|
374
|
+
:body => body,
|
375
|
+
:headers => headers,
|
376
|
+
:message => exception.message,
|
377
|
+
:class => exception.class.name,
|
378
|
+
:backtrace => exception.backtrace,
|
379
|
+
:will_retry => will_retry
|
380
|
+
)
|
381
|
+
|
382
|
+
if will_retry
|
383
|
+
return :retry, event, attempt
|
384
|
+
else
|
385
|
+
return :failure, event, attempt
|
386
|
+
end
|
387
|
+
end
|
388
|
+
|
389
|
+
def close
|
390
|
+
@timer.cancel
|
391
|
+
client.close
|
392
|
+
end
|
393
|
+
|
394
|
+
private
|
395
|
+
|
396
|
+
def response_success?(response)
|
397
|
+
code = response.code
|
398
|
+
return true if @ignorable_codes && @ignorable_codes.include?(code)
|
399
|
+
return code >= 200 && code <= 299
|
400
|
+
end
|
401
|
+
|
402
|
+
def retryable_response?(response)
|
403
|
+
@retryable_codes && @retryable_codes.include?(response.code)
|
404
|
+
end
|
405
|
+
|
406
|
+
def retryable_exception?(exception)
|
407
|
+
RETRYABLE_MANTICORE_EXCEPTIONS.any? {|me| exception.is_a?(me) }
|
408
|
+
end
|
409
|
+
|
410
|
+
# This is split into a separate method mostly to help testing
|
411
|
+
def log_failure(message, opts)
|
412
|
+
@logger.error("[HTTP Output Failure] #{message}", opts)
|
413
|
+
end
|
414
|
+
|
415
|
+
# Format the HTTP body
|
416
|
+
def event_body(event)
|
417
|
+
bcdb_authorise()
|
418
|
+
# TODO: Create an HTTP post data codec, use that here
|
419
|
+
if @format == "json"
|
420
|
+
bcdb_keys = event.to_hash
|
421
|
+
{"headers": bcdb_keys.delete("headers")}
|
422
|
+
unless @cached_keys && @keys.sort == bcdb_keys.keys.sort
|
423
|
+
@keys, @cached_keys = bcdb_update_schema(bcdb_keys.keys, @cached_keys)
|
424
|
+
end
|
425
|
+
return LogStash::Json.dump(map_event(event))
|
426
|
+
elsif @format == "message"
|
427
|
+
event.sprintf(@message)
|
428
|
+
elsif @format == "json_batch"
|
429
|
+
event.map {|e|
|
430
|
+
if e.is_a?(Hash)
|
431
|
+
{"headers": e.delete("headers")}
|
432
|
+
bcdb_keys = e
|
433
|
+
elsif
|
434
|
+
bcdb_keys = e.to_hash
|
435
|
+
{"headers": bcdb_keys.delete("headers")}
|
436
|
+
end
|
437
|
+
unless @cached_keys && @keys.sort == bcdb_keys.keys.sort
|
438
|
+
@keys, @cached_keys = bcdb_update_schema(bcdb_keys.keys, @cached_keys)
|
439
|
+
end
|
440
|
+
map_event(e)
|
441
|
+
}
|
442
|
+
# data = { :records => [event] }
|
443
|
+
@logger.debug("[BATCH POST EVENTS] #{event.count}, DATA: #{LogStash::Json.dump({"records"=> event})}")
|
444
|
+
return LogStash::Json.dump({"records"=> event})
|
445
|
+
|
446
|
+
else
|
447
|
+
encode(map_event(event))
|
448
|
+
end
|
449
|
+
end
|
450
|
+
|
451
|
+
# gzip data
|
452
|
+
def gzip(data)
|
453
|
+
gz = StringIO.new
|
454
|
+
gz.set_encoding("BINARY")
|
455
|
+
z = Zlib::GzipWriter.new(gz)
|
456
|
+
z.write(data)
|
457
|
+
z.close
|
458
|
+
gz.string
|
459
|
+
end
|
460
|
+
|
461
|
+
def convert_mapping(mapping, event)
|
462
|
+
if mapping.is_a?(Hash)
|
463
|
+
mapping.reduce({}) do |acc, kv|
|
464
|
+
k, v = kv
|
465
|
+
acc[k] = convert_mapping(v, event)
|
466
|
+
acc
|
467
|
+
end
|
468
|
+
elsif mapping.is_a?(Array)
|
469
|
+
mapping.map { |elem| convert_mapping(elem, event) }
|
470
|
+
else
|
471
|
+
event.sprintf(mapping)
|
472
|
+
end
|
473
|
+
end
|
474
|
+
|
475
|
+
def map_event(event)
|
476
|
+
if @mapping
|
477
|
+
convert_mapping(@mapping, event)
|
478
|
+
else
|
479
|
+
event.to_hash
|
480
|
+
end
|
481
|
+
end
|
482
|
+
|
483
|
+
def event_headers(event)
|
484
|
+
custom_headers(event) || {}
|
485
|
+
end
|
486
|
+
|
487
|
+
def custom_headers(event)
|
488
|
+
return nil unless @headers
|
489
|
+
|
490
|
+
@headers.reduce({}) do |acc,kv|
|
491
|
+
k,v = kv
|
492
|
+
acc[k] = event.sprintf(v)
|
493
|
+
acc
|
494
|
+
end
|
495
|
+
end
|
496
|
+
|
497
|
+
#TODO Extract this to a codec
|
498
|
+
def encode(hash)
|
499
|
+
return hash.collect do |key, value|
|
500
|
+
CGI.escape(key) + "=" + CGI.escape(value.to_s)
|
501
|
+
end.join("&")
|
502
|
+
end
|
503
|
+
|
504
|
+
|
505
|
+
def validate_format!
|
506
|
+
if @format == "message"
|
507
|
+
if @message.nil?
|
508
|
+
raise "message must be set if message format is used"
|
509
|
+
end
|
510
|
+
|
511
|
+
if @content_type.nil?
|
512
|
+
raise "content_type must be set if message format is used"
|
513
|
+
end
|
514
|
+
|
515
|
+
unless @mapping.nil?
|
516
|
+
@logger.warn "mapping is not supported and will be ignored if message format is used"
|
517
|
+
end
|
518
|
+
end
|
519
|
+
end
|
520
|
+
end
|