logstash-output-bcdb 0.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,520 @@
1
+ # encoding: utf-8
2
+ require "logstash/outputs/base"
3
+ require "logstash/namespace"
4
+ require "logstash/json"
5
+ require "uri"
6
+ require "logstash/plugin_mixins/http_client"
7
+ require "zlib"
8
+ require 'json'
9
+ require 'net/http'
10
+
11
+ class LogStash::Outputs::Bcdb < LogStash::Outputs::Base
12
+ include LogStash::PluginMixins::HttpClient
13
+
14
+ concurrency :shared
15
+
16
+ attr_accessor :is_batch
17
+
18
+ VALID_METHODS = ["put", "post", "patch", "delete", "get", "head"]
19
+
20
+ RETRYABLE_MANTICORE_EXCEPTIONS = [
21
+ ::Manticore::Timeout,
22
+ ::Manticore::SocketException,
23
+ ::Manticore::ClientProtocolException,
24
+ ::Manticore::ResolutionFailure,
25
+ ::Manticore::SocketTimeout
26
+ ]
27
+
28
+ # This output lets you send events to a
29
+ # generic HTTP(S) endpoint
30
+ #
31
+ # This output will execute up to 'pool_max' requests in parallel for performance.
32
+ # Consider this when tuning this plugin for performance.
33
+ #
34
+ # Additionally, note that when parallel execution is used strict ordering of events is not
35
+ # guaranteed!
36
+ #
37
+ # Beware, this gem does not yet support codecs. Please use the 'format' option for now.
38
+
39
+ config_name "bcdb"
40
+
41
+ config :url, :validate => :string
42
+ # BCDB data endpoint
43
+ config :base_url, :validate => :string, :required => :true
44
+
45
+ # BCDB Auth endpoint
46
+ config :auth_url, :validate => :string, :required => :true
47
+
48
+ # BCDB database entity model name
49
+ config :bcdb_entity, :validate => :string, :default => "loglines"
50
+
51
+ config :username, :validate => :string, :required => :true
52
+ config :password, :validate => :string, :required => :true
53
+ config :client_id, :validate => :string, :required => :true
54
+ config :client_secret, :validate => :string, :required => :true
55
+ config :grant_type, :validate => ["password", "authorization_code"], :default => "password"
56
+
57
+ # The HTTP Verb. One of "put", "post", "patch", "delete", "get", "head"
58
+ config :http_method, :validate => VALID_METHODS, :default => "post"
59
+
60
+ # Custom headers to use
61
+ # format is `headers => ["X-My-Header", "%{host}"]`
62
+ config :headers, :validate => :hash, :default => {}
63
+
64
+ # Content type
65
+ #
66
+ # If not specified, this defaults to the following:
67
+ #
68
+ # * if format is "json", "application/json"
69
+ # * if format is "form", "application/x-www-form-urlencoded"
70
+ config :content_type, :validate => :string
71
+
72
+ # Set this to false if you don't want this output to retry failed requests
73
+ config :retry_failed, :validate => :boolean, :default => true
74
+
75
+ # If encountered as response codes this plugin will retry these requests
76
+ config :retryable_codes, :validate => :number, :list => true, :default => [429, 500, 502, 503, 504]
77
+
78
+ # If you would like to consider some non-2xx codes to be successes
79
+ # enumerate them here. Responses returning these codes will be considered successes
80
+ config :ignorable_codes, :validate => :number, :list => true
81
+
82
+ # This lets you choose the structure and parts of the event that are sent.
83
+ #
84
+ #
85
+ # For example:
86
+ # [source,ruby]
87
+ # mapping => {"foo" => "%{host}"
88
+ # "bar" => "%{type}"}
89
+ config :mapping, :validate => :hash
90
+
91
+ # Set the format of the http body.
92
+ #
93
+ # If form, then the body will be the mapping (or whole event) converted
94
+ # into a query parameter string, e.g. `foo=bar&baz=fizz...`
95
+ #
96
+ # If message, then the body will be the result of formatting the event according to message
97
+ #
98
+ # Otherwise, the event is sent as json.
99
+ config :format, :validate => ["json", "json_batch"], :default => "json"
100
+
101
+ # Set this to true if you want to enable gzip compression for your http requests
102
+ config :http_compression, :validate => :boolean, :default => false
103
+
104
+ config :message, :validate => :string
105
+
106
+
107
+ def bcdb_authorise()
108
+ auth_uri = URI.parse(@auth_url)
109
+ auth_data = {
110
+ :username => @username,
111
+ :password => @password,
112
+ :client_id => @client_id,
113
+ :client_secret => @client_secret,
114
+ :grant_type => @grant_type
115
+ }
116
+ status = true
117
+ begin
118
+ unless (@token_oauth && (@expires_token && Time.now.utc < @expires_token))
119
+ https= Net::HTTP.new(auth_uri.host,auth_uri.port)
120
+ https.use_ssl = auth_uri.scheme == 'https'
121
+
122
+ request = Net::HTTP::Post.new(auth_uri.path)
123
+ request.set_form_data(auth_data)
124
+ request['Content-Type'] = "application/x-www-form-urlencoded"
125
+ resp = https.request(request)
126
+ bcdb_response = {}
127
+ bcdb_response = JSON.parse(resp.body) rescue bcdb_response["code"] = 5000.to_s
128
+ if resp.code == 200.to_s && bcdb_response['access_token']
129
+ @token_oauth = bcdb_response['access_token']
130
+ @headers["Authorization"] = "Bearer #{@token_oauth}"
131
+ @expires_token = Time.now.utc + bcdb_response['expires_in'].to_i
132
+ else
133
+ status = false
134
+ @logger.error("Authentification failed please check your credentials")
135
+ end
136
+ end
137
+ rescue => e
138
+ # This should never happen unless there's a flat out bug in the code
139
+ @logger.error("Error Makeing Authorization Request to BCDB",
140
+ :class => e.class.name,
141
+ :message => e.message,
142
+ :backtrace => e.backtrace)
143
+ sleep(2)
144
+ bcdb_authorise()
145
+ end
146
+ return status
147
+ end
148
+
149
+ def bcdb_update_schema(data, cached_keys=false)
150
+ bcdb_authorise()
151
+ schema_uri = URI.parse(@create_schema_url)
152
+ schema_properties = {}
153
+ data.each do |key|
154
+ schema_properties["#{key}"] = {
155
+ :"$id" => "/properties/#{schema_properties["#{key}"]}",
156
+ :type => "string",
157
+ :title => "The #{schema_properties["#{key}"]} Schema"
158
+ }
159
+ end
160
+ schema_data = {
161
+ :type => "object",
162
+ :"$id" => @bcdb_entity,
163
+ :"$schema" => "http://json-schema.org/draft-07/schema#",
164
+ :title => "The Root Schema",
165
+ :properties => schema_properties,
166
+ :autoId => true
167
+ }
168
+ body = JSON(schema_data)
169
+
170
+ if cached_keys
171
+ request = bcdb_url(schema_uri,'put', body)
172
+ else
173
+ request = bcdb_url(schema_uri,'post',body)
174
+ resp = JSON.parse(request.body)["code"] rescue @logger.error("[BCDB SCHEMA] Unexpected error")
175
+ if request.code == 403
176
+ @logger.error("Authentification failed please check your credentials")
177
+ elsif resp == 4009 || resp ==4000
178
+ request = bcdb_url(schema_uri,'put', body)
179
+ end
180
+ end
181
+ return data, true
182
+ end
183
+ def bcdb_url(uri,type,body)
184
+ bcdb_request = Net::HTTP.new(uri.host,uri.port)
185
+ bcdb_request.use_ssl = uri.scheme == 'https'
186
+ case type
187
+ when 'post'
188
+ request = Net::HTTP::Post.new(uri.path)
189
+ when 'put'
190
+ request = Net::HTTP::Put.new(uri.path)
191
+ end
192
+ request.body = body
193
+ request['Content-Type'] = "application/json"
194
+ request['authorization'] = "Bearer #{@token_oauth}"
195
+ response = bcdb_request.request(request)
196
+ return response
197
+ end
198
+
199
+ def register
200
+ @http_method = @http_method.to_sym
201
+
202
+ # We count outstanding requests with this queue
203
+ # This queue tracks the requests to create backpressure
204
+ # When this queue is empty no new requests may be sent,
205
+ # tokens must be added back by the client on success
206
+ @request_tokens = SizedQueue.new(@pool_max)
207
+ @pool_max.times {|t| @request_tokens << true }
208
+
209
+ @requests = Array.new
210
+
211
+ if @content_type.nil?
212
+ case @format
213
+ when "form" ; @content_type = "application/x-www-form-urlencoded"
214
+ when "json" ; @content_type = "application/json"
215
+ when "json_batch" ; @content_type = "application/json"
216
+ when "message" ; @content_type = "text/plain"
217
+ end
218
+ end
219
+
220
+ @is_batch = @format == "json_batch"
221
+
222
+ @headers["Content-Type"] = @content_type
223
+
224
+ validate_format!
225
+ bcdb_authorise()
226
+ @create_schema_url = "#{@base_url}" + "/data/catalog/_JsonSchema/" + "#{@bcdb_entity}"
227
+ if @format == "json_batch"
228
+ @url = "#{@base_url}" + "/data/bulkAsync/" + "#{@bcdb_entity}"
229
+ else
230
+ @url = "#{@base_url}" + "/data/" + "#{@bcdb_entity}"
231
+ end
232
+
233
+ # Run named Timer as daemon thread
234
+ @timer = java.util.Timer.new("HTTP Output #{self.params['id']}", true)
235
+ end # def register
236
+
237
+ def multi_receive(events)
238
+ return if events.empty?
239
+ send_events(events)
240
+ end
241
+
242
+ class RetryTimerTask < java.util.TimerTask
243
+ def initialize(pending, event, attempt)
244
+ @pending = pending
245
+ @event = event
246
+ @attempt = attempt
247
+ super()
248
+ end
249
+
250
+ def run
251
+ @pending << [@event, @attempt]
252
+ end
253
+ end
254
+
255
+ def log_retryable_response(response)
256
+ if (response.code == 429)
257
+ @logger.debug? && @logger.debug("Encountered a 429 response, will retry. This is not serious, just flow control via HTTP")
258
+ else
259
+ @logger.warn("Encountered a retryable HTTP request in HTTP output, will retry", :code => response.code, :body => response.body)
260
+ end
261
+ end
262
+
263
+ def log_error_response(response, url, event)
264
+ log_failure(
265
+ "Encountered non-2xx HTTP code #{response.code}",
266
+ :response_code => response.code,
267
+ :url => url,
268
+ :event => event
269
+ )
270
+ end
271
+
272
+ def send_events(events)
273
+ successes = java.util.concurrent.atomic.AtomicInteger.new(0)
274
+ failures = java.util.concurrent.atomic.AtomicInteger.new(0)
275
+ retries = java.util.concurrent.atomic.AtomicInteger.new(0)
276
+ event_count = @is_batch ? 1 : events.size
277
+
278
+ pending = Queue.new
279
+ if @is_batch
280
+ pending << [events, 0]
281
+ else
282
+ events.each {|e| pending << [e, 0]}
283
+ end
284
+
285
+ while popped = pending.pop
286
+ break if popped == :done
287
+
288
+ event, attempt = popped
289
+
290
+ action, event, attempt = send_event(event, attempt)
291
+ begin
292
+ action = :failure if action == :retry && !@retry_failed
293
+
294
+ case action
295
+ when :success
296
+ successes.incrementAndGet
297
+ when :retry
298
+ retries.incrementAndGet
299
+
300
+ next_attempt = attempt+1
301
+ sleep_for = sleep_for_attempt(next_attempt)
302
+ @logger.info("Retrying http request, will sleep for #{sleep_for} seconds")
303
+ timer_task = RetryTimerTask.new(pending, event, next_attempt)
304
+ @timer.schedule(timer_task, sleep_for*1000)
305
+ when :failure
306
+ failures.incrementAndGet
307
+ else
308
+ raise "Unknown action #{action}"
309
+ end
310
+
311
+ if action == :success || action == :failure
312
+ if successes.get+failures.get == event_count
313
+ pending << :done
314
+ end
315
+ end
316
+ rescue => e
317
+ # This should never happen unless there's a flat out bug in the code
318
+ @logger.error("Error sending HTTP Request",
319
+ :class => e.class.name,
320
+ :message => e.message,
321
+ :backtrace => e.backtrace)
322
+ failures.incrementAndGet
323
+ raise e
324
+ end
325
+ end
326
+ rescue => e
327
+ @logger.error("Error in http output loop",
328
+ :class => e.class.name,
329
+ :message => e.message,
330
+ :backtrace => e.backtrace)
331
+ raise e
332
+ end
333
+
334
+ def sleep_for_attempt(attempt)
335
+ sleep_for = attempt**2
336
+ sleep_for = sleep_for <= 60 ? sleep_for : 60
337
+ (sleep_for/2) + (rand(0..sleep_for)/2)
338
+ end
339
+
340
+ def send_event(event, attempt)
341
+ bcdb_authorise()
342
+
343
+ body = event_body(event)
344
+ # Send the request
345
+ url = @is_batch ? @url : event.sprintf(@url)
346
+ headers = @is_batch ? @headers : event_headers(event)
347
+
348
+ # Compress the body and add appropriate header
349
+ if @http_compression == true
350
+ headers["Content-Encoding"] = "gzip"
351
+ body = gzip(body)
352
+ end
353
+
354
+ # Create an async request
355
+ response = client.send(@http_method, url, :body => body, :headers => headers).call
356
+ @logger.debug("[MAKEING REQUEST] Url: #{url}, response #{response.inspect}")
357
+ if !response_success?(response)
358
+ if retryable_response?(response)
359
+ log_retryable_response(response)
360
+ return :retry, event, attempt
361
+ else
362
+ log_error_response(response, url, event)
363
+ return :failure, event, attempt
364
+ end
365
+ else
366
+ return :success, event, attempt
367
+ end
368
+
369
+ rescue => exception
370
+ will_retry = retryable_exception?(exception)
371
+ log_failure("Could not fetch URL",
372
+ :url => url,
373
+ :method => @http_method,
374
+ :body => body,
375
+ :headers => headers,
376
+ :message => exception.message,
377
+ :class => exception.class.name,
378
+ :backtrace => exception.backtrace,
379
+ :will_retry => will_retry
380
+ )
381
+
382
+ if will_retry
383
+ return :retry, event, attempt
384
+ else
385
+ return :failure, event, attempt
386
+ end
387
+ end
388
+
389
+ def close
390
+ @timer.cancel
391
+ client.close
392
+ end
393
+
394
+ private
395
+
396
+ def response_success?(response)
397
+ code = response.code
398
+ return true if @ignorable_codes && @ignorable_codes.include?(code)
399
+ return code >= 200 && code <= 299
400
+ end
401
+
402
+ def retryable_response?(response)
403
+ @retryable_codes && @retryable_codes.include?(response.code)
404
+ end
405
+
406
+ def retryable_exception?(exception)
407
+ RETRYABLE_MANTICORE_EXCEPTIONS.any? {|me| exception.is_a?(me) }
408
+ end
409
+
410
+ # This is split into a separate method mostly to help testing
411
+ def log_failure(message, opts)
412
+ @logger.error("[HTTP Output Failure] #{message}", opts)
413
+ end
414
+
415
+ # Format the HTTP body
416
+ def event_body(event)
417
+ bcdb_authorise()
418
+ # TODO: Create an HTTP post data codec, use that here
419
+ if @format == "json"
420
+ bcdb_keys = event.to_hash
421
+ {"headers": bcdb_keys.delete("headers")}
422
+ unless @cached_keys && @keys.sort == bcdb_keys.keys.sort
423
+ @keys, @cached_keys = bcdb_update_schema(bcdb_keys.keys, @cached_keys)
424
+ end
425
+ return LogStash::Json.dump(map_event(event))
426
+ elsif @format == "message"
427
+ event.sprintf(@message)
428
+ elsif @format == "json_batch"
429
+ event.map {|e|
430
+ if e.is_a?(Hash)
431
+ {"headers": e.delete("headers")}
432
+ bcdb_keys = e
433
+ elsif
434
+ bcdb_keys = e.to_hash
435
+ {"headers": bcdb_keys.delete("headers")}
436
+ end
437
+ unless @cached_keys && @keys.sort == bcdb_keys.keys.sort
438
+ @keys, @cached_keys = bcdb_update_schema(bcdb_keys.keys, @cached_keys)
439
+ end
440
+ map_event(e)
441
+ }
442
+ # data = { :records => [event] }
443
+ @logger.debug("[BATCH POST EVENTS] #{event.count}, DATA: #{LogStash::Json.dump({"records"=> event})}")
444
+ return LogStash::Json.dump({"records"=> event})
445
+
446
+ else
447
+ encode(map_event(event))
448
+ end
449
+ end
450
+
451
+ # gzip data
452
+ def gzip(data)
453
+ gz = StringIO.new
454
+ gz.set_encoding("BINARY")
455
+ z = Zlib::GzipWriter.new(gz)
456
+ z.write(data)
457
+ z.close
458
+ gz.string
459
+ end
460
+
461
+ def convert_mapping(mapping, event)
462
+ if mapping.is_a?(Hash)
463
+ mapping.reduce({}) do |acc, kv|
464
+ k, v = kv
465
+ acc[k] = convert_mapping(v, event)
466
+ acc
467
+ end
468
+ elsif mapping.is_a?(Array)
469
+ mapping.map { |elem| convert_mapping(elem, event) }
470
+ else
471
+ event.sprintf(mapping)
472
+ end
473
+ end
474
+
475
+ def map_event(event)
476
+ if @mapping
477
+ convert_mapping(@mapping, event)
478
+ else
479
+ event.to_hash
480
+ end
481
+ end
482
+
483
+ def event_headers(event)
484
+ custom_headers(event) || {}
485
+ end
486
+
487
+ def custom_headers(event)
488
+ return nil unless @headers
489
+
490
+ @headers.reduce({}) do |acc,kv|
491
+ k,v = kv
492
+ acc[k] = event.sprintf(v)
493
+ acc
494
+ end
495
+ end
496
+
497
+ #TODO Extract this to a codec
498
+ def encode(hash)
499
+ return hash.collect do |key, value|
500
+ CGI.escape(key) + "=" + CGI.escape(value.to_s)
501
+ end.join("&")
502
+ end
503
+
504
+
505
+ def validate_format!
506
+ if @format == "message"
507
+ if @message.nil?
508
+ raise "message must be set if message format is used"
509
+ end
510
+
511
+ if @content_type.nil?
512
+ raise "content_type must be set if message format is used"
513
+ end
514
+
515
+ unless @mapping.nil?
516
+ @logger.warn "mapping is not supported and will be ignored if message format is used"
517
+ end
518
+ end
519
+ end
520
+ end