logstash-output-sumologic 1.1.9 → 1.3.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -1,11 +1,11 @@
1
1
  # encoding: utf-8
2
- require "stringio"
3
- require "zlib"
4
- require "logstash/outputs/sumologic/common"
5
2
 
6
3
  module LogStash; module Outputs; class SumoLogic;
7
4
  class Compressor
8
5
 
6
+ require "stringio"
7
+ require "zlib"
8
+ require "logstash/outputs/sumologic/common"
9
9
  include LogStash::Outputs::SumoLogic::Common
10
10
 
11
11
  def initialize(config)
@@ -1,27 +1,12 @@
1
1
  # encoding: utf-8
2
- require "socket"
3
- require "logstash/outputs/sumologic/common"
4
2
 
5
3
  module LogStash; module Outputs; class SumoLogic;
6
4
  class HeaderBuilder
7
5
 
6
+ require "socket"
7
+ require "logstash/outputs/sumologic/common"
8
8
  include LogStash::Outputs::SumoLogic::Common
9
9
 
10
- CONTENT_TYPE = "Content-Type"
11
- CONTENT_TYPE_LOG = "text/plain"
12
- CONTENT_TYPE_GRAPHITE = "application/vnd.sumologic.graphite"
13
- CONTENT_TYPE_CARBON2 = "application/vnd.sumologic.carbon2"
14
- CONTENT_ENCODING = "Content-Encoding"
15
-
16
- CATEGORY_HEADER = "X-Sumo-Category"
17
- CATEGORY_HEADER_DEFAULT = "Logstash"
18
- HOST_HEADER = "X-Sumo-Host"
19
- NAME_HEADER = "X-Sumo-Name"
20
- NAME_HEADER_DEFAULT = "logstash-output-sumologic"
21
-
22
- CLIENT_HEADER = "X-Sumo-Client"
23
- CLIENT_HEADER_VALUE = "logstash-output-sumologic"
24
-
25
10
  def initialize(config)
26
11
 
27
12
  @extra_headers = config["extra_headers"] ||= {}
@@ -36,31 +21,19 @@ module LogStash; module Outputs; class SumoLogic;
36
21
 
37
22
  end # def initialize
38
23
 
39
- def build()
40
- headers = build_common()
41
- headers[CATEGORY_HEADER] = @source_category unless @source_category.blank?
42
- append_content_header(headers)
43
- headers
44
- end # def build
45
-
46
- def build_stats()
47
- headers = build_common()
48
- headers[CATEGORY_HEADER] = "#{@source_category}.stats"
49
- headers[CONTENT_TYPE] = CONTENT_TYPE_CARBON2
50
- headers
51
- end # def build_stats
52
-
53
- private
54
- def build_common()
55
- headers = Hash.new()
24
+ def build(event)
25
+ headers = Hash.new
56
26
  headers.merge!(@extra_headers)
57
27
  headers[CLIENT_HEADER] = CLIENT_HEADER_VALUE
58
- headers[HOST_HEADER] = @source_host unless @source_host.blank?
59
- headers[NAME_HEADER] = @source_name unless @source_name.blank?
28
+ headers[CATEGORY_HEADER] = event.sprintf(@source_category) unless blank?(@source_category)
29
+ headers[HOST_HEADER] = event.sprintf(@source_host) unless blank?(@source_host)
30
+ headers[NAME_HEADER] = event.sprintf(@source_name) unless blank?(@source_name)
31
+ append_content_header(headers)
60
32
  append_compress_header(headers)
61
33
  headers
62
- end # build_common
34
+ end # def build
63
35
 
36
+ private
64
37
  def append_content_header(headers)
65
38
  contentType = CONTENT_TYPE_LOG
66
39
  if @metrics || @fields_as_metrics
@@ -76,4 +49,4 @@ module LogStash; module Outputs; class SumoLogic;
76
49
  end # append_compress_header
77
50
 
78
51
  end
79
- end; end; end
52
+ end; end; end
@@ -1,28 +1,43 @@
1
1
  # encoding: utf-8
2
- require "logstash/outputs/sumologic/common"
3
- require "logstash/outputs/sumologic/statistics"
4
-
5
2
  module LogStash; module Outputs; class SumoLogic;
6
3
  class MessageQueue
7
4
 
5
+ require "logstash/outputs/sumologic/common"
6
+ require "logstash/outputs/sumologic/statistics"
7
+ include LogStash::Outputs::SumoLogic::Common
8
+
8
9
  def initialize(stats, config)
9
10
  @queue_max = (config["queue_max"] ||= 1) < 1 ? 1 : config["queue_max"]
10
11
  @queue = SizedQueue::new(@queue_max)
12
+ log_info("initialize memory queue", :max => @queue_max)
13
+ @queue_bytesize = Concurrent::AtomicFixnum.new
11
14
  @stats = stats
12
- end
15
+ end # def initialize
13
16
 
14
- def enq(obj)
15
- if (obj.bytesize > 0)
16
- @queue.enq(obj)
17
- @stats.record_enque(obj)
18
- end
19
- end # def push
17
+ def enq(batch)
18
+ batch_size = batch.payload.bytesize
19
+ if (batch_size > 0)
20
+ @queue.enq(batch)
21
+ @stats.record_enque(batch_size)
22
+ @queue_bytesize.update { |v| v + batch_size }
23
+ log_dbg("enqueue",
24
+ :objects_in_queue => size,
25
+ :bytes_in_queue => @queue_bytesize,
26
+ :size => batch_size)
27
+ end
28
+ end # def enq
20
29
 
21
30
  def deq()
22
- obj = @queue.deq()
23
- @stats.record_deque(obj)
24
- obj
25
- end # def pop
31
+ batch = @queue.deq()
32
+ batch_size = batch.payload.bytesize
33
+ @stats.record_deque(batch_size)
34
+ @queue_bytesize.update { |v| v - batch_size }
35
+ log_dbg("dequeue",
36
+ :objects_in_queue => size,
37
+ :bytes_in_queue => @queue_bytesize,
38
+ :size => batch_size)
39
+ batch
40
+ end # def deq
26
41
 
27
42
  def drain()
28
43
  @queue.size.times.map {
@@ -33,6 +48,10 @@ module LogStash; module Outputs; class SumoLogic;
33
48
  def size()
34
49
  @queue.size()
35
50
  end # size
51
+
52
+ def bytesize()
53
+ @queue_bytesize.value
54
+ end # bytesize
36
55
 
37
56
  end
38
57
  end; end; end
@@ -1,11 +1,11 @@
1
1
  # encoding: utf-8
2
- require "logstash/outputs/sumologic/common"
3
- require "logstash/outputs/sumologic/statistics"
4
- require "logstash/outputs/sumologic/message_queue"
5
2
 
6
3
  module LogStash; module Outputs; class SumoLogic;
7
4
  class Monitor
8
5
 
6
+ require "logstash/outputs/sumologic/common"
7
+ require "logstash/outputs/sumologic/statistics"
8
+ require "logstash/outputs/sumologic/message_queue"
9
9
  include LogStash::Outputs::SumoLogic::Common
10
10
 
11
11
  attr_reader :is_pile
@@ -21,6 +21,7 @@ module LogStash; module Outputs; class SumoLogic;
21
21
  end # initialize
22
22
 
23
23
  def start()
24
+ log_info("starting monitor...", :interval => @interval)
24
25
  @stopping.make_false()
25
26
  if (@enabled)
26
27
  @monitor_t = Thread.new {
@@ -37,9 +38,9 @@ module LogStash; module Outputs; class SumoLogic;
37
38
  def stop()
38
39
  @stopping.make_true()
39
40
  if (@enabled)
40
- log_info "shutting down monitor..."
41
+ log_info("shutting down monitor...")
41
42
  @monitor_t.join
42
- log_info "monitor is fully shutted down"
43
+ log_info("monitor is fully shutted down")
43
44
  end
44
45
  end # def stop
45
46
 
@@ -58,9 +59,12 @@ module LogStash; module Outputs; class SumoLogic;
58
59
  "total_response_success"
59
60
  ].map { |key|
60
61
  value = @stats.send(key).value
62
+ log_dbg("stats",
63
+ :key => key,
64
+ :value => value)
61
65
  build_metric_line(key, value, timestamp)
62
66
  }.join($/)
63
-
67
+
64
68
  "#{STATS_TAG}#{counters}"
65
69
  end # def build_stats_payload
66
70
 
@@ -1,12 +1,11 @@
1
1
  # encoding: utf-8
2
- require "logstash/json"
3
- require "logstash/event"
4
-
5
- require "logstash/outputs/sumologic/common"
6
2
 
7
3
  module LogStash; module Outputs; class SumoLogic;
8
4
  class PayloadBuilder
9
5
 
6
+ require "logstash/json"
7
+ require "logstash/event"
8
+ require "logstash/outputs/sumologic/common"
10
9
  include LogStash::Outputs::SumoLogic::Common
11
10
 
12
11
  TIMESTAMP_FIELD = "@timestamp"
@@ -128,12 +127,17 @@ module LogStash; module Outputs; class SumoLogic;
128
127
  end # def expand_hash
129
128
 
130
129
  def apply_template(template, event)
131
- if template.include? JSON_PLACEHOLDER
130
+ if template == JSON_PLACEHOLDER
131
+ hash = event2hash(event)
132
+ LogStash::Json.dump(hash)
133
+ elsif template.include? JSON_PLACEHOLDER
134
+ result = event.sprintf(template)
132
135
  hash = event2hash(event)
133
136
  dump = LogStash::Json.dump(hash)
134
- template = template.gsub(JSON_PLACEHOLDER) { dump }
137
+ result.gsub(JSON_PLACEHOLDER) { dump }
138
+ else
139
+ event.sprintf(template)
135
140
  end
136
- event.sprintf(template)
137
141
  end # def expand
138
142
 
139
143
  def get_metrics_name(event, name)
@@ -1,11 +1,11 @@
1
1
  # encoding: utf-8
2
- require "logstash/outputs/sumologic/common"
3
- require "logstash/outputs/sumologic/statistics"
4
- require "logstash/outputs/sumologic/message_queue"
5
2
 
6
3
  module LogStash; module Outputs; class SumoLogic;
7
4
  class Piler
8
5
 
6
+ require "logstash/outputs/sumologic/common"
7
+ require "logstash/outputs/sumologic/statistics"
8
+ require "logstash/outputs/sumologic/message_queue"
9
9
  include LogStash::Outputs::SumoLogic::Common
10
10
 
11
11
  attr_reader :is_pile
@@ -17,23 +17,26 @@ module LogStash; module Outputs; class SumoLogic;
17
17
  @queue = queue
18
18
  @stats = stats
19
19
  @stopping = Concurrent::AtomicBoolean.new(false)
20
+ @payload_builder = PayloadBuilder.new(@stats, config)
21
+ @header_builder = HeaderBuilder.new(config)
20
22
  @is_pile = (@interval > 0 && @pile_max > 0)
21
-
22
23
  if (@is_pile)
23
- @pile = Array.new
24
- @pile_size = 0
24
+ @pile = Hash.new("")
25
25
  @semaphore = Mutex.new
26
26
  end
27
-
27
+
28
28
  end # def initialize
29
29
 
30
30
  def start()
31
31
  @stopping.make_false()
32
32
  if (@is_pile)
33
+ log_info("starting piler...",
34
+ :max => @pile_max,
35
+ :timeout => @interval)
33
36
  @piler_t = Thread.new {
34
37
  while @stopping.false?
35
38
  Stud.stoppable_sleep(@interval) { @stopping.true? }
36
- log_dbg("timeout, enqueue pile now")
39
+ log_dbg("timeout", :timeout => @interval)
37
40
  enq_and_clear()
38
41
  end # while
39
42
  }
@@ -43,45 +46,44 @@ module LogStash; module Outputs; class SumoLogic;
43
46
  def stop()
44
47
  @stopping.make_true()
45
48
  if (@is_pile)
46
- log_info "shutting down piler..."
47
- @piler_t.join
48
- log_info "piler is fully shutted down"
49
+ log_info("shutting down piler in #{@interval * 2} secs ...")
50
+ @piler_t.join(@interval * 2)
51
+ log_info("piler is fully shutted down")
49
52
  end
50
53
  end # def stop
51
54
 
52
- def input(entry)
55
+ def input(event)
53
56
  if (@stopping.true?)
54
- log_warn "piler is shutting down, message ignored", "message" => entry
55
- elsif (@is_pile)
56
- @semaphore.synchronize {
57
- if @pile_size + entry.bytesize > @pile_max
58
- @queue.enq(@pile.join($/))
59
- @pile.clear
60
- @pile_size = 0
61
- @stats.record_clear_pile()
62
- end
63
- @pile << entry
64
- @pile_size += entry.bytesize
65
- @stats.record_input(entry)
66
- }
57
+ log_warn("piler is shutting down, event is dropped",
58
+ "event" => event)
67
59
  else
68
- @queue.enq(entry)
69
- end # if
60
+ headers = @header_builder.build(event)
61
+ payload = @payload_builder.build(event)
62
+ if (@is_pile)
63
+ @semaphore.synchronize {
64
+ content = @pile[headers]
65
+ size = content.bytesize
66
+ if size + payload.bytesize > @pile_max
67
+ @queue.enq(Batch.new(headers, content))
68
+ @pile[headers] = ""
69
+ end
70
+ @pile[headers] = blank?(@pile[headers]) ? payload : "#{@pile[headers]}\n#{payload}"
71
+ }
72
+ else
73
+ @queue.enq(Batch.new(headers, payload))
74
+ end # if
75
+ end
70
76
  end # def input
71
77
 
72
78
  private
73
79
  def enq_and_clear()
74
- if (@pile.size > 0)
75
- @semaphore.synchronize {
76
- if (@pile.size > 0)
77
- @queue.enq(@pile.join($/))
78
- @pile.clear
79
- @pile_size = 0
80
- @stats.record_clear_pile()
81
- end
82
- }
83
- end
80
+ @semaphore.synchronize {
81
+ @pile.each do |headers, content|
82
+ @queue.enq(Batch.new(headers, content))
83
+ end
84
+ @pile.clear()
85
+ }
84
86
  end # def enq_and_clear
85
87
 
86
88
  end
87
- end; end; end
89
+ end; end; end
@@ -1,19 +1,19 @@
1
1
  # encoding: utf-8
2
- require "net/https"
3
- require "socket"
4
- require "thread"
5
- require "uri"
6
- require "logstash/outputs/sumologic/common"
7
- require "logstash/outputs/sumologic/compressor"
8
- require "logstash/outputs/sumologic/header_builder"
9
- require "logstash/outputs/sumologic/statistics"
10
- require "logstash/outputs/sumologic/message_queue"
11
2
 
12
3
  module LogStash; module Outputs; class SumoLogic;
13
4
  class Sender
14
5
 
6
+ require "net/https"
7
+ require "socket"
8
+ require "thread"
9
+ require "uri"
10
+ require "logstash/outputs/sumologic/common"
11
+ require "logstash/outputs/sumologic/compressor"
12
+ require "logstash/outputs/sumologic/header_builder"
13
+ require "logstash/outputs/sumologic/statistics"
14
+ require "logstash/outputs/sumologic/message_queue"
15
15
  include LogStash::Outputs::SumoLogic::Common
16
- STOP_TAG = "PLUGIN STOPPED"
16
+
17
17
 
18
18
  def initialize(client, queue, stats, config)
19
19
  @client = client
@@ -28,37 +28,37 @@ module LogStash; module Outputs; class SumoLogic;
28
28
  @tokens = SizedQueue.new(@sender_max)
29
29
  @sender_max.times { |t| @tokens << t }
30
30
 
31
- @header_builder = LogStash::Outputs::SumoLogic::HeaderBuilder.new(config)
32
- @headers = @header_builder.build()
33
- @stats_headers = @header_builder.build_stats()
34
31
  @compressor = LogStash::Outputs::SumoLogic::Compressor.new(config)
35
32
 
36
33
  end # def initialize
37
34
 
38
35
  def start()
36
+ log_info("starting sender...",
37
+ :max => @sender_max,
38
+ :requeue => @sleep_before_requeue)
39
39
  @stopping.make_false()
40
40
  @sender_t = Thread.new {
41
41
  while @stopping.false?
42
- content = @queue.deq()
43
- send_request(content)
42
+ batch = @queue.deq()
43
+ send_request(batch)
44
44
  end # while
45
- @queue.drain().map { |content|
46
- send_request(content)
45
+ @queue.drain().map { |batch|
46
+ send_request(batch)
47
47
  }
48
- log_info "waiting messages sent out..."
48
+ log_info("waiting while senders finishing...")
49
49
  while @tokens.size < @sender_max
50
50
  sleep 1
51
51
  end # while
52
52
  }
53
- end # def start_sender
53
+ end # def start
54
54
 
55
55
  def stop()
56
- log_info "shutting down sender..."
56
+ log_info("shutting down sender...")
57
57
  @stopping.make_true()
58
- @queue.enq(STOP_TAG)
58
+ @queue.enq(Batch.new(Hash.new, STOP_TAG))
59
59
  @sender_t.join
60
- log_info "sender is fully shutted down"
61
- end # def stop_sender
60
+ log_info("sender is fully shutted down")
61
+ end # def stop
62
62
 
63
63
  def connect()
64
64
  uri = URI.parse(@url)
@@ -68,34 +68,33 @@ module LogStash; module Outputs; class SumoLogic;
68
68
  begin
69
69
  res = http.request(request)
70
70
  if res.code.to_i != 200
71
- log_err(
72
- "Server rejected the request",
71
+ log_err("ping rejected",
73
72
  :url => @url,
74
- :code => res.code
75
- )
73
+ :code => res.code,
74
+ :body => res.body)
76
75
  false
77
76
  else
78
- log_dbg(
79
- "Server accepted the request",
80
- :url => @url
81
- )
77
+ log_info("ping accepted",
78
+ :url => @url)
82
79
  true
83
80
  end
84
- rescue Exception => ex
85
- log_err(
86
- "Cannot connect to given url",
81
+ rescue Exception => exception
82
+ log_err("ping failed",
87
83
  :url => @url,
88
- :exception => ex
89
- )
84
+ :message => exception.message,
85
+ :class => exception.class.name,
86
+ :backtrace => exception.backtrace)
90
87
  false
91
88
  end
92
89
  end # def connect
93
90
 
94
91
  private
95
92
 
96
- def send_request(content)
93
+ def send_request(batch)
94
+ content = batch.payload
95
+ headers = batch.headers
97
96
  if content == STOP_TAG
98
- log_dbg "STOP_TAG is received."
97
+ log_info("STOP_TAG is received.")
99
98
  return
100
99
  end
101
100
 
@@ -103,12 +102,17 @@ module LogStash; module Outputs; class SumoLogic;
103
102
 
104
103
  if @stats_enabled && content.start_with?(STATS_TAG)
105
104
  body = @compressor.compress(content[STATS_TAG.length..-1])
106
- headers = @stats_headers
105
+ headers[CATEGORY_HEADER] = "#{headers[CATEGORY_HEADER]}.stats"
106
+ headers[CONTENT_TYPE] = CONTENT_TYPE_CARBON2
107
107
  else
108
108
  body = @compressor.compress(content)
109
- headers = @headers
110
109
  end
111
-
110
+
111
+ log_dbg("sending request",
112
+ :headers => headers,
113
+ :content_size => content.size,
114
+ :content => content[0..20],
115
+ :payload_size => body.size)
112
116
  request = @client.send(:background).send(:post, @url, :body => body, :headers => headers)
113
117
 
114
118
  request.on_complete do
@@ -118,19 +122,16 @@ module LogStash; module Outputs; class SumoLogic;
118
122
  request.on_success do |response|
119
123
  @stats.record_response_success(response.code)
120
124
  if response.code < 200 || response.code > 299
121
- log_err(
122
- "HTTP request rejected(#{response.code})",
125
+ log_err("request rejected",
123
126
  :token => token,
124
127
  :code => response.code,
125
128
  :headers => headers,
126
- :contet => content[0..20]
127
- )
129
+ :contet => content[0..20])
128
130
  if response.code == 429 || response.code == 503 || response.code == 504
129
- requeue_message(content)
131
+ requeue_message(batch)
130
132
  end
131
133
  else
132
- log_dbg(
133
- "HTTP request accepted",
134
+ log_dbg("request accepted",
134
135
  :token => token,
135
136
  :code => response.code)
136
137
  end
@@ -138,28 +139,32 @@ module LogStash; module Outputs; class SumoLogic;
138
139
 
139
140
  request.on_failure do |exception|
140
141
  @stats.record_response_failure()
141
- log_err(
142
- "Error in network transmission",
142
+ log_err("error in network transmission",
143
143
  :token => token,
144
144
  :message => exception.message,
145
145
  :class => exception.class.name,
146
- :backtrace => exception.backtrace
147
- )
148
- requeue_message(content)
146
+ :backtrace => exception.backtrace)
147
+ requeue_message(batch)
149
148
  end
150
149
 
151
150
  @stats.record_request(content.bytesize, body.bytesize)
152
151
  request.call
153
152
  end # def send_request
154
153
 
155
- def requeue_message(content)
156
- if @stopping.false? && @sleep_before_requeue >= 0
157
- log_warn(
158
- "requeue message",
154
+ def requeue_message(batch)
155
+ content = batch.payload
156
+ if @stats_enabled && content.start_with?(STATS_TAG)
157
+ log_warn("do not requeue stats payload",
158
+ :content => content)
159
+ elsif @stopping.false? && @sleep_before_requeue >= 0
160
+ log_info("requeue message",
159
161
  :after => @sleep_before_requeue,
160
- :content => content[0..20])
162
+ :queue_size => @queue.size,
163
+ :content_size => content.size,
164
+ :content => content[0..20],
165
+ :headers => batch.headers)
161
166
  Stud.stoppable_sleep(@sleep_before_requeue) { @stopping.true? }
162
- @queue.enq(content)
167
+ @queue.enq(batch)
163
168
  end
164
169
  end # def reque_message
165
170