logstash-output-sumologic 1.2.2 → 1.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +9 -1
- data/README.md +1 -1
- data/lib/logstash/outputs/sumologic.rb +7 -16
- data/lib/logstash/outputs/sumologic/batch.rb +13 -0
- data/lib/logstash/outputs/sumologic/common.rb +18 -2
- data/lib/logstash/outputs/sumologic/compressor.rb +3 -3
- data/lib/logstash/outputs/sumologic/header_builder.rb +10 -37
- data/lib/logstash/outputs/sumologic/message_queue.rb +23 -15
- data/lib/logstash/outputs/sumologic/monitor.rb +3 -3
- data/lib/logstash/outputs/sumologic/payload_builder.rb +3 -4
- data/lib/logstash/outputs/sumologic/piler.rb +34 -36
- data/lib/logstash/outputs/sumologic/sender.rb +31 -27
- data/lib/logstash/outputs/sumologic/statistics.rb +7 -31
- data/logstash-output-sumologic.gemspec +3 -3
- data/spec/outputs/sumologic/header_builder_spec.rb +48 -1
- data/spec/outputs/sumologic/message_queue_spec.rb +13 -11
- data/spec/outputs/sumologic/piler_spec.rb +67 -102
- data/spec/outputs/sumologic_spec.rb +10 -0
- metadata +22 -20
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 246eb9c67732ee8ec198f775b9dfa1bca652105c9a7c9b10504a3f3476a4468f
|
4
|
+
data.tar.gz: 06b46d47f19665b15a0b4404b8dc332c9ad29874520767353be3daddbb2920d4
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 23bf3e2632570fe4c352f461a7b144156267b394dfa095c5c909e8376b28f067e2d0a3bcb12460e2af23fd532544f3cbc425451d9a4ee93de9cdcb806449bb5e
|
7
|
+
data.tar.gz: 31dad1b3410d87e45e220b39dec13129e2ac7e7b19281697947ee273a72e76d7bae792acdfaaaaa5a442bb94a47adba8edfafccff3d0a20a537caec6d7fa035e
|
data/CHANGELOG.md
CHANGED
@@ -1,5 +1,13 @@
|
|
1
1
|
# Change Log
|
2
2
|
|
3
|
+
## 1.3.0
|
4
|
+
- [#41](https://github.com/SumoLogic/logstash-output-sumologic/pull/41) Provide Docker image with Logstash 6.6 + output plugin on docker hub
|
5
|
+
- [#41](https://github.com/SumoLogic/logstash-output-sumologic/pull/41) Kubernetes support with Logstash beats to SumoLogic
|
6
|
+
- [#41](https://github.com/SumoLogic/logstash-output-sumologic/pull/41) CI improving
|
7
|
+
- [#36](https://github.com/SumoLogic/logstash-output-sumologic/pull/36) Update sender to send in batch.
|
8
|
+
- [#36](https://github.com/SumoLogic/logstash-output-sumologic/pull/36) Support %{} field evaluation in `source_category`, `source_name`, `source_host` parameters
|
9
|
+
- [#39](https://github.com/SumoLogic/logstash-output-sumologic/pull/39) Disable cookies by default
|
10
|
+
|
3
11
|
## 1.2.2
|
4
12
|
|
5
13
|
- Bug fix: memory leak when using `%{@json}` in format
|
@@ -21,4 +29,4 @@
|
|
21
29
|
|
22
30
|
## 1.0.0
|
23
31
|
|
24
|
-
- Initial release
|
32
|
+
- Initial release
|
data/README.md
CHANGED
@@ -1,6 +1,6 @@
|
|
1
1
|
# Logstash Sumo Logic Output Plugin
|
2
2
|
|
3
|
-
[](https://travis-ci.org/SumoLogic/logstash-output-sumologic) [](https://badge.fury.io/rb/logstash-output-sumologic)
|
3
|
+
[](https://travis-ci.org/SumoLogic/logstash-output-sumologic) [](https://badge.fury.io/rb/logstash-output-sumologic) [](https://hub.docker.com/r/sumologic/logstash-output-sumologic)
|
4
4
|
|
5
5
|
This is an output plugin for [Logstash](https://github.com/elastic/logstash).
|
6
6
|
It is fully free and fully open source. The license is Apache 2.0, meaning you are pretty much free to use it however you want in whatever way.
|
@@ -14,6 +14,7 @@ require "logstash/plugin_mixins/http_client"
|
|
14
14
|
class LogStash::Outputs::SumoLogic < LogStash::Outputs::Base
|
15
15
|
declare_threadsafe!
|
16
16
|
|
17
|
+
require "logstash/outputs/sumologic/batch"
|
17
18
|
require "logstash/outputs/sumologic/common"
|
18
19
|
require "logstash/outputs/sumologic/compressor"
|
19
20
|
require "logstash/outputs/sumologic/header_builder"
|
@@ -110,13 +111,15 @@ class LogStash::Outputs::SumoLogic < LogStash::Outputs::Base
|
|
110
111
|
# Sending throughput data points every (x) seconds
|
111
112
|
config :stats_interval, :validate => :number, :default => 60
|
112
113
|
|
114
|
+
# Disable cookies by default (used in HTTP mixin)
|
115
|
+
config :cookies, :validate => :boolean, :default => false
|
116
|
+
|
113
117
|
attr_reader :stats
|
114
118
|
|
115
119
|
def register
|
116
120
|
set_logger(@logger)
|
117
|
-
@stats = Statistics.new
|
121
|
+
@stats = Statistics.new
|
118
122
|
@queue = MessageQueue.new(@stats, config)
|
119
|
-
@builder = PayloadBuilder.new(@stats, config)
|
120
123
|
@piler = Piler.new(@queue, @stats, config)
|
121
124
|
@monitor = Monitor.new(@queue, @stats, config)
|
122
125
|
@sender = Sender.new(client, @queue, @stats, config)
|
@@ -130,24 +133,12 @@ class LogStash::Outputs::SumoLogic < LogStash::Outputs::Base
|
|
130
133
|
end # def register
|
131
134
|
|
132
135
|
def multi_receive(events)
|
133
|
-
|
134
|
-
begin
|
135
|
-
content = Array(events).map { |event| @builder.build(event) }.join($/)
|
136
|
-
@queue.enq(content)
|
137
|
-
@stats.record_multi_input(events.size, content.bytesize)
|
138
|
-
rescue Exception => exception
|
139
|
-
log_err("error when processing events",
|
140
|
-
:events => events,
|
141
|
-
:message => exception.message,
|
142
|
-
:class => exception.class.name,
|
143
|
-
:backtrace => exception.backtrace)
|
144
|
-
end
|
136
|
+
Array(events).map { |event| receive(event) }
|
145
137
|
end # def multi_receive
|
146
138
|
|
147
139
|
def receive(event)
|
148
140
|
begin
|
149
|
-
|
150
|
-
@piler.input(content)
|
141
|
+
@piler.input(event)
|
151
142
|
rescue Exception => exception
|
152
143
|
log_err("error when processing event",
|
153
144
|
:event => event,
|
@@ -1,9 +1,9 @@
|
|
1
1
|
# encoding: utf-8
|
2
|
-
require "date"
|
3
|
-
|
4
2
|
module LogStash; module Outputs; class SumoLogic;
|
5
3
|
module Common
|
6
4
|
|
5
|
+
require "date"
|
6
|
+
|
7
7
|
# global constants
|
8
8
|
DEFAULT_LOG_FORMAT = "%{@timestamp} %{host} %{message}"
|
9
9
|
METRICS_NAME_PLACEHOLDER = "*"
|
@@ -12,6 +12,22 @@ module LogStash; module Outputs; class SumoLogic;
|
|
12
12
|
DEFLATE = "deflate"
|
13
13
|
GZIP = "gzip"
|
14
14
|
STATS_TAG = "STATS_TAG"
|
15
|
+
STOP_TAG = "PLUGIN STOPPED"
|
16
|
+
|
17
|
+
CONTENT_TYPE = "Content-Type"
|
18
|
+
CONTENT_TYPE_LOG = "text/plain"
|
19
|
+
CONTENT_TYPE_GRAPHITE = "application/vnd.sumologic.graphite"
|
20
|
+
CONTENT_TYPE_CARBON2 = "application/vnd.sumologic.carbon2"
|
21
|
+
CONTENT_ENCODING = "Content-Encoding"
|
22
|
+
|
23
|
+
CATEGORY_HEADER = "X-Sumo-Category"
|
24
|
+
CATEGORY_HEADER_DEFAULT = "Logstash"
|
25
|
+
HOST_HEADER = "X-Sumo-Host"
|
26
|
+
NAME_HEADER = "X-Sumo-Name"
|
27
|
+
NAME_HEADER_DEFAULT = "logstash-output-sumologic"
|
28
|
+
|
29
|
+
CLIENT_HEADER = "X-Sumo-Client"
|
30
|
+
CLIENT_HEADER_VALUE = "logstash-output-sumologic"
|
15
31
|
|
16
32
|
# for debugging test
|
17
33
|
LOG_TO_CONSOLE = false
|
@@ -1,11 +1,11 @@
|
|
1
1
|
# encoding: utf-8
|
2
|
-
require "stringio"
|
3
|
-
require "zlib"
|
4
|
-
require "logstash/outputs/sumologic/common"
|
5
2
|
|
6
3
|
module LogStash; module Outputs; class SumoLogic;
|
7
4
|
class Compressor
|
8
5
|
|
6
|
+
require "stringio"
|
7
|
+
require "zlib"
|
8
|
+
require "logstash/outputs/sumologic/common"
|
9
9
|
include LogStash::Outputs::SumoLogic::Common
|
10
10
|
|
11
11
|
def initialize(config)
|
@@ -1,27 +1,12 @@
|
|
1
1
|
# encoding: utf-8
|
2
|
-
require "socket"
|
3
|
-
require "logstash/outputs/sumologic/common"
|
4
2
|
|
5
3
|
module LogStash; module Outputs; class SumoLogic;
|
6
4
|
class HeaderBuilder
|
7
5
|
|
6
|
+
require "socket"
|
7
|
+
require "logstash/outputs/sumologic/common"
|
8
8
|
include LogStash::Outputs::SumoLogic::Common
|
9
9
|
|
10
|
-
CONTENT_TYPE = "Content-Type"
|
11
|
-
CONTENT_TYPE_LOG = "text/plain"
|
12
|
-
CONTENT_TYPE_GRAPHITE = "application/vnd.sumologic.graphite"
|
13
|
-
CONTENT_TYPE_CARBON2 = "application/vnd.sumologic.carbon2"
|
14
|
-
CONTENT_ENCODING = "Content-Encoding"
|
15
|
-
|
16
|
-
CATEGORY_HEADER = "X-Sumo-Category"
|
17
|
-
CATEGORY_HEADER_DEFAULT = "Logstash"
|
18
|
-
HOST_HEADER = "X-Sumo-Host"
|
19
|
-
NAME_HEADER = "X-Sumo-Name"
|
20
|
-
NAME_HEADER_DEFAULT = "logstash-output-sumologic"
|
21
|
-
|
22
|
-
CLIENT_HEADER = "X-Sumo-Client"
|
23
|
-
CLIENT_HEADER_VALUE = "logstash-output-sumologic"
|
24
|
-
|
25
10
|
def initialize(config)
|
26
11
|
|
27
12
|
@extra_headers = config["extra_headers"] ||= {}
|
@@ -36,31 +21,19 @@ module LogStash; module Outputs; class SumoLogic;
|
|
36
21
|
|
37
22
|
end # def initialize
|
38
23
|
|
39
|
-
def build()
|
40
|
-
headers =
|
41
|
-
headers[CATEGORY_HEADER] = @source_category unless @source_category.blank?
|
42
|
-
append_content_header(headers)
|
43
|
-
headers
|
44
|
-
end # def build
|
45
|
-
|
46
|
-
def build_stats()
|
47
|
-
headers = build_common()
|
48
|
-
headers[CATEGORY_HEADER] = "#{@source_category}.stats"
|
49
|
-
headers[CONTENT_TYPE] = CONTENT_TYPE_CARBON2
|
50
|
-
headers
|
51
|
-
end # def build_stats
|
52
|
-
|
53
|
-
private
|
54
|
-
def build_common()
|
55
|
-
headers = Hash.new()
|
24
|
+
def build(event)
|
25
|
+
headers = Hash.new
|
56
26
|
headers.merge!(@extra_headers)
|
57
27
|
headers[CLIENT_HEADER] = CLIENT_HEADER_VALUE
|
58
|
-
headers[
|
59
|
-
headers[
|
28
|
+
headers[CATEGORY_HEADER] = event.sprintf(@source_category) unless @source_category.blank?
|
29
|
+
headers[HOST_HEADER] = event.sprintf(@source_host) unless @source_host.blank?
|
30
|
+
headers[NAME_HEADER] = event.sprintf(@source_name) unless @source_name.blank?
|
31
|
+
append_content_header(headers)
|
60
32
|
append_compress_header(headers)
|
61
33
|
headers
|
62
|
-
end #
|
34
|
+
end # def build
|
63
35
|
|
36
|
+
private
|
64
37
|
def append_content_header(headers)
|
65
38
|
contentType = CONTENT_TYPE_LOG
|
66
39
|
if @metrics || @fields_as_metrics
|
@@ -1,38 +1,42 @@
|
|
1
1
|
# encoding: utf-8
|
2
|
-
require "logstash/outputs/sumologic/common"
|
3
|
-
require "logstash/outputs/sumologic/statistics"
|
4
|
-
|
5
2
|
module LogStash; module Outputs; class SumoLogic;
|
6
3
|
class MessageQueue
|
7
4
|
|
5
|
+
require "logstash/outputs/sumologic/common"
|
6
|
+
require "logstash/outputs/sumologic/statistics"
|
8
7
|
include LogStash::Outputs::SumoLogic::Common
|
9
8
|
|
10
9
|
def initialize(stats, config)
|
11
10
|
@queue_max = (config["queue_max"] ||= 1) < 1 ? 1 : config["queue_max"]
|
12
11
|
@queue = SizedQueue::new(@queue_max)
|
13
12
|
log_info("initialize memory queue", :max => @queue_max)
|
13
|
+
@queue_bytesize = Concurrent::AtomicFixnum.new
|
14
14
|
@stats = stats
|
15
15
|
end # def initialize
|
16
16
|
|
17
|
-
def enq(
|
18
|
-
|
19
|
-
|
20
|
-
@
|
17
|
+
def enq(batch)
|
18
|
+
batch_size = batch.payload.bytesize
|
19
|
+
if (batch_size > 0)
|
20
|
+
@queue.enq(batch)
|
21
|
+
@stats.record_enque(batch_size)
|
22
|
+
@queue_bytesize.update { |v| v + batch_size }
|
21
23
|
log_dbg("enqueue",
|
22
24
|
:objects_in_queue => size,
|
23
|
-
:bytes_in_queue => @
|
24
|
-
:size =>
|
25
|
-
|
25
|
+
:bytes_in_queue => @queue_bytesize,
|
26
|
+
:size => batch_size)
|
27
|
+
end
|
26
28
|
end # def enq
|
27
29
|
|
28
30
|
def deq()
|
29
|
-
|
30
|
-
|
31
|
+
batch = @queue.deq()
|
32
|
+
batch_size = batch.payload.bytesize
|
33
|
+
@stats.record_deque(batch_size)
|
34
|
+
@queue_bytesize.update { |v| v - batch_size }
|
31
35
|
log_dbg("dequeue",
|
32
36
|
:objects_in_queue => size,
|
33
|
-
:bytes_in_queue => @
|
34
|
-
:size =>
|
35
|
-
|
37
|
+
:bytes_in_queue => @queue_bytesize,
|
38
|
+
:size => batch_size)
|
39
|
+
batch
|
36
40
|
end # def deq
|
37
41
|
|
38
42
|
def drain()
|
@@ -44,6 +48,10 @@ module LogStash; module Outputs; class SumoLogic;
|
|
44
48
|
def size()
|
45
49
|
@queue.size()
|
46
50
|
end # size
|
51
|
+
|
52
|
+
def bytesize()
|
53
|
+
@queue_bytesize.value
|
54
|
+
end # bytesize
|
47
55
|
|
48
56
|
end
|
49
57
|
end; end; end
|
@@ -1,11 +1,11 @@
|
|
1
1
|
# encoding: utf-8
|
2
|
-
require "logstash/outputs/sumologic/common"
|
3
|
-
require "logstash/outputs/sumologic/statistics"
|
4
|
-
require "logstash/outputs/sumologic/message_queue"
|
5
2
|
|
6
3
|
module LogStash; module Outputs; class SumoLogic;
|
7
4
|
class Monitor
|
8
5
|
|
6
|
+
require "logstash/outputs/sumologic/common"
|
7
|
+
require "logstash/outputs/sumologic/statistics"
|
8
|
+
require "logstash/outputs/sumologic/message_queue"
|
9
9
|
include LogStash::Outputs::SumoLogic::Common
|
10
10
|
|
11
11
|
attr_reader :is_pile
|
@@ -1,12 +1,11 @@
|
|
1
1
|
# encoding: utf-8
|
2
|
-
require "logstash/json"
|
3
|
-
require "logstash/event"
|
4
|
-
|
5
|
-
require "logstash/outputs/sumologic/common"
|
6
2
|
|
7
3
|
module LogStash; module Outputs; class SumoLogic;
|
8
4
|
class PayloadBuilder
|
9
5
|
|
6
|
+
require "logstash/json"
|
7
|
+
require "logstash/event"
|
8
|
+
require "logstash/outputs/sumologic/common"
|
10
9
|
include LogStash::Outputs::SumoLogic::Common
|
11
10
|
|
12
11
|
TIMESTAMP_FIELD = "@timestamp"
|
@@ -1,11 +1,11 @@
|
|
1
1
|
# encoding: utf-8
|
2
|
-
require "logstash/outputs/sumologic/common"
|
3
|
-
require "logstash/outputs/sumologic/statistics"
|
4
|
-
require "logstash/outputs/sumologic/message_queue"
|
5
2
|
|
6
3
|
module LogStash; module Outputs; class SumoLogic;
|
7
4
|
class Piler
|
8
5
|
|
6
|
+
require "logstash/outputs/sumologic/common"
|
7
|
+
require "logstash/outputs/sumologic/statistics"
|
8
|
+
require "logstash/outputs/sumologic/message_queue"
|
9
9
|
include LogStash::Outputs::SumoLogic::Common
|
10
10
|
|
11
11
|
attr_reader :is_pile
|
@@ -17,14 +17,14 @@ module LogStash; module Outputs; class SumoLogic;
|
|
17
17
|
@queue = queue
|
18
18
|
@stats = stats
|
19
19
|
@stopping = Concurrent::AtomicBoolean.new(false)
|
20
|
+
@payload_builder = PayloadBuilder.new(@stats, config)
|
21
|
+
@header_builder = HeaderBuilder.new(config)
|
20
22
|
@is_pile = (@interval > 0 && @pile_max > 0)
|
21
|
-
|
22
23
|
if (@is_pile)
|
23
|
-
@pile =
|
24
|
-
@pile_size = 0
|
24
|
+
@pile = Hash.new("")
|
25
25
|
@semaphore = Mutex.new
|
26
26
|
end
|
27
|
-
|
27
|
+
|
28
28
|
end # def initialize
|
29
29
|
|
30
30
|
def start()
|
@@ -46,45 +46,43 @@ module LogStash; module Outputs; class SumoLogic;
|
|
46
46
|
def stop()
|
47
47
|
@stopping.make_true()
|
48
48
|
if (@is_pile)
|
49
|
-
log_info("shutting down piler...")
|
50
|
-
@piler_t.join
|
49
|
+
log_info("shutting down piler in #{@interval * 2} secs ...")
|
50
|
+
@piler_t.join(@interval * 2)
|
51
51
|
log_info("piler is fully shutted down")
|
52
52
|
end
|
53
53
|
end # def stop
|
54
54
|
|
55
|
-
def input(
|
55
|
+
def input(event)
|
56
56
|
if (@stopping.true?)
|
57
|
-
log_warn("piler is shutting down,
|
58
|
-
"
|
59
|
-
elsif (@is_pile)
|
60
|
-
@semaphore.synchronize {
|
61
|
-
if @pile_size + entry.bytesize > @pile_max
|
62
|
-
@queue.enq(@pile.join($/))
|
63
|
-
@pile.clear
|
64
|
-
@pile_size = 0
|
65
|
-
@stats.record_clear_pile()
|
66
|
-
end
|
67
|
-
@pile << entry
|
68
|
-
@pile_size += entry.bytesize
|
69
|
-
@stats.record_input(entry)
|
70
|
-
}
|
57
|
+
log_warn("piler is shutting down, event is dropped",
|
58
|
+
"event" => event)
|
71
59
|
else
|
72
|
-
@
|
73
|
-
|
60
|
+
headers = @header_builder.build(event)
|
61
|
+
payload = @payload_builder.build(event)
|
62
|
+
if (@is_pile)
|
63
|
+
@semaphore.synchronize {
|
64
|
+
content = @pile[headers]
|
65
|
+
size = content.bytesize
|
66
|
+
if size + payload.bytesize > @pile_max
|
67
|
+
@queue.enq(Batch.new(headers, content))
|
68
|
+
@pile[headers] = ""
|
69
|
+
end
|
70
|
+
@pile[headers] = @pile[headers].blank? ? payload : "#{@pile[headers]}\n#{payload}"
|
71
|
+
}
|
72
|
+
else
|
73
|
+
@queue.enq(Batch.new(headers, payload))
|
74
|
+
end # if
|
75
|
+
end
|
74
76
|
end # def input
|
75
77
|
|
76
78
|
private
|
77
79
|
def enq_and_clear()
|
78
|
-
|
79
|
-
@
|
80
|
-
|
81
|
-
|
82
|
-
|
83
|
-
|
84
|
-
@stats.record_clear_pile()
|
85
|
-
end
|
86
|
-
}
|
87
|
-
end
|
80
|
+
@semaphore.synchronize {
|
81
|
+
@pile.each do |headers, content|
|
82
|
+
@queue.enq(Batch.new(headers, content))
|
83
|
+
end
|
84
|
+
@pile.clear()
|
85
|
+
}
|
88
86
|
end # def enq_and_clear
|
89
87
|
|
90
88
|
end
|
@@ -1,19 +1,19 @@
|
|
1
1
|
# encoding: utf-8
|
2
|
-
require "net/https"
|
3
|
-
require "socket"
|
4
|
-
require "thread"
|
5
|
-
require "uri"
|
6
|
-
require "logstash/outputs/sumologic/common"
|
7
|
-
require "logstash/outputs/sumologic/compressor"
|
8
|
-
require "logstash/outputs/sumologic/header_builder"
|
9
|
-
require "logstash/outputs/sumologic/statistics"
|
10
|
-
require "logstash/outputs/sumologic/message_queue"
|
11
2
|
|
12
3
|
module LogStash; module Outputs; class SumoLogic;
|
13
4
|
class Sender
|
14
5
|
|
6
|
+
require "net/https"
|
7
|
+
require "socket"
|
8
|
+
require "thread"
|
9
|
+
require "uri"
|
10
|
+
require "logstash/outputs/sumologic/common"
|
11
|
+
require "logstash/outputs/sumologic/compressor"
|
12
|
+
require "logstash/outputs/sumologic/header_builder"
|
13
|
+
require "logstash/outputs/sumologic/statistics"
|
14
|
+
require "logstash/outputs/sumologic/message_queue"
|
15
15
|
include LogStash::Outputs::SumoLogic::Common
|
16
|
-
|
16
|
+
|
17
17
|
|
18
18
|
def initialize(client, queue, stats, config)
|
19
19
|
@client = client
|
@@ -28,9 +28,6 @@ module LogStash; module Outputs; class SumoLogic;
|
|
28
28
|
@tokens = SizedQueue.new(@sender_max)
|
29
29
|
@sender_max.times { |t| @tokens << t }
|
30
30
|
|
31
|
-
@header_builder = LogStash::Outputs::SumoLogic::HeaderBuilder.new(config)
|
32
|
-
@headers = @header_builder.build()
|
33
|
-
@stats_headers = @header_builder.build_stats()
|
34
31
|
@compressor = LogStash::Outputs::SumoLogic::Compressor.new(config)
|
35
32
|
|
36
33
|
end # def initialize
|
@@ -42,11 +39,11 @@ module LogStash; module Outputs; class SumoLogic;
|
|
42
39
|
@stopping.make_false()
|
43
40
|
@sender_t = Thread.new {
|
44
41
|
while @stopping.false?
|
45
|
-
|
46
|
-
send_request(
|
42
|
+
batch = @queue.deq()
|
43
|
+
send_request(batch)
|
47
44
|
end # while
|
48
|
-
@queue.drain().map { |
|
49
|
-
send_request(
|
45
|
+
@queue.drain().map { |batch|
|
46
|
+
send_request(batch)
|
50
47
|
}
|
51
48
|
log_info("waiting while senders finishing...")
|
52
49
|
while @tokens.size < @sender_max
|
@@ -58,7 +55,7 @@ module LogStash; module Outputs; class SumoLogic;
|
|
58
55
|
def stop()
|
59
56
|
log_info("shutting down sender...")
|
60
57
|
@stopping.make_true()
|
61
|
-
@queue.enq(STOP_TAG)
|
58
|
+
@queue.enq(Batch.new(Hash.new, STOP_TAG))
|
62
59
|
@sender_t.join
|
63
60
|
log_info("sender is fully shutted down")
|
64
61
|
end # def stop
|
@@ -93,7 +90,9 @@ module LogStash; module Outputs; class SumoLogic;
|
|
93
90
|
|
94
91
|
private
|
95
92
|
|
96
|
-
def send_request(
|
93
|
+
def send_request(batch)
|
94
|
+
content = batch.payload
|
95
|
+
headers = batch.headers
|
97
96
|
if content == STOP_TAG
|
98
97
|
log_info("STOP_TAG is received.")
|
99
98
|
return
|
@@ -103,10 +102,10 @@ module LogStash; module Outputs; class SumoLogic;
|
|
103
102
|
|
104
103
|
if @stats_enabled && content.start_with?(STATS_TAG)
|
105
104
|
body = @compressor.compress(content[STATS_TAG.length..-1])
|
106
|
-
headers =
|
105
|
+
headers[CATEGORY_HEADER] = "#{headers[CATEGORY_HEADER]}.stats"
|
106
|
+
headers[CONTENT_TYPE] = CONTENT_TYPE_CARBON2
|
107
107
|
else
|
108
108
|
body = @compressor.compress(content)
|
109
|
-
headers = @headers
|
110
109
|
end
|
111
110
|
|
112
111
|
log_dbg("sending request",
|
@@ -129,7 +128,7 @@ module LogStash; module Outputs; class SumoLogic;
|
|
129
128
|
:headers => headers,
|
130
129
|
:contet => content[0..20])
|
131
130
|
if response.code == 429 || response.code == 503 || response.code == 504
|
132
|
-
requeue_message(
|
131
|
+
requeue_message(batch)
|
133
132
|
end
|
134
133
|
else
|
135
134
|
log_dbg("request accepted",
|
@@ -145,22 +144,27 @@ module LogStash; module Outputs; class SumoLogic;
|
|
145
144
|
:message => exception.message,
|
146
145
|
:class => exception.class.name,
|
147
146
|
:backtrace => exception.backtrace)
|
148
|
-
requeue_message(
|
147
|
+
requeue_message(batch)
|
149
148
|
end
|
150
149
|
|
151
150
|
@stats.record_request(content.bytesize, body.bytesize)
|
152
151
|
request.call
|
153
152
|
end # def send_request
|
154
153
|
|
155
|
-
def requeue_message(
|
156
|
-
|
154
|
+
def requeue_message(batch)
|
155
|
+
content = batch.payload
|
156
|
+
if @stats_enabled && content.start_with?(STATS_TAG)
|
157
|
+
log_warn("do not requeue stats payload",
|
158
|
+
:content => content)
|
159
|
+
elsif @stopping.false? && @sleep_before_requeue >= 0
|
157
160
|
log_info("requeue message",
|
158
161
|
:after => @sleep_before_requeue,
|
159
162
|
:queue_size => @queue.size,
|
160
163
|
:content_size => content.size,
|
161
|
-
:content => content[0..20]
|
164
|
+
:content => content[0..20],
|
165
|
+
:headers => batch.headers)
|
162
166
|
Stud.stoppable_sleep(@sleep_before_requeue) { @stopping.true? }
|
163
|
-
@queue.enq(
|
167
|
+
@queue.enq(batch)
|
164
168
|
end
|
165
169
|
end # def reque_message
|
166
170
|
|
@@ -1,9 +1,9 @@
|
|
1
1
|
# encoding: utf-8
|
2
|
-
require "logstash/outputs/sumologic/common"
|
3
2
|
|
4
3
|
module LogStash; module Outputs; class SumoLogic;
|
5
4
|
class Statistics
|
6
5
|
|
6
|
+
require "logstash/outputs/sumologic/common"
|
7
7
|
include LogStash::Outputs::SumoLogic::Common
|
8
8
|
|
9
9
|
attr_reader :initialize_time
|
@@ -11,14 +11,10 @@ module LogStash; module Outputs; class SumoLogic;
|
|
11
11
|
attr_reader :total_input_bytes
|
12
12
|
attr_reader :total_metrics_datapoints
|
13
13
|
attr_reader :total_log_lines
|
14
|
-
attr_reader :current_pile_items
|
15
|
-
attr_reader :current_pile_bytes
|
16
14
|
attr_reader :total_enque_times
|
17
15
|
attr_reader :total_enque_bytes
|
18
16
|
attr_reader :total_deque_times
|
19
17
|
attr_reader :total_deque_bytes
|
20
|
-
attr_reader :current_queue_items
|
21
|
-
attr_reader :current_queue_bytes
|
22
18
|
attr_reader :total_output_requests
|
23
19
|
attr_reader :total_output_bytes
|
24
20
|
attr_reader :total_output_bytes_compressed
|
@@ -32,14 +28,10 @@ module LogStash; module Outputs; class SumoLogic;
|
|
32
28
|
@total_input_bytes = Concurrent::AtomicFixnum.new
|
33
29
|
@total_metrics_datapoints = Concurrent::AtomicFixnum.new
|
34
30
|
@total_log_lines = Concurrent::AtomicFixnum.new
|
35
|
-
@current_pile_items = Concurrent::AtomicFixnum.new
|
36
|
-
@current_pile_bytes = Concurrent::AtomicFixnum.new
|
37
31
|
@total_enque_times = Concurrent::AtomicFixnum.new
|
38
32
|
@total_enque_bytes = Concurrent::AtomicFixnum.new
|
39
33
|
@total_deque_times = Concurrent::AtomicFixnum.new
|
40
34
|
@total_deque_bytes = Concurrent::AtomicFixnum.new
|
41
|
-
@current_queue_items = Concurrent::AtomicFixnum.new
|
42
|
-
@current_queue_bytes = Concurrent::AtomicFixnum.new
|
43
35
|
@total_output_requests = Concurrent::AtomicFixnum.new
|
44
36
|
@total_output_bytes = Concurrent::AtomicFixnum.new
|
45
37
|
@total_output_bytes_compressed = Concurrent::AtomicFixnum.new
|
@@ -53,16 +45,9 @@ module LogStash; module Outputs; class SumoLogic;
|
|
53
45
|
@total_response.get(key) ? @total_response.get(key).value : 0
|
54
46
|
end
|
55
47
|
|
56
|
-
def
|
57
|
-
@total_input_events.update { |v| v + events }
|
58
|
-
@total_input_bytes.update { |v| v + bytesize }
|
59
|
-
end # def record_multi_input
|
60
|
-
|
61
|
-
def record_input(entry)
|
48
|
+
def record_input(size)
|
62
49
|
@total_input_events.increment()
|
63
|
-
@total_input_bytes.update { |v| v +
|
64
|
-
@current_pile_items.increment()
|
65
|
-
@current_pile_bytes.update { |v| v + entry.bytesize }
|
50
|
+
@total_input_bytes.update { |v| v + size }
|
66
51
|
end # def record_input
|
67
52
|
|
68
53
|
def record_log_process()
|
@@ -73,23 +58,14 @@ module LogStash; module Outputs; class SumoLogic;
|
|
73
58
|
@total_metrics_datapoints.update { |v| v + dps }
|
74
59
|
end # def record_metrics_process
|
75
60
|
|
76
|
-
def
|
77
|
-
@current_pile_items.value= 0
|
78
|
-
@current_pile_bytes.value= 0
|
79
|
-
end # def record_pile_clear
|
80
|
-
|
81
|
-
def record_enque(payload)
|
61
|
+
def record_enque(size)
|
82
62
|
@total_enque_times.increment()
|
83
|
-
@total_enque_bytes.update { |v| v +
|
84
|
-
@current_queue_items.increment()
|
85
|
-
@current_queue_bytes.update { |v| v + payload.bytesize }
|
63
|
+
@total_enque_bytes.update { |v| v + size }
|
86
64
|
end # def record_enque
|
87
65
|
|
88
|
-
def record_deque(
|
66
|
+
def record_deque(size)
|
89
67
|
@total_deque_times.increment()
|
90
|
-
@total_deque_bytes.update { |v| v +
|
91
|
-
@current_queue_items.decrement()
|
92
|
-
@current_queue_bytes.update { |v| v - payload.bytesize }
|
68
|
+
@total_deque_bytes.update { |v| v + size }
|
93
69
|
end # def record_deque
|
94
70
|
|
95
71
|
def record_request(size, size_compressed)
|
@@ -1,6 +1,6 @@
|
|
1
1
|
Gem::Specification.new do |s|
|
2
2
|
s.name = 'logstash-output-sumologic'
|
3
|
-
s.version = '1.
|
3
|
+
s.version = '1.3.0'
|
4
4
|
s.licenses = ['Apache-2.0']
|
5
5
|
s.summary = 'Deliever the log to Sumo Logic cloud service.'
|
6
6
|
s.description = 'This gem is a Logstash output plugin to deliver the log or metrics to Sumo Logic cloud service. Go to https://github.com/SumoLogic/logstash-output-sumologic for getting help, reporting issues, etc.'
|
@@ -20,8 +20,8 @@ Gem::Specification.new do |s|
|
|
20
20
|
# Gem dependencies
|
21
21
|
s.add_runtime_dependency 'manticore', '>= 0.5.4', '< 1.0.0'
|
22
22
|
s.add_runtime_dependency 'logstash-core-plugin-api', '>= 1.60', '<= 2.99'
|
23
|
-
s.add_runtime_dependency 'logstash-
|
24
|
-
s.add_runtime_dependency 'logstash-mixin-http_client'
|
23
|
+
s.add_runtime_dependency 'logstash-mixin-http_client', '~> 6.0'
|
25
24
|
|
25
|
+
s.add_development_dependency 'logstash-codec-plain'
|
26
26
|
s.add_development_dependency 'logstash-devutils'
|
27
27
|
end
|
@@ -5,12 +5,14 @@ require "logstash/outputs/sumologic"
|
|
5
5
|
describe LogStash::Outputs::SumoLogic::HeaderBuilder do
|
6
6
|
|
7
7
|
result = {}
|
8
|
+
event = LogStash::Event.new("foo" => "bar", "message" => "Hello world")
|
8
9
|
|
9
10
|
before :each do
|
10
|
-
result = builder.build()
|
11
|
+
result = builder.build(event)
|
11
12
|
end
|
12
13
|
|
13
14
|
context "should build headers by default" do
|
15
|
+
|
14
16
|
let(:builder) { LogStash::Outputs::SumoLogic::HeaderBuilder.new("url" => "http://localhost/1234") }
|
15
17
|
|
16
18
|
specify {
|
@@ -41,6 +43,21 @@ describe LogStash::Outputs::SumoLogic::HeaderBuilder do
|
|
41
43
|
|
42
44
|
end # context
|
43
45
|
|
46
|
+
context "should override source_category with template" do
|
47
|
+
|
48
|
+
let(:builder) {
|
49
|
+
LogStash::Outputs::SumoLogic::HeaderBuilder.new(
|
50
|
+
"url" => "http://localhost/1234",
|
51
|
+
"source_category" => "my source category %{foo}")
|
52
|
+
}
|
53
|
+
|
54
|
+
specify {
|
55
|
+
expect(result.count).to eq(5)
|
56
|
+
expect(result["X-Sumo-Category"]).to eq("my source category bar")
|
57
|
+
}
|
58
|
+
|
59
|
+
end # context
|
60
|
+
|
44
61
|
context "should override source_name" do
|
45
62
|
|
46
63
|
let(:builder) {
|
@@ -56,6 +73,21 @@ describe LogStash::Outputs::SumoLogic::HeaderBuilder do
|
|
56
73
|
|
57
74
|
end # context
|
58
75
|
|
76
|
+
context "should override source_name with template" do
|
77
|
+
|
78
|
+
let(:builder) {
|
79
|
+
LogStash::Outputs::SumoLogic::HeaderBuilder.new(
|
80
|
+
"url" => "http://localhost/1234",
|
81
|
+
"source_name" => "my source name %{foo}")
|
82
|
+
}
|
83
|
+
|
84
|
+
specify {
|
85
|
+
expect(result.count).to eq(5)
|
86
|
+
expect(result["X-Sumo-Name"]).to eq("my source name bar")
|
87
|
+
}
|
88
|
+
|
89
|
+
end # context
|
90
|
+
|
59
91
|
context "should override source_host" do
|
60
92
|
|
61
93
|
let(:builder) {
|
@@ -71,6 +103,21 @@ describe LogStash::Outputs::SumoLogic::HeaderBuilder do
|
|
71
103
|
|
72
104
|
end # context
|
73
105
|
|
106
|
+
context "should override source_host with template" do
|
107
|
+
|
108
|
+
let(:builder) {
|
109
|
+
LogStash::Outputs::SumoLogic::HeaderBuilder.new(
|
110
|
+
"url" => "http://localhost/1234",
|
111
|
+
"source_host" => "my source host %{foo}")
|
112
|
+
}
|
113
|
+
|
114
|
+
specify {
|
115
|
+
expect(result.count).to eq(5)
|
116
|
+
expect(result["X-Sumo-Host"]).to eq("my source host bar")
|
117
|
+
}
|
118
|
+
|
119
|
+
end # context
|
120
|
+
|
74
121
|
context "should hornor extra_headers" do
|
75
122
|
|
76
123
|
let(:builder) {
|
@@ -1,46 +1,48 @@
|
|
1
1
|
# encoding: utf-8
|
2
2
|
require "logstash/devutils/rspec/spec_helper"
|
3
3
|
require "logstash/outputs/sumologic"
|
4
|
+
include LogStash::Outputs
|
5
|
+
|
6
|
+
describe SumoLogic::MessageQueue do
|
4
7
|
|
5
|
-
describe LogStash::Outputs::SumoLogic::MessageQueue do
|
6
|
-
|
7
8
|
context "working in pile mode if interval > 0 && pile_max > 0" do
|
8
9
|
|
9
|
-
let(:
|
10
|
+
let(:queue) { SumoLogic::MessageQueue.new(stats, "queue_max" => 10) }
|
11
|
+
let(:stats) { SumoLogic::Statistics.new }
|
10
12
|
|
11
13
|
it "enq() correctly" do
|
12
|
-
queue = LogStash::Outputs::SumoLogic::MessageQueue.new(stats, "queue_max" => 10)
|
13
14
|
10.times { |i|
|
14
|
-
queue.enq("test
|
15
|
+
queue.enq(SumoLogic::Batch.new(Hash.new, "test - #{i}"))
|
15
16
|
expect(queue.size()).to eq(i + 1)
|
16
17
|
expect(stats.total_enque_times.value).to eq(i + 1)
|
17
18
|
}
|
19
|
+
expect(queue.bytesize()).to eq(100)
|
18
20
|
end
|
19
21
|
|
20
22
|
it "deq() correctly" do
|
21
|
-
queue = LogStash::Outputs::SumoLogic::MessageQueue.new(stats, "queue_max" => 10)
|
22
23
|
10.times { |i|
|
23
|
-
queue.enq("test
|
24
|
+
queue.enq(SumoLogic::Batch.new(Hash.new, "test - #{i}"))
|
24
25
|
}
|
25
26
|
10.times { |i|
|
26
27
|
expect(queue.size()).to eq(10 - i)
|
27
28
|
result = queue.deq()
|
28
|
-
expect(result).to eq("test
|
29
|
+
expect(result.payload).to eq("test - #{i}")
|
29
30
|
expect(stats.total_deque_times.value).to eq(i + 1)
|
30
31
|
}
|
32
|
+
expect(queue.bytesize()).to eq(0)
|
31
33
|
end
|
32
34
|
|
33
35
|
it "drain() correctly" do
|
34
|
-
queue = LogStash::Outputs::SumoLogic::MessageQueue.new(stats, "queue_max" => 10)
|
35
36
|
10.times { |i|
|
36
|
-
queue.enq("test
|
37
|
+
queue.enq(SumoLogic::Batch.new(Hash.new, "test - #{i}"))
|
37
38
|
}
|
38
39
|
result = queue.drain()
|
39
40
|
expect(queue.size()).to eq(0)
|
40
41
|
expect(stats.total_deque_times.value).to eq(10)
|
41
42
|
expect(result.size).to eq(10)
|
43
|
+
expect(queue.bytesize()).to eq(0)
|
42
44
|
10.times { |i|
|
43
|
-
expect(result[i]).to eq("test
|
45
|
+
expect(result[i].payload).to eq("test - #{i}")
|
44
46
|
}
|
45
47
|
end
|
46
48
|
|
@@ -1,9 +1,13 @@
|
|
1
1
|
# encoding: utf-8
|
2
2
|
require "logstash/devutils/rspec/spec_helper"
|
3
3
|
require "logstash/outputs/sumologic"
|
4
|
+
include LogStash::Outputs
|
4
5
|
|
5
|
-
describe
|
6
|
+
describe SumoLogic::Piler do
|
6
7
|
|
8
|
+
event = LogStash::Event.new("foo" => "bar", "message" => "This is a log line")
|
9
|
+
event_10 = LogStash::Event.new("foo" => "bar", "message" => "1234567890")
|
10
|
+
|
7
11
|
before :each do
|
8
12
|
piler.start()
|
9
13
|
end
|
@@ -15,9 +19,9 @@ describe LogStash::Outputs::SumoLogic::Piler do
|
|
15
19
|
|
16
20
|
context "working in pile mode if interval > 0 && pile_max > 0" do
|
17
21
|
let(:config) { {"queue_max" => 10, "interval" => 10, "pile_max" => 100 } }
|
18
|
-
let(:stats) {
|
19
|
-
let(:queue) {
|
20
|
-
let(:piler) {
|
22
|
+
let(:stats) { SumoLogic::Statistics.new }
|
23
|
+
let(:queue) { SumoLogic::MessageQueue.new(stats, config) }
|
24
|
+
let(:piler) { SumoLogic::Piler.new(queue, stats, config) }
|
21
25
|
specify {
|
22
26
|
expect(piler.is_pile).to be true
|
23
27
|
}
|
@@ -25,9 +29,9 @@ describe LogStash::Outputs::SumoLogic::Piler do
|
|
25
29
|
|
26
30
|
context "working in non-pile mode if interval <= 0" do
|
27
31
|
let(:config) { {"queue_max" => 10, "interval" => 0, "pile_max" => 100 } }
|
28
|
-
let(:stats) {
|
29
|
-
let(:queue) {
|
30
|
-
let(:piler) {
|
32
|
+
let(:stats) { SumoLogic::Statistics.new }
|
33
|
+
let(:queue) { SumoLogic::MessageQueue.new(stats, config) }
|
34
|
+
let(:piler) { SumoLogic::Piler.new(queue, stats, config) }
|
31
35
|
specify {
|
32
36
|
expect(piler.is_pile).to be false
|
33
37
|
}
|
@@ -35,152 +39,113 @@ describe LogStash::Outputs::SumoLogic::Piler do
|
|
35
39
|
|
36
40
|
context "working in non-pile mode if pile_max <= 0" do
|
37
41
|
let(:config) { {"queue_max" => 10, "interval" => 10, "pile_max" => 0 } }
|
38
|
-
let(:stats) {
|
39
|
-
let(:queue) {
|
40
|
-
let(:piler) {
|
42
|
+
let(:stats) { SumoLogic::Statistics.new }
|
43
|
+
let(:queue) { SumoLogic::MessageQueue.new(stats, config) }
|
44
|
+
let(:piler) { SumoLogic::Piler.new(queue, stats, config) }
|
41
45
|
specify {
|
42
46
|
expect(piler.is_pile).to be false
|
43
47
|
}
|
44
48
|
end # context
|
45
49
|
|
46
50
|
context "in non-pile mode" do
|
47
|
-
|
48
|
-
let(:
|
49
|
-
let(:
|
50
|
-
let(:
|
51
|
-
let(:piler) { LogStash::Outputs::SumoLogic::Piler.new(queue, stats, config) }
|
51
|
+
let(:config) { {"queue_max" => 10, "interval" => 0, "pile_max" => 100, "format" => "%{message}" } }
|
52
|
+
let(:stats) { SumoLogic::Statistics.new }
|
53
|
+
let(:queue) { SumoLogic::MessageQueue.new(stats, config) }
|
54
|
+
let(:piler) { SumoLogic::Piler.new(queue, stats, config) }
|
52
55
|
|
53
56
|
it "enque immediately after input" do
|
54
|
-
expect(stats.
|
55
|
-
expect(
|
56
|
-
piler.input(
|
57
|
-
expect(stats.
|
58
|
-
expect(stats.
|
59
|
-
expect(
|
57
|
+
expect(stats.total_enque_times.value).to be 0
|
58
|
+
expect(queue.size).to be 0
|
59
|
+
piler.input(event)
|
60
|
+
expect(stats.total_enque_times.value).to be 1
|
61
|
+
expect(stats.total_enque_bytes.value).to be 18
|
62
|
+
expect(queue.size).to be 1
|
63
|
+
expect(queue.bytesize).to be 18
|
60
64
|
end
|
61
65
|
|
62
66
|
it "deque correctly" do
|
63
|
-
|
64
|
-
expect(
|
65
|
-
|
66
|
-
expect(
|
67
|
-
expect(stats.current_queue_items.value).to be 1
|
68
|
-
expect(stats.current_queue_bytes.value).to be 18
|
69
|
-
expect(stats.total_deque_times.value).to be 0
|
70
|
-
expect(queue.deq()).to eq "This is a log line"
|
71
|
-
expect(stats.current_queue_items.value).to be 0
|
72
|
-
expect(stats.current_queue_bytes.value).to be 0
|
67
|
+
piler.input(event)
|
68
|
+
expect(queue.deq().payload).to eq "This is a log line"
|
69
|
+
expect(queue.size).to be 0
|
70
|
+
expect(queue.bytesize).to be 0
|
73
71
|
expect(stats.total_deque_times.value).to be 1
|
72
|
+
expect(stats.total_deque_bytes.value).to be 18
|
74
73
|
end
|
75
74
|
|
76
75
|
end # context
|
77
76
|
|
78
77
|
context "in pile mode" do
|
79
78
|
|
80
|
-
let(:config) { {"queue_max" => 10, "interval" => 5, "pile_max" => 25 } }
|
81
|
-
let(:stats) {
|
82
|
-
let(:queue) {
|
83
|
-
let(:piler) {
|
84
|
-
|
85
|
-
it "wait in pile before size reach pile_max" do
|
86
|
-
expect(stats.current_pile_items.value).to be 0
|
87
|
-
expect(stats.current_pile_bytes.value).to be 0
|
88
|
-
expect(stats.current_queue_items.value).to be 0
|
89
|
-
piler.input("1234567890")
|
90
|
-
expect(stats.current_pile_items.value).to be 1
|
91
|
-
expect(stats.current_pile_bytes.value).to be 10
|
92
|
-
expect(stats.current_queue_items.value).to be 0
|
93
|
-
piler.input("1234567890")
|
94
|
-
expect(stats.current_pile_items.value).to be 2
|
95
|
-
expect(stats.current_pile_bytes.value).to be 20
|
96
|
-
expect(stats.current_queue_items.value).to be 0
|
97
|
-
end
|
79
|
+
let(:config) { {"queue_max" => 10, "interval" => 5, "pile_max" => 25, "format" => "%{message}" } }
|
80
|
+
let(:stats) { SumoLogic::Statistics.new }
|
81
|
+
let(:queue) { SumoLogic::MessageQueue.new(stats, config) }
|
82
|
+
let(:piler) { SumoLogic::Piler.new(queue, stats, config) }
|
98
83
|
|
99
84
|
it "enqueue content from pile when reach pile_max" do
|
100
|
-
expect(
|
101
|
-
|
102
|
-
expect(
|
103
|
-
piler.input(
|
104
|
-
|
105
|
-
|
106
|
-
|
107
|
-
expect(stats.current_pile_items.value).to be 1
|
108
|
-
expect(stats.current_pile_bytes.value).to be 10
|
109
|
-
expect(stats.current_queue_items.value).to be 1
|
110
|
-
expect(stats.current_queue_bytes.value).to be 21
|
85
|
+
expect(queue.size).to be 0
|
86
|
+
piler.input(event_10)
|
87
|
+
expect(queue.size).to be 0
|
88
|
+
piler.input(event_10)
|
89
|
+
expect(queue.size).to be 0
|
90
|
+
piler.input(event_10)
|
91
|
+
expect(queue.size).to be 1
|
111
92
|
end
|
112
93
|
|
113
94
|
it "enqueue content from pile when reach interval" do
|
114
|
-
expect(
|
115
|
-
|
116
|
-
|
117
|
-
piler.input(
|
118
|
-
|
119
|
-
|
120
|
-
expect(stats.current_pile_items.value).to be 0
|
121
|
-
expect(stats.current_pile_bytes.value).to be 0
|
122
|
-
expect(stats.current_queue_items.value).to be 1
|
123
|
-
expect(stats.current_queue_bytes.value).to be 21
|
95
|
+
expect(queue.size).to be 0
|
96
|
+
piler.input(event_10)
|
97
|
+
expect(queue.size).to be 0
|
98
|
+
piler.input(event_10)
|
99
|
+
sleep(10)
|
100
|
+
expect(queue.size).to be 1
|
124
101
|
end
|
125
102
|
|
126
103
|
end # context
|
127
104
|
|
128
105
|
context "pile to message queue" do
|
129
106
|
|
130
|
-
let(:config) { {"queue_max" => 5, "interval" =>
|
131
|
-
let(:stats) {
|
132
|
-
let(:queue) {
|
133
|
-
let(:piler) {
|
134
|
-
|
135
|
-
it "enqueue payloads from pile before reach queue_max" do
|
136
|
-
expect(stats.current_queue_items.value).to be 0
|
137
|
-
piler.input("1234567890")
|
138
|
-
expect(stats.current_queue_items.value).to be 0
|
139
|
-
expect(stats.current_queue_bytes.value).to be 0
|
140
|
-
piler.input("2234567890")
|
141
|
-
expect(stats.current_queue_items.value).to be 1
|
142
|
-
expect(stats.current_queue_bytes.value).to be 10
|
143
|
-
piler.input("3234567890")
|
144
|
-
expect(stats.current_queue_items.value).to be 2
|
145
|
-
expect(stats.current_queue_bytes.value).to be 20
|
146
|
-
end
|
107
|
+
let(:config) { {"queue_max" => 5, "interval" => 3, "pile_max" => 5, "format" => "%{message}"} }
|
108
|
+
let(:stats) { SumoLogic::Statistics.new }
|
109
|
+
let(:queue) { SumoLogic::MessageQueue.new(stats, config) }
|
110
|
+
let(:piler) { SumoLogic::Piler.new(queue, stats, config) }
|
147
111
|
|
148
112
|
it "block input thread if queue is full" do
|
149
113
|
input_t = Thread.new {
|
150
|
-
for i in 0..
|
151
|
-
piler.input(
|
114
|
+
for i in 0..10 do
|
115
|
+
piler.input(event_10)
|
152
116
|
end
|
153
117
|
}
|
154
118
|
sleep(3)
|
155
|
-
expect(
|
156
|
-
expect(
|
157
|
-
queue.drain()
|
119
|
+
expect(queue.size).to be 5
|
120
|
+
expect(queue.bytesize).to be 50
|
158
121
|
piler.stop()
|
159
|
-
|
160
|
-
|
161
|
-
expect(stats.total_deque_bytes.value).to be 50
|
122
|
+
queue.drain()
|
123
|
+
input_t.kill()
|
162
124
|
end
|
163
125
|
|
164
126
|
it "resume input thread if queue is drained" do
|
165
127
|
input_t = Thread.new {
|
166
|
-
for i in 0..
|
167
|
-
piler.input(
|
128
|
+
for i in 0..10 do
|
129
|
+
piler.input(event_10)
|
168
130
|
end
|
169
131
|
}
|
170
|
-
sleep(
|
132
|
+
sleep(5)
|
171
133
|
expect(stats.total_deque_times.value).to be 0
|
134
|
+
expect(queue.size).to be 5
|
172
135
|
expect(stats.total_enque_times.value).to be 5
|
173
136
|
queue.deq()
|
174
|
-
sleep(
|
137
|
+
sleep(3)
|
175
138
|
expect(stats.total_deque_times.value).to be 1
|
139
|
+
expect(queue.size).to be 5
|
176
140
|
expect(stats.total_enque_times.value).to be 6
|
177
141
|
queue.deq()
|
178
|
-
sleep(
|
142
|
+
sleep(3)
|
179
143
|
expect(stats.total_deque_times.value).to be 2
|
144
|
+
expect(queue.size).to be 5
|
180
145
|
expect(stats.total_enque_times.value).to be 7
|
181
|
-
queue.drain()
|
182
146
|
piler.stop()
|
183
|
-
|
147
|
+
queue.drain()
|
148
|
+
input_t.kill()
|
184
149
|
end
|
185
150
|
|
186
151
|
end # context
|
@@ -13,6 +13,16 @@ describe LogStash::Outputs::SumoLogic, :unless => (ENV["sumo_url"].to_s.empty?)
|
|
13
13
|
plugin.close()
|
14
14
|
end
|
15
15
|
|
16
|
+
context "default configuration" do
|
17
|
+
let(:plugin) {
|
18
|
+
LogStash::Outputs::SumoLogic.new("url" => ENV["sumo_url"])
|
19
|
+
}
|
20
|
+
|
21
|
+
it "cookies is by default disabled" do
|
22
|
+
expect(plugin.cookies).to be false
|
23
|
+
end
|
24
|
+
end
|
25
|
+
|
16
26
|
context "no pile" do
|
17
27
|
context "single sender" do
|
18
28
|
context "send log in json" do
|
metadata
CHANGED
@@ -1,17 +1,16 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: logstash-output-sumologic
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 1.
|
4
|
+
version: 1.3.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Sumo Logic
|
8
|
-
autorequire:
|
8
|
+
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date:
|
11
|
+
date: 2020-01-14 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
|
-
name: manticore
|
15
14
|
requirement: !ruby/object:Gem::Requirement
|
16
15
|
requirements:
|
17
16
|
- - ">="
|
@@ -20,8 +19,9 @@ dependencies:
|
|
20
19
|
- - "<"
|
21
20
|
- !ruby/object:Gem::Version
|
22
21
|
version: 1.0.0
|
23
|
-
|
22
|
+
name: manticore
|
24
23
|
prerelease: false
|
24
|
+
type: :runtime
|
25
25
|
version_requirements: !ruby/object:Gem::Requirement
|
26
26
|
requirements:
|
27
27
|
- - ">="
|
@@ -31,7 +31,6 @@ dependencies:
|
|
31
31
|
- !ruby/object:Gem::Version
|
32
32
|
version: 1.0.0
|
33
33
|
- !ruby/object:Gem::Dependency
|
34
|
-
name: logstash-core-plugin-api
|
35
34
|
requirement: !ruby/object:Gem::Requirement
|
36
35
|
requirements:
|
37
36
|
- - ">="
|
@@ -40,8 +39,9 @@ dependencies:
|
|
40
39
|
- - "<="
|
41
40
|
- !ruby/object:Gem::Version
|
42
41
|
version: '2.99'
|
43
|
-
|
42
|
+
name: logstash-core-plugin-api
|
44
43
|
prerelease: false
|
44
|
+
type: :runtime
|
45
45
|
version_requirements: !ruby/object:Gem::Requirement
|
46
46
|
requirements:
|
47
47
|
- - ">="
|
@@ -51,42 +51,42 @@ dependencies:
|
|
51
51
|
- !ruby/object:Gem::Version
|
52
52
|
version: '2.99'
|
53
53
|
- !ruby/object:Gem::Dependency
|
54
|
-
name: logstash-codec-plain
|
55
54
|
requirement: !ruby/object:Gem::Requirement
|
56
55
|
requirements:
|
57
|
-
- - "
|
56
|
+
- - "~>"
|
58
57
|
- !ruby/object:Gem::Version
|
59
|
-
version: '0'
|
60
|
-
|
58
|
+
version: '6.0'
|
59
|
+
name: logstash-mixin-http_client
|
61
60
|
prerelease: false
|
61
|
+
type: :runtime
|
62
62
|
version_requirements: !ruby/object:Gem::Requirement
|
63
63
|
requirements:
|
64
|
-
- - "
|
64
|
+
- - "~>"
|
65
65
|
- !ruby/object:Gem::Version
|
66
|
-
version: '0'
|
66
|
+
version: '6.0'
|
67
67
|
- !ruby/object:Gem::Dependency
|
68
|
-
name: logstash-mixin-http_client
|
69
68
|
requirement: !ruby/object:Gem::Requirement
|
70
69
|
requirements:
|
71
70
|
- - ">="
|
72
71
|
- !ruby/object:Gem::Version
|
73
72
|
version: '0'
|
74
|
-
|
73
|
+
name: logstash-codec-plain
|
75
74
|
prerelease: false
|
75
|
+
type: :development
|
76
76
|
version_requirements: !ruby/object:Gem::Requirement
|
77
77
|
requirements:
|
78
78
|
- - ">="
|
79
79
|
- !ruby/object:Gem::Version
|
80
80
|
version: '0'
|
81
81
|
- !ruby/object:Gem::Dependency
|
82
|
-
name: logstash-devutils
|
83
82
|
requirement: !ruby/object:Gem::Requirement
|
84
83
|
requirements:
|
85
84
|
- - ">="
|
86
85
|
- !ruby/object:Gem::Version
|
87
86
|
version: '0'
|
88
|
-
|
87
|
+
name: logstash-devutils
|
89
88
|
prerelease: false
|
89
|
+
type: :development
|
90
90
|
version_requirements: !ruby/object:Gem::Requirement
|
91
91
|
requirements:
|
92
92
|
- - ">="
|
@@ -106,6 +106,7 @@ files:
|
|
106
106
|
- LICENSE
|
107
107
|
- README.md
|
108
108
|
- lib/logstash/outputs/sumologic.rb
|
109
|
+
- lib/logstash/outputs/sumologic/batch.rb
|
109
110
|
- lib/logstash/outputs/sumologic/common.rb
|
110
111
|
- lib/logstash/outputs/sumologic/compressor.rb
|
111
112
|
- lib/logstash/outputs/sumologic/header_builder.rb
|
@@ -130,7 +131,7 @@ licenses:
|
|
130
131
|
metadata:
|
131
132
|
logstash_plugin: 'true'
|
132
133
|
logstash_group: output
|
133
|
-
post_install_message:
|
134
|
+
post_install_message:
|
134
135
|
rdoc_options: []
|
135
136
|
require_paths:
|
136
137
|
- lib
|
@@ -145,8 +146,9 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
145
146
|
- !ruby/object:Gem::Version
|
146
147
|
version: '0'
|
147
148
|
requirements: []
|
148
|
-
|
149
|
-
|
149
|
+
rubyforge_project:
|
150
|
+
rubygems_version: 2.6.13
|
151
|
+
signing_key:
|
150
152
|
specification_version: 4
|
151
153
|
summary: Deliever the log to Sumo Logic cloud service.
|
152
154
|
test_files:
|