logstash-output-sumologic 1.2.0 → 1.3.2
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/CHANGELOG.md +27 -1
- data/DEVELOPER.md +19 -3
- data/LICENSE +191 -13
- data/README.md +10 -6
- data/lib/logstash/outputs/sumologic.rb +11 -22
- data/lib/logstash/outputs/sumologic/batch.rb +13 -0
- data/lib/logstash/outputs/sumologic/common.rb +34 -7
- data/lib/logstash/outputs/sumologic/compressor.rb +3 -3
- data/lib/logstash/outputs/sumologic/header_builder.rb +19 -33
- data/lib/logstash/outputs/sumologic/message_queue.rb +33 -14
- data/lib/logstash/outputs/sumologic/monitor.rb +14 -9
- data/lib/logstash/outputs/sumologic/payload_builder.rb +11 -7
- data/lib/logstash/outputs/sumologic/piler.rb +40 -38
- data/lib/logstash/outputs/sumologic/sender.rb +63 -60
- data/lib/logstash/outputs/sumologic/statistics.rb +7 -31
- data/logstash-output-sumologic.gemspec +3 -3
- data/spec/outputs/sumologic/header_builder_spec.rb +134 -1
- data/spec/outputs/sumologic/message_queue_spec.rb +13 -11
- data/spec/outputs/sumologic/payload_builder_spec.rb +2 -3
- data/spec/outputs/sumologic/piler_spec.rb +67 -102
- data/spec/outputs/sumologic_spec.rb +10 -0
- metadata +15 -9
@@ -1,9 +1,9 @@
|
|
1
1
|
# encoding: utf-8
|
2
|
-
require "logstash/outputs/sumologic/common"
|
3
2
|
|
4
3
|
module LogStash; module Outputs; class SumoLogic;
|
5
4
|
class Statistics
|
6
5
|
|
6
|
+
require "logstash/outputs/sumologic/common"
|
7
7
|
include LogStash::Outputs::SumoLogic::Common
|
8
8
|
|
9
9
|
attr_reader :initialize_time
|
@@ -11,14 +11,10 @@ module LogStash; module Outputs; class SumoLogic;
|
|
11
11
|
attr_reader :total_input_bytes
|
12
12
|
attr_reader :total_metrics_datapoints
|
13
13
|
attr_reader :total_log_lines
|
14
|
-
attr_reader :current_pile_items
|
15
|
-
attr_reader :current_pile_bytes
|
16
14
|
attr_reader :total_enque_times
|
17
15
|
attr_reader :total_enque_bytes
|
18
16
|
attr_reader :total_deque_times
|
19
17
|
attr_reader :total_deque_bytes
|
20
|
-
attr_reader :current_queue_items
|
21
|
-
attr_reader :current_queue_bytes
|
22
18
|
attr_reader :total_output_requests
|
23
19
|
attr_reader :total_output_bytes
|
24
20
|
attr_reader :total_output_bytes_compressed
|
@@ -32,14 +28,10 @@ module LogStash; module Outputs; class SumoLogic;
|
|
32
28
|
@total_input_bytes = Concurrent::AtomicFixnum.new
|
33
29
|
@total_metrics_datapoints = Concurrent::AtomicFixnum.new
|
34
30
|
@total_log_lines = Concurrent::AtomicFixnum.new
|
35
|
-
@current_pile_items = Concurrent::AtomicFixnum.new
|
36
|
-
@current_pile_bytes = Concurrent::AtomicFixnum.new
|
37
31
|
@total_enque_times = Concurrent::AtomicFixnum.new
|
38
32
|
@total_enque_bytes = Concurrent::AtomicFixnum.new
|
39
33
|
@total_deque_times = Concurrent::AtomicFixnum.new
|
40
34
|
@total_deque_bytes = Concurrent::AtomicFixnum.new
|
41
|
-
@current_queue_items = Concurrent::AtomicFixnum.new
|
42
|
-
@current_queue_bytes = Concurrent::AtomicFixnum.new
|
43
35
|
@total_output_requests = Concurrent::AtomicFixnum.new
|
44
36
|
@total_output_bytes = Concurrent::AtomicFixnum.new
|
45
37
|
@total_output_bytes_compressed = Concurrent::AtomicFixnum.new
|
@@ -53,16 +45,9 @@ module LogStash; module Outputs; class SumoLogic;
|
|
53
45
|
@total_response.get(key) ? @total_response.get(key).value : 0
|
54
46
|
end
|
55
47
|
|
56
|
-
def
|
57
|
-
@total_input_events.update { |v| v + events }
|
58
|
-
@total_input_bytes.update { |v| v + bytesize }
|
59
|
-
end # def record_multi_input
|
60
|
-
|
61
|
-
def record_input(entry)
|
48
|
+
def record_input(size)
|
62
49
|
@total_input_events.increment()
|
63
|
-
@total_input_bytes.update { |v| v +
|
64
|
-
@current_pile_items.increment()
|
65
|
-
@current_pile_bytes.update { |v| v + entry.bytesize }
|
50
|
+
@total_input_bytes.update { |v| v + size }
|
66
51
|
end # def record_input
|
67
52
|
|
68
53
|
def record_log_process()
|
@@ -73,23 +58,14 @@ module LogStash; module Outputs; class SumoLogic;
|
|
73
58
|
@total_metrics_datapoints.update { |v| v + dps }
|
74
59
|
end # def record_metrics_process
|
75
60
|
|
76
|
-
def
|
77
|
-
@current_pile_items.value= 0
|
78
|
-
@current_pile_bytes.value= 0
|
79
|
-
end # def record_pile_clear
|
80
|
-
|
81
|
-
def record_enque(payload)
|
61
|
+
def record_enque(size)
|
82
62
|
@total_enque_times.increment()
|
83
|
-
@total_enque_bytes.update { |v| v +
|
84
|
-
@current_queue_items.increment()
|
85
|
-
@current_queue_bytes.update { |v| v + payload.bytesize }
|
63
|
+
@total_enque_bytes.update { |v| v + size }
|
86
64
|
end # def record_enque
|
87
65
|
|
88
|
-
def record_deque(
|
66
|
+
def record_deque(size)
|
89
67
|
@total_deque_times.increment()
|
90
|
-
@total_deque_bytes.update { |v| v +
|
91
|
-
@current_queue_items.decrement()
|
92
|
-
@current_queue_bytes.update { |v| v - payload.bytesize }
|
68
|
+
@total_deque_bytes.update { |v| v + size }
|
93
69
|
end # def record_deque
|
94
70
|
|
95
71
|
def record_request(size, size_compressed)
|
@@ -1,6 +1,6 @@
|
|
1
1
|
Gem::Specification.new do |s|
|
2
2
|
s.name = 'logstash-output-sumologic'
|
3
|
-
s.version = '1.2
|
3
|
+
s.version = '1.3.2'
|
4
4
|
s.licenses = ['Apache-2.0']
|
5
5
|
s.summary = 'Deliever the log to Sumo Logic cloud service.'
|
6
6
|
s.description = 'This gem is a Logstash output plugin to deliver the log or metrics to Sumo Logic cloud service. Go to https://github.com/SumoLogic/logstash-output-sumologic for getting help, reporting issues, etc.'
|
@@ -20,8 +20,8 @@ Gem::Specification.new do |s|
|
|
20
20
|
# Gem dependencies
|
21
21
|
s.add_runtime_dependency 'manticore', '>= 0.5.4', '< 1.0.0'
|
22
22
|
s.add_runtime_dependency 'logstash-core-plugin-api', '>= 1.60', '<= 2.99'
|
23
|
-
s.add_runtime_dependency 'logstash-
|
24
|
-
s.add_runtime_dependency 'logstash-mixin-http_client'
|
23
|
+
s.add_runtime_dependency 'logstash-mixin-http_client', '>= 6', '< 8'
|
25
24
|
|
25
|
+
s.add_development_dependency 'logstash-codec-plain'
|
26
26
|
s.add_development_dependency 'logstash-devutils'
|
27
27
|
end
|
@@ -5,12 +5,14 @@ require "logstash/outputs/sumologic"
|
|
5
5
|
describe LogStash::Outputs::SumoLogic::HeaderBuilder do
|
6
6
|
|
7
7
|
result = {}
|
8
|
+
event = LogStash::Event.new("foo" => "bar", "message" => "Hello world")
|
8
9
|
|
9
10
|
before :each do
|
10
|
-
result = builder.build()
|
11
|
+
result = builder.build(event)
|
11
12
|
end
|
12
13
|
|
13
14
|
context "should build headers by default" do
|
15
|
+
|
14
16
|
let(:builder) { LogStash::Outputs::SumoLogic::HeaderBuilder.new("url" => "http://localhost/1234") }
|
15
17
|
|
16
18
|
specify {
|
@@ -41,6 +43,21 @@ describe LogStash::Outputs::SumoLogic::HeaderBuilder do
|
|
41
43
|
|
42
44
|
end # context
|
43
45
|
|
46
|
+
context "should override source_category with template" do
|
47
|
+
|
48
|
+
let(:builder) {
|
49
|
+
LogStash::Outputs::SumoLogic::HeaderBuilder.new(
|
50
|
+
"url" => "http://localhost/1234",
|
51
|
+
"source_category" => "my source category %{foo}")
|
52
|
+
}
|
53
|
+
|
54
|
+
specify {
|
55
|
+
expect(result.count).to eq(5)
|
56
|
+
expect(result["X-Sumo-Category"]).to eq("my source category bar")
|
57
|
+
}
|
58
|
+
|
59
|
+
end # context
|
60
|
+
|
44
61
|
context "should override source_name" do
|
45
62
|
|
46
63
|
let(:builder) {
|
@@ -56,6 +73,21 @@ describe LogStash::Outputs::SumoLogic::HeaderBuilder do
|
|
56
73
|
|
57
74
|
end # context
|
58
75
|
|
76
|
+
context "should override source_name with template" do
|
77
|
+
|
78
|
+
let(:builder) {
|
79
|
+
LogStash::Outputs::SumoLogic::HeaderBuilder.new(
|
80
|
+
"url" => "http://localhost/1234",
|
81
|
+
"source_name" => "my source name %{foo}")
|
82
|
+
}
|
83
|
+
|
84
|
+
specify {
|
85
|
+
expect(result.count).to eq(5)
|
86
|
+
expect(result["X-Sumo-Name"]).to eq("my source name bar")
|
87
|
+
}
|
88
|
+
|
89
|
+
end # context
|
90
|
+
|
59
91
|
context "should override source_host" do
|
60
92
|
|
61
93
|
let(:builder) {
|
@@ -71,6 +103,21 @@ describe LogStash::Outputs::SumoLogic::HeaderBuilder do
|
|
71
103
|
|
72
104
|
end # context
|
73
105
|
|
106
|
+
context "should override source_host with template" do
|
107
|
+
|
108
|
+
let(:builder) {
|
109
|
+
LogStash::Outputs::SumoLogic::HeaderBuilder.new(
|
110
|
+
"url" => "http://localhost/1234",
|
111
|
+
"source_host" => "my source host %{foo}")
|
112
|
+
}
|
113
|
+
|
114
|
+
specify {
|
115
|
+
expect(result.count).to eq(5)
|
116
|
+
expect(result["X-Sumo-Host"]).to eq("my source host bar")
|
117
|
+
}
|
118
|
+
|
119
|
+
end # context
|
120
|
+
|
74
121
|
context "should hornor extra_headers" do
|
75
122
|
|
76
123
|
let(:builder) {
|
@@ -194,4 +241,90 @@ describe LogStash::Outputs::SumoLogic::HeaderBuilder do
|
|
194
241
|
|
195
242
|
end # context
|
196
243
|
|
244
|
+
context "should build headers for stats" do
|
245
|
+
let(:builder) {
|
246
|
+
LogStash::Outputs::SumoLogic::HeaderBuilder.new("url" => "http://localhost/1234")
|
247
|
+
}
|
248
|
+
|
249
|
+
specify {
|
250
|
+
stats_result = builder.build_stats()
|
251
|
+
expected = {
|
252
|
+
"X-Sumo-Client" => "logstash-output-sumologic",
|
253
|
+
"X-Sumo-Name" => "logstash-output-sumologic",
|
254
|
+
"X-Sumo-Host" => Socket.gethostname,
|
255
|
+
"X-Sumo-Category" => "Logstash.stats",
|
256
|
+
"Content-Type" => "application/vnd.sumologic.carbon2"
|
257
|
+
}
|
258
|
+
expect(stats_result).to eq(expected)
|
259
|
+
}
|
260
|
+
end
|
261
|
+
|
262
|
+
context "should build headers for stats with overridden source category" do
|
263
|
+
let(:builder) {
|
264
|
+
LogStash::Outputs::SumoLogic::HeaderBuilder.new("url" => "http://localhost/1234", "stats_category" => "custom")
|
265
|
+
}
|
266
|
+
|
267
|
+
specify {
|
268
|
+
stats_result = builder.build_stats()
|
269
|
+
expect(stats_result["X-Sumo-Category"]).to eq("custom")
|
270
|
+
}
|
271
|
+
end
|
272
|
+
|
273
|
+
context "should build headers for stats with compression" do
|
274
|
+
let(:builder) {
|
275
|
+
LogStash::Outputs::SumoLogic::HeaderBuilder.new("url" => "http://localhost/1234", "compress" => true, "compress_encoding" => "gzip")
|
276
|
+
}
|
277
|
+
|
278
|
+
specify {
|
279
|
+
stats_result = builder.build_stats()
|
280
|
+
expect(stats_result["Content-Encoding"]).to eq("gzip")
|
281
|
+
}
|
282
|
+
end
|
283
|
+
|
284
|
+
context "should build headers for stats with extra_headers" do
|
285
|
+
let(:builder) {
|
286
|
+
LogStash::Outputs::SumoLogic::HeaderBuilder.new(
|
287
|
+
"url" => "http://localhost/1234",
|
288
|
+
"extra_headers" => {
|
289
|
+
"foo" => "bar"
|
290
|
+
})
|
291
|
+
}
|
292
|
+
|
293
|
+
specify {
|
294
|
+
stats_result = builder.build_stats()
|
295
|
+
expect(stats_result.count).to eq(6)
|
296
|
+
expect(stats_result["foo"]).to eq("bar")
|
297
|
+
}
|
298
|
+
end
|
299
|
+
|
300
|
+
context "should build headers for stats with extra_headers but never overwrite pre-defined headers" do
|
301
|
+
|
302
|
+
let(:builder) {
|
303
|
+
LogStash::Outputs::SumoLogic::HeaderBuilder.new(
|
304
|
+
"url" => "http://localhost/1234",
|
305
|
+
"extra_headers" => {
|
306
|
+
"foo" => "bar",
|
307
|
+
"X-Sumo-Client" => "a",
|
308
|
+
"X-Sumo-Name" => "b",
|
309
|
+
"X-Sumo-Host" => "c",
|
310
|
+
"X-Sumo-Category" => "d",
|
311
|
+
"Content-Type" => "e"
|
312
|
+
})
|
313
|
+
}
|
314
|
+
|
315
|
+
specify {
|
316
|
+
stats_result = builder.build_stats()
|
317
|
+
expected = {
|
318
|
+
"foo" => "bar",
|
319
|
+
"X-Sumo-Client" => "logstash-output-sumologic",
|
320
|
+
"X-Sumo-Name" => "logstash-output-sumologic",
|
321
|
+
"X-Sumo-Host" => Socket.gethostname,
|
322
|
+
"X-Sumo-Category" => "Logstash.stats",
|
323
|
+
"Content-Type" => "application/vnd.sumologic.carbon2"
|
324
|
+
}
|
325
|
+
expect(stats_result).to eq(expected)
|
326
|
+
}
|
327
|
+
|
328
|
+
end # context
|
329
|
+
|
197
330
|
end # describe
|
@@ -1,46 +1,48 @@
|
|
1
1
|
# encoding: utf-8
|
2
2
|
require "logstash/devutils/rspec/spec_helper"
|
3
3
|
require "logstash/outputs/sumologic"
|
4
|
+
include LogStash::Outputs
|
5
|
+
|
6
|
+
describe SumoLogic::MessageQueue do
|
4
7
|
|
5
|
-
describe LogStash::Outputs::SumoLogic::MessageQueue do
|
6
|
-
|
7
8
|
context "working in pile mode if interval > 0 && pile_max > 0" do
|
8
9
|
|
9
|
-
let(:
|
10
|
+
let(:queue) { SumoLogic::MessageQueue.new(stats, "queue_max" => 10) }
|
11
|
+
let(:stats) { SumoLogic::Statistics.new }
|
10
12
|
|
11
13
|
it "enq() correctly" do
|
12
|
-
queue = LogStash::Outputs::SumoLogic::MessageQueue.new(stats, "queue_max" => 10)
|
13
14
|
10.times { |i|
|
14
|
-
queue.enq("test
|
15
|
+
queue.enq(SumoLogic::Batch.new(Hash.new, "test - #{i}"))
|
15
16
|
expect(queue.size()).to eq(i + 1)
|
16
17
|
expect(stats.total_enque_times.value).to eq(i + 1)
|
17
18
|
}
|
19
|
+
expect(queue.bytesize()).to eq(100)
|
18
20
|
end
|
19
21
|
|
20
22
|
it "deq() correctly" do
|
21
|
-
queue = LogStash::Outputs::SumoLogic::MessageQueue.new(stats, "queue_max" => 10)
|
22
23
|
10.times { |i|
|
23
|
-
queue.enq("test
|
24
|
+
queue.enq(SumoLogic::Batch.new(Hash.new, "test - #{i}"))
|
24
25
|
}
|
25
26
|
10.times { |i|
|
26
27
|
expect(queue.size()).to eq(10 - i)
|
27
28
|
result = queue.deq()
|
28
|
-
expect(result).to eq("test
|
29
|
+
expect(result.payload).to eq("test - #{i}")
|
29
30
|
expect(stats.total_deque_times.value).to eq(i + 1)
|
30
31
|
}
|
32
|
+
expect(queue.bytesize()).to eq(0)
|
31
33
|
end
|
32
34
|
|
33
35
|
it "drain() correctly" do
|
34
|
-
queue = LogStash::Outputs::SumoLogic::MessageQueue.new(stats, "queue_max" => 10)
|
35
36
|
10.times { |i|
|
36
|
-
queue.enq("test
|
37
|
+
queue.enq(SumoLogic::Batch.new(Hash.new, "test - #{i}"))
|
37
38
|
}
|
38
39
|
result = queue.drain()
|
39
40
|
expect(queue.size()).to eq(0)
|
40
41
|
expect(stats.total_deque_times.value).to eq(10)
|
41
42
|
expect(result.size).to eq(10)
|
43
|
+
expect(queue.bytesize()).to eq(0)
|
42
44
|
10.times { |i|
|
43
|
-
expect(result[i]).to eq("test
|
45
|
+
expect(result[i].payload).to eq("test - #{i}")
|
44
46
|
}
|
45
47
|
end
|
46
48
|
|
@@ -69,14 +69,13 @@ describe LogStash::Outputs::SumoLogic::PayloadBuilder do
|
|
69
69
|
"format" => "%{host} %{@json}",
|
70
70
|
"json_mapping" => {
|
71
71
|
"foo" => "%{foo}",
|
72
|
-
"bar" => "%{bar}"
|
73
|
-
"%{foo}" => "%{bar}"
|
72
|
+
"bar" => "%{bar}"
|
74
73
|
})
|
75
74
|
}
|
76
75
|
let(:event) { LogStash::Event.new("host" => "myHost", "foo" => "fancy", "bar" => 24) }
|
77
76
|
|
78
77
|
specify {
|
79
|
-
expect(result).to eq("myHost {\"foo\":\"fancy\",\"bar\":\"24\"
|
78
|
+
expect(result).to eq("myHost {\"foo\":\"fancy\",\"bar\":\"24\"}")
|
80
79
|
}
|
81
80
|
|
82
81
|
end # context
|
@@ -1,9 +1,13 @@
|
|
1
1
|
# encoding: utf-8
|
2
2
|
require "logstash/devutils/rspec/spec_helper"
|
3
3
|
require "logstash/outputs/sumologic"
|
4
|
+
include LogStash::Outputs
|
4
5
|
|
5
|
-
describe
|
6
|
+
describe SumoLogic::Piler do
|
6
7
|
|
8
|
+
event = LogStash::Event.new("foo" => "bar", "message" => "This is a log line")
|
9
|
+
event_10 = LogStash::Event.new("foo" => "bar", "message" => "1234567890")
|
10
|
+
|
7
11
|
before :each do
|
8
12
|
piler.start()
|
9
13
|
end
|
@@ -15,9 +19,9 @@ describe LogStash::Outputs::SumoLogic::Piler do
|
|
15
19
|
|
16
20
|
context "working in pile mode if interval > 0 && pile_max > 0" do
|
17
21
|
let(:config) { {"queue_max" => 10, "interval" => 10, "pile_max" => 100 } }
|
18
|
-
let(:stats) {
|
19
|
-
let(:queue) {
|
20
|
-
let(:piler) {
|
22
|
+
let(:stats) { SumoLogic::Statistics.new }
|
23
|
+
let(:queue) { SumoLogic::MessageQueue.new(stats, config) }
|
24
|
+
let(:piler) { SumoLogic::Piler.new(queue, stats, config) }
|
21
25
|
specify {
|
22
26
|
expect(piler.is_pile).to be true
|
23
27
|
}
|
@@ -25,9 +29,9 @@ describe LogStash::Outputs::SumoLogic::Piler do
|
|
25
29
|
|
26
30
|
context "working in non-pile mode if interval <= 0" do
|
27
31
|
let(:config) { {"queue_max" => 10, "interval" => 0, "pile_max" => 100 } }
|
28
|
-
let(:stats) {
|
29
|
-
let(:queue) {
|
30
|
-
let(:piler) {
|
32
|
+
let(:stats) { SumoLogic::Statistics.new }
|
33
|
+
let(:queue) { SumoLogic::MessageQueue.new(stats, config) }
|
34
|
+
let(:piler) { SumoLogic::Piler.new(queue, stats, config) }
|
31
35
|
specify {
|
32
36
|
expect(piler.is_pile).to be false
|
33
37
|
}
|
@@ -35,152 +39,113 @@ describe LogStash::Outputs::SumoLogic::Piler do
|
|
35
39
|
|
36
40
|
context "working in non-pile mode if pile_max <= 0" do
|
37
41
|
let(:config) { {"queue_max" => 10, "interval" => 10, "pile_max" => 0 } }
|
38
|
-
let(:stats) {
|
39
|
-
let(:queue) {
|
40
|
-
let(:piler) {
|
42
|
+
let(:stats) { SumoLogic::Statistics.new }
|
43
|
+
let(:queue) { SumoLogic::MessageQueue.new(stats, config) }
|
44
|
+
let(:piler) { SumoLogic::Piler.new(queue, stats, config) }
|
41
45
|
specify {
|
42
46
|
expect(piler.is_pile).to be false
|
43
47
|
}
|
44
48
|
end # context
|
45
49
|
|
46
50
|
context "in non-pile mode" do
|
47
|
-
|
48
|
-
let(:
|
49
|
-
let(:
|
50
|
-
let(:
|
51
|
-
let(:piler) { LogStash::Outputs::SumoLogic::Piler.new(queue, stats, config) }
|
51
|
+
let(:config) { {"queue_max" => 10, "interval" => 0, "pile_max" => 100, "format" => "%{message}" } }
|
52
|
+
let(:stats) { SumoLogic::Statistics.new }
|
53
|
+
let(:queue) { SumoLogic::MessageQueue.new(stats, config) }
|
54
|
+
let(:piler) { SumoLogic::Piler.new(queue, stats, config) }
|
52
55
|
|
53
56
|
it "enque immediately after input" do
|
54
|
-
expect(stats.
|
55
|
-
expect(
|
56
|
-
piler.input(
|
57
|
-
expect(stats.
|
58
|
-
expect(stats.
|
59
|
-
expect(
|
57
|
+
expect(stats.total_enque_times.value).to be 0
|
58
|
+
expect(queue.size).to be 0
|
59
|
+
piler.input(event)
|
60
|
+
expect(stats.total_enque_times.value).to be 1
|
61
|
+
expect(stats.total_enque_bytes.value).to be 18
|
62
|
+
expect(queue.size).to be 1
|
63
|
+
expect(queue.bytesize).to be 18
|
60
64
|
end
|
61
65
|
|
62
66
|
it "deque correctly" do
|
63
|
-
|
64
|
-
expect(
|
65
|
-
|
66
|
-
expect(
|
67
|
-
expect(stats.current_queue_items.value).to be 1
|
68
|
-
expect(stats.current_queue_bytes.value).to be 18
|
69
|
-
expect(stats.total_deque_times.value).to be 0
|
70
|
-
expect(queue.deq()).to eq "This is a log line"
|
71
|
-
expect(stats.current_queue_items.value).to be 0
|
72
|
-
expect(stats.current_queue_bytes.value).to be 0
|
67
|
+
piler.input(event)
|
68
|
+
expect(queue.deq().payload).to eq "This is a log line"
|
69
|
+
expect(queue.size).to be 0
|
70
|
+
expect(queue.bytesize).to be 0
|
73
71
|
expect(stats.total_deque_times.value).to be 1
|
72
|
+
expect(stats.total_deque_bytes.value).to be 18
|
74
73
|
end
|
75
74
|
|
76
75
|
end # context
|
77
76
|
|
78
77
|
context "in pile mode" do
|
79
78
|
|
80
|
-
let(:config) { {"queue_max" => 10, "interval" => 5, "pile_max" => 25 } }
|
81
|
-
let(:stats) {
|
82
|
-
let(:queue) {
|
83
|
-
let(:piler) {
|
84
|
-
|
85
|
-
it "wait in pile before size reach pile_max" do
|
86
|
-
expect(stats.current_pile_items.value).to be 0
|
87
|
-
expect(stats.current_pile_bytes.value).to be 0
|
88
|
-
expect(stats.current_queue_items.value).to be 0
|
89
|
-
piler.input("1234567890")
|
90
|
-
expect(stats.current_pile_items.value).to be 1
|
91
|
-
expect(stats.current_pile_bytes.value).to be 10
|
92
|
-
expect(stats.current_queue_items.value).to be 0
|
93
|
-
piler.input("1234567890")
|
94
|
-
expect(stats.current_pile_items.value).to be 2
|
95
|
-
expect(stats.current_pile_bytes.value).to be 20
|
96
|
-
expect(stats.current_queue_items.value).to be 0
|
97
|
-
end
|
79
|
+
let(:config) { {"queue_max" => 10, "interval" => 5, "pile_max" => 25, "format" => "%{message}" } }
|
80
|
+
let(:stats) { SumoLogic::Statistics.new }
|
81
|
+
let(:queue) { SumoLogic::MessageQueue.new(stats, config) }
|
82
|
+
let(:piler) { SumoLogic::Piler.new(queue, stats, config) }
|
98
83
|
|
99
84
|
it "enqueue content from pile when reach pile_max" do
|
100
|
-
expect(
|
101
|
-
|
102
|
-
expect(
|
103
|
-
piler.input(
|
104
|
-
|
105
|
-
|
106
|
-
|
107
|
-
expect(stats.current_pile_items.value).to be 1
|
108
|
-
expect(stats.current_pile_bytes.value).to be 10
|
109
|
-
expect(stats.current_queue_items.value).to be 1
|
110
|
-
expect(stats.current_queue_bytes.value).to be 21
|
85
|
+
expect(queue.size).to be 0
|
86
|
+
piler.input(event_10)
|
87
|
+
expect(queue.size).to be 0
|
88
|
+
piler.input(event_10)
|
89
|
+
expect(queue.size).to be 0
|
90
|
+
piler.input(event_10)
|
91
|
+
expect(queue.size).to be 1
|
111
92
|
end
|
112
93
|
|
113
94
|
it "enqueue content from pile when reach interval" do
|
114
|
-
expect(
|
115
|
-
|
116
|
-
|
117
|
-
piler.input(
|
118
|
-
|
119
|
-
|
120
|
-
expect(stats.current_pile_items.value).to be 0
|
121
|
-
expect(stats.current_pile_bytes.value).to be 0
|
122
|
-
expect(stats.current_queue_items.value).to be 1
|
123
|
-
expect(stats.current_queue_bytes.value).to be 21
|
95
|
+
expect(queue.size).to be 0
|
96
|
+
piler.input(event_10)
|
97
|
+
expect(queue.size).to be 0
|
98
|
+
piler.input(event_10)
|
99
|
+
sleep(10)
|
100
|
+
expect(queue.size).to be 1
|
124
101
|
end
|
125
102
|
|
126
103
|
end # context
|
127
104
|
|
128
105
|
context "pile to message queue" do
|
129
106
|
|
130
|
-
let(:config) { {"queue_max" => 5, "interval" =>
|
131
|
-
let(:stats) {
|
132
|
-
let(:queue) {
|
133
|
-
let(:piler) {
|
134
|
-
|
135
|
-
it "enqueue payloads from pile before reach queue_max" do
|
136
|
-
expect(stats.current_queue_items.value).to be 0
|
137
|
-
piler.input("1234567890")
|
138
|
-
expect(stats.current_queue_items.value).to be 0
|
139
|
-
expect(stats.current_queue_bytes.value).to be 0
|
140
|
-
piler.input("2234567890")
|
141
|
-
expect(stats.current_queue_items.value).to be 1
|
142
|
-
expect(stats.current_queue_bytes.value).to be 10
|
143
|
-
piler.input("3234567890")
|
144
|
-
expect(stats.current_queue_items.value).to be 2
|
145
|
-
expect(stats.current_queue_bytes.value).to be 20
|
146
|
-
end
|
107
|
+
let(:config) { {"queue_max" => 5, "interval" => 3, "pile_max" => 5, "format" => "%{message}"} }
|
108
|
+
let(:stats) { SumoLogic::Statistics.new }
|
109
|
+
let(:queue) { SumoLogic::MessageQueue.new(stats, config) }
|
110
|
+
let(:piler) { SumoLogic::Piler.new(queue, stats, config) }
|
147
111
|
|
148
112
|
it "block input thread if queue is full" do
|
149
113
|
input_t = Thread.new {
|
150
|
-
for i in 0..
|
151
|
-
piler.input(
|
114
|
+
for i in 0..10 do
|
115
|
+
piler.input(event_10)
|
152
116
|
end
|
153
117
|
}
|
154
118
|
sleep(3)
|
155
|
-
expect(
|
156
|
-
expect(
|
157
|
-
queue.drain()
|
119
|
+
expect(queue.size).to be 5
|
120
|
+
expect(queue.bytesize).to be 50
|
158
121
|
piler.stop()
|
159
|
-
|
160
|
-
|
161
|
-
expect(stats.total_deque_bytes.value).to be 50
|
122
|
+
queue.drain()
|
123
|
+
input_t.kill()
|
162
124
|
end
|
163
125
|
|
164
126
|
it "resume input thread if queue is drained" do
|
165
127
|
input_t = Thread.new {
|
166
|
-
for i in 0..
|
167
|
-
piler.input(
|
128
|
+
for i in 0..10 do
|
129
|
+
piler.input(event_10)
|
168
130
|
end
|
169
131
|
}
|
170
|
-
sleep(
|
132
|
+
sleep(5)
|
171
133
|
expect(stats.total_deque_times.value).to be 0
|
134
|
+
expect(queue.size).to be 5
|
172
135
|
expect(stats.total_enque_times.value).to be 5
|
173
136
|
queue.deq()
|
174
|
-
sleep(
|
137
|
+
sleep(3)
|
175
138
|
expect(stats.total_deque_times.value).to be 1
|
139
|
+
expect(queue.size).to be 5
|
176
140
|
expect(stats.total_enque_times.value).to be 6
|
177
141
|
queue.deq()
|
178
|
-
sleep(
|
142
|
+
sleep(3)
|
179
143
|
expect(stats.total_deque_times.value).to be 2
|
144
|
+
expect(queue.size).to be 5
|
180
145
|
expect(stats.total_enque_times.value).to be 7
|
181
|
-
queue.drain()
|
182
146
|
piler.stop()
|
183
|
-
|
147
|
+
queue.drain()
|
148
|
+
input_t.kill()
|
184
149
|
end
|
185
150
|
|
186
151
|
end # context
|