logstash-output-sumologic 1.1.9 → 1.3.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -1,9 +1,9 @@
1
1
  # encoding: utf-8
2
- require "logstash/outputs/sumologic/common"
3
2
 
4
3
  module LogStash; module Outputs; class SumoLogic;
5
4
  class Statistics
6
5
 
6
+ require "logstash/outputs/sumologic/common"
7
7
  include LogStash::Outputs::SumoLogic::Common
8
8
 
9
9
  attr_reader :initialize_time
@@ -11,14 +11,10 @@ module LogStash; module Outputs; class SumoLogic;
11
11
  attr_reader :total_input_bytes
12
12
  attr_reader :total_metrics_datapoints
13
13
  attr_reader :total_log_lines
14
- attr_reader :current_pile_items
15
- attr_reader :current_pile_bytes
16
14
  attr_reader :total_enque_times
17
15
  attr_reader :total_enque_bytes
18
16
  attr_reader :total_deque_times
19
17
  attr_reader :total_deque_bytes
20
- attr_reader :current_queue_items
21
- attr_reader :current_queue_bytes
22
18
  attr_reader :total_output_requests
23
19
  attr_reader :total_output_bytes
24
20
  attr_reader :total_output_bytes_compressed
@@ -32,14 +28,10 @@ module LogStash; module Outputs; class SumoLogic;
32
28
  @total_input_bytes = Concurrent::AtomicFixnum.new
33
29
  @total_metrics_datapoints = Concurrent::AtomicFixnum.new
34
30
  @total_log_lines = Concurrent::AtomicFixnum.new
35
- @current_pile_items = Concurrent::AtomicFixnum.new
36
- @current_pile_bytes = Concurrent::AtomicFixnum.new
37
31
  @total_enque_times = Concurrent::AtomicFixnum.new
38
32
  @total_enque_bytes = Concurrent::AtomicFixnum.new
39
33
  @total_deque_times = Concurrent::AtomicFixnum.new
40
34
  @total_deque_bytes = Concurrent::AtomicFixnum.new
41
- @current_queue_items = Concurrent::AtomicFixnum.new
42
- @current_queue_bytes = Concurrent::AtomicFixnum.new
43
35
  @total_output_requests = Concurrent::AtomicFixnum.new
44
36
  @total_output_bytes = Concurrent::AtomicFixnum.new
45
37
  @total_output_bytes_compressed = Concurrent::AtomicFixnum.new
@@ -53,16 +45,9 @@ module LogStash; module Outputs; class SumoLogic;
53
45
  @total_response.get(key) ? @total_response.get(key).value : 0
54
46
  end
55
47
 
56
- def record_multi_input(events, bytesize)
57
- @total_input_events.update { |v| v + events }
58
- @total_input_bytes.update { |v| v + bytesize }
59
- end # def record_multi_input
60
-
61
- def record_input(entry)
48
+ def record_input(size)
62
49
  @total_input_events.increment()
63
- @total_input_bytes.update { |v| v + entry.bytesize }
64
- @current_pile_items.increment()
65
- @current_pile_bytes.update { |v| v + entry.bytesize }
50
+ @total_input_bytes.update { |v| v + size }
66
51
  end # def record_input
67
52
 
68
53
  def record_log_process()
@@ -73,23 +58,14 @@ module LogStash; module Outputs; class SumoLogic;
73
58
  @total_metrics_datapoints.update { |v| v + dps }
74
59
  end # def record_metrics_process
75
60
 
76
- def record_clear_pile()
77
- @current_pile_items.value= 0
78
- @current_pile_bytes.value= 0
79
- end # def record_pile_clear
80
-
81
- def record_enque(payload)
61
+ def record_enque(size)
82
62
  @total_enque_times.increment()
83
- @total_enque_bytes.update { |v| v + payload.bytesize }
84
- @current_queue_items.increment()
85
- @current_queue_bytes.update { |v| v + payload.bytesize }
63
+ @total_enque_bytes.update { |v| v + size }
86
64
  end # def record_enque
87
65
 
88
- def record_deque(payload)
66
+ def record_deque(size)
89
67
  @total_deque_times.increment()
90
- @total_deque_bytes.update { |v| v + payload.bytesize }
91
- @current_queue_items.decrement()
92
- @current_queue_bytes.update { |v| v - payload.bytesize }
68
+ @total_deque_bytes.update { |v| v + size }
93
69
  end # def record_deque
94
70
 
95
71
  def record_request(size, size_compressed)
@@ -1,6 +1,6 @@
1
1
  Gem::Specification.new do |s|
2
2
  s.name = 'logstash-output-sumologic'
3
- s.version = '1.1.9'
3
+ s.version = '1.3.1'
4
4
  s.licenses = ['Apache-2.0']
5
5
  s.summary = 'Deliever the log to Sumo Logic cloud service.'
6
6
  s.description = 'This gem is a Logstash output plugin to deliver the log or metrics to Sumo Logic cloud service. Go to https://github.com/SumoLogic/logstash-output-sumologic for getting help, reporting issues, etc.'
@@ -20,8 +20,8 @@ Gem::Specification.new do |s|
20
20
  # Gem dependencies
21
21
  s.add_runtime_dependency 'manticore', '>= 0.5.4', '< 1.0.0'
22
22
  s.add_runtime_dependency 'logstash-core-plugin-api', '>= 1.60', '<= 2.99'
23
- s.add_runtime_dependency 'logstash-codec-plain'
24
- s.add_runtime_dependency 'logstash-mixin-http_client'
23
+ s.add_runtime_dependency 'logstash-mixin-http_client', '>= 6', '< 8'
25
24
 
25
+ s.add_development_dependency 'logstash-codec-plain'
26
26
  s.add_development_dependency 'logstash-devutils'
27
27
  end
@@ -5,12 +5,14 @@ require "logstash/outputs/sumologic"
5
5
  describe LogStash::Outputs::SumoLogic::HeaderBuilder do
6
6
 
7
7
  result = {}
8
+ event = LogStash::Event.new("foo" => "bar", "message" => "Hello world")
8
9
 
9
10
  before :each do
10
- result = builder.build()
11
+ result = builder.build(event)
11
12
  end
12
13
 
13
14
  context "should build headers by default" do
15
+
14
16
  let(:builder) { LogStash::Outputs::SumoLogic::HeaderBuilder.new("url" => "http://localhost/1234") }
15
17
 
16
18
  specify {
@@ -41,6 +43,21 @@ describe LogStash::Outputs::SumoLogic::HeaderBuilder do
41
43
 
42
44
  end # context
43
45
 
46
+ context "should override source_category with template" do
47
+
48
+ let(:builder) {
49
+ LogStash::Outputs::SumoLogic::HeaderBuilder.new(
50
+ "url" => "http://localhost/1234",
51
+ "source_category" => "my source category %{foo}")
52
+ }
53
+
54
+ specify {
55
+ expect(result.count).to eq(5)
56
+ expect(result["X-Sumo-Category"]).to eq("my source category bar")
57
+ }
58
+
59
+ end # context
60
+
44
61
  context "should override source_name" do
45
62
 
46
63
  let(:builder) {
@@ -56,6 +73,21 @@ describe LogStash::Outputs::SumoLogic::HeaderBuilder do
56
73
 
57
74
  end # context
58
75
 
76
+ context "should override source_name with template" do
77
+
78
+ let(:builder) {
79
+ LogStash::Outputs::SumoLogic::HeaderBuilder.new(
80
+ "url" => "http://localhost/1234",
81
+ "source_name" => "my source name %{foo}")
82
+ }
83
+
84
+ specify {
85
+ expect(result.count).to eq(5)
86
+ expect(result["X-Sumo-Name"]).to eq("my source name bar")
87
+ }
88
+
89
+ end # context
90
+
59
91
  context "should override source_host" do
60
92
 
61
93
  let(:builder) {
@@ -71,6 +103,21 @@ describe LogStash::Outputs::SumoLogic::HeaderBuilder do
71
103
 
72
104
  end # context
73
105
 
106
+ context "should override source_host with template" do
107
+
108
+ let(:builder) {
109
+ LogStash::Outputs::SumoLogic::HeaderBuilder.new(
110
+ "url" => "http://localhost/1234",
111
+ "source_host" => "my source host %{foo}")
112
+ }
113
+
114
+ specify {
115
+ expect(result.count).to eq(5)
116
+ expect(result["X-Sumo-Host"]).to eq("my source host bar")
117
+ }
118
+
119
+ end # context
120
+
74
121
  context "should hornor extra_headers" do
75
122
 
76
123
  let(:builder) {
@@ -1,46 +1,48 @@
1
1
  # encoding: utf-8
2
2
  require "logstash/devutils/rspec/spec_helper"
3
3
  require "logstash/outputs/sumologic"
4
+ include LogStash::Outputs
5
+
6
+ describe SumoLogic::MessageQueue do
4
7
 
5
- describe LogStash::Outputs::SumoLogic::MessageQueue do
6
-
7
8
  context "working in pile mode if interval > 0 && pile_max > 0" do
8
9
 
9
- let(:stats) { LogStash::Outputs::SumoLogic::Statistics.new() }
10
+ let(:queue) { SumoLogic::MessageQueue.new(stats, "queue_max" => 10) }
11
+ let(:stats) { SumoLogic::Statistics.new }
10
12
 
11
13
  it "enq() correctly" do
12
- queue = LogStash::Outputs::SumoLogic::MessageQueue.new(stats, "queue_max" => 10)
13
14
  10.times { |i|
14
- queue.enq("test - #{i}")
15
+ queue.enq(SumoLogic::Batch.new(Hash.new, "test - #{i}"))
15
16
  expect(queue.size()).to eq(i + 1)
16
17
  expect(stats.total_enque_times.value).to eq(i + 1)
17
18
  }
19
+ expect(queue.bytesize()).to eq(100)
18
20
  end
19
21
 
20
22
  it "deq() correctly" do
21
- queue = LogStash::Outputs::SumoLogic::MessageQueue.new(stats, "queue_max" => 10)
22
23
  10.times { |i|
23
- queue.enq("test - #{i}")
24
+ queue.enq(SumoLogic::Batch.new(Hash.new, "test - #{i}"))
24
25
  }
25
26
  10.times { |i|
26
27
  expect(queue.size()).to eq(10 - i)
27
28
  result = queue.deq()
28
- expect(result).to eq("test - #{i}")
29
+ expect(result.payload).to eq("test - #{i}")
29
30
  expect(stats.total_deque_times.value).to eq(i + 1)
30
31
  }
32
+ expect(queue.bytesize()).to eq(0)
31
33
  end
32
34
 
33
35
  it "drain() correctly" do
34
- queue = LogStash::Outputs::SumoLogic::MessageQueue.new(stats, "queue_max" => 10)
35
36
  10.times { |i|
36
- queue.enq("test - #{i}")
37
+ queue.enq(SumoLogic::Batch.new(Hash.new, "test - #{i}"))
37
38
  }
38
39
  result = queue.drain()
39
40
  expect(queue.size()).to eq(0)
40
41
  expect(stats.total_deque_times.value).to eq(10)
41
42
  expect(result.size).to eq(10)
43
+ expect(queue.bytesize()).to eq(0)
42
44
  10.times { |i|
43
- expect(result[i]).to eq("test - #{i}")
45
+ expect(result[i].payload).to eq("test - #{i}")
44
46
  }
45
47
  end
46
48
 
@@ -69,14 +69,13 @@ describe LogStash::Outputs::SumoLogic::PayloadBuilder do
69
69
  "format" => "%{host} %{@json}",
70
70
  "json_mapping" => {
71
71
  "foo" => "%{foo}",
72
- "bar" => "%{bar}",
73
- "%{foo}" => "%{bar}"
72
+ "bar" => "%{bar}"
74
73
  })
75
74
  }
76
75
  let(:event) { LogStash::Event.new("host" => "myHost", "foo" => "fancy", "bar" => 24) }
77
76
 
78
77
  specify {
79
- expect(result).to eq("myHost {\"foo\":\"fancy\",\"bar\":\"24\",\"fancy\":\"24\"}")
78
+ expect(result).to eq("myHost {\"foo\":\"fancy\",\"bar\":\"24\"}")
80
79
  }
81
80
 
82
81
  end # context
@@ -1,9 +1,13 @@
1
1
  # encoding: utf-8
2
2
  require "logstash/devutils/rspec/spec_helper"
3
3
  require "logstash/outputs/sumologic"
4
+ include LogStash::Outputs
4
5
 
5
- describe LogStash::Outputs::SumoLogic::Piler do
6
+ describe SumoLogic::Piler do
6
7
 
8
+ event = LogStash::Event.new("foo" => "bar", "message" => "This is a log line")
9
+ event_10 = LogStash::Event.new("foo" => "bar", "message" => "1234567890")
10
+
7
11
  before :each do
8
12
  piler.start()
9
13
  end
@@ -15,9 +19,9 @@ describe LogStash::Outputs::SumoLogic::Piler do
15
19
 
16
20
  context "working in pile mode if interval > 0 && pile_max > 0" do
17
21
  let(:config) { {"queue_max" => 10, "interval" => 10, "pile_max" => 100 } }
18
- let(:stats) { LogStash::Outputs::SumoLogic::Statistics.new() }
19
- let(:queue) { LogStash::Outputs::SumoLogic::MessageQueue.new(stats, config) }
20
- let(:piler) { LogStash::Outputs::SumoLogic::Piler.new(queue, stats, config) }
22
+ let(:stats) { SumoLogic::Statistics.new }
23
+ let(:queue) { SumoLogic::MessageQueue.new(stats, config) }
24
+ let(:piler) { SumoLogic::Piler.new(queue, stats, config) }
21
25
  specify {
22
26
  expect(piler.is_pile).to be true
23
27
  }
@@ -25,9 +29,9 @@ describe LogStash::Outputs::SumoLogic::Piler do
25
29
 
26
30
  context "working in non-pile mode if interval <= 0" do
27
31
  let(:config) { {"queue_max" => 10, "interval" => 0, "pile_max" => 100 } }
28
- let(:stats) { LogStash::Outputs::SumoLogic::Statistics.new() }
29
- let(:queue) { LogStash::Outputs::SumoLogic::MessageQueue.new(stats, config) }
30
- let(:piler) { LogStash::Outputs::SumoLogic::Piler.new(queue, stats, config) }
32
+ let(:stats) { SumoLogic::Statistics.new }
33
+ let(:queue) { SumoLogic::MessageQueue.new(stats, config) }
34
+ let(:piler) { SumoLogic::Piler.new(queue, stats, config) }
31
35
  specify {
32
36
  expect(piler.is_pile).to be false
33
37
  }
@@ -35,152 +39,113 @@ describe LogStash::Outputs::SumoLogic::Piler do
35
39
 
36
40
  context "working in non-pile mode if pile_max <= 0" do
37
41
  let(:config) { {"queue_max" => 10, "interval" => 10, "pile_max" => 0 } }
38
- let(:stats) { LogStash::Outputs::SumoLogic::Statistics.new() }
39
- let(:queue) { LogStash::Outputs::SumoLogic::MessageQueue.new(stats, config) }
40
- let(:piler) { LogStash::Outputs::SumoLogic::Piler.new(queue, stats, config) }
42
+ let(:stats) { SumoLogic::Statistics.new }
43
+ let(:queue) { SumoLogic::MessageQueue.new(stats, config) }
44
+ let(:piler) { SumoLogic::Piler.new(queue, stats, config) }
41
45
  specify {
42
46
  expect(piler.is_pile).to be false
43
47
  }
44
48
  end # context
45
49
 
46
50
  context "in non-pile mode" do
47
-
48
- let(:config) { {"queue_max" => 10, "interval" => 0, "pile_max" => 100 } }
49
- let(:stats) { LogStash::Outputs::SumoLogic::Statistics.new() }
50
- let(:queue) { LogStash::Outputs::SumoLogic::MessageQueue.new(stats, config) }
51
- let(:piler) { LogStash::Outputs::SumoLogic::Piler.new(queue, stats, config) }
51
+ let(:config) { {"queue_max" => 10, "interval" => 0, "pile_max" => 100, "format" => "%{message}" } }
52
+ let(:stats) { SumoLogic::Statistics.new }
53
+ let(:queue) { SumoLogic::MessageQueue.new(stats, config) }
54
+ let(:piler) { SumoLogic::Piler.new(queue, stats, config) }
52
55
 
53
56
  it "enque immediately after input" do
54
- expect(stats.current_pile_items.value).to be 0
55
- expect(stats.current_queue_items.value).to be 0
56
- piler.input("This is a log line")
57
- expect(stats.current_pile_items.value).to be 0
58
- expect(stats.current_queue_items.value).to be 1
59
- expect(stats.current_queue_bytes.value).to be 18
57
+ expect(stats.total_enque_times.value).to be 0
58
+ expect(queue.size).to be 0
59
+ piler.input(event)
60
+ expect(stats.total_enque_times.value).to be 1
61
+ expect(stats.total_enque_bytes.value).to be 18
62
+ expect(queue.size).to be 1
63
+ expect(queue.bytesize).to be 18
60
64
  end
61
65
 
62
66
  it "deque correctly" do
63
- expect(stats.current_queue_items.value).to be 0
64
- expect(stats.total_enque_times.value).to be 0
65
- piler.input("This is a log line")
66
- expect(stats.total_enque_times.value).to be 1
67
- expect(stats.current_queue_items.value).to be 1
68
- expect(stats.current_queue_bytes.value).to be 18
69
- expect(stats.total_deque_times.value).to be 0
70
- expect(queue.deq()).to eq "This is a log line"
71
- expect(stats.current_queue_items.value).to be 0
72
- expect(stats.current_queue_bytes.value).to be 0
67
+ piler.input(event)
68
+ expect(queue.deq().payload).to eq "This is a log line"
69
+ expect(queue.size).to be 0
70
+ expect(queue.bytesize).to be 0
73
71
  expect(stats.total_deque_times.value).to be 1
72
+ expect(stats.total_deque_bytes.value).to be 18
74
73
  end
75
74
 
76
75
  end # context
77
76
 
78
77
  context "in pile mode" do
79
78
 
80
- let(:config) { {"queue_max" => 10, "interval" => 5, "pile_max" => 25 } }
81
- let(:stats) { LogStash::Outputs::SumoLogic::Statistics.new() }
82
- let(:queue) { LogStash::Outputs::SumoLogic::MessageQueue.new(stats, config) }
83
- let(:piler) { LogStash::Outputs::SumoLogic::Piler.new(queue, stats, config) }
84
-
85
- it "wait in pile before size reach pile_max" do
86
- expect(stats.current_pile_items.value).to be 0
87
- expect(stats.current_pile_bytes.value).to be 0
88
- expect(stats.current_queue_items.value).to be 0
89
- piler.input("1234567890")
90
- expect(stats.current_pile_items.value).to be 1
91
- expect(stats.current_pile_bytes.value).to be 10
92
- expect(stats.current_queue_items.value).to be 0
93
- piler.input("1234567890")
94
- expect(stats.current_pile_items.value).to be 2
95
- expect(stats.current_pile_bytes.value).to be 20
96
- expect(stats.current_queue_items.value).to be 0
97
- end
79
+ let(:config) { {"queue_max" => 10, "interval" => 5, "pile_max" => 25, "format" => "%{message}" } }
80
+ let(:stats) { SumoLogic::Statistics.new }
81
+ let(:queue) { SumoLogic::MessageQueue.new(stats, config) }
82
+ let(:piler) { SumoLogic::Piler.new(queue, stats, config) }
98
83
 
99
84
  it "enqueue content from pile when reach pile_max" do
100
- expect(stats.current_pile_items.value).to be 0
101
- expect(stats.current_pile_bytes.value).to be 0
102
- expect(stats.current_queue_items.value).to be 0
103
- piler.input("1234567890")
104
- piler.input("1234567890")
105
- expect(stats.current_queue_items.value).to be 0
106
- piler.input("1234567890")
107
- expect(stats.current_pile_items.value).to be 1
108
- expect(stats.current_pile_bytes.value).to be 10
109
- expect(stats.current_queue_items.value).to be 1
110
- expect(stats.current_queue_bytes.value).to be 21
85
+ expect(queue.size).to be 0
86
+ piler.input(event_10)
87
+ expect(queue.size).to be 0
88
+ piler.input(event_10)
89
+ expect(queue.size).to be 0
90
+ piler.input(event_10)
91
+ expect(queue.size).to be 1
111
92
  end
112
93
 
113
94
  it "enqueue content from pile when reach interval" do
114
- expect(stats.current_pile_items.value).to be 0
115
- expect(stats.current_pile_bytes.value).to be 0
116
- piler.input("1234567890")
117
- piler.input("1234567890")
118
- expect(stats.current_queue_items.value).to be 0
119
- sleep(8)
120
- expect(stats.current_pile_items.value).to be 0
121
- expect(stats.current_pile_bytes.value).to be 0
122
- expect(stats.current_queue_items.value).to be 1
123
- expect(stats.current_queue_bytes.value).to be 21
95
+ expect(queue.size).to be 0
96
+ piler.input(event_10)
97
+ expect(queue.size).to be 0
98
+ piler.input(event_10)
99
+ sleep(10)
100
+ expect(queue.size).to be 1
124
101
  end
125
102
 
126
103
  end # context
127
104
 
128
105
  context "pile to message queue" do
129
106
 
130
- let(:config) { {"queue_max" => 5, "interval" => 500, "pile_max" => 5} }
131
- let(:stats) { LogStash::Outputs::SumoLogic::Statistics.new() }
132
- let(:queue) { LogStash::Outputs::SumoLogic::MessageQueue.new(stats, config) }
133
- let(:piler) { LogStash::Outputs::SumoLogic::Piler.new(queue, stats, config) }
134
-
135
- it "enqueue payloads from pile before reach queue_max" do
136
- expect(stats.current_queue_items.value).to be 0
137
- piler.input("1234567890")
138
- expect(stats.current_queue_items.value).to be 0
139
- expect(stats.current_queue_bytes.value).to be 0
140
- piler.input("2234567890")
141
- expect(stats.current_queue_items.value).to be 1
142
- expect(stats.current_queue_bytes.value).to be 10
143
- piler.input("3234567890")
144
- expect(stats.current_queue_items.value).to be 2
145
- expect(stats.current_queue_bytes.value).to be 20
146
- end
107
+ let(:config) { {"queue_max" => 5, "interval" => 3, "pile_max" => 5, "format" => "%{message}"} }
108
+ let(:stats) { SumoLogic::Statistics.new }
109
+ let(:queue) { SumoLogic::MessageQueue.new(stats, config) }
110
+ let(:piler) { SumoLogic::Piler.new(queue, stats, config) }
147
111
 
148
112
  it "block input thread if queue is full" do
149
113
  input_t = Thread.new {
150
- for i in 0..9 do
151
- piler.input("#{i}234567890")
114
+ for i in 0..10 do
115
+ piler.input(event_10)
152
116
  end
153
117
  }
154
118
  sleep(3)
155
- expect(stats.current_queue_items.value).to be 5
156
- expect(stats.current_queue_bytes.value).to be 50
157
- queue.drain()
119
+ expect(queue.size).to be 5
120
+ expect(queue.bytesize).to be 50
158
121
  piler.stop()
159
- input_t.join
160
- expect(stats.total_deque_times.value).to be 5
161
- expect(stats.total_deque_bytes.value).to be 50
122
+ queue.drain()
123
+ input_t.kill()
162
124
  end
163
125
 
164
126
  it "resume input thread if queue is drained" do
165
127
  input_t = Thread.new {
166
- for i in 0..9 do
167
- piler.input("#{i}234567890")
128
+ for i in 0..10 do
129
+ piler.input(event_10)
168
130
  end
169
131
  }
170
- sleep(3)
132
+ sleep(5)
171
133
  expect(stats.total_deque_times.value).to be 0
134
+ expect(queue.size).to be 5
172
135
  expect(stats.total_enque_times.value).to be 5
173
136
  queue.deq()
174
- sleep(1)
137
+ sleep(3)
175
138
  expect(stats.total_deque_times.value).to be 1
139
+ expect(queue.size).to be 5
176
140
  expect(stats.total_enque_times.value).to be 6
177
141
  queue.deq()
178
- sleep(1)
142
+ sleep(3)
179
143
  expect(stats.total_deque_times.value).to be 2
144
+ expect(queue.size).to be 5
180
145
  expect(stats.total_enque_times.value).to be 7
181
- queue.drain()
182
146
  piler.stop()
183
- input_t.join
147
+ queue.drain()
148
+ input_t.kill()
184
149
  end
185
150
 
186
151
  end # context