logstash-output-loki 1.0.0 → 1.0.3

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: 9dd08c392ec88148a6f410e159169b3a19f877a25174fa3f43b2fdeb7fe16128
4
- data.tar.gz: 34aaf03576d8fbbb97aab02013113254734984ee1744c2ab4e8438bd51657569
3
+ metadata.gz: a76f3826de04d39e57f06c522125adfc150e1b9c44dee915cb5a220e829b5ae6
4
+ data.tar.gz: d4f881165097a6c04dcef15eca32f3570f0d35453a16713e5da1a50e022876d0
5
5
  SHA512:
6
- metadata.gz: 3520ab5618092976cfe771ad4f9ddea4aca3b96efe75d322c2f505e7fd6e642f3cee4775eb1ef0514c9db360a3c36b138e9f1426f6de9b625945cafdafb0b27d
7
- data.tar.gz: 6c7bb11d79fe0828e629bc27771a4f63584ea76af2ddb69eac950e5c7e3b115b62b674be28e8a828c97d8d3d851406748f0c83d49d9147080f0cc593ad1a076f
6
+ metadata.gz: 706a86a1852e2b2e97bbd340904b1a0a72751beca4ca4257bfbd66683e0c3a7fe5a54d273666c2da98947b84270b4e9544fe36b6e1742877ccffcbf67fe090bc
7
+ data.tar.gz: 5710b40ac392fa8e82c29343f22b444ce12284023497da3d0cb9c28049b81811ece3bcf431e5b948b7c60dd29c941dcd3c508289fc9a2d49faf5331eda893f69
data/Gemfile CHANGED
@@ -2,10 +2,13 @@ source 'https://rubygems.org'
2
2
 
3
3
  gemspec
4
4
 
5
- logstash_path = ENV["LOGSTASH_PATH"] || "logstash-libs"
6
- use_logstash_source = ENV["LOGSTASH_SOURCE"] && ENV["LOGSTASH_SOURCE"].to_s == "1"
5
+ logstash_path = ENV["LOGSTASH_PATH"] || "./logstash"
7
6
 
8
- if Dir.exist?(logstash_path) && use_logstash_source
7
+ if Dir.exist?(logstash_path)
9
8
  gem 'logstash-core', :path => "#{logstash_path}/logstash-core"
10
9
  gem 'logstash-core-plugin-api', :path => "#{logstash_path}/logstash-core-plugin-api"
10
+ else
11
+ raise 'missing logstash vendoring'
11
12
  end
13
+
14
+ gem "webmock", "~> 3.8"
data/README.md CHANGED
@@ -1,10 +1,12 @@
1
- # Loki Logstash Output Plugin
1
+ # Contributing to Loki Logstash Output Plugin
2
2
 
3
- Logstash plugin to send logstash aggregated logs to Loki.
3
+ For information about how to use this plugin see this [documentation](../../docs/sources/clients/logstash/_index.md).
4
4
 
5
5
  ## Install dependencies
6
6
 
7
- First you need to setup JRuby environment to build this plugin. Refer https://github.com/rbenv/rbenv for setting up your rbenv environment.
7
+ First, make sure you have JDK version `8` or `11` installed and you have set the `JAVA_HOME` environment variable.
8
+
9
+ You need to setup JRuby environment to build this plugin. Refer https://github.com/rbenv/rbenv for setting up your rbenv environment.
8
10
 
9
11
  After setting up `rbenv`. Install JRuby
10
12
 
@@ -20,7 +22,7 @@ ruby --version
20
22
  jruby 9.2.10
21
23
  ```
22
24
 
23
- You should use make sure you are running jruby and not ruby. If the command below still shows ruby and not jruby, check that PATH contains `$HOME/.rbenv/shims` and `$HOME/.rbenv/bin`. Also verify that you have this in your bash profile:
25
+ You should make sure you are running `jruby` and not `ruby`. If the command `ruby --version` still shows `ruby` and not `jruby`, check that PATH contains `$HOME/.rbenv/shims` and `$HOME/.rbenv/bin`. Also verify that you have this in your bash profile:
24
26
 
25
27
  ```bash
26
28
  export PATH="$HOME/.rbenv/bin:$PATH"
@@ -32,7 +34,7 @@ Then install bundler
32
34
 
33
35
  Follow those instructions to [install logstash](https://www.elastic.co/guide/en/logstash/current/installing-logstash.html) before moving to the next section.
34
36
 
35
- ## Install dependencies and Build plugin
37
+ ## Build and test the plugin
36
38
 
37
39
  ### Install required packages
38
40
 
@@ -41,11 +43,11 @@ git clone git@github.com:elastic/logstash.git
41
43
  cd logstash
42
44
  git checkout tags/v7.6.2
43
45
  export LOGSTASH_PATH=`pwd`
44
- export LOGSTASH_SOURCE="1"
45
- export GEM_PATH=$LOGSTASH_PATH/vendor/bundle/
46
- export GEM_HOME=$LOGSTASH_PATH/vendor/bundle/
46
+ export GEM_PATH=$LOGSTASH_PATH/vendor/bundle/jruby/2.5.0
47
+ export GEM_HOME=$LOGSTASH_PATH/vendor/bundle/jruby/2.5.0
48
+ ./gradlew assemble
47
49
  cd ..
48
- ruby -S bundle install --path
50
+ ruby -S bundle install
49
51
  ruby -S bundle exec rake vendor
50
52
  ```
51
53
 
@@ -55,7 +57,15 @@ ruby -S bundle exec rake vendor
55
57
 
56
58
  ### Test
57
59
 
58
- `bundle exec rspec`
60
+ `ruby -S bundle exec rspec`
61
+
62
+ Alternatively if you don't want to install JRuby. Enter inside logstash-loki container.
63
+
64
+ ```bash
65
+ docker build -t logstash-loki ./
66
+ docker run -v `pwd`/spec:/home/logstash/spec -it --rm --entrypoint /bin/sh logstash-loki
67
+ bundle exec rspec
68
+ ```
59
69
 
60
70
  ## Install plugin to local logstash
61
71
 
@@ -1,47 +1,63 @@
1
1
  require 'time'
2
2
 
3
- module LogStash
4
- module Outputs
5
- class Loki
6
- class Batch
7
- attr_reader :streams
8
- def initialize(e)
9
- @bytes = 0
10
- @createdAt = Time.now
11
- @streams = {}
12
- add(e)
13
- end
14
-
15
- def size_bytes()
16
- return @bytes
17
- end
18
-
19
- def add(e)
20
- @bytes = @bytes + e.entry['line'].length
21
-
22
- # Append the entry to an already existing stream (if any)
23
- labels = e.labels.to_s
24
- if @streams.has_key?(labels)
25
- stream = @streams[labels]
26
- stream['entries'] = stream['entries'] + e.entry
27
- return
28
- else
29
- # Add the entry as a new stream
30
- @streams[labels] = {
31
- "labels" => e.labels,
32
- "entries" => [e.entry],
33
- }
34
- end
35
- end
36
-
37
- def size_bytes_after(line)
38
- return @bytes + line.length
39
- end
40
-
41
- def age()
42
- return Time.now - @createdAt
43
- end
44
- end
45
- end
46
- end
47
- end
3
+ module Loki
4
+ class Batch
5
+ attr_reader :streams
6
+ def initialize(e)
7
+ @bytes = 0
8
+ @createdAt = Time.now
9
+ @streams = {}
10
+ add(e)
11
+ end
12
+
13
+ def size_bytes
14
+ return @bytes
15
+ end
16
+
17
+ def add(e)
18
+ @bytes = @bytes + e.entry['line'].length
19
+
20
+ # Append the entry to an already existing stream (if any)
21
+ labels = e.labels.sort.to_h
22
+ labelkey = labels.to_s
23
+ if @streams.has_key?(labelkey)
24
+ stream = @streams[labelkey]
25
+ stream['entries'].append(e.entry)
26
+ return
27
+ else
28
+ # Add the entry as a new stream
29
+ @streams[labelkey] = {
30
+ "labels" => labels,
31
+ "entries" => [e.entry],
32
+ }
33
+ end
34
+ end
35
+
36
+ def size_bytes_after(line)
37
+ return @bytes + line.length
38
+ end
39
+
40
+ def age()
41
+ return Time.now - @createdAt
42
+ end
43
+
44
+ def to_json
45
+ streams = []
46
+ @streams.each { |_ , stream|
47
+ streams.append(build_stream(stream))
48
+ }
49
+ return {"streams"=>streams}.to_json
50
+ end
51
+
52
+ def build_stream(stream)
53
+ values = []
54
+ stream['entries'].each { |entry|
55
+ values.append([entry['ts'].to_s, entry['line']])
56
+ }
57
+ return {
58
+ 'stream'=>stream['labels'],
59
+ 'values' => values
60
+ }
61
+ end
62
+ end
63
+ end
@@ -1,13 +1,25 @@
1
- module LogStash
2
- module Outputs
3
- class Loki
4
- class Entry
5
- attr_reader :labels, :entry
6
- def initialize(labels, entry)
7
- @labels = labels
8
- @entry = entry
9
- end
10
- end
11
- end
12
- end
13
- end
1
+ module Loki
2
+ def to_ns(s)
3
+ (s.to_f * (10**9)).to_i
4
+ end
5
+ class Entry
6
+ include Loki
7
+ attr_reader :labels, :entry
8
+ def initialize(event,message_field)
9
+ @entry = {
10
+ "ts" => to_ns(event.get("@timestamp")),
11
+ "line" => event.get(message_field).to_s
12
+ }
13
+ event = event.clone()
14
+ event.remove(message_field)
15
+ event.remove("@timestamp")
16
+
17
+ @labels = {}
18
+ event.to_hash.each { |key,value|
19
+ next if key.start_with?('@')
20
+ next if value.is_a?(Hash)
21
+ @labels[key] = value.to_s
22
+ }
23
+ end
24
+ end
25
+ end
@@ -1,16 +1,15 @@
1
1
  # encoding: utf-8
2
2
  require "logstash/outputs/base"
3
+ require "logstash/outputs/loki/entry"
4
+ require "logstash/outputs/loki/batch"
3
5
  require "logstash/namespace"
4
6
  require 'net/http'
5
- require 'concurrent-edge'
6
7
  require 'time'
7
8
  require 'uri'
8
9
  require 'json'
9
10
 
10
11
  class LogStash::Outputs::Loki < LogStash::Outputs::Base
11
- require 'logstash/outputs/loki/batch'
12
- require 'logstash/outputs/loki/entry'
13
-
12
+ include Loki
14
13
  config_name "loki"
15
14
 
16
15
  ## 'A single instance of the Output will be shared among the pipeline worker threads'
@@ -30,6 +29,9 @@ class LogStash::Outputs::Loki < LogStash::Outputs::Base
30
29
  ## 'TLS'
31
30
  config :ca_cert, :validate => :path, :required => false
32
31
 
32
+ ## 'Disable server certificate verification'
33
+ config :insecure_skip_verify, :validate => :boolean, :default => false, :required => false
34
+
33
35
  ## 'Loki Tenant ID'
34
36
  config :tenant_id, :validate => :string, :required => false
35
37
 
@@ -39,24 +41,19 @@ class LogStash::Outputs::Loki < LogStash::Outputs::Base
39
41
  ## 'Interval in seconds to wait before pushing a batch of records to loki. Defaults to 1 second'
40
42
  config :batch_wait, :validate => :number, :default => 1, :required => false
41
43
 
42
- ## 'Array of label names to include in all logstreams'
43
- config :include_labels, :validate => :array, :default => [], :required => true
44
-
45
- ## 'Extra labels to add to all log streams'
46
- config :external_labels, :validate => :hash, :default => {}, :required => false
47
-
48
44
  ## 'Log line field to pick from logstash. Defaults to "message"'
49
45
  config :message_field, :validate => :string, :default => "message", :required => false
50
46
 
51
47
  ## 'Backoff configuration. Initial backoff time between retries. Default 1s'
52
48
  config :min_delay, :validate => :number, :default => 1, :required => false
53
49
 
54
- ## 'Backoff configuration. Maximum backoff time between retries. Default 300s'
55
- config :max_delay, :validate => :number, :default => 300, :required => false
50
+ ## 'Backoff configuration. Maximum backoff time between retries. Default 300s'
51
+ config :max_delay, :validate => :number, :default => 300, :required => false
56
52
 
57
53
  ## 'Backoff configuration. Maximum number of retries to do'
58
54
  config :retries, :validate => :number, :default => 10, :required => false
59
55
 
56
+ attr_reader :batch
60
57
  public
61
58
  def register
62
59
  @uri = URI.parse(@url)
@@ -64,22 +61,16 @@ class LogStash::Outputs::Loki < LogStash::Outputs::Base
64
61
  raise LogStash::ConfigurationError, "url parameter must be valid HTTP, currently '#{@url}'"
65
62
  end
66
63
 
67
- if @include_labels.empty?
68
- raise LogStash::ConfigurationError, "include_labels should contain atleast one label, currently '#{@include_labels}'"
69
- end
70
-
71
64
  if @min_delay > @max_delay
72
65
  raise LogStash::ConfigurationError, "Min delay should be less than Max delay, currently 'Min delay is #{@min_delay} and Max delay is #{@max_delay}'"
73
66
  end
74
67
 
75
68
  @logger.info("Loki output plugin", :class => self.class.name)
76
69
 
77
- # intialize channels
78
- @Channel = Concurrent::Channel
79
- @entries = @Channel.new
80
-
81
- # excluded message and timestamp from labels
82
- @exclude_labels = ["message", "@timestamp"]
70
+ # initialize Queue and Mutex
71
+ @entries = Queue.new
72
+ @mutex = Mutex.new
73
+ @stop = false
83
74
 
84
75
  # create nil batch object.
85
76
  @batch = nil
@@ -90,7 +81,52 @@ class LogStash::Outputs::Loki < LogStash::Outputs::Base
90
81
  validate_ssl_key
91
82
  end
92
83
 
93
- @Channel.go{run()}
84
+ # start batch_max_wait and batch_max_size threads
85
+ @batch_wait_thread = Thread.new{max_batch_wait()}
86
+ @batch_size_thread = Thread.new{max_batch_size()}
87
+ end
88
+
89
+ def max_batch_size
90
+ loop do
91
+ @mutex.synchronize do
92
+ return if @stop
93
+ end
94
+
95
+ e = @entries.deq
96
+ return if e.nil?
97
+
98
+ @mutex.synchronize do
99
+ if !add_entry_to_batch(e)
100
+ @logger.debug("Max batch_size is reached. Sending batch to loki")
101
+ send(@batch)
102
+ @batch = Batch.new(e)
103
+ end
104
+ end
105
+ end
106
+ end
107
+
108
+ def max_batch_wait
109
+ # minimum wait frequency is 10 milliseconds
110
+ min_wait_checkfrequency = 1/100
111
+ max_wait_checkfrequency = @batch_wait
112
+ if max_wait_checkfrequency < min_wait_checkfrequency
113
+ max_wait_checkfrequency = min_wait_checkfrequency
114
+ end
115
+
116
+ loop do
117
+ @mutex.synchronize do
118
+ return if @stop
119
+ end
120
+
121
+ sleep(max_wait_checkfrequency)
122
+ if is_batch_expired
123
+ @mutex.synchronize do
124
+ @logger.debug("Max batch_wait time is reached. Sending batch to loki")
125
+ send(@batch)
126
+ @batch = nil
127
+ end
128
+ end
129
+ end
94
130
  end
95
131
 
96
132
  def ssl_cert?
@@ -113,6 +149,13 @@ class LogStash::Outputs::Loki < LogStash::Outputs::Base
113
149
  use_ssl: uri.scheme == 'https'
114
150
  }
115
151
 
152
+ # disable server certificate verification
153
+ if @insecure_skip_verify
154
+ opts = opts.merge(
155
+ verify_mode: OpenSSL::SSL::VERIFY_NONE
156
+ )
157
+ end
158
+
116
159
  if !@cert.nil? && !@key.nil?
117
160
  opts = opts.merge(
118
161
  verify_mode: OpenSSL::SSL::VERIFY_PEER,
@@ -129,126 +172,65 @@ class LogStash::Outputs::Loki < LogStash::Outputs::Base
129
172
  opts
130
173
  end
131
174
 
132
- def run()
133
- min_wait_checkfrequency = 1/1000 #1 millisecond
134
- max_wait_checkfrequency = @batch_wait
135
- if max_wait_checkfrequency < min_wait_checkfrequency
136
- max_wait_checkfrequency = min_wait_checkfrequency
175
+ # Add an entry to the current batch returns false if the batch is full
176
+ # and the entry can't be added.
177
+ def add_entry_to_batch(e)
178
+ line = e.entry['line']
179
+ # we don't want to send empty lines.
180
+ return true if line.to_s.strip.empty?
181
+
182
+ if @batch.nil?
183
+ @batch = Batch.new(e)
184
+ return true
137
185
  end
138
186
 
139
- @max_wait_check = Concurrent::Channel.tick(max_wait_checkfrequency)
140
- loop do
141
- Concurrent::Channel.select do |s|
142
- s.take(@entries) { |e|
143
- if @batch.nil?
144
- @batch = Batch.new(e)
145
- next
146
- end
147
-
148
- line = e.entry['line']
149
- if @batch.size_bytes_after(line) > @batch_size
150
- @logger.debug("Max batch_size is reached. Sending batch to loki")
151
- send(@tenant_id, @batch)
152
- @batch = Batch.new(e)
153
- next
154
- end
155
- @batch.add(e)
156
- }
157
- s.take(@max_wait_check) {
158
- # Send batch if max wait time has been reached
159
- if !@batch.nil?
160
- if @batch.age() < @batch_wait
161
- next
162
- end
163
-
164
- @logger.debug("Max batch_wait time is reached. Sending batch to loki")
165
- send(@tenant_id, @batch)
166
- @batch = nil
167
- end
168
- }
169
- end
187
+ if @batch.size_bytes_after(line) > @batch_size
188
+ return false
170
189
  end
190
+ @batch.add(e)
191
+ return true
192
+ end
193
+
194
+ def is_batch_expired
195
+ return !@batch.nil? && @batch.age() >= @batch_wait
171
196
  end
172
197
 
173
198
  ## Receives logstash events
174
199
  public
175
200
  def receive(event)
176
- labels = {}
177
- event_hash = event.to_hash
178
- lbls = handle_labels(event_hash, labels, "")
179
-
180
- data_labels, entry_hash = build_entry(lbls, event)
181
- @entries << Entry.new(data_labels, entry_hash)
182
-
201
+ @entries << Entry.new(event, @message_field)
183
202
  end
184
203
 
185
204
  def close
186
- @logger.info("Closing loki output plugin. Flushing all pending batches")
187
- send(@tenant_id, @batch) if !@batch.nil?
188
205
  @entries.close
189
- @max_wait_check.close if !@max_wait_check.nil?
190
- end
191
-
192
- def build_entry(lbls, event)
193
- labels = lbls.merge(@external_labels)
194
- entry_hash = {
195
- "ts" => event.get("@timestamp").to_i * (10**9),
196
- "line" => event.get(@message_field).to_s
197
- }
198
- return labels, entry_hash
199
- end
200
-
201
- def handle_labels(event_hash, labels, parent_key)
202
- event_hash.each{ |key,value|
203
- if !@exclude_labels.include?(key)
204
- if value.is_a?(Hash)
205
- if parent_key != ""
206
- handle_labels(value, labels, parent_key + "_" + key)
207
- else
208
- handle_labels(value, labels, key)
209
- end
210
- else
211
- if parent_key != ""
212
- labels[parent_key + "_" + key] = value.to_s
213
- else
214
- labels[key] = value.to_s
215
- end
216
- end
217
- end
218
- }
219
- return extract_labels(labels)
220
- end
206
+ @mutex.synchronize do
207
+ @stop = true
208
+ end
209
+ @batch_wait_thread.join
210
+ @batch_size_thread.join
221
211
 
222
- def extract_labels(extracted_labels)
223
- labels = {}
224
- extracted_labels.each { |key, value|
225
- if @include_labels.include?(key)
226
- key = key.gsub("@", '')
227
- labels[key] = value
228
- end
229
- }
230
- return labels
212
+ # if by any chance we still have a forming batch, we need to send it.
213
+ send(@batch) if !@batch.nil?
214
+ @batch = nil
231
215
  end
232
216
 
233
- def send(tenant_id, batch)
234
- payload = build_payload(batch)
235
- res = loki_http_request(tenant_id, payload, @min_delay, @max_delay, @retries)
236
-
217
+ def send(batch)
218
+ payload = batch.to_json
219
+ res = loki_http_request(payload)
237
220
  if res.is_a?(Net::HTTPSuccess)
238
221
  @logger.debug("Successfully pushed data to loki")
239
- return
240
222
  else
241
- @logger.error("failed to write post to ", :uri => @uri, :code => res.code, :body => res.body, :message => res.message) if !res.nil?
242
- @logger.debug("Payload object ", :payload => payload)
223
+ @logger.debug("failed payload", :payload => payload)
243
224
  end
244
225
  end
245
226
 
246
- def loki_http_request(tenant_id, payload, min_delay, max_delay, retries)
227
+ def loki_http_request(payload)
247
228
  req = Net::HTTP::Post.new(
248
229
  @uri.request_uri
249
230
  )
250
231
  req.add_field('Content-Type', 'application/json')
251
- req.add_field('X-Scope-OrgID', tenant_id) if tenant_id
232
+ req.add_field('X-Scope-OrgID', @tenant_id) if @tenant_id
233
+ req['User-Agent']= 'loki-logstash'
252
234
  req.basic_auth(@username, @password) if @username
253
235
  req.body = payload
254
236
 
@@ -256,53 +238,28 @@ class LogStash::Outputs::Loki < LogStash::Outputs::Base
256
238
 
257
239
  @logger.debug("sending #{req.body.length} bytes to loki")
258
240
  retry_count = 0
259
- delay = min_delay
241
+ delay = @min_delay
260
242
  begin
261
- res = Net::HTTP.start(@uri.host, @uri.port, **opts) { |http| http.request(req) }
262
- rescue Net::HTTPTooManyRequests, Net::HTTPServerError, Errno::ECONNREFUSED => e
263
- unless retry_count < retries
264
- @logger.error("Error while sending data to loki. Tried #{retry_count} times\n. :error => #{e}")
265
- return res
266
- end
267
-
243
+ res = Net::HTTP.start(@uri.host, @uri.port, **opts) { |http|
244
+ http.request(req)
245
+ }
246
+ return res if !res.nil? && res.code.to_i != 429 && res.code.to_i.div(100) != 5
247
+ raise StandardError.new res
248
+ rescue StandardError => e
268
249
  retry_count += 1
269
- @logger.warn("Trying to send again. Attempt number: #{retry_count}. Retrying in #{delay}s")
270
- sleep delay
271
-
272
- if (delay * 2 - delay) > max_delay
273
- delay = delay
250
+ @logger.warn("Failed to send batch, attempt: #{retry_count}/#{@retries}", :error_inspect => e.inspect, :error => e)
251
+ if retry_count < @retries
252
+ sleep delay
253
+ if delay * 2 <= @max_delay
254
+ delay = delay * 2
255
+ else
256
+ delay = @max_delay
257
+ end
258
+ retry
274
259
  else
275
- delay = delay * 2
260
+ @logger.error("Failed to send batch", :error_inspect => e.inspect, :error => e)
261
+ return res
276
262
  end
277
-
278
- retry
279
- rescue StandardError => e
280
- @logger.error("Error while connecting to loki server ", :error_inspect => e.inspect, :error => e)
281
- return res
282
263
  end
283
- return res
284
- end
285
-
286
- def build_payload(batch)
287
- payload = {}
288
- payload['streams'] = []
289
- batch.streams.each { |labels, stream|
290
- stream_obj = get_stream_obj(stream)
291
- payload['streams'].push(stream_obj)
292
- }
293
- return payload.to_json
294
- end
295
-
296
- def get_stream_obj(stream)
297
- stream_obj = {}
298
- stream_obj['stream'] = stream['labels']
299
- stream_obj['values'] = []
300
- values = []
301
- stream['entries'].each { |entry|
302
- values.push(entry['ts'].to_s)
303
- values.push(entry['line'])
304
- }
305
- stream_obj['values'].push(values)
306
- return stream_obj
307
264
  end
308
265
  end
@@ -1,8 +1,8 @@
1
1
  Gem::Specification.new do |s|
2
- s.name = 'logstash-output-loki'
3
- s.version = '1.0.0'
4
- s.authors = ['Aditya C S']
5
- s.email = ['aditya.gnu@gmail.com']
2
+ s.name = 'logstash-output-loki'
3
+ s.version = '1.0.3'
4
+ s.authors = ['Aditya C S','Cyril Tovena']
5
+ s.email = ['aditya.gnu@gmail.com','cyril.tovena@grafana.com']
6
6
 
7
7
  s.summary = 'Output plugin to ship logs to a Grafana Loki server'
8
8
  s.description = 'Output plugin to ship logs to a Grafana Loki server'
@@ -11,7 +11,7 @@ Gem::Specification.new do |s|
11
11
  s.require_paths = ["lib"]
12
12
 
13
13
  # Files
14
- s.files = Dir['lib/**/*','spec/**/*','vendor/**/*','*.gemspec','*.md','CONTRIBUTORS','Gemfile','LICENSE','NOTICE.TXT']
14
+ s.files = Dir['lib/**/*','spec/**/*','vendor/**/*','*.gemspec','*.md','CONTRIBUTORS','Gemfile']
15
15
  # Tests
16
16
  s.test_files = s.files.grep(%r{^(test|spec|features)/})
17
17
 
@@ -22,6 +22,5 @@ Gem::Specification.new do |s|
22
22
  #
23
23
  s.add_runtime_dependency "logstash-core-plugin-api", ">= 1.60", "<= 2.99"
24
24
  s.add_runtime_dependency "logstash-codec-plain", "3.0.6"
25
- s.add_runtime_dependency "concurrent-ruby-edge", "0.6.0"
26
25
  s.add_development_dependency 'logstash-devutils', "2.0.2"
27
26
  end
@@ -0,0 +1,59 @@
1
+ # encoding: utf-8
2
+ require "logstash/devutils/rspec/spec_helper"
3
+ require "logstash/outputs/loki"
4
+ require "logstash/codecs/plain"
5
+ require "logstash/event"
6
+ require "net/http"
7
+ include Loki
8
+
9
+ describe Loki::Entry do
10
+ context 'test entry generation' do
11
+ let (:event) {
12
+ LogStash::Event.new(
13
+ {
14
+ 'message' => 'hello',
15
+ '@metadata' => {'foo'=>'bar'},
16
+ '@version' => '1',
17
+ 'foo' => 5,
18
+ 'agent' => 'filebeat',
19
+ 'log' => {
20
+ 'file' =>
21
+ {'@path' => '/path/to/file.log'},
22
+ },
23
+ 'host' => '172.0.0.1',
24
+ '@timestamp' => Time.now
25
+ }
26
+ )
27
+ }
28
+
29
+ it 'labels extracted should not contains object and metadata or timestamp' do
30
+ entry = Entry.new(event,"message")
31
+ expect(entry.labels).to eql({ 'agent' => 'filebeat', 'host' => '172.0.0.1', 'foo'=>'5'})
32
+ expect(entry.entry['ts']).to eql to_ns(event.get("@timestamp"))
33
+ expect(entry.entry['line']).to eql 'hello'
34
+ end
35
+ end
36
+
37
+ context 'test batch generation with label order' do
38
+ let (:entries) {[
39
+ Entry.new(LogStash::Event.new({"message"=>"foobuzz","buzz"=>"bar","cluster"=>"us-central1","@timestamp"=>Time.at(1)}),"message"),
40
+ Entry.new(LogStash::Event.new({"log"=>"foobar","bar"=>"bar","@timestamp"=>Time.at(2)}),"log"),
41
+ Entry.new(LogStash::Event.new({"cluster"=>"us-central1","message"=>"foobuzz","buzz"=>"bar","@timestamp"=>Time.at(3)}),"message"),
42
+
43
+ ]}
44
+ let (:expected) {
45
+ {"streams" => [
46
+ {"stream"=> {"buzz"=>"bar","cluster"=>"us-central1"}, "values" => [[to_ns(Time.at(1)).to_s,"foobuzz"],[to_ns(Time.at(3)).to_s,"foobuzz"]]},
47
+ {"stream"=> {"bar"=>"bar"}, "values"=>[[to_ns(Time.at(2)).to_s,"foobar"]]},
48
+ ] }
49
+ }
50
+
51
+ it 'to_json' do
52
+ @batch = Loki::Batch.new(entries.first)
53
+ entries.drop(1).each { |e| @batch.add(e)}
54
+ expect(JSON.parse(@batch.to_json)).to eql expected
55
+ end
56
+ end
57
+
58
+
59
+ end
@@ -4,9 +4,12 @@ require "logstash/outputs/loki"
4
4
  require "logstash/codecs/plain"
5
5
  require "logstash/event"
6
6
  require "net/http"
7
+ require 'webmock/rspec'
8
+ include Loki
7
9
 
8
10
  describe LogStash::Outputs::Loki do
9
- let (:simple_loki_config) {{'url' => 'http://localhost:3100', 'include_labels' => ["test_key", "other_key"], 'external_labels' => {"test" => "value"}}}
11
+
12
+ let (:simple_loki_config) { {'url' => 'http://localhost:3100'} }
10
13
 
11
14
  context 'when initializing' do
12
15
  it "should register" do
@@ -14,100 +17,232 @@ describe LogStash::Outputs::Loki do
14
17
  expect { loki.register }.to_not raise_error
15
18
  end
16
19
 
17
- it 'should populate loki config with default or intialized values' do
20
+ it 'should populate loki config with default or initialized values' do
18
21
  loki = LogStash::Outputs::Loki.new(simple_loki_config)
19
22
  expect(loki.url).to eql 'http://localhost:3100'
20
23
  expect(loki.tenant_id).to eql nil
21
24
  expect(loki.batch_size).to eql 102400
22
25
  expect(loki.batch_wait).to eql 1
23
- expect(loki.include_labels).to eql ["test_key", "other_key"]
24
- expect(loki.external_labels).to include("test" => "value")
25
26
  end
26
27
  end
27
28
 
28
- context 'test labels' do
29
- let (:simple_loki_config) {{'url' => 'http://localhost:3100', 'include_labels' => ["@version", 'log_file_@path', 'host']}}
30
- let (:event) { LogStash::Event.new({'message' => 'hello', '@version' => '1', 'agent' => 'filebeat', 'log' => {'file' => {'@path' => '/path/to/file.log'}}, 'host' => '172.0.0.1',
31
- '@timestamp' => LogStash::Timestamp.now}) }
32
- let(:loki) { LogStash::Plugin.lookup("output", "loki").new(simple_loki_config) }
29
+ context 'when adding en entry to the batch' do
30
+ let (:simple_loki_config) {{'url' => 'http://localhost:3100'}}
31
+ let (:entry) {Entry.new(LogStash::Event.new({"message"=>"foobuzz","buzz"=>"bar","cluster"=>"us-central1","@timestamp"=>Time.at(1)}),"message")}
32
+ let (:lbs) { {"buzz"=>"bar","cluster"=>"us-central1"}.sort.to_h}
33
33
 
34
- before do
35
- loki.register
36
- loki.close
37
- end
34
+ it 'should not add empty line' do
35
+ plugin = LogStash::Plugin.lookup("output", "loki").new(simple_loki_config)
36
+ emptyEntry = Entry.new(LogStash::Event.new({"message"=>"foobuzz","buzz"=>"bar","cluster"=>"us-central1","@timestamp"=>Time.at(1)}),"foo")
37
+ expect(plugin.add_entry_to_batch(emptyEntry)).to eql true
38
+ expect(plugin.batch).to eql nil
39
+ end
38
40
 
39
- it 'labels extracted should have only included labels' do
40
- labels = {}
41
- event_hash = event.to_hash
42
- expected_labels = {"version" => "1", "host" => "172.0.0.1", "log_file_path" => '/path/to/file.log'}
43
- expect(loki.handle_labels(event_hash, labels, "")).to eql expected_labels
44
- end
41
+ it 'should add entry' do
42
+ plugin = LogStash::Plugin.lookup("output", "loki").new(simple_loki_config)
43
+ expect(plugin.batch).to eql nil
44
+ expect(plugin.add_entry_to_batch(entry)).to eql true
45
+ expect(plugin.add_entry_to_batch(entry)).to eql true
46
+ expect(plugin.batch).not_to be_nil
47
+ expect(plugin.batch.streams.length).to eq 1
48
+ expect(plugin.batch.streams[lbs.to_s]['entries'].length).to eq 2
49
+ expect(plugin.batch.streams[lbs.to_s]['labels']).to eq lbs
50
+ expect(plugin.batch.size_bytes).to eq 14
51
+ end
52
+
53
+ it 'should not add if full' do
54
+ plugin = LogStash::Plugin.lookup("output", "loki").new(simple_loki_config.merge!({'batch_size'=>10}))
55
+ expect(plugin.batch).to eql nil
56
+ expect(plugin.add_entry_to_batch(entry)).to eql true # first entry is fine.
57
+ expect(plugin.batch).not_to be_nil
58
+ expect(plugin.batch.streams.length).to eq 1
59
+ expect(plugin.batch.streams[lbs.to_s]['entries'].length).to eq 1
60
+ expect(plugin.batch.streams[lbs.to_s]['labels']).to eq lbs
61
+ expect(plugin.batch.size_bytes).to eq 7
62
+ expect(plugin.add_entry_to_batch(entry)).to eql false # second entry goes over the limit.
63
+ expect(plugin.batch).not_to be_nil
64
+ expect(plugin.batch.streams.length).to eq 1
65
+ expect(plugin.batch.streams[lbs.to_s]['entries'].length).to eq 1
66
+ expect(plugin.batch.streams[lbs.to_s]['labels']).to eq lbs
67
+ expect(plugin.batch.size_bytes).to eq 7
68
+ end
45
69
  end
46
70
 
47
- context 'validate entries' do
48
- let(:timestamp) {LogStash::Timestamp.now}
49
- let (:simple_loki_config) {{'url' => 'http://localhost:3100', 'include_labels' => ["version", "host", "test"], 'external_labels' => {"test" => "value"}}}
50
- let (:event) { LogStash::Event.new({'message' => 'hello', '@version' => '1', 'agent' => 'filebeat', 'host' => '172.0.0.1',
51
- '@timestamp' => timestamp}) }
52
- let(:loki) { LogStash::Plugin.lookup("output", "loki").new(simple_loki_config) }
71
+ context 'batch expiration' do
72
+ let (:entry) {Entry.new(LogStash::Event.new({"message"=>"foobuzz","buzz"=>"bar","cluster"=>"us-central1","@timestamp"=>Time.at(1)}),"message")}
53
73
 
54
- before do
55
- loki.register
56
- loki.close
74
+ it 'should not expire if empty' do
75
+ loki = LogStash::Outputs::Loki.new(simple_loki_config.merge!({'batch_wait'=>0.5}))
76
+ sleep(1)
77
+ expect(loki.is_batch_expired).to be false
57
78
  end
58
-
59
- it 'validate expected entries are added to entries stream' do
60
- labels = {"version" => "1", "host" => "172.0.0.1"}
61
- expected_labels = {"version" => "1", "host" => "172.0.0.1", "test" => "value"}
62
- expected_entry_hash = {
63
- "ts" => timestamp.to_i * (10**9),
64
- "line" => "hello".to_s
65
- }
66
- expected_labels_and_entry_hash = [{"version" => "1", "host" => "172.0.0.1", "test" => "value"}, expected_entry_hash]
67
- expect(loki.build_entry(labels, event)).to eq(expected_labels_and_entry_hash)
79
+ it 'should not expire batch if not old' do
80
+ loki = LogStash::Outputs::Loki.new(simple_loki_config.merge!({'batch_wait'=>0.5}))
81
+ expect(loki.add_entry_to_batch(entry)).to eql true
82
+ expect(loki.is_batch_expired).to be false
83
+ end
84
+ it 'should expire if old' do
85
+ loki = LogStash::Outputs::Loki.new(simple_loki_config.merge!({'batch_wait'=>0.5}))
86
+ expect(loki.add_entry_to_batch(entry)).to eql true
87
+ sleep(1)
88
+ expect(loki.is_batch_expired).to be true
68
89
  end
69
90
  end
70
91
 
71
- context 'test http requests' do
72
- let (:simple_loki_config) {{'url' => 'http://localhost:3100', 'include_labels' => ["@version", "host", "test"],}}
73
- let (:event) { LogStash::Event.new({'message' => 'hello', '@version' => '1', 'host' => '172.0.0.1',
74
- '@timestamp' => LogStash::Timestamp.now}) }
75
- let(:loki) { LogStash::Plugin.lookup("output", "loki").new(simple_loki_config) }
92
+ context 'channel' do
93
+ let (:event) {LogStash::Event.new({"message"=>"foobuzz","buzz"=>"bar","cluster"=>"us-central1","@timestamp"=>Time.at(1)})}
76
94
 
77
- before do
95
+ it 'should send entry if batch size reached with no tenant' do
96
+ loki = LogStash::Outputs::Loki.new(simple_loki_config.merge!({'batch_wait'=>0.5,'batch_size'=>10}))
78
97
  loki.register
98
+ sent = Queue.new
99
+ allow(loki).to receive(:send) do |batch|
100
+ Thread.new do
101
+ sent << batch
102
+ end
103
+ end
104
+ loki.receive(event)
105
+ loki.receive(event)
106
+ sent.deq
107
+ sent.deq
79
108
  loki.close
80
109
  end
110
+ it 'should send entry while closing' do
111
+ loki = LogStash::Outputs::Loki.new(simple_loki_config.merge!({'batch_wait'=>10,'batch_size'=>10}))
112
+ loki.register
113
+ sent = Queue.new
114
+ allow(loki).to receive(:send) do | batch|
115
+ Thread.new do
116
+ sent << batch
117
+ end
118
+ end
119
+ loki.receive(event)
120
+ loki.close
121
+ sent.deq
122
+ end
123
+ it 'should send entry when batch is expiring' do
124
+ loki = LogStash::Outputs::Loki.new(simple_loki_config.merge!({'batch_wait'=>0.5,'batch_size'=>10}))
125
+ loki.register
126
+ sent = Queue.new
127
+ allow(loki).to receive(:send) do | batch|
128
+ Thread.new do
129
+ sent << batch
130
+ end
131
+ end
132
+ loki.receive(event)
133
+ sent.deq
134
+ sleep(0.01) # Adding a minimal sleep. In few cases @batch=nil might happen after evaluating for nil
135
+ expect(loki.batch).to be_nil
136
+ loki.close
137
+ end
138
+ end
81
139
 
82
- it 'test http requests and raise_error when requests are not successful' do
83
- labels = {}
84
- event_hash = event.to_hash
85
- lbls = loki.handle_labels(event_hash, labels, "")
86
- entry_hash = {
87
- "ts" => event.get("@timestamp").to_i * (10**9),
88
- "line" => event.get("message").to_s
89
- }
90
- e = LogStash::Outputs::Loki::Entry.new(lbls, entry_hash)
91
- batch = LogStash::Outputs::Loki::Batch.new(e)
92
- payload = loki.build_payload(batch)
93
-
94
- # response should be nil on connection error
95
- expect(loki.loki_http_request("fake", payload, 1, 2, 3)).to eql nil
96
-
97
- success = Net::HTTPSuccess.new(1.0, 200, 'OK')
98
- allow(loki).to receive(:loki_http_request) { success }
99
- allow(success).to receive(:payload).and_return('fake body')
100
- expect(loki.loki_http_request("fake", batch, 1, 300, 10).class).to eql Net::HTTPSuccess
140
+ context 'http requests' do
141
+ let (:entry) {Entry.new(LogStash::Event.new({"message"=>"foobuzz","buzz"=>"bar","cluster"=>"us-central1","@timestamp"=>Time.at(1)}),"message")}
101
142
 
102
- too_many_requests = Net::HTTPTooManyRequests.new(1.0, 429, 'OK')
103
- allow(loki).to receive(:loki_http_request) { too_many_requests }
104
- allow(too_many_requests).to receive(:payload).and_return('fake body')
105
- expect(loki.loki_http_request("fake", batch, 1, 300, 10).class).to eql Net::HTTPTooManyRequests
143
+ it 'should send credentials' do
144
+ conf = {
145
+ 'url'=>'http://localhost:3100/loki/api/v1/push',
146
+ 'username' => 'foo',
147
+ 'password' => 'bar',
148
+ 'tenant_id' => 't'
149
+ }
150
+ loki = LogStash::Outputs::Loki.new(conf)
151
+ loki.register
152
+ b = Batch.new(entry)
153
+ post = stub_request(:post, "http://localhost:3100/loki/api/v1/push").with(
154
+ basic_auth: ['foo', 'bar'],
155
+ body: b.to_json,
156
+ headers:{
157
+ 'Content-Type' => 'application/json' ,
158
+ 'User-Agent' => 'loki-logstash',
159
+ 'X-Scope-OrgID'=>'t',
160
+ 'Accept'=>'*/*',
161
+ 'Accept-Encoding'=>'gzip;q=1.0,deflate;q=0.6,identity;q=0.3',
162
+ }
163
+ )
164
+ loki.send(b)
165
+ expect(post).to have_been_requested.times(1)
166
+ end
106
167
 
107
- server_error = Net::HTTPServerError.new(1.0, 429, 'OK')
108
- allow(loki).to receive(:loki_http_request) { server_error }
109
- allow(server_error).to receive(:payload).and_return('fake body')
110
- expect(loki.loki_http_request("fake", batch, 1, 300, 10).class).to eql Net::HTTPServerError
168
+ it 'should not send credentials' do
169
+ conf = {
170
+ 'url'=>'http://foo.com/loki/api/v1/push',
171
+ }
172
+ loki = LogStash::Outputs::Loki.new(conf)
173
+ loki.register
174
+ b = Batch.new(entry)
175
+ post = stub_request(:post, "http://foo.com/loki/api/v1/push").with(
176
+ body: b.to_json,
177
+ headers:{
178
+ 'Content-Type' => 'application/json' ,
179
+ 'User-Agent' => 'loki-logstash',
180
+ 'Accept'=>'*/*',
181
+ 'Accept-Encoding'=>'gzip;q=1.0,deflate;q=0.6,identity;q=0.3',
182
+ }
183
+ )
184
+ loki.send(b)
185
+ expect(post).to have_been_requested.times(1)
186
+ end
187
+ it 'should retry 500' do
188
+ conf = {
189
+ 'url'=>'http://foo.com/loki/api/v1/push',
190
+ 'retries' => 3,
191
+ }
192
+ loki = LogStash::Outputs::Loki.new(conf)
193
+ loki.register
194
+ b = Batch.new(entry)
195
+ post = stub_request(:post, "http://foo.com/loki/api/v1/push").with(
196
+ body: b.to_json,
197
+ ).to_return(status: [500, "Internal Server Error"])
198
+ loki.send(b)
199
+ loki.close
200
+ expect(post).to have_been_requested.times(3)
201
+ end
202
+ it 'should retry 429' do
203
+ conf = {
204
+ 'url'=>'http://foo.com/loki/api/v1/push',
205
+ 'retries' => 2,
206
+ }
207
+ loki = LogStash::Outputs::Loki.new(conf)
208
+ loki.register
209
+ b = Batch.new(entry)
210
+ post = stub_request(:post, "http://foo.com/loki/api/v1/push").with(
211
+ body: b.to_json,
212
+ ).to_return(status: [429, "stop spamming"])
213
+ loki.send(b)
214
+ loki.close
215
+ expect(post).to have_been_requested.times(2)
216
+ end
217
+ it 'should not retry 400' do
218
+ conf = {
219
+ 'url'=>'http://foo.com/loki/api/v1/push',
220
+ 'retries' => 11,
221
+ }
222
+ loki = LogStash::Outputs::Loki.new(conf)
223
+ loki.register
224
+ b = Batch.new(entry)
225
+ post = stub_request(:post, "http://foo.com/loki/api/v1/push").with(
226
+ body: b.to_json,
227
+ ).to_return(status: [400, "bad request"])
228
+ loki.send(b)
229
+ loki.close
230
+ expect(post).to have_been_requested.times(1)
231
+ end
232
+ it 'should retry exception' do
233
+ conf = {
234
+ 'url'=>'http://foo.com/loki/api/v1/push',
235
+ 'retries' => 11,
236
+ }
237
+ loki = LogStash::Outputs::Loki.new(conf)
238
+ loki.register
239
+ b = Batch.new(entry)
240
+ post = stub_request(:post, "http://foo.com/loki/api/v1/push").with(
241
+ body: b.to_json,
242
+ ).to_raise("some error").then.to_return(status: [200, "fine !"])
243
+ loki.send(b)
244
+ loki.close
245
+ expect(post).to have_been_requested.times(2)
111
246
  end
112
247
  end
113
248
  end
metadata CHANGED
@@ -1,16 +1,18 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: logstash-output-loki
3
3
  version: !ruby/object:Gem::Version
4
- version: 1.0.0
4
+ version: 1.0.3
5
5
  platform: ruby
6
6
  authors:
7
7
  - Aditya C S
8
- autorequire:
8
+ - Cyril Tovena
9
+ autorequire:
9
10
  bindir: bin
10
11
  cert_chain: []
11
- date: 2020-07-13 00:00:00.000000000 Z
12
+ date: 2020-11-13 00:00:00.000000000 Z
12
13
  dependencies:
13
14
  - !ruby/object:Gem::Dependency
15
+ name: logstash-core-plugin-api
14
16
  requirement: !ruby/object:Gem::Requirement
15
17
  requirements:
16
18
  - - ">="
@@ -19,7 +21,6 @@ dependencies:
19
21
  - - "<="
20
22
  - !ruby/object:Gem::Version
21
23
  version: '2.99'
22
- name: logstash-core-plugin-api
23
24
  type: :runtime
24
25
  prerelease: false
25
26
  version_requirements: !ruby/object:Gem::Requirement
@@ -31,12 +32,12 @@ dependencies:
31
32
  - !ruby/object:Gem::Version
32
33
  version: '2.99'
33
34
  - !ruby/object:Gem::Dependency
35
+ name: logstash-codec-plain
34
36
  requirement: !ruby/object:Gem::Requirement
35
37
  requirements:
36
38
  - - '='
37
39
  - !ruby/object:Gem::Version
38
40
  version: 3.0.6
39
- name: logstash-codec-plain
40
41
  type: :runtime
41
42
  prerelease: false
42
43
  version_requirements: !ruby/object:Gem::Requirement
@@ -45,26 +46,12 @@ dependencies:
45
46
  - !ruby/object:Gem::Version
46
47
  version: 3.0.6
47
48
  - !ruby/object:Gem::Dependency
48
- requirement: !ruby/object:Gem::Requirement
49
- requirements:
50
- - - '='
51
- - !ruby/object:Gem::Version
52
- version: 0.6.0
53
- name: concurrent-ruby-edge
54
- type: :runtime
55
- prerelease: false
56
- version_requirements: !ruby/object:Gem::Requirement
57
- requirements:
58
- - - '='
59
- - !ruby/object:Gem::Version
60
- version: 0.6.0
61
- - !ruby/object:Gem::Dependency
49
+ name: logstash-devutils
62
50
  requirement: !ruby/object:Gem::Requirement
63
51
  requirements:
64
52
  - - '='
65
53
  - !ruby/object:Gem::Version
66
54
  version: 2.0.2
67
- name: logstash-devutils
68
55
  type: :development
69
56
  prerelease: false
70
57
  version_requirements: !ruby/object:Gem::Requirement
@@ -75,6 +62,7 @@ dependencies:
75
62
  description: Output plugin to ship logs to a Grafana Loki server
76
63
  email:
77
64
  - aditya.gnu@gmail.com
65
+ - cyril.tovena@grafana.com
78
66
  executables: []
79
67
  extensions: []
80
68
  extra_rdoc_files: []
@@ -85,6 +73,7 @@ files:
85
73
  - lib/logstash/outputs/loki/batch.rb
86
74
  - lib/logstash/outputs/loki/entry.rb
87
75
  - logstash-output-loki.gemspec
76
+ - spec/outputs/loki/entry_spec.rb
88
77
  - spec/outputs/loki_spec.rb
89
78
  homepage: https://github.com/grafana/loki/
90
79
  licenses:
@@ -92,7 +81,7 @@ licenses:
92
81
  metadata:
93
82
  logstash_plugin: 'true'
94
83
  logstash_group: output
95
- post_install_message:
84
+ post_install_message:
96
85
  rdoc_options: []
97
86
  require_paths:
98
87
  - lib
@@ -107,9 +96,10 @@ required_rubygems_version: !ruby/object:Gem::Requirement
107
96
  - !ruby/object:Gem::Version
108
97
  version: '0'
109
98
  requirements: []
110
- rubygems_version: 3.0.6
111
- signing_key:
99
+ rubygems_version: 3.0.3
100
+ signing_key:
112
101
  specification_version: 4
113
102
  summary: Output plugin to ship logs to a Grafana Loki server
114
103
  test_files:
104
+ - spec/outputs/loki/entry_spec.rb
115
105
  - spec/outputs/loki_spec.rb