logstash-output-loki 1.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/Gemfile +11 -0
- data/README.md +66 -0
- data/lib/logstash/outputs/loki.rb +308 -0
- data/lib/logstash/outputs/loki/batch.rb +47 -0
- data/lib/logstash/outputs/loki/entry.rb +13 -0
- data/logstash-output-loki.gemspec +27 -0
- data/spec/outputs/loki_spec.rb +113 -0
- metadata +115 -0
checksums.yaml
ADDED
@@ -0,0 +1,7 @@
|
|
1
|
+
---
|
2
|
+
SHA256:
|
3
|
+
metadata.gz: 9dd08c392ec88148a6f410e159169b3a19f877a25174fa3f43b2fdeb7fe16128
|
4
|
+
data.tar.gz: 34aaf03576d8fbbb97aab02013113254734984ee1744c2ab4e8438bd51657569
|
5
|
+
SHA512:
|
6
|
+
metadata.gz: 3520ab5618092976cfe771ad4f9ddea4aca3b96efe75d322c2f505e7fd6e642f3cee4775eb1ef0514c9db360a3c36b138e9f1426f6de9b625945cafdafb0b27d
|
7
|
+
data.tar.gz: 6c7bb11d79fe0828e629bc27771a4f63584ea76af2ddb69eac950e5c7e3b115b62b674be28e8a828c97d8d3d851406748f0c83d49d9147080f0cc593ad1a076f
|
data/Gemfile
ADDED
@@ -0,0 +1,11 @@
|
|
1
|
+
source 'https://rubygems.org'
|
2
|
+
|
3
|
+
gemspec
|
4
|
+
|
5
|
+
logstash_path = ENV["LOGSTASH_PATH"] || "logstash-libs"
|
6
|
+
use_logstash_source = ENV["LOGSTASH_SOURCE"] && ENV["LOGSTASH_SOURCE"].to_s == "1"
|
7
|
+
|
8
|
+
if Dir.exist?(logstash_path) && use_logstash_source
|
9
|
+
gem 'logstash-core', :path => "#{logstash_path}/logstash-core"
|
10
|
+
gem 'logstash-core-plugin-api', :path => "#{logstash_path}/logstash-core-plugin-api"
|
11
|
+
end
|
data/README.md
ADDED
@@ -0,0 +1,66 @@
|
|
1
|
+
# Loki Logstash Output Plugin
|
2
|
+
|
3
|
+
Logstash plugin to send logstash aggregated logs to Loki.
|
4
|
+
|
5
|
+
## Install dependencies
|
6
|
+
|
7
|
+
First you need to setup JRuby environment to build this plugin. Refer https://github.com/rbenv/rbenv for setting up your rbenv environment.
|
8
|
+
|
9
|
+
After setting up `rbenv`. Install JRuby
|
10
|
+
|
11
|
+
```bash
|
12
|
+
rbenv install jruby-9.2.10.0
|
13
|
+
rbenv local jruby-9.2.10.0
|
14
|
+
```
|
15
|
+
|
16
|
+
Check that the environment is configured
|
17
|
+
|
18
|
+
```bash
|
19
|
+
ruby --version
|
20
|
+
jruby 9.2.10
|
21
|
+
```
|
22
|
+
|
23
|
+
You should use make sure you are running jruby and not ruby. If the command below still shows ruby and not jruby, check that PATH contains `$HOME/.rbenv/shims` and `$HOME/.rbenv/bin`. Also verify that you have this in your bash profile:
|
24
|
+
|
25
|
+
```bash
|
26
|
+
export PATH="$HOME/.rbenv/bin:$PATH"
|
27
|
+
eval "$(rbenv init -)"
|
28
|
+
```
|
29
|
+
|
30
|
+
Then install bundler
|
31
|
+
`gem install bundler:2.1.4`
|
32
|
+
|
33
|
+
Follow those instructions to [install logstash](https://www.elastic.co/guide/en/logstash/current/installing-logstash.html) before moving to the next section.
|
34
|
+
|
35
|
+
## Install dependencies and Build plugin
|
36
|
+
|
37
|
+
### Install required packages
|
38
|
+
|
39
|
+
```bash
|
40
|
+
git clone git@github.com:elastic/logstash.git
|
41
|
+
cd logstash
|
42
|
+
git checkout tags/v7.6.2
|
43
|
+
export LOGSTASH_PATH=`pwd`
|
44
|
+
export LOGSTASH_SOURCE="1"
|
45
|
+
export GEM_PATH=$LOGSTASH_PATH/vendor/bundle/
|
46
|
+
export GEM_HOME=$LOGSTASH_PATH/vendor/bundle/
|
47
|
+
cd ..
|
48
|
+
ruby -S bundle install --path
|
49
|
+
ruby -S bundle exec rake vendor
|
50
|
+
```
|
51
|
+
|
52
|
+
### Build the plugin
|
53
|
+
|
54
|
+
`gem build logstash-output-loki.gemspec`
|
55
|
+
|
56
|
+
### Test
|
57
|
+
|
58
|
+
`bundle exec rspec`
|
59
|
+
|
60
|
+
## Install plugin to local logstash
|
61
|
+
|
62
|
+
`bin/logstash-plugin install --no-verify --local logstash-output-loki-1.0.0.gem`
|
63
|
+
|
64
|
+
## Send sample event and check plugin is working
|
65
|
+
|
66
|
+
`bin/logstash -f loki.conf`
|
@@ -0,0 +1,308 @@
|
|
1
|
+
# encoding: utf-8
|
2
|
+
require "logstash/outputs/base"
|
3
|
+
require "logstash/namespace"
|
4
|
+
require 'net/http'
|
5
|
+
require 'concurrent-edge'
|
6
|
+
require 'time'
|
7
|
+
require 'uri'
|
8
|
+
require 'json'
|
9
|
+
|
10
|
+
class LogStash::Outputs::Loki < LogStash::Outputs::Base
|
11
|
+
require 'logstash/outputs/loki/batch'
|
12
|
+
require 'logstash/outputs/loki/entry'
|
13
|
+
|
14
|
+
config_name "loki"
|
15
|
+
|
16
|
+
## 'A single instance of the Output will be shared among the pipeline worker threads'
|
17
|
+
concurrency :single
|
18
|
+
|
19
|
+
## 'Loki URL'
|
20
|
+
config :url, :validate => :string, :required => true
|
21
|
+
|
22
|
+
## 'BasicAuth credentials'
|
23
|
+
config :username, :validate => :string, :required => false
|
24
|
+
config :password, :validate => :string, secret: true, :required => false
|
25
|
+
|
26
|
+
## 'Client certificate'
|
27
|
+
config :cert, :validate => :path, :required => false
|
28
|
+
config :key, :validate => :path, :required => false
|
29
|
+
|
30
|
+
## 'TLS'
|
31
|
+
config :ca_cert, :validate => :path, :required => false
|
32
|
+
|
33
|
+
## 'Loki Tenant ID'
|
34
|
+
config :tenant_id, :validate => :string, :required => false
|
35
|
+
|
36
|
+
## 'Maximum batch size to accrue before pushing to loki. Defaults to 102400 bytes'
|
37
|
+
config :batch_size, :validate => :number, :default => 102400, :required => false
|
38
|
+
|
39
|
+
## 'Interval in seconds to wait before pushing a batch of records to loki. Defaults to 1 second'
|
40
|
+
config :batch_wait, :validate => :number, :default => 1, :required => false
|
41
|
+
|
42
|
+
## 'Array of label names to include in all logstreams'
|
43
|
+
config :include_labels, :validate => :array, :default => [], :required => true
|
44
|
+
|
45
|
+
## 'Extra labels to add to all log streams'
|
46
|
+
config :external_labels, :validate => :hash, :default => {}, :required => false
|
47
|
+
|
48
|
+
## 'Log line field to pick from logstash. Defaults to "message"'
|
49
|
+
config :message_field, :validate => :string, :default => "message", :required => false
|
50
|
+
|
51
|
+
## 'Backoff configuration. Initial backoff time between retries. Default 1s'
|
52
|
+
config :min_delay, :validate => :number, :default => 1, :required => false
|
53
|
+
|
54
|
+
## 'Backoff configuration. Maximum backoff time between retries. Default 300s'
|
55
|
+
config :max_delay, :validate => :number, :default => 300, :required => false
|
56
|
+
|
57
|
+
## 'Backoff configuration. Maximum number of retries to do'
|
58
|
+
config :retries, :validate => :number, :default => 10, :required => false
|
59
|
+
|
60
|
+
public
|
61
|
+
def register
|
62
|
+
@uri = URI.parse(@url)
|
63
|
+
unless @uri.is_a?(URI::HTTP) || @uri.is_a?(URI::HTTPS)
|
64
|
+
raise LogStash::ConfigurationError, "url parameter must be valid HTTP, currently '#{@url}'"
|
65
|
+
end
|
66
|
+
|
67
|
+
if @include_labels.empty?
|
68
|
+
raise LogStash::ConfigurationError, "include_labels should contain atleast one label, currently '#{@include_labels}'"
|
69
|
+
end
|
70
|
+
|
71
|
+
if @min_delay > @max_delay
|
72
|
+
raise LogStash::ConfigurationError, "Min delay should be less than Max delay, currently 'Min delay is #{@min_delay} and Max delay is #{@max_delay}'"
|
73
|
+
end
|
74
|
+
|
75
|
+
@logger.info("Loki output plugin", :class => self.class.name)
|
76
|
+
|
77
|
+
# intialize channels
|
78
|
+
@Channel = Concurrent::Channel
|
79
|
+
@entries = @Channel.new
|
80
|
+
|
81
|
+
# excluded message and timestamp from labels
|
82
|
+
@exclude_labels = ["message", "@timestamp"]
|
83
|
+
|
84
|
+
# create nil batch object.
|
85
|
+
@batch = nil
|
86
|
+
|
87
|
+
# validate certs
|
88
|
+
if ssl_cert?
|
89
|
+
load_ssl
|
90
|
+
validate_ssl_key
|
91
|
+
end
|
92
|
+
|
93
|
+
@Channel.go{run()}
|
94
|
+
end
|
95
|
+
|
96
|
+
def ssl_cert?
|
97
|
+
!@key.nil? && !@cert.nil?
|
98
|
+
end
|
99
|
+
|
100
|
+
def load_ssl
|
101
|
+
@cert = OpenSSL::X509::Certificate.new(File.read(@cert)) if @cert
|
102
|
+
@key = OpenSSL::PKey.read(File.read(@key)) if @key
|
103
|
+
end
|
104
|
+
|
105
|
+
def validate_ssl_key
|
106
|
+
if !@key.is_a?(OpenSSL::PKey::RSA) && !@key.is_a?(OpenSSL::PKey::DSA)
|
107
|
+
raise LogStash::ConfigurationError, "Unsupported private key type '#{@key.class}''"
|
108
|
+
end
|
109
|
+
end
|
110
|
+
|
111
|
+
def ssl_opts(uri)
|
112
|
+
opts = {
|
113
|
+
use_ssl: uri.scheme == 'https'
|
114
|
+
}
|
115
|
+
|
116
|
+
if !@cert.nil? && !@key.nil?
|
117
|
+
opts = opts.merge(
|
118
|
+
verify_mode: OpenSSL::SSL::VERIFY_PEER,
|
119
|
+
cert: @cert,
|
120
|
+
key: @key
|
121
|
+
)
|
122
|
+
end
|
123
|
+
|
124
|
+
unless @ca_cert.nil?
|
125
|
+
opts = opts.merge(
|
126
|
+
ca_file: @ca_cert
|
127
|
+
)
|
128
|
+
end
|
129
|
+
opts
|
130
|
+
end
|
131
|
+
|
132
|
+
def run()
|
133
|
+
min_wait_checkfrequency = 1/1000 #1 millisecond
|
134
|
+
max_wait_checkfrequency = @batch_wait
|
135
|
+
if max_wait_checkfrequency < min_wait_checkfrequency
|
136
|
+
max_wait_checkfrequency = min_wait_checkfrequency
|
137
|
+
end
|
138
|
+
|
139
|
+
@max_wait_check = Concurrent::Channel.tick(max_wait_checkfrequency)
|
140
|
+
loop do
|
141
|
+
Concurrent::Channel.select do |s|
|
142
|
+
s.take(@entries) { |e|
|
143
|
+
if @batch.nil?
|
144
|
+
@batch = Batch.new(e)
|
145
|
+
next
|
146
|
+
end
|
147
|
+
|
148
|
+
line = e.entry['line']
|
149
|
+
if @batch.size_bytes_after(line) > @batch_size
|
150
|
+
@logger.debug("Max batch_size is reached. Sending batch to loki")
|
151
|
+
send(@tenant_id, @batch)
|
152
|
+
@batch = Batch.new(e)
|
153
|
+
next
|
154
|
+
end
|
155
|
+
@batch.add(e)
|
156
|
+
}
|
157
|
+
s.take(@max_wait_check) {
|
158
|
+
# Send batch if max wait time has been reached
|
159
|
+
if !@batch.nil?
|
160
|
+
if @batch.age() < @batch_wait
|
161
|
+
next
|
162
|
+
end
|
163
|
+
|
164
|
+
@logger.debug("Max batch_wait time is reached. Sending batch to loki")
|
165
|
+
send(@tenant_id, @batch)
|
166
|
+
@batch = nil
|
167
|
+
end
|
168
|
+
}
|
169
|
+
end
|
170
|
+
end
|
171
|
+
end
|
172
|
+
|
173
|
+
## Receives logstash events
|
174
|
+
public
|
175
|
+
def receive(event)
|
176
|
+
labels = {}
|
177
|
+
event_hash = event.to_hash
|
178
|
+
lbls = handle_labels(event_hash, labels, "")
|
179
|
+
|
180
|
+
data_labels, entry_hash = build_entry(lbls, event)
|
181
|
+
@entries << Entry.new(data_labels, entry_hash)
|
182
|
+
|
183
|
+
end
|
184
|
+
|
185
|
+
def close
|
186
|
+
@logger.info("Closing loki output plugin. Flushing all pending batches")
|
187
|
+
send(@tenant_id, @batch) if !@batch.nil?
|
188
|
+
@entries.close
|
189
|
+
@max_wait_check.close if !@max_wait_check.nil?
|
190
|
+
end
|
191
|
+
|
192
|
+
def build_entry(lbls, event)
|
193
|
+
labels = lbls.merge(@external_labels)
|
194
|
+
entry_hash = {
|
195
|
+
"ts" => event.get("@timestamp").to_i * (10**9),
|
196
|
+
"line" => event.get(@message_field).to_s
|
197
|
+
}
|
198
|
+
return labels, entry_hash
|
199
|
+
end
|
200
|
+
|
201
|
+
def handle_labels(event_hash, labels, parent_key)
|
202
|
+
event_hash.each{ |key,value|
|
203
|
+
if !@exclude_labels.include?(key)
|
204
|
+
if value.is_a?(Hash)
|
205
|
+
if parent_key != ""
|
206
|
+
handle_labels(value, labels, parent_key + "_" + key)
|
207
|
+
else
|
208
|
+
handle_labels(value, labels, key)
|
209
|
+
end
|
210
|
+
else
|
211
|
+
if parent_key != ""
|
212
|
+
labels[parent_key + "_" + key] = value.to_s
|
213
|
+
else
|
214
|
+
labels[key] = value.to_s
|
215
|
+
end
|
216
|
+
end
|
217
|
+
end
|
218
|
+
}
|
219
|
+
return extract_labels(labels)
|
220
|
+
end
|
221
|
+
|
222
|
+
def extract_labels(extracted_labels)
|
223
|
+
labels = {}
|
224
|
+
extracted_labels.each { |key, value|
|
225
|
+
if @include_labels.include?(key)
|
226
|
+
key = key.gsub("@", '')
|
227
|
+
labels[key] = value
|
228
|
+
end
|
229
|
+
}
|
230
|
+
return labels
|
231
|
+
end
|
232
|
+
|
233
|
+
def send(tenant_id, batch)
|
234
|
+
payload = build_payload(batch)
|
235
|
+
res = loki_http_request(tenant_id, payload, @min_delay, @max_delay, @retries)
|
236
|
+
|
237
|
+
if res.is_a?(Net::HTTPSuccess)
|
238
|
+
@logger.debug("Successfully pushed data to loki")
|
239
|
+
return
|
240
|
+
else
|
241
|
+
@logger.error("failed to write post to ", :uri => @uri, :code => res.code, :body => res.body, :message => res.message) if !res.nil?
|
242
|
+
@logger.debug("Payload object ", :payload => payload)
|
243
|
+
end
|
244
|
+
end
|
245
|
+
|
246
|
+
def loki_http_request(tenant_id, payload, min_delay, max_delay, retries)
|
247
|
+
req = Net::HTTP::Post.new(
|
248
|
+
@uri.request_uri
|
249
|
+
)
|
250
|
+
req.add_field('Content-Type', 'application/json')
|
251
|
+
req.add_field('X-Scope-OrgID', tenant_id) if tenant_id
|
252
|
+
req.basic_auth(@username, @password) if @username
|
253
|
+
req.body = payload
|
254
|
+
|
255
|
+
opts = ssl_opts(@uri)
|
256
|
+
|
257
|
+
@logger.debug("sending #{req.body.length} bytes to loki")
|
258
|
+
retry_count = 0
|
259
|
+
delay = min_delay
|
260
|
+
begin
|
261
|
+
res = Net::HTTP.start(@uri.host, @uri.port, **opts) { |http| http.request(req) }
|
262
|
+
rescue Net::HTTPTooManyRequests, Net::HTTPServerError, Errno::ECONNREFUSED => e
|
263
|
+
unless retry_count < retries
|
264
|
+
@logger.error("Error while sending data to loki. Tried #{retry_count} times\n. :error => #{e}")
|
265
|
+
return res
|
266
|
+
end
|
267
|
+
|
268
|
+
retry_count += 1
|
269
|
+
@logger.warn("Trying to send again. Attempt number: #{retry_count}. Retrying in #{delay}s")
|
270
|
+
sleep delay
|
271
|
+
|
272
|
+
if (delay * 2 - delay) > max_delay
|
273
|
+
delay = delay
|
274
|
+
else
|
275
|
+
delay = delay * 2
|
276
|
+
end
|
277
|
+
|
278
|
+
retry
|
279
|
+
rescue StandardError => e
|
280
|
+
@logger.error("Error while connecting to loki server ", :error_inspect => e.inspect, :error => e)
|
281
|
+
return res
|
282
|
+
end
|
283
|
+
return res
|
284
|
+
end
|
285
|
+
|
286
|
+
def build_payload(batch)
|
287
|
+
payload = {}
|
288
|
+
payload['streams'] = []
|
289
|
+
batch.streams.each { |labels, stream|
|
290
|
+
stream_obj = get_stream_obj(stream)
|
291
|
+
payload['streams'].push(stream_obj)
|
292
|
+
}
|
293
|
+
return payload.to_json
|
294
|
+
end
|
295
|
+
|
296
|
+
def get_stream_obj(stream)
|
297
|
+
stream_obj = {}
|
298
|
+
stream_obj['stream'] = stream['labels']
|
299
|
+
stream_obj['values'] = []
|
300
|
+
values = []
|
301
|
+
stream['entries'].each { |entry|
|
302
|
+
values.push(entry['ts'].to_s)
|
303
|
+
values.push(entry['line'])
|
304
|
+
}
|
305
|
+
stream_obj['values'].push(values)
|
306
|
+
return stream_obj
|
307
|
+
end
|
308
|
+
end
|
@@ -0,0 +1,47 @@
|
|
1
|
+
require 'time'
|
2
|
+
|
3
|
+
module LogStash
|
4
|
+
module Outputs
|
5
|
+
class Loki
|
6
|
+
class Batch
|
7
|
+
attr_reader :streams
|
8
|
+
def initialize(e)
|
9
|
+
@bytes = 0
|
10
|
+
@createdAt = Time.now
|
11
|
+
@streams = {}
|
12
|
+
add(e)
|
13
|
+
end
|
14
|
+
|
15
|
+
def size_bytes()
|
16
|
+
return @bytes
|
17
|
+
end
|
18
|
+
|
19
|
+
def add(e)
|
20
|
+
@bytes = @bytes + e.entry['line'].length
|
21
|
+
|
22
|
+
# Append the entry to an already existing stream (if any)
|
23
|
+
labels = e.labels.to_s
|
24
|
+
if @streams.has_key?(labels)
|
25
|
+
stream = @streams[labels]
|
26
|
+
stream['entries'] = stream['entries'] + e.entry
|
27
|
+
return
|
28
|
+
else
|
29
|
+
# Add the entry as a new stream
|
30
|
+
@streams[labels] = {
|
31
|
+
"labels" => e.labels,
|
32
|
+
"entries" => [e.entry],
|
33
|
+
}
|
34
|
+
end
|
35
|
+
end
|
36
|
+
|
37
|
+
def size_bytes_after(line)
|
38
|
+
return @bytes + line.length
|
39
|
+
end
|
40
|
+
|
41
|
+
def age()
|
42
|
+
return Time.now - @createdAt
|
43
|
+
end
|
44
|
+
end
|
45
|
+
end
|
46
|
+
end
|
47
|
+
end
|
@@ -0,0 +1,27 @@
|
|
1
|
+
Gem::Specification.new do |s|
|
2
|
+
s.name = 'logstash-output-loki'
|
3
|
+
s.version = '1.0.0'
|
4
|
+
s.authors = ['Aditya C S']
|
5
|
+
s.email = ['aditya.gnu@gmail.com']
|
6
|
+
|
7
|
+
s.summary = 'Output plugin to ship logs to a Grafana Loki server'
|
8
|
+
s.description = 'Output plugin to ship logs to a Grafana Loki server'
|
9
|
+
s.homepage = 'https://github.com/grafana/loki/'
|
10
|
+
s.license = 'Apache-2.0'
|
11
|
+
s.require_paths = ["lib"]
|
12
|
+
|
13
|
+
# Files
|
14
|
+
s.files = Dir['lib/**/*','spec/**/*','vendor/**/*','*.gemspec','*.md','CONTRIBUTORS','Gemfile','LICENSE','NOTICE.TXT']
|
15
|
+
# Tests
|
16
|
+
s.test_files = s.files.grep(%r{^(test|spec|features)/})
|
17
|
+
|
18
|
+
# Special flag to let us know this is actually a logstash plugin
|
19
|
+
s.metadata = { "logstash_plugin" => "true", "logstash_group" => "output" }
|
20
|
+
|
21
|
+
# Gem dependencies
|
22
|
+
#
|
23
|
+
s.add_runtime_dependency "logstash-core-plugin-api", ">= 1.60", "<= 2.99"
|
24
|
+
s.add_runtime_dependency "logstash-codec-plain", "3.0.6"
|
25
|
+
s.add_runtime_dependency "concurrent-ruby-edge", "0.6.0"
|
26
|
+
s.add_development_dependency 'logstash-devutils', "2.0.2"
|
27
|
+
end
|
@@ -0,0 +1,113 @@
|
|
1
|
+
# encoding: utf-8
|
2
|
+
require "logstash/devutils/rspec/spec_helper"
|
3
|
+
require "logstash/outputs/loki"
|
4
|
+
require "logstash/codecs/plain"
|
5
|
+
require "logstash/event"
|
6
|
+
require "net/http"
|
7
|
+
|
8
|
+
describe LogStash::Outputs::Loki do
|
9
|
+
let (:simple_loki_config) {{'url' => 'http://localhost:3100', 'include_labels' => ["test_key", "other_key"], 'external_labels' => {"test" => "value"}}}
|
10
|
+
|
11
|
+
context 'when initializing' do
|
12
|
+
it "should register" do
|
13
|
+
loki = LogStash::Plugin.lookup("output", "loki").new(simple_loki_config)
|
14
|
+
expect { loki.register }.to_not raise_error
|
15
|
+
end
|
16
|
+
|
17
|
+
it 'should populate loki config with default or intialized values' do
|
18
|
+
loki = LogStash::Outputs::Loki.new(simple_loki_config)
|
19
|
+
expect(loki.url).to eql 'http://localhost:3100'
|
20
|
+
expect(loki.tenant_id).to eql nil
|
21
|
+
expect(loki.batch_size).to eql 102400
|
22
|
+
expect(loki.batch_wait).to eql 1
|
23
|
+
expect(loki.include_labels).to eql ["test_key", "other_key"]
|
24
|
+
expect(loki.external_labels).to include("test" => "value")
|
25
|
+
end
|
26
|
+
end
|
27
|
+
|
28
|
+
context 'test labels' do
|
29
|
+
let (:simple_loki_config) {{'url' => 'http://localhost:3100', 'include_labels' => ["@version", 'log_file_@path', 'host']}}
|
30
|
+
let (:event) { LogStash::Event.new({'message' => 'hello', '@version' => '1', 'agent' => 'filebeat', 'log' => {'file' => {'@path' => '/path/to/file.log'}}, 'host' => '172.0.0.1',
|
31
|
+
'@timestamp' => LogStash::Timestamp.now}) }
|
32
|
+
let(:loki) { LogStash::Plugin.lookup("output", "loki").new(simple_loki_config) }
|
33
|
+
|
34
|
+
before do
|
35
|
+
loki.register
|
36
|
+
loki.close
|
37
|
+
end
|
38
|
+
|
39
|
+
it 'labels extracted should have only included labels' do
|
40
|
+
labels = {}
|
41
|
+
event_hash = event.to_hash
|
42
|
+
expected_labels = {"version" => "1", "host" => "172.0.0.1", "log_file_path" => '/path/to/file.log'}
|
43
|
+
expect(loki.handle_labels(event_hash, labels, "")).to eql expected_labels
|
44
|
+
end
|
45
|
+
end
|
46
|
+
|
47
|
+
context 'validate entries' do
|
48
|
+
let(:timestamp) {LogStash::Timestamp.now}
|
49
|
+
let (:simple_loki_config) {{'url' => 'http://localhost:3100', 'include_labels' => ["version", "host", "test"], 'external_labels' => {"test" => "value"}}}
|
50
|
+
let (:event) { LogStash::Event.new({'message' => 'hello', '@version' => '1', 'agent' => 'filebeat', 'host' => '172.0.0.1',
|
51
|
+
'@timestamp' => timestamp}) }
|
52
|
+
let(:loki) { LogStash::Plugin.lookup("output", "loki").new(simple_loki_config) }
|
53
|
+
|
54
|
+
before do
|
55
|
+
loki.register
|
56
|
+
loki.close
|
57
|
+
end
|
58
|
+
|
59
|
+
it 'validate expected entries are added to entries stream' do
|
60
|
+
labels = {"version" => "1", "host" => "172.0.0.1"}
|
61
|
+
expected_labels = {"version" => "1", "host" => "172.0.0.1", "test" => "value"}
|
62
|
+
expected_entry_hash = {
|
63
|
+
"ts" => timestamp.to_i * (10**9),
|
64
|
+
"line" => "hello".to_s
|
65
|
+
}
|
66
|
+
expected_labels_and_entry_hash = [{"version" => "1", "host" => "172.0.0.1", "test" => "value"}, expected_entry_hash]
|
67
|
+
expect(loki.build_entry(labels, event)).to eq(expected_labels_and_entry_hash)
|
68
|
+
end
|
69
|
+
end
|
70
|
+
|
71
|
+
context 'test http requests' do
|
72
|
+
let (:simple_loki_config) {{'url' => 'http://localhost:3100', 'include_labels' => ["@version", "host", "test"],}}
|
73
|
+
let (:event) { LogStash::Event.new({'message' => 'hello', '@version' => '1', 'host' => '172.0.0.1',
|
74
|
+
'@timestamp' => LogStash::Timestamp.now}) }
|
75
|
+
let(:loki) { LogStash::Plugin.lookup("output", "loki").new(simple_loki_config) }
|
76
|
+
|
77
|
+
before do
|
78
|
+
loki.register
|
79
|
+
loki.close
|
80
|
+
end
|
81
|
+
|
82
|
+
it 'test http requests and raise_error when requests are not successful' do
|
83
|
+
labels = {}
|
84
|
+
event_hash = event.to_hash
|
85
|
+
lbls = loki.handle_labels(event_hash, labels, "")
|
86
|
+
entry_hash = {
|
87
|
+
"ts" => event.get("@timestamp").to_i * (10**9),
|
88
|
+
"line" => event.get("message").to_s
|
89
|
+
}
|
90
|
+
e = LogStash::Outputs::Loki::Entry.new(lbls, entry_hash)
|
91
|
+
batch = LogStash::Outputs::Loki::Batch.new(e)
|
92
|
+
payload = loki.build_payload(batch)
|
93
|
+
|
94
|
+
# response should be nil on connection error
|
95
|
+
expect(loki.loki_http_request("fake", payload, 1, 2, 3)).to eql nil
|
96
|
+
|
97
|
+
success = Net::HTTPSuccess.new(1.0, 200, 'OK')
|
98
|
+
allow(loki).to receive(:loki_http_request) { success }
|
99
|
+
allow(success).to receive(:payload).and_return('fake body')
|
100
|
+
expect(loki.loki_http_request("fake", batch, 1, 300, 10).class).to eql Net::HTTPSuccess
|
101
|
+
|
102
|
+
too_many_requests = Net::HTTPTooManyRequests.new(1.0, 429, 'OK')
|
103
|
+
allow(loki).to receive(:loki_http_request) { too_many_requests }
|
104
|
+
allow(too_many_requests).to receive(:payload).and_return('fake body')
|
105
|
+
expect(loki.loki_http_request("fake", batch, 1, 300, 10).class).to eql Net::HTTPTooManyRequests
|
106
|
+
|
107
|
+
server_error = Net::HTTPServerError.new(1.0, 429, 'OK')
|
108
|
+
allow(loki).to receive(:loki_http_request) { server_error }
|
109
|
+
allow(server_error).to receive(:payload).and_return('fake body')
|
110
|
+
expect(loki.loki_http_request("fake", batch, 1, 300, 10).class).to eql Net::HTTPServerError
|
111
|
+
end
|
112
|
+
end
|
113
|
+
end
|
metadata
ADDED
@@ -0,0 +1,115 @@
|
|
1
|
+
--- !ruby/object:Gem::Specification
|
2
|
+
name: logstash-output-loki
|
3
|
+
version: !ruby/object:Gem::Version
|
4
|
+
version: 1.0.0
|
5
|
+
platform: ruby
|
6
|
+
authors:
|
7
|
+
- Aditya C S
|
8
|
+
autorequire:
|
9
|
+
bindir: bin
|
10
|
+
cert_chain: []
|
11
|
+
date: 2020-07-13 00:00:00.000000000 Z
|
12
|
+
dependencies:
|
13
|
+
- !ruby/object:Gem::Dependency
|
14
|
+
requirement: !ruby/object:Gem::Requirement
|
15
|
+
requirements:
|
16
|
+
- - ">="
|
17
|
+
- !ruby/object:Gem::Version
|
18
|
+
version: '1.60'
|
19
|
+
- - "<="
|
20
|
+
- !ruby/object:Gem::Version
|
21
|
+
version: '2.99'
|
22
|
+
name: logstash-core-plugin-api
|
23
|
+
type: :runtime
|
24
|
+
prerelease: false
|
25
|
+
version_requirements: !ruby/object:Gem::Requirement
|
26
|
+
requirements:
|
27
|
+
- - ">="
|
28
|
+
- !ruby/object:Gem::Version
|
29
|
+
version: '1.60'
|
30
|
+
- - "<="
|
31
|
+
- !ruby/object:Gem::Version
|
32
|
+
version: '2.99'
|
33
|
+
- !ruby/object:Gem::Dependency
|
34
|
+
requirement: !ruby/object:Gem::Requirement
|
35
|
+
requirements:
|
36
|
+
- - '='
|
37
|
+
- !ruby/object:Gem::Version
|
38
|
+
version: 3.0.6
|
39
|
+
name: logstash-codec-plain
|
40
|
+
type: :runtime
|
41
|
+
prerelease: false
|
42
|
+
version_requirements: !ruby/object:Gem::Requirement
|
43
|
+
requirements:
|
44
|
+
- - '='
|
45
|
+
- !ruby/object:Gem::Version
|
46
|
+
version: 3.0.6
|
47
|
+
- !ruby/object:Gem::Dependency
|
48
|
+
requirement: !ruby/object:Gem::Requirement
|
49
|
+
requirements:
|
50
|
+
- - '='
|
51
|
+
- !ruby/object:Gem::Version
|
52
|
+
version: 0.6.0
|
53
|
+
name: concurrent-ruby-edge
|
54
|
+
type: :runtime
|
55
|
+
prerelease: false
|
56
|
+
version_requirements: !ruby/object:Gem::Requirement
|
57
|
+
requirements:
|
58
|
+
- - '='
|
59
|
+
- !ruby/object:Gem::Version
|
60
|
+
version: 0.6.0
|
61
|
+
- !ruby/object:Gem::Dependency
|
62
|
+
requirement: !ruby/object:Gem::Requirement
|
63
|
+
requirements:
|
64
|
+
- - '='
|
65
|
+
- !ruby/object:Gem::Version
|
66
|
+
version: 2.0.2
|
67
|
+
name: logstash-devutils
|
68
|
+
type: :development
|
69
|
+
prerelease: false
|
70
|
+
version_requirements: !ruby/object:Gem::Requirement
|
71
|
+
requirements:
|
72
|
+
- - '='
|
73
|
+
- !ruby/object:Gem::Version
|
74
|
+
version: 2.0.2
|
75
|
+
description: Output plugin to ship logs to a Grafana Loki server
|
76
|
+
email:
|
77
|
+
- aditya.gnu@gmail.com
|
78
|
+
executables: []
|
79
|
+
extensions: []
|
80
|
+
extra_rdoc_files: []
|
81
|
+
files:
|
82
|
+
- Gemfile
|
83
|
+
- README.md
|
84
|
+
- lib/logstash/outputs/loki.rb
|
85
|
+
- lib/logstash/outputs/loki/batch.rb
|
86
|
+
- lib/logstash/outputs/loki/entry.rb
|
87
|
+
- logstash-output-loki.gemspec
|
88
|
+
- spec/outputs/loki_spec.rb
|
89
|
+
homepage: https://github.com/grafana/loki/
|
90
|
+
licenses:
|
91
|
+
- Apache-2.0
|
92
|
+
metadata:
|
93
|
+
logstash_plugin: 'true'
|
94
|
+
logstash_group: output
|
95
|
+
post_install_message:
|
96
|
+
rdoc_options: []
|
97
|
+
require_paths:
|
98
|
+
- lib
|
99
|
+
required_ruby_version: !ruby/object:Gem::Requirement
|
100
|
+
requirements:
|
101
|
+
- - ">="
|
102
|
+
- !ruby/object:Gem::Version
|
103
|
+
version: '0'
|
104
|
+
required_rubygems_version: !ruby/object:Gem::Requirement
|
105
|
+
requirements:
|
106
|
+
- - ">="
|
107
|
+
- !ruby/object:Gem::Version
|
108
|
+
version: '0'
|
109
|
+
requirements: []
|
110
|
+
rubygems_version: 3.0.6
|
111
|
+
signing_key:
|
112
|
+
specification_version: 4
|
113
|
+
summary: Output plugin to ship logs to a Grafana Loki server
|
114
|
+
test_files:
|
115
|
+
- spec/outputs/loki_spec.rb
|