logstash-output-loki 1.0.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +7 -0
- data/Gemfile +11 -0
- data/README.md +66 -0
- data/lib/logstash/outputs/loki.rb +308 -0
- data/lib/logstash/outputs/loki/batch.rb +47 -0
- data/lib/logstash/outputs/loki/entry.rb +13 -0
- data/logstash-output-loki.gemspec +27 -0
- data/spec/outputs/loki_spec.rb +113 -0
- metadata +115 -0
checksums.yaml
ADDED
@@ -0,0 +1,7 @@
|
|
1
|
+
---
|
2
|
+
SHA256:
|
3
|
+
metadata.gz: 9dd08c392ec88148a6f410e159169b3a19f877a25174fa3f43b2fdeb7fe16128
|
4
|
+
data.tar.gz: 34aaf03576d8fbbb97aab02013113254734984ee1744c2ab4e8438bd51657569
|
5
|
+
SHA512:
|
6
|
+
metadata.gz: 3520ab5618092976cfe771ad4f9ddea4aca3b96efe75d322c2f505e7fd6e642f3cee4775eb1ef0514c9db360a3c36b138e9f1426f6de9b625945cafdafb0b27d
|
7
|
+
data.tar.gz: 6c7bb11d79fe0828e629bc27771a4f63584ea76af2ddb69eac950e5c7e3b115b62b674be28e8a828c97d8d3d851406748f0c83d49d9147080f0cc593ad1a076f
|
data/Gemfile
ADDED
@@ -0,0 +1,11 @@
|
|
1
|
+
source 'https://rubygems.org'
|
2
|
+
|
3
|
+
gemspec
|
4
|
+
|
5
|
+
logstash_path = ENV["LOGSTASH_PATH"] || "logstash-libs"
|
6
|
+
use_logstash_source = ENV["LOGSTASH_SOURCE"] && ENV["LOGSTASH_SOURCE"].to_s == "1"
|
7
|
+
|
8
|
+
if Dir.exist?(logstash_path) && use_logstash_source
|
9
|
+
gem 'logstash-core', :path => "#{logstash_path}/logstash-core"
|
10
|
+
gem 'logstash-core-plugin-api', :path => "#{logstash_path}/logstash-core-plugin-api"
|
11
|
+
end
|
data/README.md
ADDED
@@ -0,0 +1,66 @@
|
|
1
|
+
# Loki Logstash Output Plugin
|
2
|
+
|
3
|
+
Logstash plugin to send logstash aggregated logs to Loki.
|
4
|
+
|
5
|
+
## Install dependencies
|
6
|
+
|
7
|
+
First you need to setup JRuby environment to build this plugin. Refer https://github.com/rbenv/rbenv for setting up your rbenv environment.
|
8
|
+
|
9
|
+
After setting up `rbenv`. Install JRuby
|
10
|
+
|
11
|
+
```bash
|
12
|
+
rbenv install jruby-9.2.10.0
|
13
|
+
rbenv local jruby-9.2.10.0
|
14
|
+
```
|
15
|
+
|
16
|
+
Check that the environment is configured
|
17
|
+
|
18
|
+
```bash
|
19
|
+
ruby --version
|
20
|
+
jruby 9.2.10
|
21
|
+
```
|
22
|
+
|
23
|
+
You should use make sure you are running jruby and not ruby. If the command below still shows ruby and not jruby, check that PATH contains `$HOME/.rbenv/shims` and `$HOME/.rbenv/bin`. Also verify that you have this in your bash profile:
|
24
|
+
|
25
|
+
```bash
|
26
|
+
export PATH="$HOME/.rbenv/bin:$PATH"
|
27
|
+
eval "$(rbenv init -)"
|
28
|
+
```
|
29
|
+
|
30
|
+
Then install bundler
|
31
|
+
`gem install bundler:2.1.4`
|
32
|
+
|
33
|
+
Follow those instructions to [install logstash](https://www.elastic.co/guide/en/logstash/current/installing-logstash.html) before moving to the next section.
|
34
|
+
|
35
|
+
## Install dependencies and Build plugin
|
36
|
+
|
37
|
+
### Install required packages
|
38
|
+
|
39
|
+
```bash
|
40
|
+
git clone git@github.com:elastic/logstash.git
|
41
|
+
cd logstash
|
42
|
+
git checkout tags/v7.6.2
|
43
|
+
export LOGSTASH_PATH=`pwd`
|
44
|
+
export LOGSTASH_SOURCE="1"
|
45
|
+
export GEM_PATH=$LOGSTASH_PATH/vendor/bundle/
|
46
|
+
export GEM_HOME=$LOGSTASH_PATH/vendor/bundle/
|
47
|
+
cd ..
|
48
|
+
ruby -S bundle install --path
|
49
|
+
ruby -S bundle exec rake vendor
|
50
|
+
```
|
51
|
+
|
52
|
+
### Build the plugin
|
53
|
+
|
54
|
+
`gem build logstash-output-loki.gemspec`
|
55
|
+
|
56
|
+
### Test
|
57
|
+
|
58
|
+
`bundle exec rspec`
|
59
|
+
|
60
|
+
## Install plugin to local logstash
|
61
|
+
|
62
|
+
`bin/logstash-plugin install --no-verify --local logstash-output-loki-1.0.0.gem`
|
63
|
+
|
64
|
+
## Send sample event and check plugin is working
|
65
|
+
|
66
|
+
`bin/logstash -f loki.conf`
|
@@ -0,0 +1,308 @@
|
|
1
|
+
# encoding: utf-8
|
2
|
+
require "logstash/outputs/base"
|
3
|
+
require "logstash/namespace"
|
4
|
+
require 'net/http'
|
5
|
+
require 'concurrent-edge'
|
6
|
+
require 'time'
|
7
|
+
require 'uri'
|
8
|
+
require 'json'
|
9
|
+
|
10
|
+
class LogStash::Outputs::Loki < LogStash::Outputs::Base
|
11
|
+
require 'logstash/outputs/loki/batch'
|
12
|
+
require 'logstash/outputs/loki/entry'
|
13
|
+
|
14
|
+
config_name "loki"
|
15
|
+
|
16
|
+
## 'A single instance of the Output will be shared among the pipeline worker threads'
|
17
|
+
concurrency :single
|
18
|
+
|
19
|
+
## 'Loki URL'
|
20
|
+
config :url, :validate => :string, :required => true
|
21
|
+
|
22
|
+
## 'BasicAuth credentials'
|
23
|
+
config :username, :validate => :string, :required => false
|
24
|
+
config :password, :validate => :string, secret: true, :required => false
|
25
|
+
|
26
|
+
## 'Client certificate'
|
27
|
+
config :cert, :validate => :path, :required => false
|
28
|
+
config :key, :validate => :path, :required => false
|
29
|
+
|
30
|
+
## 'TLS'
|
31
|
+
config :ca_cert, :validate => :path, :required => false
|
32
|
+
|
33
|
+
## 'Loki Tenant ID'
|
34
|
+
config :tenant_id, :validate => :string, :required => false
|
35
|
+
|
36
|
+
## 'Maximum batch size to accrue before pushing to loki. Defaults to 102400 bytes'
|
37
|
+
config :batch_size, :validate => :number, :default => 102400, :required => false
|
38
|
+
|
39
|
+
## 'Interval in seconds to wait before pushing a batch of records to loki. Defaults to 1 second'
|
40
|
+
config :batch_wait, :validate => :number, :default => 1, :required => false
|
41
|
+
|
42
|
+
## 'Array of label names to include in all logstreams'
|
43
|
+
config :include_labels, :validate => :array, :default => [], :required => true
|
44
|
+
|
45
|
+
## 'Extra labels to add to all log streams'
|
46
|
+
config :external_labels, :validate => :hash, :default => {}, :required => false
|
47
|
+
|
48
|
+
## 'Log line field to pick from logstash. Defaults to "message"'
|
49
|
+
config :message_field, :validate => :string, :default => "message", :required => false
|
50
|
+
|
51
|
+
## 'Backoff configuration. Initial backoff time between retries. Default 1s'
|
52
|
+
config :min_delay, :validate => :number, :default => 1, :required => false
|
53
|
+
|
54
|
+
## 'Backoff configuration. Maximum backoff time between retries. Default 300s'
|
55
|
+
config :max_delay, :validate => :number, :default => 300, :required => false
|
56
|
+
|
57
|
+
## 'Backoff configuration. Maximum number of retries to do'
|
58
|
+
config :retries, :validate => :number, :default => 10, :required => false
|
59
|
+
|
60
|
+
public
|
61
|
+
def register
|
62
|
+
@uri = URI.parse(@url)
|
63
|
+
unless @uri.is_a?(URI::HTTP) || @uri.is_a?(URI::HTTPS)
|
64
|
+
raise LogStash::ConfigurationError, "url parameter must be valid HTTP, currently '#{@url}'"
|
65
|
+
end
|
66
|
+
|
67
|
+
if @include_labels.empty?
|
68
|
+
raise LogStash::ConfigurationError, "include_labels should contain atleast one label, currently '#{@include_labels}'"
|
69
|
+
end
|
70
|
+
|
71
|
+
if @min_delay > @max_delay
|
72
|
+
raise LogStash::ConfigurationError, "Min delay should be less than Max delay, currently 'Min delay is #{@min_delay} and Max delay is #{@max_delay}'"
|
73
|
+
end
|
74
|
+
|
75
|
+
@logger.info("Loki output plugin", :class => self.class.name)
|
76
|
+
|
77
|
+
# intialize channels
|
78
|
+
@Channel = Concurrent::Channel
|
79
|
+
@entries = @Channel.new
|
80
|
+
|
81
|
+
# excluded message and timestamp from labels
|
82
|
+
@exclude_labels = ["message", "@timestamp"]
|
83
|
+
|
84
|
+
# create nil batch object.
|
85
|
+
@batch = nil
|
86
|
+
|
87
|
+
# validate certs
|
88
|
+
if ssl_cert?
|
89
|
+
load_ssl
|
90
|
+
validate_ssl_key
|
91
|
+
end
|
92
|
+
|
93
|
+
@Channel.go{run()}
|
94
|
+
end
|
95
|
+
|
96
|
+
def ssl_cert?
|
97
|
+
!@key.nil? && !@cert.nil?
|
98
|
+
end
|
99
|
+
|
100
|
+
def load_ssl
|
101
|
+
@cert = OpenSSL::X509::Certificate.new(File.read(@cert)) if @cert
|
102
|
+
@key = OpenSSL::PKey.read(File.read(@key)) if @key
|
103
|
+
end
|
104
|
+
|
105
|
+
def validate_ssl_key
|
106
|
+
if !@key.is_a?(OpenSSL::PKey::RSA) && !@key.is_a?(OpenSSL::PKey::DSA)
|
107
|
+
raise LogStash::ConfigurationError, "Unsupported private key type '#{@key.class}''"
|
108
|
+
end
|
109
|
+
end
|
110
|
+
|
111
|
+
def ssl_opts(uri)
|
112
|
+
opts = {
|
113
|
+
use_ssl: uri.scheme == 'https'
|
114
|
+
}
|
115
|
+
|
116
|
+
if !@cert.nil? && !@key.nil?
|
117
|
+
opts = opts.merge(
|
118
|
+
verify_mode: OpenSSL::SSL::VERIFY_PEER,
|
119
|
+
cert: @cert,
|
120
|
+
key: @key
|
121
|
+
)
|
122
|
+
end
|
123
|
+
|
124
|
+
unless @ca_cert.nil?
|
125
|
+
opts = opts.merge(
|
126
|
+
ca_file: @ca_cert
|
127
|
+
)
|
128
|
+
end
|
129
|
+
opts
|
130
|
+
end
|
131
|
+
|
132
|
+
def run()
|
133
|
+
min_wait_checkfrequency = 1/1000 #1 millisecond
|
134
|
+
max_wait_checkfrequency = @batch_wait
|
135
|
+
if max_wait_checkfrequency < min_wait_checkfrequency
|
136
|
+
max_wait_checkfrequency = min_wait_checkfrequency
|
137
|
+
end
|
138
|
+
|
139
|
+
@max_wait_check = Concurrent::Channel.tick(max_wait_checkfrequency)
|
140
|
+
loop do
|
141
|
+
Concurrent::Channel.select do |s|
|
142
|
+
s.take(@entries) { |e|
|
143
|
+
if @batch.nil?
|
144
|
+
@batch = Batch.new(e)
|
145
|
+
next
|
146
|
+
end
|
147
|
+
|
148
|
+
line = e.entry['line']
|
149
|
+
if @batch.size_bytes_after(line) > @batch_size
|
150
|
+
@logger.debug("Max batch_size is reached. Sending batch to loki")
|
151
|
+
send(@tenant_id, @batch)
|
152
|
+
@batch = Batch.new(e)
|
153
|
+
next
|
154
|
+
end
|
155
|
+
@batch.add(e)
|
156
|
+
}
|
157
|
+
s.take(@max_wait_check) {
|
158
|
+
# Send batch if max wait time has been reached
|
159
|
+
if !@batch.nil?
|
160
|
+
if @batch.age() < @batch_wait
|
161
|
+
next
|
162
|
+
end
|
163
|
+
|
164
|
+
@logger.debug("Max batch_wait time is reached. Sending batch to loki")
|
165
|
+
send(@tenant_id, @batch)
|
166
|
+
@batch = nil
|
167
|
+
end
|
168
|
+
}
|
169
|
+
end
|
170
|
+
end
|
171
|
+
end
|
172
|
+
|
173
|
+
## Receives logstash events
|
174
|
+
public
|
175
|
+
def receive(event)
|
176
|
+
labels = {}
|
177
|
+
event_hash = event.to_hash
|
178
|
+
lbls = handle_labels(event_hash, labels, "")
|
179
|
+
|
180
|
+
data_labels, entry_hash = build_entry(lbls, event)
|
181
|
+
@entries << Entry.new(data_labels, entry_hash)
|
182
|
+
|
183
|
+
end
|
184
|
+
|
185
|
+
def close
|
186
|
+
@logger.info("Closing loki output plugin. Flushing all pending batches")
|
187
|
+
send(@tenant_id, @batch) if !@batch.nil?
|
188
|
+
@entries.close
|
189
|
+
@max_wait_check.close if !@max_wait_check.nil?
|
190
|
+
end
|
191
|
+
|
192
|
+
def build_entry(lbls, event)
|
193
|
+
labels = lbls.merge(@external_labels)
|
194
|
+
entry_hash = {
|
195
|
+
"ts" => event.get("@timestamp").to_i * (10**9),
|
196
|
+
"line" => event.get(@message_field).to_s
|
197
|
+
}
|
198
|
+
return labels, entry_hash
|
199
|
+
end
|
200
|
+
|
201
|
+
def handle_labels(event_hash, labels, parent_key)
|
202
|
+
event_hash.each{ |key,value|
|
203
|
+
if !@exclude_labels.include?(key)
|
204
|
+
if value.is_a?(Hash)
|
205
|
+
if parent_key != ""
|
206
|
+
handle_labels(value, labels, parent_key + "_" + key)
|
207
|
+
else
|
208
|
+
handle_labels(value, labels, key)
|
209
|
+
end
|
210
|
+
else
|
211
|
+
if parent_key != ""
|
212
|
+
labels[parent_key + "_" + key] = value.to_s
|
213
|
+
else
|
214
|
+
labels[key] = value.to_s
|
215
|
+
end
|
216
|
+
end
|
217
|
+
end
|
218
|
+
}
|
219
|
+
return extract_labels(labels)
|
220
|
+
end
|
221
|
+
|
222
|
+
def extract_labels(extracted_labels)
|
223
|
+
labels = {}
|
224
|
+
extracted_labels.each { |key, value|
|
225
|
+
if @include_labels.include?(key)
|
226
|
+
key = key.gsub("@", '')
|
227
|
+
labels[key] = value
|
228
|
+
end
|
229
|
+
}
|
230
|
+
return labels
|
231
|
+
end
|
232
|
+
|
233
|
+
def send(tenant_id, batch)
|
234
|
+
payload = build_payload(batch)
|
235
|
+
res = loki_http_request(tenant_id, payload, @min_delay, @max_delay, @retries)
|
236
|
+
|
237
|
+
if res.is_a?(Net::HTTPSuccess)
|
238
|
+
@logger.debug("Successfully pushed data to loki")
|
239
|
+
return
|
240
|
+
else
|
241
|
+
@logger.error("failed to write post to ", :uri => @uri, :code => res.code, :body => res.body, :message => res.message) if !res.nil?
|
242
|
+
@logger.debug("Payload object ", :payload => payload)
|
243
|
+
end
|
244
|
+
end
|
245
|
+
|
246
|
+
def loki_http_request(tenant_id, payload, min_delay, max_delay, retries)
|
247
|
+
req = Net::HTTP::Post.new(
|
248
|
+
@uri.request_uri
|
249
|
+
)
|
250
|
+
req.add_field('Content-Type', 'application/json')
|
251
|
+
req.add_field('X-Scope-OrgID', tenant_id) if tenant_id
|
252
|
+
req.basic_auth(@username, @password) if @username
|
253
|
+
req.body = payload
|
254
|
+
|
255
|
+
opts = ssl_opts(@uri)
|
256
|
+
|
257
|
+
@logger.debug("sending #{req.body.length} bytes to loki")
|
258
|
+
retry_count = 0
|
259
|
+
delay = min_delay
|
260
|
+
begin
|
261
|
+
res = Net::HTTP.start(@uri.host, @uri.port, **opts) { |http| http.request(req) }
|
262
|
+
rescue Net::HTTPTooManyRequests, Net::HTTPServerError, Errno::ECONNREFUSED => e
|
263
|
+
unless retry_count < retries
|
264
|
+
@logger.error("Error while sending data to loki. Tried #{retry_count} times\n. :error => #{e}")
|
265
|
+
return res
|
266
|
+
end
|
267
|
+
|
268
|
+
retry_count += 1
|
269
|
+
@logger.warn("Trying to send again. Attempt number: #{retry_count}. Retrying in #{delay}s")
|
270
|
+
sleep delay
|
271
|
+
|
272
|
+
if (delay * 2 - delay) > max_delay
|
273
|
+
delay = delay
|
274
|
+
else
|
275
|
+
delay = delay * 2
|
276
|
+
end
|
277
|
+
|
278
|
+
retry
|
279
|
+
rescue StandardError => e
|
280
|
+
@logger.error("Error while connecting to loki server ", :error_inspect => e.inspect, :error => e)
|
281
|
+
return res
|
282
|
+
end
|
283
|
+
return res
|
284
|
+
end
|
285
|
+
|
286
|
+
def build_payload(batch)
|
287
|
+
payload = {}
|
288
|
+
payload['streams'] = []
|
289
|
+
batch.streams.each { |labels, stream|
|
290
|
+
stream_obj = get_stream_obj(stream)
|
291
|
+
payload['streams'].push(stream_obj)
|
292
|
+
}
|
293
|
+
return payload.to_json
|
294
|
+
end
|
295
|
+
|
296
|
+
def get_stream_obj(stream)
|
297
|
+
stream_obj = {}
|
298
|
+
stream_obj['stream'] = stream['labels']
|
299
|
+
stream_obj['values'] = []
|
300
|
+
values = []
|
301
|
+
stream['entries'].each { |entry|
|
302
|
+
values.push(entry['ts'].to_s)
|
303
|
+
values.push(entry['line'])
|
304
|
+
}
|
305
|
+
stream_obj['values'].push(values)
|
306
|
+
return stream_obj
|
307
|
+
end
|
308
|
+
end
|
@@ -0,0 +1,47 @@
|
|
1
|
+
require 'time'
|
2
|
+
|
3
|
+
module LogStash
|
4
|
+
module Outputs
|
5
|
+
class Loki
|
6
|
+
class Batch
|
7
|
+
attr_reader :streams
|
8
|
+
def initialize(e)
|
9
|
+
@bytes = 0
|
10
|
+
@createdAt = Time.now
|
11
|
+
@streams = {}
|
12
|
+
add(e)
|
13
|
+
end
|
14
|
+
|
15
|
+
def size_bytes()
|
16
|
+
return @bytes
|
17
|
+
end
|
18
|
+
|
19
|
+
def add(e)
|
20
|
+
@bytes = @bytes + e.entry['line'].length
|
21
|
+
|
22
|
+
# Append the entry to an already existing stream (if any)
|
23
|
+
labels = e.labels.to_s
|
24
|
+
if @streams.has_key?(labels)
|
25
|
+
stream = @streams[labels]
|
26
|
+
stream['entries'] = stream['entries'] + e.entry
|
27
|
+
return
|
28
|
+
else
|
29
|
+
# Add the entry as a new stream
|
30
|
+
@streams[labels] = {
|
31
|
+
"labels" => e.labels,
|
32
|
+
"entries" => [e.entry],
|
33
|
+
}
|
34
|
+
end
|
35
|
+
end
|
36
|
+
|
37
|
+
def size_bytes_after(line)
|
38
|
+
return @bytes + line.length
|
39
|
+
end
|
40
|
+
|
41
|
+
def age()
|
42
|
+
return Time.now - @createdAt
|
43
|
+
end
|
44
|
+
end
|
45
|
+
end
|
46
|
+
end
|
47
|
+
end
|
@@ -0,0 +1,27 @@
|
|
1
|
+
Gem::Specification.new do |s|
|
2
|
+
s.name = 'logstash-output-loki'
|
3
|
+
s.version = '1.0.0'
|
4
|
+
s.authors = ['Aditya C S']
|
5
|
+
s.email = ['aditya.gnu@gmail.com']
|
6
|
+
|
7
|
+
s.summary = 'Output plugin to ship logs to a Grafana Loki server'
|
8
|
+
s.description = 'Output plugin to ship logs to a Grafana Loki server'
|
9
|
+
s.homepage = 'https://github.com/grafana/loki/'
|
10
|
+
s.license = 'Apache-2.0'
|
11
|
+
s.require_paths = ["lib"]
|
12
|
+
|
13
|
+
# Files
|
14
|
+
s.files = Dir['lib/**/*','spec/**/*','vendor/**/*','*.gemspec','*.md','CONTRIBUTORS','Gemfile','LICENSE','NOTICE.TXT']
|
15
|
+
# Tests
|
16
|
+
s.test_files = s.files.grep(%r{^(test|spec|features)/})
|
17
|
+
|
18
|
+
# Special flag to let us know this is actually a logstash plugin
|
19
|
+
s.metadata = { "logstash_plugin" => "true", "logstash_group" => "output" }
|
20
|
+
|
21
|
+
# Gem dependencies
|
22
|
+
#
|
23
|
+
s.add_runtime_dependency "logstash-core-plugin-api", ">= 1.60", "<= 2.99"
|
24
|
+
s.add_runtime_dependency "logstash-codec-plain", "3.0.6"
|
25
|
+
s.add_runtime_dependency "concurrent-ruby-edge", "0.6.0"
|
26
|
+
s.add_development_dependency 'logstash-devutils', "2.0.2"
|
27
|
+
end
|
@@ -0,0 +1,113 @@
|
|
1
|
+
# encoding: utf-8
|
2
|
+
require "logstash/devutils/rspec/spec_helper"
|
3
|
+
require "logstash/outputs/loki"
|
4
|
+
require "logstash/codecs/plain"
|
5
|
+
require "logstash/event"
|
6
|
+
require "net/http"
|
7
|
+
|
8
|
+
describe LogStash::Outputs::Loki do
|
9
|
+
let (:simple_loki_config) {{'url' => 'http://localhost:3100', 'include_labels' => ["test_key", "other_key"], 'external_labels' => {"test" => "value"}}}
|
10
|
+
|
11
|
+
context 'when initializing' do
|
12
|
+
it "should register" do
|
13
|
+
loki = LogStash::Plugin.lookup("output", "loki").new(simple_loki_config)
|
14
|
+
expect { loki.register }.to_not raise_error
|
15
|
+
end
|
16
|
+
|
17
|
+
it 'should populate loki config with default or intialized values' do
|
18
|
+
loki = LogStash::Outputs::Loki.new(simple_loki_config)
|
19
|
+
expect(loki.url).to eql 'http://localhost:3100'
|
20
|
+
expect(loki.tenant_id).to eql nil
|
21
|
+
expect(loki.batch_size).to eql 102400
|
22
|
+
expect(loki.batch_wait).to eql 1
|
23
|
+
expect(loki.include_labels).to eql ["test_key", "other_key"]
|
24
|
+
expect(loki.external_labels).to include("test" => "value")
|
25
|
+
end
|
26
|
+
end
|
27
|
+
|
28
|
+
context 'test labels' do
|
29
|
+
let (:simple_loki_config) {{'url' => 'http://localhost:3100', 'include_labels' => ["@version", 'log_file_@path', 'host']}}
|
30
|
+
let (:event) { LogStash::Event.new({'message' => 'hello', '@version' => '1', 'agent' => 'filebeat', 'log' => {'file' => {'@path' => '/path/to/file.log'}}, 'host' => '172.0.0.1',
|
31
|
+
'@timestamp' => LogStash::Timestamp.now}) }
|
32
|
+
let(:loki) { LogStash::Plugin.lookup("output", "loki").new(simple_loki_config) }
|
33
|
+
|
34
|
+
before do
|
35
|
+
loki.register
|
36
|
+
loki.close
|
37
|
+
end
|
38
|
+
|
39
|
+
it 'labels extracted should have only included labels' do
|
40
|
+
labels = {}
|
41
|
+
event_hash = event.to_hash
|
42
|
+
expected_labels = {"version" => "1", "host" => "172.0.0.1", "log_file_path" => '/path/to/file.log'}
|
43
|
+
expect(loki.handle_labels(event_hash, labels, "")).to eql expected_labels
|
44
|
+
end
|
45
|
+
end
|
46
|
+
|
47
|
+
context 'validate entries' do
|
48
|
+
let(:timestamp) {LogStash::Timestamp.now}
|
49
|
+
let (:simple_loki_config) {{'url' => 'http://localhost:3100', 'include_labels' => ["version", "host", "test"], 'external_labels' => {"test" => "value"}}}
|
50
|
+
let (:event) { LogStash::Event.new({'message' => 'hello', '@version' => '1', 'agent' => 'filebeat', 'host' => '172.0.0.1',
|
51
|
+
'@timestamp' => timestamp}) }
|
52
|
+
let(:loki) { LogStash::Plugin.lookup("output", "loki").new(simple_loki_config) }
|
53
|
+
|
54
|
+
before do
|
55
|
+
loki.register
|
56
|
+
loki.close
|
57
|
+
end
|
58
|
+
|
59
|
+
it 'validate expected entries are added to entries stream' do
|
60
|
+
labels = {"version" => "1", "host" => "172.0.0.1"}
|
61
|
+
expected_labels = {"version" => "1", "host" => "172.0.0.1", "test" => "value"}
|
62
|
+
expected_entry_hash = {
|
63
|
+
"ts" => timestamp.to_i * (10**9),
|
64
|
+
"line" => "hello".to_s
|
65
|
+
}
|
66
|
+
expected_labels_and_entry_hash = [{"version" => "1", "host" => "172.0.0.1", "test" => "value"}, expected_entry_hash]
|
67
|
+
expect(loki.build_entry(labels, event)).to eq(expected_labels_and_entry_hash)
|
68
|
+
end
|
69
|
+
end
|
70
|
+
|
71
|
+
context 'test http requests' do
|
72
|
+
let (:simple_loki_config) {{'url' => 'http://localhost:3100', 'include_labels' => ["@version", "host", "test"],}}
|
73
|
+
let (:event) { LogStash::Event.new({'message' => 'hello', '@version' => '1', 'host' => '172.0.0.1',
|
74
|
+
'@timestamp' => LogStash::Timestamp.now}) }
|
75
|
+
let(:loki) { LogStash::Plugin.lookup("output", "loki").new(simple_loki_config) }
|
76
|
+
|
77
|
+
before do
|
78
|
+
loki.register
|
79
|
+
loki.close
|
80
|
+
end
|
81
|
+
|
82
|
+
it 'test http requests and raise_error when requests are not successful' do
|
83
|
+
labels = {}
|
84
|
+
event_hash = event.to_hash
|
85
|
+
lbls = loki.handle_labels(event_hash, labels, "")
|
86
|
+
entry_hash = {
|
87
|
+
"ts" => event.get("@timestamp").to_i * (10**9),
|
88
|
+
"line" => event.get("message").to_s
|
89
|
+
}
|
90
|
+
e = LogStash::Outputs::Loki::Entry.new(lbls, entry_hash)
|
91
|
+
batch = LogStash::Outputs::Loki::Batch.new(e)
|
92
|
+
payload = loki.build_payload(batch)
|
93
|
+
|
94
|
+
# response should be nil on connection error
|
95
|
+
expect(loki.loki_http_request("fake", payload, 1, 2, 3)).to eql nil
|
96
|
+
|
97
|
+
success = Net::HTTPSuccess.new(1.0, 200, 'OK')
|
98
|
+
allow(loki).to receive(:loki_http_request) { success }
|
99
|
+
allow(success).to receive(:payload).and_return('fake body')
|
100
|
+
expect(loki.loki_http_request("fake", batch, 1, 300, 10).class).to eql Net::HTTPSuccess
|
101
|
+
|
102
|
+
too_many_requests = Net::HTTPTooManyRequests.new(1.0, 429, 'OK')
|
103
|
+
allow(loki).to receive(:loki_http_request) { too_many_requests }
|
104
|
+
allow(too_many_requests).to receive(:payload).and_return('fake body')
|
105
|
+
expect(loki.loki_http_request("fake", batch, 1, 300, 10).class).to eql Net::HTTPTooManyRequests
|
106
|
+
|
107
|
+
server_error = Net::HTTPServerError.new(1.0, 429, 'OK')
|
108
|
+
allow(loki).to receive(:loki_http_request) { server_error }
|
109
|
+
allow(server_error).to receive(:payload).and_return('fake body')
|
110
|
+
expect(loki.loki_http_request("fake", batch, 1, 300, 10).class).to eql Net::HTTPServerError
|
111
|
+
end
|
112
|
+
end
|
113
|
+
end
|
metadata
ADDED
@@ -0,0 +1,115 @@
|
|
1
|
+
--- !ruby/object:Gem::Specification
|
2
|
+
name: logstash-output-loki
|
3
|
+
version: !ruby/object:Gem::Version
|
4
|
+
version: 1.0.0
|
5
|
+
platform: ruby
|
6
|
+
authors:
|
7
|
+
- Aditya C S
|
8
|
+
autorequire:
|
9
|
+
bindir: bin
|
10
|
+
cert_chain: []
|
11
|
+
date: 2020-07-13 00:00:00.000000000 Z
|
12
|
+
dependencies:
|
13
|
+
- !ruby/object:Gem::Dependency
|
14
|
+
requirement: !ruby/object:Gem::Requirement
|
15
|
+
requirements:
|
16
|
+
- - ">="
|
17
|
+
- !ruby/object:Gem::Version
|
18
|
+
version: '1.60'
|
19
|
+
- - "<="
|
20
|
+
- !ruby/object:Gem::Version
|
21
|
+
version: '2.99'
|
22
|
+
name: logstash-core-plugin-api
|
23
|
+
type: :runtime
|
24
|
+
prerelease: false
|
25
|
+
version_requirements: !ruby/object:Gem::Requirement
|
26
|
+
requirements:
|
27
|
+
- - ">="
|
28
|
+
- !ruby/object:Gem::Version
|
29
|
+
version: '1.60'
|
30
|
+
- - "<="
|
31
|
+
- !ruby/object:Gem::Version
|
32
|
+
version: '2.99'
|
33
|
+
- !ruby/object:Gem::Dependency
|
34
|
+
requirement: !ruby/object:Gem::Requirement
|
35
|
+
requirements:
|
36
|
+
- - '='
|
37
|
+
- !ruby/object:Gem::Version
|
38
|
+
version: 3.0.6
|
39
|
+
name: logstash-codec-plain
|
40
|
+
type: :runtime
|
41
|
+
prerelease: false
|
42
|
+
version_requirements: !ruby/object:Gem::Requirement
|
43
|
+
requirements:
|
44
|
+
- - '='
|
45
|
+
- !ruby/object:Gem::Version
|
46
|
+
version: 3.0.6
|
47
|
+
- !ruby/object:Gem::Dependency
|
48
|
+
requirement: !ruby/object:Gem::Requirement
|
49
|
+
requirements:
|
50
|
+
- - '='
|
51
|
+
- !ruby/object:Gem::Version
|
52
|
+
version: 0.6.0
|
53
|
+
name: concurrent-ruby-edge
|
54
|
+
type: :runtime
|
55
|
+
prerelease: false
|
56
|
+
version_requirements: !ruby/object:Gem::Requirement
|
57
|
+
requirements:
|
58
|
+
- - '='
|
59
|
+
- !ruby/object:Gem::Version
|
60
|
+
version: 0.6.0
|
61
|
+
- !ruby/object:Gem::Dependency
|
62
|
+
requirement: !ruby/object:Gem::Requirement
|
63
|
+
requirements:
|
64
|
+
- - '='
|
65
|
+
- !ruby/object:Gem::Version
|
66
|
+
version: 2.0.2
|
67
|
+
name: logstash-devutils
|
68
|
+
type: :development
|
69
|
+
prerelease: false
|
70
|
+
version_requirements: !ruby/object:Gem::Requirement
|
71
|
+
requirements:
|
72
|
+
- - '='
|
73
|
+
- !ruby/object:Gem::Version
|
74
|
+
version: 2.0.2
|
75
|
+
description: Output plugin to ship logs to a Grafana Loki server
|
76
|
+
email:
|
77
|
+
- aditya.gnu@gmail.com
|
78
|
+
executables: []
|
79
|
+
extensions: []
|
80
|
+
extra_rdoc_files: []
|
81
|
+
files:
|
82
|
+
- Gemfile
|
83
|
+
- README.md
|
84
|
+
- lib/logstash/outputs/loki.rb
|
85
|
+
- lib/logstash/outputs/loki/batch.rb
|
86
|
+
- lib/logstash/outputs/loki/entry.rb
|
87
|
+
- logstash-output-loki.gemspec
|
88
|
+
- spec/outputs/loki_spec.rb
|
89
|
+
homepage: https://github.com/grafana/loki/
|
90
|
+
licenses:
|
91
|
+
- Apache-2.0
|
92
|
+
metadata:
|
93
|
+
logstash_plugin: 'true'
|
94
|
+
logstash_group: output
|
95
|
+
post_install_message:
|
96
|
+
rdoc_options: []
|
97
|
+
require_paths:
|
98
|
+
- lib
|
99
|
+
required_ruby_version: !ruby/object:Gem::Requirement
|
100
|
+
requirements:
|
101
|
+
- - ">="
|
102
|
+
- !ruby/object:Gem::Version
|
103
|
+
version: '0'
|
104
|
+
required_rubygems_version: !ruby/object:Gem::Requirement
|
105
|
+
requirements:
|
106
|
+
- - ">="
|
107
|
+
- !ruby/object:Gem::Version
|
108
|
+
version: '0'
|
109
|
+
requirements: []
|
110
|
+
rubygems_version: 3.0.6
|
111
|
+
signing_key:
|
112
|
+
specification_version: 4
|
113
|
+
summary: Output plugin to ship logs to a Grafana Loki server
|
114
|
+
test_files:
|
115
|
+
- spec/outputs/loki_spec.rb
|