logstash-output-loki 1.0.3 → 1.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/README.md +23 -11
- data/lib/logstash/outputs/loki/entry.rb +2 -1
- data/lib/logstash/outputs/loki.rb +4 -1
- data/logstash-output-loki.gemspec +2 -2
- data/spec/outputs/loki/entry_spec.rb +11 -4
- data/spec/outputs/loki_spec.rb +20 -5
- metadata +14 -14
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 5c6817d07ada63cde10011f1e490f803ddbb0a7a54d75b9472780f4f180c65fe
|
4
|
+
data.tar.gz: e457589a713ed3bc0f74120f3671bfb6b9b9760393cb86c18764e4d5a1fa2bd7
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 358e07d5c964179b7a003e6ac840bb83df74c03e4de0f1916da310ed07e1c669a672d9363afd841c6bdfd49bb1d525ffbc815801ba4f3a8536659dd449173012
|
7
|
+
data.tar.gz: 89360e4ffcb276b9406c017c5c019c43f839398a93a0cda46e0761e45a6a851ef2e9250c9fec5ba409062aef1f12576c9fcaa939a4cba0c0c9d0db15fa698f24
|
data/README.md
CHANGED
@@ -29,8 +29,11 @@ export PATH="$HOME/.rbenv/bin:$PATH"
|
|
29
29
|
eval "$(rbenv init -)"
|
30
30
|
```
|
31
31
|
|
32
|
-
Then install bundler
|
33
|
-
|
32
|
+
Then install bundler:
|
33
|
+
|
34
|
+
```bash
|
35
|
+
gem install bundler:2.1.4
|
36
|
+
```
|
34
37
|
|
35
38
|
Follow those instructions to [install logstash](https://www.elastic.co/guide/en/logstash/current/installing-logstash.html) before moving to the next section.
|
36
39
|
|
@@ -41,36 +44,45 @@ Follow those instructions to [install logstash](https://www.elastic.co/guide/en/
|
|
41
44
|
```bash
|
42
45
|
git clone git@github.com:elastic/logstash.git
|
43
46
|
cd logstash
|
44
|
-
git checkout tags/v7.
|
45
|
-
export LOGSTASH_PATH
|
46
|
-
export GEM_PATH
|
47
|
-
export GEM_HOME
|
47
|
+
git checkout tags/v7.16.1
|
48
|
+
export LOGSTASH_PATH="$(pwd)"
|
49
|
+
export GEM_PATH="$LOGSTASH_PATH/vendor/bundle/jruby/2.5.0"
|
50
|
+
export GEM_HOME="$LOGSTASH_PATH/vendor/bundle/jruby/2.5.0"
|
48
51
|
./gradlew assemble
|
49
52
|
cd ..
|
53
|
+
ruby -S bundle config set --local path "$LOGSTASH_PATH/vendor/bundle"
|
50
54
|
ruby -S bundle install
|
51
55
|
ruby -S bundle exec rake vendor
|
52
56
|
```
|
53
57
|
|
54
58
|
### Build the plugin
|
55
59
|
|
56
|
-
|
60
|
+
```bash
|
61
|
+
gem build logstash-output-loki.gemspec
|
62
|
+
```
|
57
63
|
|
58
64
|
### Test
|
59
65
|
|
60
|
-
|
66
|
+
```bash
|
67
|
+
ruby -S bundle exec rspec
|
68
|
+
```
|
61
69
|
|
62
70
|
Alternatively if you don't want to install JRuby. Enter inside logstash-loki container.
|
63
71
|
|
64
72
|
```bash
|
65
73
|
docker build -t logstash-loki ./
|
66
|
-
docker run -v
|
74
|
+
docker run -v $(pwd)/spec:/home/logstash/spec -it --rm --entrypoint /bin/sh logstash-loki
|
67
75
|
bundle exec rspec
|
68
76
|
```
|
69
77
|
|
70
78
|
## Install plugin to local logstash
|
71
79
|
|
72
|
-
|
80
|
+
```bash
|
81
|
+
bin/logstash-plugin install --no-verify --local logstash-output-loki-1.0.0.gem
|
82
|
+
```
|
73
83
|
|
74
84
|
## Send sample event and check plugin is working
|
75
85
|
|
76
|
-
|
86
|
+
```bash
|
87
|
+
bin/logstash -f loki.conf
|
88
|
+
```
|
@@ -5,7 +5,7 @@ module Loki
|
|
5
5
|
class Entry
|
6
6
|
include Loki
|
7
7
|
attr_reader :labels, :entry
|
8
|
-
def initialize(event,message_field)
|
8
|
+
def initialize(event,message_field,include_fields)
|
9
9
|
@entry = {
|
10
10
|
"ts" => to_ns(event.get("@timestamp")),
|
11
11
|
"line" => event.get(message_field).to_s
|
@@ -18,6 +18,7 @@ module Loki
|
|
18
18
|
event.to_hash.each { |key,value|
|
19
19
|
next if key.start_with?('@')
|
20
20
|
next if value.is_a?(Hash)
|
21
|
+
next if include_fields.length() > 0 and not include_fields.include?(key)
|
21
22
|
@labels[key] = value.to_s
|
22
23
|
}
|
23
24
|
end
|
@@ -47,6 +47,9 @@ class LogStash::Outputs::Loki < LogStash::Outputs::Base
|
|
47
47
|
## 'Backoff configuration. Initial backoff time between retries. Default 1s'
|
48
48
|
config :min_delay, :validate => :number, :default => 1, :required => false
|
49
49
|
|
50
|
+
## 'An array of fields to map to labels, if defined only fields in this list will be mapped.'
|
51
|
+
config :include_fields, :validate => :array, :default => [], :required => false
|
52
|
+
|
50
53
|
## 'Backoff configuration. Maximum backoff time between retries. Default 300s'
|
51
54
|
config :max_delay, :validate => :number, :default => 300, :required => false
|
52
55
|
|
@@ -198,7 +201,7 @@ class LogStash::Outputs::Loki < LogStash::Outputs::Base
|
|
198
201
|
## Receives logstash events
|
199
202
|
public
|
200
203
|
def receive(event)
|
201
|
-
@entries << Entry.new(event, @message_field)
|
204
|
+
@entries << Entry.new(event, @message_field, @include_fields)
|
202
205
|
end
|
203
206
|
|
204
207
|
def close
|
@@ -1,6 +1,6 @@
|
|
1
1
|
Gem::Specification.new do |s|
|
2
2
|
s.name = 'logstash-output-loki'
|
3
|
-
s.version = '1.0
|
3
|
+
s.version = '1.1.0'
|
4
4
|
s.authors = ['Aditya C S','Cyril Tovena']
|
5
5
|
s.email = ['aditya.gnu@gmail.com','cyril.tovena@grafana.com']
|
6
6
|
|
@@ -21,6 +21,6 @@ Gem::Specification.new do |s|
|
|
21
21
|
# Gem dependencies
|
22
22
|
#
|
23
23
|
s.add_runtime_dependency "logstash-core-plugin-api", ">= 1.60", "<= 2.99"
|
24
|
-
s.add_runtime_dependency "logstash-codec-plain", "3.0
|
24
|
+
s.add_runtime_dependency "logstash-codec-plain", "3.1.0"
|
25
25
|
s.add_development_dependency 'logstash-devutils', "2.0.2"
|
26
26
|
end
|
@@ -27,18 +27,25 @@ describe Loki::Entry do
|
|
27
27
|
}
|
28
28
|
|
29
29
|
it 'labels extracted should not contains object and metadata or timestamp' do
|
30
|
-
entry = Entry.new(event,"message")
|
30
|
+
entry = Entry.new(event,"message", [])
|
31
31
|
expect(entry.labels).to eql({ 'agent' => 'filebeat', 'host' => '172.0.0.1', 'foo'=>'5'})
|
32
32
|
expect(entry.entry['ts']).to eql to_ns(event.get("@timestamp"))
|
33
33
|
expect(entry.entry['line']).to eql 'hello'
|
34
34
|
end
|
35
|
+
|
36
|
+
it 'labels extracted should only contain allowlisted labels' do
|
37
|
+
entry = Entry.new(event, "message", %w[agent foo])
|
38
|
+
expect(entry.labels).to eql({ 'agent' => 'filebeat', 'foo'=>'5'})
|
39
|
+
expect(entry.entry['ts']).to eql to_ns(event.get("@timestamp"))
|
40
|
+
expect(entry.entry['line']).to eql 'hello'
|
41
|
+
end
|
35
42
|
end
|
36
43
|
|
37
44
|
context 'test batch generation with label order' do
|
38
45
|
let (:entries) {[
|
39
|
-
Entry.new(LogStash::Event.new({"message"=>"foobuzz","buzz"=>"bar","cluster"=>"us-central1","@timestamp"=>Time.at(1)}),"message"),
|
40
|
-
Entry.new(LogStash::Event.new({"log"=>"foobar","bar"=>"bar","@timestamp"=>Time.at(2)}),"log"),
|
41
|
-
Entry.new(LogStash::Event.new({"cluster"=>"us-central1","message"=>"foobuzz","buzz"=>"bar","@timestamp"=>Time.at(3)}),"message"),
|
46
|
+
Entry.new(LogStash::Event.new({"message"=>"foobuzz","buzz"=>"bar","cluster"=>"us-central1","@timestamp"=>Time.at(1)}),"message", []),
|
47
|
+
Entry.new(LogStash::Event.new({"log"=>"foobar","bar"=>"bar","@timestamp"=>Time.at(2)}),"log", []),
|
48
|
+
Entry.new(LogStash::Event.new({"cluster"=>"us-central1","message"=>"foobuzz","buzz"=>"bar","@timestamp"=>Time.at(3)}),"message", []),
|
42
49
|
|
43
50
|
]}
|
44
51
|
let (:expected) {
|
data/spec/outputs/loki_spec.rb
CHANGED
@@ -28,12 +28,15 @@ describe LogStash::Outputs::Loki do
|
|
28
28
|
|
29
29
|
context 'when adding en entry to the batch' do
|
30
30
|
let (:simple_loki_config) {{'url' => 'http://localhost:3100'}}
|
31
|
-
let (:entry) {Entry.new(LogStash::Event.new({"message"=>"foobuzz","buzz"=>"bar","cluster"=>"us-central1","@timestamp"=>Time.at(1)}),"message")}
|
32
|
-
let (:lbs) {
|
31
|
+
let (:entry) {Entry.new(LogStash::Event.new({"message"=>"foobuzz","buzz"=>"bar","cluster"=>"us-central1","@timestamp"=>Time.at(1)}),"message", [])}
|
32
|
+
let (:lbs) {{"buzz"=>"bar","cluster"=>"us-central1"}.sort.to_h}
|
33
|
+
let (:include_loki_config) {{ 'url' => 'http://localhost:3100', 'include_fields' => ["cluster"] }}
|
34
|
+
let (:include_entry) {Entry.new(LogStash::Event.new({"message"=>"foobuzz","buzz"=>"bar","cluster"=>"us-central1","@timestamp"=>Time.at(1)}),"message", ["cluster"])}
|
35
|
+
let (:include_lbs) {{"cluster"=>"us-central1"}.sort.to_h}
|
33
36
|
|
34
37
|
it 'should not add empty line' do
|
35
38
|
plugin = LogStash::Plugin.lookup("output", "loki").new(simple_loki_config)
|
36
|
-
emptyEntry = Entry.new(LogStash::Event.new({"message"=>"foobuzz","buzz"=>"bar","cluster"=>"us-central1","@timestamp"=>Time.at(1)}),"foo")
|
39
|
+
emptyEntry = Entry.new(LogStash::Event.new({"message"=>"foobuzz","buzz"=>"bar","cluster"=>"us-central1","@timestamp"=>Time.at(1)}),"foo", [])
|
37
40
|
expect(plugin.add_entry_to_batch(emptyEntry)).to eql true
|
38
41
|
expect(plugin.batch).to eql nil
|
39
42
|
end
|
@@ -50,6 +53,18 @@ describe LogStash::Outputs::Loki do
|
|
50
53
|
expect(plugin.batch.size_bytes).to eq 14
|
51
54
|
end
|
52
55
|
|
56
|
+
it 'should only allowed labels defined in include_fields' do
|
57
|
+
plugin = LogStash::Plugin.lookup("output", "loki").new(include_loki_config)
|
58
|
+
expect(plugin.batch).to eql nil
|
59
|
+
expect(plugin.add_entry_to_batch(include_entry)).to eql true
|
60
|
+
expect(plugin.add_entry_to_batch(include_entry)).to eql true
|
61
|
+
expect(plugin.batch).not_to be_nil
|
62
|
+
expect(plugin.batch.streams.length).to eq 1
|
63
|
+
expect(plugin.batch.streams[include_lbs.to_s]['entries'].length).to eq 2
|
64
|
+
expect(plugin.batch.streams[include_lbs.to_s]['labels']).to eq include_lbs
|
65
|
+
expect(plugin.batch.size_bytes).to eq 14
|
66
|
+
end
|
67
|
+
|
53
68
|
it 'should not add if full' do
|
54
69
|
plugin = LogStash::Plugin.lookup("output", "loki").new(simple_loki_config.merge!({'batch_size'=>10}))
|
55
70
|
expect(plugin.batch).to eql nil
|
@@ -69,7 +84,7 @@ describe LogStash::Outputs::Loki do
|
|
69
84
|
end
|
70
85
|
|
71
86
|
context 'batch expiration' do
|
72
|
-
let (:entry) {Entry.new(LogStash::Event.new({"message"=>"foobuzz","buzz"=>"bar","cluster"=>"us-central1","@timestamp"=>Time.at(1)}),"message")}
|
87
|
+
let (:entry) {Entry.new(LogStash::Event.new({"message"=>"foobuzz","buzz"=>"bar","cluster"=>"us-central1","@timestamp"=>Time.at(1)}),"message", [])}
|
73
88
|
|
74
89
|
it 'should not expire if empty' do
|
75
90
|
loki = LogStash::Outputs::Loki.new(simple_loki_config.merge!({'batch_wait'=>0.5}))
|
@@ -138,7 +153,7 @@ describe LogStash::Outputs::Loki do
|
|
138
153
|
end
|
139
154
|
|
140
155
|
context 'http requests' do
|
141
|
-
let (:entry) {Entry.new(LogStash::Event.new({"message"=>"foobuzz","buzz"=>"bar","cluster"=>"us-central1","@timestamp"=>Time.at(1)}),"message")}
|
156
|
+
let (:entry) {Entry.new(LogStash::Event.new({"message"=>"foobuzz","buzz"=>"bar","cluster"=>"us-central1","@timestamp"=>Time.at(1)}),"message", [])}
|
142
157
|
|
143
158
|
it 'should send credentials' do
|
144
159
|
conf = {
|
metadata
CHANGED
@@ -1,18 +1,17 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: logstash-output-loki
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 1.0
|
4
|
+
version: 1.1.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Aditya C S
|
8
8
|
- Cyril Tovena
|
9
|
-
autorequire:
|
9
|
+
autorequire:
|
10
10
|
bindir: bin
|
11
11
|
cert_chain: []
|
12
|
-
date:
|
12
|
+
date: 2022-01-27 00:00:00.000000000 Z
|
13
13
|
dependencies:
|
14
14
|
- !ruby/object:Gem::Dependency
|
15
|
-
name: logstash-core-plugin-api
|
16
15
|
requirement: !ruby/object:Gem::Requirement
|
17
16
|
requirements:
|
18
17
|
- - ">="
|
@@ -21,8 +20,9 @@ dependencies:
|
|
21
20
|
- - "<="
|
22
21
|
- !ruby/object:Gem::Version
|
23
22
|
version: '2.99'
|
24
|
-
|
23
|
+
name: logstash-core-plugin-api
|
25
24
|
prerelease: false
|
25
|
+
type: :runtime
|
26
26
|
version_requirements: !ruby/object:Gem::Requirement
|
27
27
|
requirements:
|
28
28
|
- - ">="
|
@@ -32,28 +32,28 @@ dependencies:
|
|
32
32
|
- !ruby/object:Gem::Version
|
33
33
|
version: '2.99'
|
34
34
|
- !ruby/object:Gem::Dependency
|
35
|
-
name: logstash-codec-plain
|
36
35
|
requirement: !ruby/object:Gem::Requirement
|
37
36
|
requirements:
|
38
37
|
- - '='
|
39
38
|
- !ruby/object:Gem::Version
|
40
|
-
version: 3.0
|
41
|
-
|
39
|
+
version: 3.1.0
|
40
|
+
name: logstash-codec-plain
|
42
41
|
prerelease: false
|
42
|
+
type: :runtime
|
43
43
|
version_requirements: !ruby/object:Gem::Requirement
|
44
44
|
requirements:
|
45
45
|
- - '='
|
46
46
|
- !ruby/object:Gem::Version
|
47
|
-
version: 3.0
|
47
|
+
version: 3.1.0
|
48
48
|
- !ruby/object:Gem::Dependency
|
49
|
-
name: logstash-devutils
|
50
49
|
requirement: !ruby/object:Gem::Requirement
|
51
50
|
requirements:
|
52
51
|
- - '='
|
53
52
|
- !ruby/object:Gem::Version
|
54
53
|
version: 2.0.2
|
55
|
-
|
54
|
+
name: logstash-devutils
|
56
55
|
prerelease: false
|
56
|
+
type: :development
|
57
57
|
version_requirements: !ruby/object:Gem::Requirement
|
58
58
|
requirements:
|
59
59
|
- - '='
|
@@ -81,7 +81,7 @@ licenses:
|
|
81
81
|
metadata:
|
82
82
|
logstash_plugin: 'true'
|
83
83
|
logstash_group: output
|
84
|
-
post_install_message:
|
84
|
+
post_install_message:
|
85
85
|
rdoc_options: []
|
86
86
|
require_paths:
|
87
87
|
- lib
|
@@ -96,8 +96,8 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
96
96
|
- !ruby/object:Gem::Version
|
97
97
|
version: '0'
|
98
98
|
requirements: []
|
99
|
-
rubygems_version: 3.
|
100
|
-
signing_key:
|
99
|
+
rubygems_version: 3.1.6
|
100
|
+
signing_key:
|
101
101
|
specification_version: 4
|
102
102
|
summary: Output plugin to ship logs to a Grafana Loki server
|
103
103
|
test_files:
|