logstash-output-loki 1.1.0 → 1.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/lib/logstash/outputs/loki/batch.rb +13 -1
- data/lib/logstash/outputs/loki/entry.rb +17 -1
- data/lib/logstash/outputs/loki.rb +8 -5
- data/logstash-output-loki.gemspec +1 -1
- data/spec/outputs/loki/entry_spec.rb +15 -6
- data/spec/outputs/loki_spec.rb +49 -6
- metadata +2 -2
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: a2345ce18e6ef9aad96401c5b7afb5dca4f5b7dcc71c7cf8a21a1b28496e78fc
|
4
|
+
data.tar.gz: 3fedc6382bd23200cf2f9afda2e6ec4d0222e72921ccee4563671af72250032a
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: ea1a970fe497c812132c999dbf73638c6baa20021b4bac22bb20114c525910d963eef3621ec5b69a3740732668de88ad4c9c09fa9a5569e7008030685d67b337
|
7
|
+
data.tar.gz: d03d649be3e5ed202f5e27aeb307e5db39c036b6c089748a982482d1b23df3e46c1563e61c20b081bdca7d62699b871e14ee569269a77d6d00760f23e70ea12d
|
@@ -52,7 +52,19 @@ module Loki
|
|
52
52
|
def build_stream(stream)
|
53
53
|
values = []
|
54
54
|
stream['entries'].each { |entry|
|
55
|
-
|
55
|
+
if entry.key?('metadata')
|
56
|
+
sorted_metadata = entry['metadata'].sort.to_h
|
57
|
+
values.append([
|
58
|
+
entry['ts'].to_s,
|
59
|
+
entry['line'],
|
60
|
+
sorted_metadata
|
61
|
+
])
|
62
|
+
else
|
63
|
+
values.append([
|
64
|
+
entry['ts'].to_s,
|
65
|
+
entry['line']
|
66
|
+
])
|
67
|
+
end
|
56
68
|
}
|
57
69
|
return {
|
58
70
|
'stream'=>stream['labels'],
|
@@ -5,7 +5,7 @@ module Loki
|
|
5
5
|
class Entry
|
6
6
|
include Loki
|
7
7
|
attr_reader :labels, :entry
|
8
|
-
def initialize(event,message_field,include_fields)
|
8
|
+
def initialize(event,message_field,include_fields,metadata_fields)
|
9
9
|
@entry = {
|
10
10
|
"ts" => to_ns(event.get("@timestamp")),
|
11
11
|
"line" => event.get(message_field).to_s
|
@@ -21,6 +21,22 @@ module Loki
|
|
21
21
|
next if include_fields.length() > 0 and not include_fields.include?(key)
|
22
22
|
@labels[key] = value.to_s
|
23
23
|
}
|
24
|
+
|
25
|
+
# Unlike include_fields we should skip if no metadata_fields provided
|
26
|
+
if metadata_fields.length() > 0
|
27
|
+
@metadata = {}
|
28
|
+
event.to_hash.each { |key,value|
|
29
|
+
next if key.start_with?('@')
|
30
|
+
next if value.is_a?(Hash)
|
31
|
+
next if metadata_fields.length() > 0 and not metadata_fields.include?(key)
|
32
|
+
@metadata[key] = value.to_s
|
33
|
+
}
|
34
|
+
|
35
|
+
# Add @metadata to @entry if there was a match
|
36
|
+
if @metadata.size > 0
|
37
|
+
@entry.merge!('metadata' => @metadata)
|
38
|
+
end
|
39
|
+
end
|
24
40
|
end
|
25
41
|
end
|
26
42
|
end
|
@@ -50,6 +50,9 @@ class LogStash::Outputs::Loki < LogStash::Outputs::Base
|
|
50
50
|
## 'An array of fields to map to labels, if defined only fields in this list will be mapped.'
|
51
51
|
config :include_fields, :validate => :array, :default => [], :required => false
|
52
52
|
|
53
|
+
## 'An array of fields to map to structure metadata, if defined only fields in this list will be mapped.'
|
54
|
+
config :metadata_fields, :validate => :array, :default => [], :required => false
|
55
|
+
|
53
56
|
## 'Backoff configuration. Maximum backoff time between retries. Default 300s'
|
54
57
|
config :max_delay, :validate => :number, :default => 300, :required => false
|
55
58
|
|
@@ -71,7 +74,7 @@ class LogStash::Outputs::Loki < LogStash::Outputs::Base
|
|
71
74
|
@logger.info("Loki output plugin", :class => self.class.name)
|
72
75
|
|
73
76
|
# initialize Queue and Mutex
|
74
|
-
@entries = Queue.new
|
77
|
+
@entries = Queue.new
|
75
78
|
@mutex = Mutex.new
|
76
79
|
@stop = false
|
77
80
|
|
@@ -94,7 +97,7 @@ class LogStash::Outputs::Loki < LogStash::Outputs::Base
|
|
94
97
|
@mutex.synchronize do
|
95
98
|
return if @stop
|
96
99
|
end
|
97
|
-
|
100
|
+
|
98
101
|
e = @entries.deq
|
99
102
|
return if e.nil?
|
100
103
|
|
@@ -201,13 +204,13 @@ class LogStash::Outputs::Loki < LogStash::Outputs::Base
|
|
201
204
|
## Receives logstash events
|
202
205
|
public
|
203
206
|
def receive(event)
|
204
|
-
@entries << Entry.new(event, @message_field, @include_fields)
|
207
|
+
@entries << Entry.new(event, @message_field, @include_fields, @metadata_fields)
|
205
208
|
end
|
206
209
|
|
207
210
|
def close
|
208
211
|
@entries.close
|
209
|
-
@mutex.synchronize do
|
210
|
-
@stop = true
|
212
|
+
@mutex.synchronize do
|
213
|
+
@stop = true
|
211
214
|
end
|
212
215
|
@batch_wait_thread.join
|
213
216
|
@batch_size_thread.join
|
@@ -21,31 +21,40 @@ describe Loki::Entry do
|
|
21
21
|
{'@path' => '/path/to/file.log'},
|
22
22
|
},
|
23
23
|
'host' => '172.0.0.1',
|
24
|
+
'trace_id' => 'trace_001',
|
24
25
|
'@timestamp' => Time.now
|
25
26
|
}
|
26
27
|
)
|
27
28
|
}
|
28
29
|
|
29
30
|
it 'labels extracted should not contains object and metadata or timestamp' do
|
30
|
-
entry = Entry.new(event,"message", [])
|
31
|
-
expect(entry.labels).to eql({ 'agent' => 'filebeat', 'host' => '172.0.0.1', 'foo'=>'5'})
|
31
|
+
entry = Entry.new(event,"message", [], [])
|
32
|
+
expect(entry.labels).to eql({ 'agent' => 'filebeat', 'host' => '172.0.0.1', 'foo'=>'5', 'trace_id'=>'trace_001'})
|
32
33
|
expect(entry.entry['ts']).to eql to_ns(event.get("@timestamp"))
|
33
34
|
expect(entry.entry['line']).to eql 'hello'
|
34
35
|
end
|
35
36
|
|
36
37
|
it 'labels extracted should only contain allowlisted labels' do
|
37
|
-
entry = Entry.new(event, "message", %w[agent foo])
|
38
|
+
entry = Entry.new(event, "message", %w[agent foo], [])
|
38
39
|
expect(entry.labels).to eql({ 'agent' => 'filebeat', 'foo'=>'5'})
|
39
40
|
expect(entry.entry['ts']).to eql to_ns(event.get("@timestamp"))
|
40
41
|
expect(entry.entry['line']).to eql 'hello'
|
41
42
|
end
|
43
|
+
|
44
|
+
it 'labels and structured metadata extracted should only contain allow listed labels and metadata' do
|
45
|
+
entry = Entry.new(event, "message", %w[agent foo], %w[trace_id])
|
46
|
+
expect(entry.labels).to eql({ 'agent' => 'filebeat', 'foo'=>'5'})
|
47
|
+
expect(entry.entry['ts']).to eql to_ns(event.get("@timestamp"))
|
48
|
+
expect(entry.entry['line']).to eql 'hello'
|
49
|
+
expect(entry.entry['metadata']).to eql({'trace_id' => 'trace_001'})
|
50
|
+
end
|
42
51
|
end
|
43
52
|
|
44
53
|
context 'test batch generation with label order' do
|
45
54
|
let (:entries) {[
|
46
|
-
Entry.new(LogStash::Event.new({"message"=>"foobuzz","buzz"=>"bar","cluster"=>"us-central1","@timestamp"=>Time.at(1)}),"message", []),
|
47
|
-
Entry.new(LogStash::Event.new({"log"=>"foobar","bar"=>"bar","@timestamp"=>Time.at(2)}),"log", []),
|
48
|
-
Entry.new(LogStash::Event.new({"cluster"=>"us-central1","message"=>"foobuzz","buzz"=>"bar","@timestamp"=>Time.at(3)}),"message", []),
|
55
|
+
Entry.new(LogStash::Event.new({"message"=>"foobuzz","buzz"=>"bar","cluster"=>"us-central1","@timestamp"=>Time.at(1)}),"message", [], []),
|
56
|
+
Entry.new(LogStash::Event.new({"log"=>"foobar","bar"=>"bar","@timestamp"=>Time.at(2)}),"log", [], []),
|
57
|
+
Entry.new(LogStash::Event.new({"cluster"=>"us-central1","message"=>"foobuzz","buzz"=>"bar","@timestamp"=>Time.at(3)}),"message", [], []),
|
49
58
|
|
50
59
|
]}
|
51
60
|
let (:expected) {
|
data/spec/outputs/loki_spec.rb
CHANGED
@@ -28,15 +28,15 @@ describe LogStash::Outputs::Loki do
|
|
28
28
|
|
29
29
|
context 'when adding en entry to the batch' do
|
30
30
|
let (:simple_loki_config) {{'url' => 'http://localhost:3100'}}
|
31
|
-
let (:entry) {Entry.new(LogStash::Event.new({"message"=>"foobuzz","buzz"=>"bar","cluster"=>"us-central1","@timestamp"=>Time.at(1)}),"message", [])}
|
31
|
+
let (:entry) {Entry.new(LogStash::Event.new({"message"=>"foobuzz","buzz"=>"bar","cluster"=>"us-central1","@timestamp"=>Time.at(1)}),"message", [], [])}
|
32
32
|
let (:lbs) {{"buzz"=>"bar","cluster"=>"us-central1"}.sort.to_h}
|
33
33
|
let (:include_loki_config) {{ 'url' => 'http://localhost:3100', 'include_fields' => ["cluster"] }}
|
34
|
-
let (:include_entry) {Entry.new(LogStash::Event.new({"message"=>"foobuzz","buzz"=>"bar","cluster"=>"us-central1","@timestamp"=>Time.at(1)}),"message", ["cluster"])}
|
34
|
+
let (:include_entry) {Entry.new(LogStash::Event.new({"message"=>"foobuzz","buzz"=>"bar","cluster"=>"us-central1","@timestamp"=>Time.at(1)}),"message", ["cluster"], [])}
|
35
35
|
let (:include_lbs) {{"cluster"=>"us-central1"}.sort.to_h}
|
36
36
|
|
37
37
|
it 'should not add empty line' do
|
38
38
|
plugin = LogStash::Plugin.lookup("output", "loki").new(simple_loki_config)
|
39
|
-
emptyEntry = Entry.new(LogStash::Event.new({"message"=>"foobuzz","buzz"=>"bar","cluster"=>"us-central1","@timestamp"=>Time.at(1)}),"foo", [])
|
39
|
+
emptyEntry = Entry.new(LogStash::Event.new({"message"=>"foobuzz","buzz"=>"bar","cluster"=>"us-central1","@timestamp"=>Time.at(1)}),"foo", [], [])
|
40
40
|
expect(plugin.add_entry_to_batch(emptyEntry)).to eql true
|
41
41
|
expect(plugin.batch).to eql nil
|
42
42
|
end
|
@@ -83,8 +83,51 @@ describe LogStash::Outputs::Loki do
|
|
83
83
|
end
|
84
84
|
end
|
85
85
|
|
86
|
+
context 'when building json from batch to send' do
|
87
|
+
let (:basic_loki_config) {{'url' => 'http://localhost:3100'}}
|
88
|
+
let (:basic_entry) {Entry.new(LogStash::Event.new({"message"=>"foobuzz","buzz"=>"bar","cluster"=>"us-central1","trace_id"=>"trace_001","@timestamp"=>Time.at(1)}),"message", [], [])}
|
89
|
+
let (:include_loki_config) {{ 'url' => 'http://localhost:3100', 'include_fields' => ["cluster"] }}
|
90
|
+
let (:include_entry) {Entry.new(LogStash::Event.new({"message"=>"foobuzz","buzz"=>"bar","cluster"=>"us-central1","trace_id"=>"trace_001","@timestamp"=>Time.at(1)}),"message", ["cluster"], [])}
|
91
|
+
let (:metadata_loki_config) {{ 'url' => 'http://localhost:3100', 'include_fields' => ["cluster"], 'metadata_fields' => ["trace_id"] }}
|
92
|
+
let (:metadata_entry) {Entry.new(LogStash::Event.new({"message"=>"foobuzz","buzz"=>"bar","cluster"=>"us-central1","trace_id"=>"trace_001","@timestamp"=>Time.at(1)}),"message", ["cluster"], ["trace_id"])}
|
93
|
+
let (:metadata_multi_loki_config) {{ 'url' => 'http://localhost:3100', 'include_fields' => ["cluster"], 'metadata_fields' => ["trace_id", "user_id"] }}
|
94
|
+
let (:metadata_multi_entry) {Entry.new(LogStash::Event.new({"message"=>"foobuzz","buzz"=>"bar","cluster"=>"us-central1","trace_id"=>"trace_001","user_id"=>"user_001","@timestamp"=>Time.at(1)}),"message", ["cluster"], ["trace_id", "user_id"])}
|
95
|
+
|
96
|
+
it 'should not include labels or metadata' do
|
97
|
+
plugin = LogStash::Plugin.lookup("output", "loki").new(basic_loki_config)
|
98
|
+
expect(plugin.batch).to eql nil
|
99
|
+
expect(plugin.add_entry_to_batch(basic_entry)).to eql true
|
100
|
+
expect(plugin.batch).not_to be_nil
|
101
|
+
expect(plugin.batch.to_json).to eq '{"streams":[{"stream":{"buzz":"bar","cluster":"us-central1","trace_id":"trace_001"},"values":[["1000000000","foobuzz"]]}]}'
|
102
|
+
end
|
103
|
+
|
104
|
+
it 'should include metadata with no labels' do
|
105
|
+
plugin = LogStash::Plugin.lookup("output", "loki").new(metadata_loki_config)
|
106
|
+
expect(plugin.batch).to eql nil
|
107
|
+
expect(plugin.add_entry_to_batch(metadata_entry)).to eql true
|
108
|
+
expect(plugin.batch).not_to be_nil
|
109
|
+
expect(plugin.batch.to_json).to eq '{"streams":[{"stream":{"cluster":"us-central1"},"values":[["1000000000","foobuzz",{"trace_id":"trace_001"}]]}]}'
|
110
|
+
end
|
111
|
+
|
112
|
+
it 'should include labels with no metadata' do
|
113
|
+
plugin = LogStash::Plugin.lookup("output", "loki").new(include_loki_config)
|
114
|
+
expect(plugin.batch).to eql nil
|
115
|
+
expect(plugin.add_entry_to_batch(include_entry)).to eql true
|
116
|
+
expect(plugin.batch).not_to be_nil
|
117
|
+
expect(plugin.batch.to_json).to eq '{"streams":[{"stream":{"cluster":"us-central1"},"values":[["1000000000","foobuzz"]]}]}'
|
118
|
+
end
|
119
|
+
|
120
|
+
it 'should include labels with multiple metadata' do
|
121
|
+
plugin = LogStash::Plugin.lookup("output", "loki").new(metadata_multi_loki_config)
|
122
|
+
expect(plugin.batch).to eql nil
|
123
|
+
expect(plugin.add_entry_to_batch(metadata_multi_entry)).to eql true
|
124
|
+
expect(plugin.batch).not_to be_nil
|
125
|
+
expect(plugin.batch.to_json).to eq '{"streams":[{"stream":{"cluster":"us-central1"},"values":[["1000000000","foobuzz",{"trace_id":"trace_001","user_id":"user_001"}]]}]}'
|
126
|
+
end
|
127
|
+
end
|
128
|
+
|
86
129
|
context 'batch expiration' do
|
87
|
-
let (:entry) {Entry.new(LogStash::Event.new({"message"=>"foobuzz","buzz"=>"bar","cluster"=>"us-central1","@timestamp"=>Time.at(1)}),"message", [])}
|
130
|
+
let (:entry) {Entry.new(LogStash::Event.new({"message"=>"foobuzz","buzz"=>"bar","cluster"=>"us-central1","@timestamp"=>Time.at(1)}),"message", [], [])}
|
88
131
|
|
89
132
|
it 'should not expire if empty' do
|
90
133
|
loki = LogStash::Outputs::Loki.new(simple_loki_config.merge!({'batch_wait'=>0.5}))
|
@@ -147,13 +190,13 @@ describe LogStash::Outputs::Loki do
|
|
147
190
|
loki.receive(event)
|
148
191
|
sent.deq
|
149
192
|
sleep(0.01) # Adding a minimal sleep. In few cases @batch=nil might happen after evaluating for nil
|
150
|
-
expect(loki.batch).to be_nil
|
193
|
+
expect(loki.batch).to be_nil
|
151
194
|
loki.close
|
152
195
|
end
|
153
196
|
end
|
154
197
|
|
155
198
|
context 'http requests' do
|
156
|
-
let (:entry) {Entry.new(LogStash::Event.new({"message"=>"foobuzz","buzz"=>"bar","cluster"=>"us-central1","@timestamp"=>Time.at(1)}),"message", [])}
|
199
|
+
let (:entry) {Entry.new(LogStash::Event.new({"message"=>"foobuzz","buzz"=>"bar","cluster"=>"us-central1","@timestamp"=>Time.at(1)}),"message", [], [])}
|
157
200
|
|
158
201
|
it 'should send credentials' do
|
159
202
|
conf = {
|
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: logstash-output-loki
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 1.
|
4
|
+
version: 1.2.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Aditya C S
|
@@ -9,7 +9,7 @@ authors:
|
|
9
9
|
autorequire:
|
10
10
|
bindir: bin
|
11
11
|
cert_chain: []
|
12
|
-
date:
|
12
|
+
date: 2024-09-24 00:00:00.000000000 Z
|
13
13
|
dependencies:
|
14
14
|
- !ruby/object:Gem::Dependency
|
15
15
|
requirement: !ruby/object:Gem::Requirement
|