logstash-output-loki 1.0.3 → 1.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: a76f3826de04d39e57f06c522125adfc150e1b9c44dee915cb5a220e829b5ae6
4
- data.tar.gz: d4f881165097a6c04dcef15eca32f3570f0d35453a16713e5da1a50e022876d0
3
+ metadata.gz: 5c6817d07ada63cde10011f1e490f803ddbb0a7a54d75b9472780f4f180c65fe
4
+ data.tar.gz: e457589a713ed3bc0f74120f3671bfb6b9b9760393cb86c18764e4d5a1fa2bd7
5
5
  SHA512:
6
- metadata.gz: 706a86a1852e2b2e97bbd340904b1a0a72751beca4ca4257bfbd66683e0c3a7fe5a54d273666c2da98947b84270b4e9544fe36b6e1742877ccffcbf67fe090bc
7
- data.tar.gz: 5710b40ac392fa8e82c29343f22b444ce12284023497da3d0cb9c28049b81811ece3bcf431e5b948b7c60dd29c941dcd3c508289fc9a2d49faf5331eda893f69
6
+ metadata.gz: 358e07d5c964179b7a003e6ac840bb83df74c03e4de0f1916da310ed07e1c669a672d9363afd841c6bdfd49bb1d525ffbc815801ba4f3a8536659dd449173012
7
+ data.tar.gz: 89360e4ffcb276b9406c017c5c019c43f839398a93a0cda46e0761e45a6a851ef2e9250c9fec5ba409062aef1f12576c9fcaa939a4cba0c0c9d0db15fa698f24
data/README.md CHANGED
@@ -29,8 +29,11 @@ export PATH="$HOME/.rbenv/bin:$PATH"
29
29
  eval "$(rbenv init -)"
30
30
  ```
31
31
 
32
- Then install bundler
33
- `gem install bundler:2.1.4`
32
+ Then install bundler:
33
+
34
+ ```bash
35
+ gem install bundler:2.1.4
36
+ ```
34
37
 
35
38
  Follow those instructions to [install logstash](https://www.elastic.co/guide/en/logstash/current/installing-logstash.html) before moving to the next section.
36
39
 
@@ -41,36 +44,45 @@ Follow those instructions to [install logstash](https://www.elastic.co/guide/en/
41
44
  ```bash
42
45
  git clone git@github.com:elastic/logstash.git
43
46
  cd logstash
44
- git checkout tags/v7.6.2
45
- export LOGSTASH_PATH=`pwd`
46
- export GEM_PATH=$LOGSTASH_PATH/vendor/bundle/jruby/2.5.0
47
- export GEM_HOME=$LOGSTASH_PATH/vendor/bundle/jruby/2.5.0
47
+ git checkout tags/v7.16.1
48
+ export LOGSTASH_PATH="$(pwd)"
49
+ export GEM_PATH="$LOGSTASH_PATH/vendor/bundle/jruby/2.5.0"
50
+ export GEM_HOME="$LOGSTASH_PATH/vendor/bundle/jruby/2.5.0"
48
51
  ./gradlew assemble
49
52
  cd ..
53
+ ruby -S bundle config set --local path "$LOGSTASH_PATH/vendor/bundle"
50
54
  ruby -S bundle install
51
55
  ruby -S bundle exec rake vendor
52
56
  ```
53
57
 
54
58
  ### Build the plugin
55
59
 
56
- `gem build logstash-output-loki.gemspec`
60
+ ```bash
61
+ gem build logstash-output-loki.gemspec
62
+ ```
57
63
 
58
64
  ### Test
59
65
 
60
- `ruby -S bundle exec rspec`
66
+ ```bash
67
+ ruby -S bundle exec rspec
68
+ ```
61
69
 
62
70
  Alternatively if you don't want to install JRuby. Enter inside logstash-loki container.
63
71
 
64
72
  ```bash
65
73
  docker build -t logstash-loki ./
66
- docker run -v `pwd`/spec:/home/logstash/spec -it --rm --entrypoint /bin/sh logstash-loki
74
+ docker run -v $(pwd)/spec:/home/logstash/spec -it --rm --entrypoint /bin/sh logstash-loki
67
75
  bundle exec rspec
68
76
  ```
69
77
 
70
78
  ## Install plugin to local logstash
71
79
 
72
- `bin/logstash-plugin install --no-verify --local logstash-output-loki-1.0.0.gem`
80
+ ```bash
81
+ bin/logstash-plugin install --no-verify --local logstash-output-loki-1.0.0.gem
82
+ ```
73
83
 
74
84
  ## Send sample event and check plugin is working
75
85
 
76
- `bin/logstash -f loki.conf`
86
+ ```bash
87
+ bin/logstash -f loki.conf
88
+ ```
@@ -5,7 +5,7 @@ module Loki
5
5
  class Entry
6
6
  include Loki
7
7
  attr_reader :labels, :entry
8
- def initialize(event,message_field)
8
+ def initialize(event,message_field,include_fields)
9
9
  @entry = {
10
10
  "ts" => to_ns(event.get("@timestamp")),
11
11
  "line" => event.get(message_field).to_s
@@ -18,6 +18,7 @@ module Loki
18
18
  event.to_hash.each { |key,value|
19
19
  next if key.start_with?('@')
20
20
  next if value.is_a?(Hash)
21
+ next if include_fields.length() > 0 and not include_fields.include?(key)
21
22
  @labels[key] = value.to_s
22
23
  }
23
24
  end
@@ -47,6 +47,9 @@ class LogStash::Outputs::Loki < LogStash::Outputs::Base
47
47
  ## 'Backoff configuration. Initial backoff time between retries. Default 1s'
48
48
  config :min_delay, :validate => :number, :default => 1, :required => false
49
49
 
50
+ ## 'An array of fields to map to labels, if defined only fields in this list will be mapped.'
51
+ config :include_fields, :validate => :array, :default => [], :required => false
52
+
50
53
  ## 'Backoff configuration. Maximum backoff time between retries. Default 300s'
51
54
  config :max_delay, :validate => :number, :default => 300, :required => false
52
55
 
@@ -198,7 +201,7 @@ class LogStash::Outputs::Loki < LogStash::Outputs::Base
198
201
  ## Receives logstash events
199
202
  public
200
203
  def receive(event)
201
- @entries << Entry.new(event, @message_field)
204
+ @entries << Entry.new(event, @message_field, @include_fields)
202
205
  end
203
206
 
204
207
  def close
@@ -1,6 +1,6 @@
1
1
  Gem::Specification.new do |s|
2
2
  s.name = 'logstash-output-loki'
3
- s.version = '1.0.3'
3
+ s.version = '1.1.0'
4
4
  s.authors = ['Aditya C S','Cyril Tovena']
5
5
  s.email = ['aditya.gnu@gmail.com','cyril.tovena@grafana.com']
6
6
 
@@ -21,6 +21,6 @@ Gem::Specification.new do |s|
21
21
  # Gem dependencies
22
22
  #
23
23
  s.add_runtime_dependency "logstash-core-plugin-api", ">= 1.60", "<= 2.99"
24
- s.add_runtime_dependency "logstash-codec-plain", "3.0.6"
24
+ s.add_runtime_dependency "logstash-codec-plain", "3.1.0"
25
25
  s.add_development_dependency 'logstash-devutils', "2.0.2"
26
26
  end
@@ -27,18 +27,25 @@ describe Loki::Entry do
27
27
  }
28
28
 
29
29
  it 'labels extracted should not contains object and metadata or timestamp' do
30
- entry = Entry.new(event,"message")
30
+ entry = Entry.new(event,"message", [])
31
31
  expect(entry.labels).to eql({ 'agent' => 'filebeat', 'host' => '172.0.0.1', 'foo'=>'5'})
32
32
  expect(entry.entry['ts']).to eql to_ns(event.get("@timestamp"))
33
33
  expect(entry.entry['line']).to eql 'hello'
34
34
  end
35
+
36
+ it 'labels extracted should only contain allowlisted labels' do
37
+ entry = Entry.new(event, "message", %w[agent foo])
38
+ expect(entry.labels).to eql({ 'agent' => 'filebeat', 'foo'=>'5'})
39
+ expect(entry.entry['ts']).to eql to_ns(event.get("@timestamp"))
40
+ expect(entry.entry['line']).to eql 'hello'
41
+ end
35
42
  end
36
43
 
37
44
  context 'test batch generation with label order' do
38
45
  let (:entries) {[
39
- Entry.new(LogStash::Event.new({"message"=>"foobuzz","buzz"=>"bar","cluster"=>"us-central1","@timestamp"=>Time.at(1)}),"message"),
40
- Entry.new(LogStash::Event.new({"log"=>"foobar","bar"=>"bar","@timestamp"=>Time.at(2)}),"log"),
41
- Entry.new(LogStash::Event.new({"cluster"=>"us-central1","message"=>"foobuzz","buzz"=>"bar","@timestamp"=>Time.at(3)}),"message"),
46
+ Entry.new(LogStash::Event.new({"message"=>"foobuzz","buzz"=>"bar","cluster"=>"us-central1","@timestamp"=>Time.at(1)}),"message", []),
47
+ Entry.new(LogStash::Event.new({"log"=>"foobar","bar"=>"bar","@timestamp"=>Time.at(2)}),"log", []),
48
+ Entry.new(LogStash::Event.new({"cluster"=>"us-central1","message"=>"foobuzz","buzz"=>"bar","@timestamp"=>Time.at(3)}),"message", []),
42
49
 
43
50
  ]}
44
51
  let (:expected) {
@@ -28,12 +28,15 @@ describe LogStash::Outputs::Loki do
28
28
 
29
29
  context 'when adding en entry to the batch' do
30
30
  let (:simple_loki_config) {{'url' => 'http://localhost:3100'}}
31
- let (:entry) {Entry.new(LogStash::Event.new({"message"=>"foobuzz","buzz"=>"bar","cluster"=>"us-central1","@timestamp"=>Time.at(1)}),"message")}
32
- let (:lbs) { {"buzz"=>"bar","cluster"=>"us-central1"}.sort.to_h}
31
+ let (:entry) {Entry.new(LogStash::Event.new({"message"=>"foobuzz","buzz"=>"bar","cluster"=>"us-central1","@timestamp"=>Time.at(1)}),"message", [])}
32
+ let (:lbs) {{"buzz"=>"bar","cluster"=>"us-central1"}.sort.to_h}
33
+ let (:include_loki_config) {{ 'url' => 'http://localhost:3100', 'include_fields' => ["cluster"] }}
34
+ let (:include_entry) {Entry.new(LogStash::Event.new({"message"=>"foobuzz","buzz"=>"bar","cluster"=>"us-central1","@timestamp"=>Time.at(1)}),"message", ["cluster"])}
35
+ let (:include_lbs) {{"cluster"=>"us-central1"}.sort.to_h}
33
36
 
34
37
  it 'should not add empty line' do
35
38
  plugin = LogStash::Plugin.lookup("output", "loki").new(simple_loki_config)
36
- emptyEntry = Entry.new(LogStash::Event.new({"message"=>"foobuzz","buzz"=>"bar","cluster"=>"us-central1","@timestamp"=>Time.at(1)}),"foo")
39
+ emptyEntry = Entry.new(LogStash::Event.new({"message"=>"foobuzz","buzz"=>"bar","cluster"=>"us-central1","@timestamp"=>Time.at(1)}),"foo", [])
37
40
  expect(plugin.add_entry_to_batch(emptyEntry)).to eql true
38
41
  expect(plugin.batch).to eql nil
39
42
  end
@@ -50,6 +53,18 @@ describe LogStash::Outputs::Loki do
50
53
  expect(plugin.batch.size_bytes).to eq 14
51
54
  end
52
55
 
56
+ it 'should only allowed labels defined in include_fields' do
57
+ plugin = LogStash::Plugin.lookup("output", "loki").new(include_loki_config)
58
+ expect(plugin.batch).to eql nil
59
+ expect(plugin.add_entry_to_batch(include_entry)).to eql true
60
+ expect(plugin.add_entry_to_batch(include_entry)).to eql true
61
+ expect(plugin.batch).not_to be_nil
62
+ expect(plugin.batch.streams.length).to eq 1
63
+ expect(plugin.batch.streams[include_lbs.to_s]['entries'].length).to eq 2
64
+ expect(plugin.batch.streams[include_lbs.to_s]['labels']).to eq include_lbs
65
+ expect(plugin.batch.size_bytes).to eq 14
66
+ end
67
+
53
68
  it 'should not add if full' do
54
69
  plugin = LogStash::Plugin.lookup("output", "loki").new(simple_loki_config.merge!({'batch_size'=>10}))
55
70
  expect(plugin.batch).to eql nil
@@ -69,7 +84,7 @@ describe LogStash::Outputs::Loki do
69
84
  end
70
85
 
71
86
  context 'batch expiration' do
72
- let (:entry) {Entry.new(LogStash::Event.new({"message"=>"foobuzz","buzz"=>"bar","cluster"=>"us-central1","@timestamp"=>Time.at(1)}),"message")}
87
+ let (:entry) {Entry.new(LogStash::Event.new({"message"=>"foobuzz","buzz"=>"bar","cluster"=>"us-central1","@timestamp"=>Time.at(1)}),"message", [])}
73
88
 
74
89
  it 'should not expire if empty' do
75
90
  loki = LogStash::Outputs::Loki.new(simple_loki_config.merge!({'batch_wait'=>0.5}))
@@ -138,7 +153,7 @@ describe LogStash::Outputs::Loki do
138
153
  end
139
154
 
140
155
  context 'http requests' do
141
- let (:entry) {Entry.new(LogStash::Event.new({"message"=>"foobuzz","buzz"=>"bar","cluster"=>"us-central1","@timestamp"=>Time.at(1)}),"message")}
156
+ let (:entry) {Entry.new(LogStash::Event.new({"message"=>"foobuzz","buzz"=>"bar","cluster"=>"us-central1","@timestamp"=>Time.at(1)}),"message", [])}
142
157
 
143
158
  it 'should send credentials' do
144
159
  conf = {
metadata CHANGED
@@ -1,18 +1,17 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: logstash-output-loki
3
3
  version: !ruby/object:Gem::Version
4
- version: 1.0.3
4
+ version: 1.1.0
5
5
  platform: ruby
6
6
  authors:
7
7
  - Aditya C S
8
8
  - Cyril Tovena
9
- autorequire:
9
+ autorequire:
10
10
  bindir: bin
11
11
  cert_chain: []
12
- date: 2020-11-13 00:00:00.000000000 Z
12
+ date: 2022-01-27 00:00:00.000000000 Z
13
13
  dependencies:
14
14
  - !ruby/object:Gem::Dependency
15
- name: logstash-core-plugin-api
16
15
  requirement: !ruby/object:Gem::Requirement
17
16
  requirements:
18
17
  - - ">="
@@ -21,8 +20,9 @@ dependencies:
21
20
  - - "<="
22
21
  - !ruby/object:Gem::Version
23
22
  version: '2.99'
24
- type: :runtime
23
+ name: logstash-core-plugin-api
25
24
  prerelease: false
25
+ type: :runtime
26
26
  version_requirements: !ruby/object:Gem::Requirement
27
27
  requirements:
28
28
  - - ">="
@@ -32,28 +32,28 @@ dependencies:
32
32
  - !ruby/object:Gem::Version
33
33
  version: '2.99'
34
34
  - !ruby/object:Gem::Dependency
35
- name: logstash-codec-plain
36
35
  requirement: !ruby/object:Gem::Requirement
37
36
  requirements:
38
37
  - - '='
39
38
  - !ruby/object:Gem::Version
40
- version: 3.0.6
41
- type: :runtime
39
+ version: 3.1.0
40
+ name: logstash-codec-plain
42
41
  prerelease: false
42
+ type: :runtime
43
43
  version_requirements: !ruby/object:Gem::Requirement
44
44
  requirements:
45
45
  - - '='
46
46
  - !ruby/object:Gem::Version
47
- version: 3.0.6
47
+ version: 3.1.0
48
48
  - !ruby/object:Gem::Dependency
49
- name: logstash-devutils
50
49
  requirement: !ruby/object:Gem::Requirement
51
50
  requirements:
52
51
  - - '='
53
52
  - !ruby/object:Gem::Version
54
53
  version: 2.0.2
55
- type: :development
54
+ name: logstash-devutils
56
55
  prerelease: false
56
+ type: :development
57
57
  version_requirements: !ruby/object:Gem::Requirement
58
58
  requirements:
59
59
  - - '='
@@ -81,7 +81,7 @@ licenses:
81
81
  metadata:
82
82
  logstash_plugin: 'true'
83
83
  logstash_group: output
84
- post_install_message:
84
+ post_install_message:
85
85
  rdoc_options: []
86
86
  require_paths:
87
87
  - lib
@@ -96,8 +96,8 @@ required_rubygems_version: !ruby/object:Gem::Requirement
96
96
  - !ruby/object:Gem::Version
97
97
  version: '0'
98
98
  requirements: []
99
- rubygems_version: 3.0.3
100
- signing_key:
99
+ rubygems_version: 3.1.6
100
+ signing_key:
101
101
  specification_version: 4
102
102
  summary: Output plugin to ship logs to a Grafana Loki server
103
103
  test_files: