logstash-output-loki 1.0.3 → 1.1.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: a76f3826de04d39e57f06c522125adfc150e1b9c44dee915cb5a220e829b5ae6
-  data.tar.gz: d4f881165097a6c04dcef15eca32f3570f0d35453a16713e5da1a50e022876d0
+  metadata.gz: 5c6817d07ada63cde10011f1e490f803ddbb0a7a54d75b9472780f4f180c65fe
+  data.tar.gz: e457589a713ed3bc0f74120f3671bfb6b9b9760393cb86c18764e4d5a1fa2bd7
 SHA512:
-  metadata.gz: 706a86a1852e2b2e97bbd340904b1a0a72751beca4ca4257bfbd66683e0c3a7fe5a54d273666c2da98947b84270b4e9544fe36b6e1742877ccffcbf67fe090bc
-  data.tar.gz: 5710b40ac392fa8e82c29343f22b444ce12284023497da3d0cb9c28049b81811ece3bcf431e5b948b7c60dd29c941dcd3c508289fc9a2d49faf5331eda893f69
+  metadata.gz: 358e07d5c964179b7a003e6ac840bb83df74c03e4de0f1916da310ed07e1c669a672d9363afd841c6bdfd49bb1d525ffbc815801ba4f3a8536659dd449173012
+  data.tar.gz: 89360e4ffcb276b9406c017c5c019c43f839398a93a0cda46e0761e45a6a851ef2e9250c9fec5ba409062aef1f12576c9fcaa939a4cba0c0c9d0db15fa698f24

data/README.md

@@ -29,8 +29,11 @@ export PATH="$HOME/.rbenv/bin:$PATH"
 eval "$(rbenv init -)"
 ```
 
-Then install bundler
-`gem install bundler:2.1.4`
+Then install bundler:
+
+```bash
+gem install bundler:2.1.4
+```
 
 Follow those instructions to [install logstash](https://www.elastic.co/guide/en/logstash/current/installing-logstash.html) before moving to the next section.
 
@@ -41,36 +44,45 @@ Follow those instructions to [install logstash](https://www.elastic.co/guide/en/
 ```bash
 git clone git@github.com:elastic/logstash.git
 cd logstash
-git checkout tags/v7.6.2
-export LOGSTASH_PATH=`pwd`
-export GEM_PATH=$LOGSTASH_PATH/vendor/bundle/jruby/2.5.0
-export GEM_HOME=$LOGSTASH_PATH/vendor/bundle/jruby/2.5.0
+git checkout tags/v7.16.1
+export LOGSTASH_PATH="$(pwd)"
+export GEM_PATH="$LOGSTASH_PATH/vendor/bundle/jruby/2.5.0"
+export GEM_HOME="$LOGSTASH_PATH/vendor/bundle/jruby/2.5.0"
 ./gradlew assemble
 cd ..
+ruby -S bundle config set --local path "$LOGSTASH_PATH/vendor/bundle"
 ruby -S bundle install
 ruby -S bundle exec rake vendor
 ```
 
 ### Build the plugin
 
-`gem build logstash-output-loki.gemspec`
+```bash
+gem build logstash-output-loki.gemspec
+```
 
 ### Test
 
-`ruby -S bundle exec rspec`
+```bash
+ruby -S bundle exec rspec
+```
 
 Alternatively if you don't want to install JRuby. Enter inside logstash-loki container.
 
 ```bash
 docker build -t logstash-loki ./
-docker run -v  `pwd`/spec:/home/logstash/spec -it --rm --entrypoint /bin/sh logstash-loki
+docker run -v  $(pwd)/spec:/home/logstash/spec -it --rm --entrypoint /bin/sh logstash-loki
 bundle exec rspec
 ```
 
 ## Install plugin to local logstash
 
-`bin/logstash-plugin install --no-verify --local logstash-output-loki-1.0.0.gem`
+```bash
+bin/logstash-plugin install --no-verify --local logstash-output-loki-1.0.0.gem
+```
 
 ## Send sample event and check plugin is working
 
-`bin/logstash -f loki.conf`
+```bash
+bin/logstash -f loki.conf
+```

data/lib/logstash/outputs/loki/entry.rb

@@ -5,7 +5,7 @@ module Loki
     class Entry
         include Loki
         attr_reader :labels, :entry
-        def initialize(event,message_field)
+        def initialize(event,message_field,include_fields)
             @entry = {
                 "ts" => to_ns(event.get("@timestamp")),
                 "line" => event.get(message_field).to_s
@@ -18,6 +18,7 @@ module Loki
             event.to_hash.each { |key,value|
                 next if key.start_with?('@')
                 next if value.is_a?(Hash)
+                next if include_fields.length() > 0 and not include_fields.include?(key)
                 @labels[key] = value.to_s
             }
         end

data/lib/logstash/outputs/loki.rb

@@ -47,6 +47,9 @@ class LogStash::Outputs::Loki < LogStash::Outputs::Base
   ## 'Backoff configuration. Initial backoff time between retries. Default 1s'
   config :min_delay, :validate => :number, :default => 1, :required => false
 
+  ## 'An array of fields to map to labels, if defined only fields in this list will be mapped.'
+  config :include_fields, :validate => :array, :default => [], :required => false
+
   ## 'Backoff configuration. Maximum backoff time between retries. Default 300s'
   config :max_delay, :validate => :number, :default => 300, :required => false
 
@@ -198,7 +201,7 @@ class LogStash::Outputs::Loki < LogStash::Outputs::Base
   ## Receives logstash events
   public
   def receive(event)
-    @entries << Entry.new(event, @message_field)
+    @entries << Entry.new(event, @message_field, @include_fields)
   end
 
   def close

data/logstash-output-loki.gemspec

@@ -1,6 +1,6 @@
 Gem::Specification.new do |s|
   s.name    = 'logstash-output-loki'
-  s.version = '1.0.3'
+  s.version = '1.1.0'
   s.authors = ['Aditya C S','Cyril Tovena']
   s.email   = ['aditya.gnu@gmail.com','cyril.tovena@grafana.com']
 
@@ -21,6 +21,6 @@ Gem::Specification.new do |s|
   # Gem dependencies
   #
   s.add_runtime_dependency "logstash-core-plugin-api", ">= 1.60", "<= 2.99"
-  s.add_runtime_dependency "logstash-codec-plain", "3.0.6"
+  s.add_runtime_dependency "logstash-codec-plain", "3.1.0"
   s.add_development_dependency 'logstash-devutils', "2.0.2"
 end

data/spec/outputs/loki/entry_spec.rb

@@ -27,18 +27,25 @@ describe Loki::Entry do
       }
 
       it 'labels extracted should not contains object and metadata or timestamp' do
-        entry = Entry.new(event,"message")
+        entry = Entry.new(event,"message", [])
         expect(entry.labels).to eql({ 'agent' => 'filebeat', 'host' => '172.0.0.1', 'foo'=>'5'})
         expect(entry.entry['ts']).to eql to_ns(event.get("@timestamp"))
         expect(entry.entry['line']).to eql 'hello'
       end
+
+      it 'labels extracted should only contain allowlisted labels' do
+        entry = Entry.new(event, "message", %w[agent foo])
+        expect(entry.labels).to eql({ 'agent' => 'filebeat', 'foo'=>'5'})
+        expect(entry.entry['ts']).to eql to_ns(event.get("@timestamp"))
+        expect(entry.entry['line']).to eql 'hello'
+      end
   end
 
   context 'test batch generation with label order' do
     let (:entries)  {[
-        Entry.new(LogStash::Event.new({"message"=>"foobuzz","buzz"=>"bar","cluster"=>"us-central1","@timestamp"=>Time.at(1)}),"message"),
-        Entry.new(LogStash::Event.new({"log"=>"foobar","bar"=>"bar","@timestamp"=>Time.at(2)}),"log"),
-        Entry.new(LogStash::Event.new({"cluster"=>"us-central1","message"=>"foobuzz","buzz"=>"bar","@timestamp"=>Time.at(3)}),"message"),
+        Entry.new(LogStash::Event.new({"message"=>"foobuzz","buzz"=>"bar","cluster"=>"us-central1","@timestamp"=>Time.at(1)}),"message", []),
+        Entry.new(LogStash::Event.new({"log"=>"foobar","bar"=>"bar","@timestamp"=>Time.at(2)}),"log", []),
+        Entry.new(LogStash::Event.new({"cluster"=>"us-central1","message"=>"foobuzz","buzz"=>"bar","@timestamp"=>Time.at(3)}),"message", []),
 
     ]}
     let (:expected) {

data/spec/outputs/loki_spec.rb

@@ -28,12 +28,15 @@ describe LogStash::Outputs::Loki do
 
   context 'when adding en entry to the batch' do
     let (:simple_loki_config) {{'url' => 'http://localhost:3100'}}
-    let (:entry) {Entry.new(LogStash::Event.new({"message"=>"foobuzz","buzz"=>"bar","cluster"=>"us-central1","@timestamp"=>Time.at(1)}),"message")}
-    let (:lbs) { {"buzz"=>"bar","cluster"=>"us-central1"}.sort.to_h}
+    let (:entry) {Entry.new(LogStash::Event.new({"message"=>"foobuzz","buzz"=>"bar","cluster"=>"us-central1","@timestamp"=>Time.at(1)}),"message", [])}
+    let (:lbs) {{"buzz"=>"bar","cluster"=>"us-central1"}.sort.to_h}
+    let (:include_loki_config) {{ 'url' => 'http://localhost:3100', 'include_fields' => ["cluster"] }}
+    let (:include_entry) {Entry.new(LogStash::Event.new({"message"=>"foobuzz","buzz"=>"bar","cluster"=>"us-central1","@timestamp"=>Time.at(1)}),"message", ["cluster"])}
+    let (:include_lbs) {{"cluster"=>"us-central1"}.sort.to_h}
 
     it 'should not add empty line' do
       plugin = LogStash::Plugin.lookup("output", "loki").new(simple_loki_config)
-      emptyEntry = Entry.new(LogStash::Event.new({"message"=>"foobuzz","buzz"=>"bar","cluster"=>"us-central1","@timestamp"=>Time.at(1)}),"foo")
+      emptyEntry = Entry.new(LogStash::Event.new({"message"=>"foobuzz","buzz"=>"bar","cluster"=>"us-central1","@timestamp"=>Time.at(1)}),"foo", [])
       expect(plugin.add_entry_to_batch(emptyEntry)).to eql true
       expect(plugin.batch).to eql nil
     end
@@ -50,6 +53,18 @@ describe LogStash::Outputs::Loki do
       expect(plugin.batch.size_bytes).to eq 14
     end
 
+    it 'should only allowed labels defined in include_fields' do
+      plugin = LogStash::Plugin.lookup("output", "loki").new(include_loki_config)
+      expect(plugin.batch).to eql nil
+      expect(plugin.add_entry_to_batch(include_entry)).to eql true
+      expect(plugin.add_entry_to_batch(include_entry)).to eql true
+      expect(plugin.batch).not_to be_nil
+      expect(plugin.batch.streams.length).to eq 1
+      expect(plugin.batch.streams[include_lbs.to_s]['entries'].length).to eq 2
+      expect(plugin.batch.streams[include_lbs.to_s]['labels']).to eq include_lbs
+      expect(plugin.batch.size_bytes).to eq 14
+    end
+
     it 'should not add if full' do
       plugin = LogStash::Plugin.lookup("output", "loki").new(simple_loki_config.merge!({'batch_size'=>10}))
       expect(plugin.batch).to eql nil
@@ -69,7 +84,7 @@ describe LogStash::Outputs::Loki do
   end
 
   context 'batch expiration' do
-    let (:entry) {Entry.new(LogStash::Event.new({"message"=>"foobuzz","buzz"=>"bar","cluster"=>"us-central1","@timestamp"=>Time.at(1)}),"message")}
+    let (:entry) {Entry.new(LogStash::Event.new({"message"=>"foobuzz","buzz"=>"bar","cluster"=>"us-central1","@timestamp"=>Time.at(1)}),"message", [])}
 
     it 'should not expire if empty' do
       loki = LogStash::Outputs::Loki.new(simple_loki_config.merge!({'batch_wait'=>0.5}))
@@ -138,7 +153,7 @@ describe LogStash::Outputs::Loki do
   end
 
   context 'http requests' do
-    let (:entry) {Entry.new(LogStash::Event.new({"message"=>"foobuzz","buzz"=>"bar","cluster"=>"us-central1","@timestamp"=>Time.at(1)}),"message")}
+    let (:entry) {Entry.new(LogStash::Event.new({"message"=>"foobuzz","buzz"=>"bar","cluster"=>"us-central1","@timestamp"=>Time.at(1)}),"message", [])}
 
     it 'should send credentials' do
       conf = {

metadata

@@ -1,18 +1,17 @@
 --- !ruby/object:Gem::Specification
 name: logstash-output-loki
 version: !ruby/object:Gem::Version
-  version: 1.0.3
+  version: 1.1.0
 platform: ruby
 authors:
 - Aditya C S
 - Cyril Tovena
-autorequire: 
+autorequire:
 bindir: bin
 cert_chain: []
-date: 2020-11-13 00:00:00.000000000 Z
+date: 2022-01-27 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
-  name: logstash-core-plugin-api
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
@@ -21,8 +20,9 @@ dependencies:
     - - "<="
       - !ruby/object:Gem::Version
         version: '2.99'
-  type: :runtime
+  name: logstash-core-plugin-api
   prerelease: false
+  type: :runtime
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
@@ -32,28 +32,28 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '2.99'
 - !ruby/object:Gem::Dependency
-  name: logstash-codec-plain
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - '='
       - !ruby/object:Gem::Version
-        version: 3.0.6
-  type: :runtime
+        version: 3.1.0
+  name: logstash-codec-plain
   prerelease: false
+  type: :runtime
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - '='
       - !ruby/object:Gem::Version
-        version: 3.0.6
+        version: 3.1.0
 - !ruby/object:Gem::Dependency
-  name: logstash-devutils
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - '='
       - !ruby/object:Gem::Version
         version: 2.0.2
-  type: :development
+  name: logstash-devutils
   prerelease: false
+  type: :development
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - '='
@@ -81,7 +81,7 @@ licenses:
 metadata:
   logstash_plugin: 'true'
   logstash_group: output
-post_install_message: 
+post_install_message:
 rdoc_options: []
 require_paths:
 - lib
@@ -96,8 +96,8 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.0.3
-signing_key: 
+rubygems_version: 3.1.6
+signing_key:
 specification_version: 4
 summary: Output plugin to ship logs to a Grafana Loki server
 test_files: