logstash-output-loki 1.0.4 → 1.1.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 1b8d307f78ff3b06b017c8e44c8a814e48a8a29248244055559abffa7727ebc5
-  data.tar.gz: 9218364927282f2271517ce49326b67a6c38a5cdbd28b54010e5cf4b0ad9d37e
+  metadata.gz: 5c6817d07ada63cde10011f1e490f803ddbb0a7a54d75b9472780f4f180c65fe
+  data.tar.gz: e457589a713ed3bc0f74120f3671bfb6b9b9760393cb86c18764e4d5a1fa2bd7
 SHA512:
-  metadata.gz: 63a598ad87b1d96e308db1773fc724e599c06933479f3c9e37ed7bd667f6ea719a9ba27ce3c2c4d4f9fb65f24bc11c893f0a04e34263262ba6c9181cbc6be2af
-  data.tar.gz: 2da45f13f4d77ee10e4224dbe4e350916e1f8cecb2fbcedd105660d7f7e2d9bc4fabfd5e602c39f7965c8ebb016599266bf1f4734db0add4323c15a9e1d6054b
+  metadata.gz: 358e07d5c964179b7a003e6ac840bb83df74c03e4de0f1916da310ed07e1c669a672d9363afd841c6bdfd49bb1d525ffbc815801ba4f3a8536659dd449173012
+  data.tar.gz: 89360e4ffcb276b9406c017c5c019c43f839398a93a0cda46e0761e45a6a851ef2e9250c9fec5ba409062aef1f12576c9fcaa939a4cba0c0c9d0db15fa698f24

data/lib/logstash/outputs/loki/entry.rb

@@ -5,7 +5,7 @@ module Loki
     class Entry
         include Loki
         attr_reader :labels, :entry
-        def initialize(event,message_field)
+        def initialize(event,message_field,include_fields)
             @entry = {
                 "ts" => to_ns(event.get("@timestamp")),
                 "line" => event.get(message_field).to_s
@@ -18,6 +18,7 @@ module Loki
             event.to_hash.each { |key,value|
                 next if key.start_with?('@')
                 next if value.is_a?(Hash)
+                next if include_fields.length() > 0 and not include_fields.include?(key)
                 @labels[key] = value.to_s
             }
         end

data/lib/logstash/outputs/loki.rb

@@ -47,6 +47,9 @@ class LogStash::Outputs::Loki < LogStash::Outputs::Base
   ## 'Backoff configuration. Initial backoff time between retries. Default 1s'
   config :min_delay, :validate => :number, :default => 1, :required => false
 
+  ## 'An array of fields to map to labels, if defined only fields in this list will be mapped.'
+  config :include_fields, :validate => :array, :default => [], :required => false
+
   ## 'Backoff configuration. Maximum backoff time between retries. Default 300s'
   config :max_delay, :validate => :number, :default => 300, :required => false
 
@@ -198,7 +201,7 @@ class LogStash::Outputs::Loki < LogStash::Outputs::Base
   ## Receives logstash events
   public
   def receive(event)
-    @entries << Entry.new(event, @message_field)
+    @entries << Entry.new(event, @message_field, @include_fields)
   end
 
   def close

data/logstash-output-loki.gemspec

@@ -1,6 +1,6 @@
 Gem::Specification.new do |s|
   s.name    = 'logstash-output-loki'
-  s.version = '1.0.4'
+  s.version = '1.1.0'
   s.authors = ['Aditya C S','Cyril Tovena']
   s.email   = ['aditya.gnu@gmail.com','cyril.tovena@grafana.com']
 

data/spec/outputs/loki/entry_spec.rb

@@ -27,18 +27,25 @@ describe Loki::Entry do
       }
 
       it 'labels extracted should not contains object and metadata or timestamp' do
-        entry = Entry.new(event,"message")
+        entry = Entry.new(event,"message", [])
         expect(entry.labels).to eql({ 'agent' => 'filebeat', 'host' => '172.0.0.1', 'foo'=>'5'})
         expect(entry.entry['ts']).to eql to_ns(event.get("@timestamp"))
         expect(entry.entry['line']).to eql 'hello'
       end
+
+      it 'labels extracted should only contain allowlisted labels' do
+        entry = Entry.new(event, "message", %w[agent foo])
+        expect(entry.labels).to eql({ 'agent' => 'filebeat', 'foo'=>'5'})
+        expect(entry.entry['ts']).to eql to_ns(event.get("@timestamp"))
+        expect(entry.entry['line']).to eql 'hello'
+      end
   end
 
   context 'test batch generation with label order' do
     let (:entries)  {[
-        Entry.new(LogStash::Event.new({"message"=>"foobuzz","buzz"=>"bar","cluster"=>"us-central1","@timestamp"=>Time.at(1)}),"message"),
-        Entry.new(LogStash::Event.new({"log"=>"foobar","bar"=>"bar","@timestamp"=>Time.at(2)}),"log"),
-        Entry.new(LogStash::Event.new({"cluster"=>"us-central1","message"=>"foobuzz","buzz"=>"bar","@timestamp"=>Time.at(3)}),"message"),
+        Entry.new(LogStash::Event.new({"message"=>"foobuzz","buzz"=>"bar","cluster"=>"us-central1","@timestamp"=>Time.at(1)}),"message", []),
+        Entry.new(LogStash::Event.new({"log"=>"foobar","bar"=>"bar","@timestamp"=>Time.at(2)}),"log", []),
+        Entry.new(LogStash::Event.new({"cluster"=>"us-central1","message"=>"foobuzz","buzz"=>"bar","@timestamp"=>Time.at(3)}),"message", []),
 
     ]}
     let (:expected) {

data/spec/outputs/loki_spec.rb

@@ -28,12 +28,15 @@ describe LogStash::Outputs::Loki do
 
   context 'when adding en entry to the batch' do
     let (:simple_loki_config) {{'url' => 'http://localhost:3100'}}
-    let (:entry) {Entry.new(LogStash::Event.new({"message"=>"foobuzz","buzz"=>"bar","cluster"=>"us-central1","@timestamp"=>Time.at(1)}),"message")}
-    let (:lbs) { {"buzz"=>"bar","cluster"=>"us-central1"}.sort.to_h}
+    let (:entry) {Entry.new(LogStash::Event.new({"message"=>"foobuzz","buzz"=>"bar","cluster"=>"us-central1","@timestamp"=>Time.at(1)}),"message", [])}
+    let (:lbs) {{"buzz"=>"bar","cluster"=>"us-central1"}.sort.to_h}
+    let (:include_loki_config) {{ 'url' => 'http://localhost:3100', 'include_fields' => ["cluster"] }}
+    let (:include_entry) {Entry.new(LogStash::Event.new({"message"=>"foobuzz","buzz"=>"bar","cluster"=>"us-central1","@timestamp"=>Time.at(1)}),"message", ["cluster"])}
+    let (:include_lbs) {{"cluster"=>"us-central1"}.sort.to_h}
 
     it 'should not add empty line' do
       plugin = LogStash::Plugin.lookup("output", "loki").new(simple_loki_config)
-      emptyEntry = Entry.new(LogStash::Event.new({"message"=>"foobuzz","buzz"=>"bar","cluster"=>"us-central1","@timestamp"=>Time.at(1)}),"foo")
+      emptyEntry = Entry.new(LogStash::Event.new({"message"=>"foobuzz","buzz"=>"bar","cluster"=>"us-central1","@timestamp"=>Time.at(1)}),"foo", [])
       expect(plugin.add_entry_to_batch(emptyEntry)).to eql true
       expect(plugin.batch).to eql nil
     end
@@ -50,6 +53,18 @@ describe LogStash::Outputs::Loki do
       expect(plugin.batch.size_bytes).to eq 14
     end
 
+    it 'should only allowed labels defined in include_fields' do
+      plugin = LogStash::Plugin.lookup("output", "loki").new(include_loki_config)
+      expect(plugin.batch).to eql nil
+      expect(plugin.add_entry_to_batch(include_entry)).to eql true
+      expect(plugin.add_entry_to_batch(include_entry)).to eql true
+      expect(plugin.batch).not_to be_nil
+      expect(plugin.batch.streams.length).to eq 1
+      expect(plugin.batch.streams[include_lbs.to_s]['entries'].length).to eq 2
+      expect(plugin.batch.streams[include_lbs.to_s]['labels']).to eq include_lbs
+      expect(plugin.batch.size_bytes).to eq 14
+    end
+
     it 'should not add if full' do
       plugin = LogStash::Plugin.lookup("output", "loki").new(simple_loki_config.merge!({'batch_size'=>10}))
       expect(plugin.batch).to eql nil
@@ -69,7 +84,7 @@ describe LogStash::Outputs::Loki do
   end
 
   context 'batch expiration' do
-    let (:entry) {Entry.new(LogStash::Event.new({"message"=>"foobuzz","buzz"=>"bar","cluster"=>"us-central1","@timestamp"=>Time.at(1)}),"message")}
+    let (:entry) {Entry.new(LogStash::Event.new({"message"=>"foobuzz","buzz"=>"bar","cluster"=>"us-central1","@timestamp"=>Time.at(1)}),"message", [])}
 
     it 'should not expire if empty' do
       loki = LogStash::Outputs::Loki.new(simple_loki_config.merge!({'batch_wait'=>0.5}))
@@ -138,7 +153,7 @@ describe LogStash::Outputs::Loki do
   end
 
   context 'http requests' do
-    let (:entry) {Entry.new(LogStash::Event.new({"message"=>"foobuzz","buzz"=>"bar","cluster"=>"us-central1","@timestamp"=>Time.at(1)}),"message")}
+    let (:entry) {Entry.new(LogStash::Event.new({"message"=>"foobuzz","buzz"=>"bar","cluster"=>"us-central1","@timestamp"=>Time.at(1)}),"message", [])}
 
     it 'should send credentials' do
       conf = {

metadata

@@ -1,18 +1,17 @@
 --- !ruby/object:Gem::Specification
 name: logstash-output-loki
 version: !ruby/object:Gem::Version
-  version: 1.0.4
+  version: 1.1.0
 platform: ruby
 authors:
 - Aditya C S
 - Cyril Tovena
-autorequire: 
+autorequire:
 bindir: bin
 cert_chain: []
-date: 2022-01-25 00:00:00.000000000 Z
+date: 2022-01-27 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
-  name: logstash-core-plugin-api
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
@@ -21,8 +20,9 @@ dependencies:
     - - "<="
       - !ruby/object:Gem::Version
         version: '2.99'
-  type: :runtime
+  name: logstash-core-plugin-api
   prerelease: false
+  type: :runtime
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
@@ -32,28 +32,28 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '2.99'
 - !ruby/object:Gem::Dependency
-  name: logstash-codec-plain
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - '='
       - !ruby/object:Gem::Version
         version: 3.1.0
-  type: :runtime
+  name: logstash-codec-plain
   prerelease: false
+  type: :runtime
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - '='
       - !ruby/object:Gem::Version
         version: 3.1.0
 - !ruby/object:Gem::Dependency
-  name: logstash-devutils
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - '='
       - !ruby/object:Gem::Version
         version: 2.0.2
-  type: :development
+  name: logstash-devutils
   prerelease: false
+  type: :development
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - '='
@@ -81,7 +81,7 @@ licenses:
 metadata:
   logstash_plugin: 'true'
   logstash_group: output
-post_install_message: 
+post_install_message:
 rdoc_options: []
 require_paths:
 - lib
@@ -96,8 +96,8 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.0.3.1
-signing_key: 
+rubygems_version: 3.1.6
+signing_key:
 specification_version: 4
 summary: Output plugin to ship logs to a Grafana Loki server
 test_files: