logstash-output-loki 1.1.0 → 1.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 5c6817d07ada63cde10011f1e490f803ddbb0a7a54d75b9472780f4f180c65fe
-  data.tar.gz: e457589a713ed3bc0f74120f3671bfb6b9b9760393cb86c18764e4d5a1fa2bd7
+  metadata.gz: a2345ce18e6ef9aad96401c5b7afb5dca4f5b7dcc71c7cf8a21a1b28496e78fc
+  data.tar.gz: 3fedc6382bd23200cf2f9afda2e6ec4d0222e72921ccee4563671af72250032a
 SHA512:
-  metadata.gz: 358e07d5c964179b7a003e6ac840bb83df74c03e4de0f1916da310ed07e1c669a672d9363afd841c6bdfd49bb1d525ffbc815801ba4f3a8536659dd449173012
-  data.tar.gz: 89360e4ffcb276b9406c017c5c019c43f839398a93a0cda46e0761e45a6a851ef2e9250c9fec5ba409062aef1f12576c9fcaa939a4cba0c0c9d0db15fa698f24
+  metadata.gz: ea1a970fe497c812132c999dbf73638c6baa20021b4bac22bb20114c525910d963eef3621ec5b69a3740732668de88ad4c9c09fa9a5569e7008030685d67b337
+  data.tar.gz: d03d649be3e5ed202f5e27aeb307e5db39c036b6c089748a982482d1b23df3e46c1563e61c20b081bdca7d62699b871e14ee569269a77d6d00760f23e70ea12d

data/lib/logstash/outputs/loki/batch.rb

@@ -52,7 +52,19 @@ module Loki
         def build_stream(stream)
             values = []
             stream['entries'].each { |entry|
-                values.append([entry['ts'].to_s, entry['line']])
+                if entry.key?('metadata')
+                    sorted_metadata = entry['metadata'].sort.to_h
+                    values.append([
+                        entry['ts'].to_s,
+                        entry['line'],
+                        sorted_metadata
+                    ])
+                else
+                    values.append([
+                        entry['ts'].to_s,
+                        entry['line']
+                    ])
+                end
             }
             return {
                 'stream'=>stream['labels'],

data/lib/logstash/outputs/loki/entry.rb

@@ -5,7 +5,7 @@ module Loki
     class Entry
         include Loki
         attr_reader :labels, :entry
-        def initialize(event,message_field,include_fields)
+        def initialize(event,message_field,include_fields,metadata_fields)
             @entry = {
                 "ts" => to_ns(event.get("@timestamp")),
                 "line" => event.get(message_field).to_s
@@ -21,6 +21,22 @@ module Loki
                 next if include_fields.length() > 0 and not include_fields.include?(key)
                 @labels[key] = value.to_s
             }
+
+            # Unlike include_fields we should skip if no metadata_fields provided
+            if metadata_fields.length() > 0
+                @metadata = {}
+                event.to_hash.each { |key,value|
+                    next if key.start_with?('@')
+                    next if value.is_a?(Hash)
+                    next if metadata_fields.length() > 0 and not metadata_fields.include?(key)
+                    @metadata[key] = value.to_s
+                }
+
+                # Add @metadata to @entry if there was a match
+                if @metadata.size > 0
+                    @entry.merge!('metadata' => @metadata)
+                end
+            end
         end
     end
 end

data/lib/logstash/outputs/loki.rb

@@ -50,6 +50,9 @@ class LogStash::Outputs::Loki < LogStash::Outputs::Base
   ## 'An array of fields to map to labels, if defined only fields in this list will be mapped.'
   config :include_fields, :validate => :array, :default => [], :required => false
 
+  ## 'An array of fields to map to structure metadata, if defined only fields in this list will be mapped.'
+  config :metadata_fields, :validate => :array, :default => [], :required => false
+
   ## 'Backoff configuration. Maximum backoff time between retries. Default 300s'
   config :max_delay, :validate => :number, :default => 300, :required => false
 
@@ -71,7 +74,7 @@ class LogStash::Outputs::Loki < LogStash::Outputs::Base
     @logger.info("Loki output plugin", :class => self.class.name)
 
     # initialize Queue and Mutex
-    @entries = Queue.new 
+    @entries = Queue.new
     @mutex = Mutex.new
     @stop = false
 
@@ -94,7 +97,7 @@ class LogStash::Outputs::Loki < LogStash::Outputs::Base
       @mutex.synchronize do
         return if @stop
       end
-  
+
       e = @entries.deq
       return if e.nil?
 
@@ -201,13 +204,13 @@ class LogStash::Outputs::Loki < LogStash::Outputs::Base
   ## Receives logstash events
   public
   def receive(event)
-    @entries << Entry.new(event, @message_field, @include_fields)
+    @entries << Entry.new(event, @message_field, @include_fields, @metadata_fields)
   end
 
   def close
     @entries.close
-    @mutex.synchronize do 
-      @stop = true 
+    @mutex.synchronize do
+      @stop = true
     end
     @batch_wait_thread.join
     @batch_size_thread.join

data/logstash-output-loki.gemspec

@@ -1,6 +1,6 @@
 Gem::Specification.new do |s|
   s.name    = 'logstash-output-loki'
-  s.version = '1.1.0'
+  s.version = '1.2.0'
   s.authors = ['Aditya C S','Cyril Tovena']
   s.email   = ['aditya.gnu@gmail.com','cyril.tovena@grafana.com']
 

data/spec/outputs/loki/entry_spec.rb

@@ -21,31 +21,40 @@ describe Loki::Entry do
                         {'@path' => '/path/to/file.log'},
                    },
               'host' => '172.0.0.1',
+              'trace_id' => 'trace_001',
               '@timestamp' => Time.now
             }
         )
       }
 
       it 'labels extracted should not contains object and metadata or timestamp' do
-        entry = Entry.new(event,"message", [])
-        expect(entry.labels).to eql({ 'agent' => 'filebeat', 'host' => '172.0.0.1', 'foo'=>'5'})
+        entry = Entry.new(event,"message", [], [])
+        expect(entry.labels).to eql({ 'agent' => 'filebeat', 'host' => '172.0.0.1', 'foo'=>'5', 'trace_id'=>'trace_001'})
         expect(entry.entry['ts']).to eql to_ns(event.get("@timestamp"))
         expect(entry.entry['line']).to eql 'hello'
       end
 
       it 'labels extracted should only contain allowlisted labels' do
-        entry = Entry.new(event, "message", %w[agent foo])
+        entry = Entry.new(event, "message", %w[agent foo], [])
         expect(entry.labels).to eql({ 'agent' => 'filebeat', 'foo'=>'5'})
         expect(entry.entry['ts']).to eql to_ns(event.get("@timestamp"))
         expect(entry.entry['line']).to eql 'hello'
       end
+
+      it 'labels and structured metadata extracted should only contain allow listed labels and metadata' do
+        entry = Entry.new(event, "message", %w[agent foo], %w[trace_id])
+        expect(entry.labels).to eql({ 'agent' => 'filebeat', 'foo'=>'5'})
+        expect(entry.entry['ts']).to eql to_ns(event.get("@timestamp"))
+        expect(entry.entry['line']).to eql 'hello'
+        expect(entry.entry['metadata']).to eql({'trace_id' => 'trace_001'})
+      end
   end
 
   context 'test batch generation with label order' do
     let (:entries)  {[
-        Entry.new(LogStash::Event.new({"message"=>"foobuzz","buzz"=>"bar","cluster"=>"us-central1","@timestamp"=>Time.at(1)}),"message", []),
-        Entry.new(LogStash::Event.new({"log"=>"foobar","bar"=>"bar","@timestamp"=>Time.at(2)}),"log", []),
-        Entry.new(LogStash::Event.new({"cluster"=>"us-central1","message"=>"foobuzz","buzz"=>"bar","@timestamp"=>Time.at(3)}),"message", []),
+        Entry.new(LogStash::Event.new({"message"=>"foobuzz","buzz"=>"bar","cluster"=>"us-central1","@timestamp"=>Time.at(1)}),"message", [], []),
+        Entry.new(LogStash::Event.new({"log"=>"foobar","bar"=>"bar","@timestamp"=>Time.at(2)}),"log", [], []),
+        Entry.new(LogStash::Event.new({"cluster"=>"us-central1","message"=>"foobuzz","buzz"=>"bar","@timestamp"=>Time.at(3)}),"message", [], []),
 
     ]}
     let (:expected) {

data/spec/outputs/loki_spec.rb

@@ -28,15 +28,15 @@ describe LogStash::Outputs::Loki do
 
   context 'when adding en entry to the batch' do
     let (:simple_loki_config) {{'url' => 'http://localhost:3100'}}
-    let (:entry) {Entry.new(LogStash::Event.new({"message"=>"foobuzz","buzz"=>"bar","cluster"=>"us-central1","@timestamp"=>Time.at(1)}),"message", [])}
+    let (:entry) {Entry.new(LogStash::Event.new({"message"=>"foobuzz","buzz"=>"bar","cluster"=>"us-central1","@timestamp"=>Time.at(1)}),"message", [], [])}
     let (:lbs) {{"buzz"=>"bar","cluster"=>"us-central1"}.sort.to_h}
     let (:include_loki_config) {{ 'url' => 'http://localhost:3100', 'include_fields' => ["cluster"] }}
-    let (:include_entry) {Entry.new(LogStash::Event.new({"message"=>"foobuzz","buzz"=>"bar","cluster"=>"us-central1","@timestamp"=>Time.at(1)}),"message", ["cluster"])}
+    let (:include_entry) {Entry.new(LogStash::Event.new({"message"=>"foobuzz","buzz"=>"bar","cluster"=>"us-central1","@timestamp"=>Time.at(1)}),"message", ["cluster"], [])}
     let (:include_lbs) {{"cluster"=>"us-central1"}.sort.to_h}
 
     it 'should not add empty line' do
       plugin = LogStash::Plugin.lookup("output", "loki").new(simple_loki_config)
-      emptyEntry = Entry.new(LogStash::Event.new({"message"=>"foobuzz","buzz"=>"bar","cluster"=>"us-central1","@timestamp"=>Time.at(1)}),"foo", [])
+      emptyEntry = Entry.new(LogStash::Event.new({"message"=>"foobuzz","buzz"=>"bar","cluster"=>"us-central1","@timestamp"=>Time.at(1)}),"foo", [], [])
       expect(plugin.add_entry_to_batch(emptyEntry)).to eql true
       expect(plugin.batch).to eql nil
     end
@@ -83,8 +83,51 @@ describe LogStash::Outputs::Loki do
     end
   end
 
+  context 'when building json from batch to send' do
+    let (:basic_loki_config) {{'url' => 'http://localhost:3100'}}
+    let (:basic_entry) {Entry.new(LogStash::Event.new({"message"=>"foobuzz","buzz"=>"bar","cluster"=>"us-central1","trace_id"=>"trace_001","@timestamp"=>Time.at(1)}),"message", [], [])}
+    let (:include_loki_config) {{ 'url' => 'http://localhost:3100', 'include_fields' => ["cluster"] }}
+    let (:include_entry) {Entry.new(LogStash::Event.new({"message"=>"foobuzz","buzz"=>"bar","cluster"=>"us-central1","trace_id"=>"trace_001","@timestamp"=>Time.at(1)}),"message", ["cluster"], [])}
+    let (:metadata_loki_config) {{ 'url' => 'http://localhost:3100', 'include_fields' => ["cluster"], 'metadata_fields' => ["trace_id"] }}
+    let (:metadata_entry) {Entry.new(LogStash::Event.new({"message"=>"foobuzz","buzz"=>"bar","cluster"=>"us-central1","trace_id"=>"trace_001","@timestamp"=>Time.at(1)}),"message", ["cluster"], ["trace_id"])}
+    let (:metadata_multi_loki_config) {{ 'url' => 'http://localhost:3100', 'include_fields' => ["cluster"], 'metadata_fields' => ["trace_id", "user_id"] }}
+    let (:metadata_multi_entry) {Entry.new(LogStash::Event.new({"message"=>"foobuzz","buzz"=>"bar","cluster"=>"us-central1","trace_id"=>"trace_001","user_id"=>"user_001","@timestamp"=>Time.at(1)}),"message", ["cluster"], ["trace_id", "user_id"])}
+
+    it 'should not include labels or metadata' do
+      plugin = LogStash::Plugin.lookup("output", "loki").new(basic_loki_config)
+      expect(plugin.batch).to eql nil
+      expect(plugin.add_entry_to_batch(basic_entry)).to eql true
+      expect(plugin.batch).not_to be_nil
+      expect(plugin.batch.to_json).to eq '{"streams":[{"stream":{"buzz":"bar","cluster":"us-central1","trace_id":"trace_001"},"values":[["1000000000","foobuzz"]]}]}'
+    end
+
+    it 'should include metadata with no labels' do
+      plugin = LogStash::Plugin.lookup("output", "loki").new(metadata_loki_config)
+      expect(plugin.batch).to eql nil
+      expect(plugin.add_entry_to_batch(metadata_entry)).to eql true
+      expect(plugin.batch).not_to be_nil
+      expect(plugin.batch.to_json).to eq '{"streams":[{"stream":{"cluster":"us-central1"},"values":[["1000000000","foobuzz",{"trace_id":"trace_001"}]]}]}'
+    end
+
+    it 'should include labels with no metadata' do
+      plugin = LogStash::Plugin.lookup("output", "loki").new(include_loki_config)
+      expect(plugin.batch).to eql nil
+      expect(plugin.add_entry_to_batch(include_entry)).to eql true
+      expect(plugin.batch).not_to be_nil
+      expect(plugin.batch.to_json).to eq '{"streams":[{"stream":{"cluster":"us-central1"},"values":[["1000000000","foobuzz"]]}]}'
+    end
+
+    it 'should include labels with multiple metadata' do
+      plugin = LogStash::Plugin.lookup("output", "loki").new(metadata_multi_loki_config)
+      expect(plugin.batch).to eql nil
+      expect(plugin.add_entry_to_batch(metadata_multi_entry)).to eql true
+      expect(plugin.batch).not_to be_nil
+      expect(plugin.batch.to_json).to eq '{"streams":[{"stream":{"cluster":"us-central1"},"values":[["1000000000","foobuzz",{"trace_id":"trace_001","user_id":"user_001"}]]}]}'
+    end
+  end
+
   context 'batch expiration' do
-    let (:entry) {Entry.new(LogStash::Event.new({"message"=>"foobuzz","buzz"=>"bar","cluster"=>"us-central1","@timestamp"=>Time.at(1)}),"message", [])}
+    let (:entry) {Entry.new(LogStash::Event.new({"message"=>"foobuzz","buzz"=>"bar","cluster"=>"us-central1","@timestamp"=>Time.at(1)}),"message", [], [])}
 
     it 'should not expire if empty' do
       loki = LogStash::Outputs::Loki.new(simple_loki_config.merge!({'batch_wait'=>0.5}))
@@ -147,13 +190,13 @@ describe LogStash::Outputs::Loki do
       loki.receive(event)
       sent.deq
       sleep(0.01) # Adding a minimal sleep. In few cases @batch=nil might happen after evaluating for nil
-      expect(loki.batch).to be_nil 
+      expect(loki.batch).to be_nil
       loki.close
     end
   end
 
   context 'http requests' do
-    let (:entry) {Entry.new(LogStash::Event.new({"message"=>"foobuzz","buzz"=>"bar","cluster"=>"us-central1","@timestamp"=>Time.at(1)}),"message", [])}
+    let (:entry) {Entry.new(LogStash::Event.new({"message"=>"foobuzz","buzz"=>"bar","cluster"=>"us-central1","@timestamp"=>Time.at(1)}),"message", [], [])}
 
     it 'should send credentials' do
       conf = {

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: logstash-output-loki
 version: !ruby/object:Gem::Version
-  version: 1.1.0
+  version: 1.2.0
 platform: ruby
 authors:
 - Aditya C S
@@ -9,7 +9,7 @@ authors:
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2022-01-27 00:00:00.000000000 Z
+date: 2024-09-24 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement