logstash-output-loki-tenants 1.0.3 → 1.0.4

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 468a3f120f85bb99690179384555a9cc4d0b090f027fa6abdbc2e40200973964
-  data.tar.gz: 58532c550dff9474933c6df25b635586d0fd745014753babf8dfaf300e653939
+  metadata.gz: 3f63ff2d7beb2921dd555f62616813a045cca6dff4d0ca44ab290f38b4ff593f
+  data.tar.gz: 3d5864cb9c48e62b563c609c55d4b59e75976b5ce9542d17cc9d6ca7ac3c29ad
 SHA512:
-  metadata.gz: 45db85447deeda78d894384604a1dd470b1b873d5ea854c6ab26390bdbfdc33640aaef7833af7aa66188ed96ad174414a85e0b90ad5ed186c0bbd7ac21ed4932
-  data.tar.gz: '005299b0c0541b42806f35ae6d0aa7f55967066f8851642c9ec791f58b2d5387065c4600e51bc91fb7830c082abddbf14de51c5c9a82e3ca9d806c6a896de5e1'
+  metadata.gz: c37c14272d5241ee466626eced3e9fe22ed85a8b38dd25e67e000c39e70cea5c49aefb354465cb8068b2b504afd73d8c2ce4bcf60086ecb4b03961aa4a671c23
+  data.tar.gz: 8eccc504913b8d3e62edeb2fc5cd57034511e50b523af3ec820646214d474e2a19f57e6ef03c3693d237d96022b750e9e84caa8737c0894cffcef8f09e39b385

data/README.md

@@ -1,4 +1,21 @@
-# Contributing to Loki Logstash Output Plugin
+# Fork of Loki Logstash Output Plugin
+
+Added features:
+
+* split batches by the tenant attribute. Use ‘default’ if attribute not set.
+* add  X-Scope-OrgID header based on ‘tenant' message field.
+* DO not set header if 'tenant’ attribute is empty.
+
+Available from <https://rubygems.org/gems/logstash-output-loki-tenants>
+
+## Building and pushing gem
+1. `gem build logstash-output-loki.gemspec`
+2. Push desired build version `gem push logstash-output-loki-tenants-{VERSION}.gem`
+    - In case of massage 'Repushing of gem versions is not allowed.' Raise the plugin version in logstash-output-loki.gemspec
+    - Rebuild the plugin
+    - Push proper version
+
+## Contributing to Loki Logstash Output Plugin
 
 For information about how to use this plugin see this [documentation](../../docs/sources/clients/logstash/_index.md).
 

data/lib/logstash/outputs/loki.rb

@@ -69,7 +69,7 @@ class LogStash::Outputs::Loki < LogStash::Outputs::Base
     @logger.info("Loki output plugin", :class => self.class.name)
 
     # initialize Queue and Mutex
-    @entries = Queue.new
+    @entries = Queue.new 
     @mutex = Mutex.new
     @stop = false
 
@@ -88,10 +88,10 @@ class LogStash::Outputs::Loki < LogStash::Outputs::Base
 
   end
 
-  def batch(tenant = 'fake')
+  def batch(tenant = 'default')
     return nil if @batches.nil?
     return @batches[tenant] if !tenant.nil? && !tenant.empty? && @batches.key?(tenant)
-    return @batches['fake'] if @batches.key?('fake')
+    return @batches['default'] if @batches.key?('default')
     return nil
   end
 
@@ -100,13 +100,13 @@ class LogStash::Outputs::Loki < LogStash::Outputs::Base
       @mutex.synchronize do
         return if @stop
       end
-
+  
       e = @entries.deq
       return if e.nil?
 
       tenant = nil
       tenant = e.labels['tenant'] if !e.labels.nil? && e.labels.key?('tenant')
-      tenant = 'fake' if tenant.nil? or tenant.empty?
+      tenant = 'default' if tenant.nil? or tenant.empty?
 
       @mutex.synchronize do
         if !add_entry_to_batch(e, tenant)
@@ -190,12 +190,12 @@ class LogStash::Outputs::Loki < LogStash::Outputs::Base
 
   # Add an entry to the current batch returns false if the batch is full
   # and the entry can't be added.
-  def add_entry_to_batch(e, tenant = 'fake')
+  def add_entry_to_batch(e, tenant = 'default')
     line = e.entry['line']
     # we don't want to send empty lines.
     return true if line.to_s.strip.empty?
 
-    tenant = 'fake' if tenant.nil? or tenant.empty?
+    tenant = 'default' if tenant.nil? or tenant.empty?
 
     if @batches.nil?
       @batches = Hash.new
@@ -214,8 +214,8 @@ class LogStash::Outputs::Loki < LogStash::Outputs::Base
     return true
   end
 
-  def is_batch_expired(tenant = 'fake')
-    tenant = 'fake' if tenant.nil? or tenant.empty?
+  def is_batch_expired(tenant = 'default')
+    tenant = 'default' if tenant.nil? or tenant.empty?
     return !@batches.nil? && @batches.key?(tenant) && @batches[tenant].age() >= @batch_wait
   end
 
@@ -231,8 +231,8 @@ class LogStash::Outputs::Loki < LogStash::Outputs::Base
 
   def close
     @entries.close
-    @mutex.synchronize do
-      @stop = true
+    @mutex.synchronize do 
+      @stop = true 
     end
     @batch_wait_thread.join
     @batch_size_thread.join
@@ -245,7 +245,7 @@ class LogStash::Outputs::Loki < LogStash::Outputs::Base
     @batches = nil
   end
 
-  def send(batch, tenant = 'fake')
+  def send(batch, tenant = 'default')
     payload = batch.to_json
     res = loki_http_request(payload, tenant)
     if res.is_a?(Net::HTTPSuccess)
@@ -255,12 +255,12 @@ class LogStash::Outputs::Loki < LogStash::Outputs::Base
     end
   end
 
-  def loki_http_request(payload, tenant = 'fake')
+  def loki_http_request(payload, tenant = 'default')
     req = Net::HTTP::Post.new(
       @uri.request_uri
     )
     req.add_field('Content-Type', 'application/json')
-    if !tenant.nil? && !tenant.empty? && !tenant.eql?('fake')
+    if !tenant.nil? && !tenant.empty? && !tenant.eql?('default')
       req.add_field('X-Scope-OrgID', tenant)
     elsif !@tenant_id.nil? && !@tenant_id.empty?
       req.add_field('X-Scope-OrgID', @tenant_id)

data/logstash-output-loki.gemspec

@@ -1,6 +1,6 @@
 Gem::Specification.new do |s|
   s.name    = 'logstash-output-loki-tenants'
-  s.version = '1.0.3'
+  s.version = '1.0.4'
   s.authors = ['Co.brick']
   s.email   = ['aiops.dev@cobrick.com']
 

data/spec/outputs/loki_spec.rb

@@ -35,9 +35,10 @@ describe LogStash::Outputs::Loki do
       plugin = LogStash::Plugin.lookup("output", "loki").new(simple_loki_config)
       expect(plugin.batch).to eql nil
       expect(plugin.add_entry_to_batch(entry, "a")).to eql true
-      expect(plugin.add_entry_to_batch(entry, "b")).to eql true
+      expect(plugin.add_entry_to_batch(entry, "fake")).to eql true
       expect(plugin.add_entry_to_batch(entry, nil)).to eql true
       expect(plugin.add_entry_to_batch(entry, "")).to eql true
+      expect(plugin.add_entry_to_batch(entry, "default")).to eql true
       expect(plugin.batches.keys.length).to eq 3
     end
 
@@ -58,6 +59,20 @@ describe LogStash::Outputs::Loki do
       expect(plugin.batch.streams[lbs.to_s]['entries'].length).to eq 2
       expect(plugin.batch.streams[lbs.to_s]['labels']).to eq lbs
       expect(plugin.batch.size_bytes).to eq 14
+      expect(plugin.batch("default").size_bytes).to eq 14
+    end
+
+    it 'should add entry to tenant fake' do
+      plugin = LogStash::Plugin.lookup("output", "loki").new(simple_loki_config)
+      expect(plugin.batch("fake")).to eql nil
+      expect(plugin.add_entry_to_batch(entry, "fake")).to eql true
+      expect(plugin.add_entry_to_batch(entry, "fake")).to eql true
+      expect(plugin.batch("fake")).not_to be_nil
+      expect(plugin.batch).to be_nil
+      expect(plugin.batch("fake").streams.length).to eq 1
+      expect(plugin.batch("fake").streams[lbs.to_s]['entries'].length).to eq 2
+      expect(plugin.batch("fake").streams[lbs.to_s]['labels']).to eq lbs
+      expect(plugin.batch("fake").size_bytes).to eq 14
     end
 
     it 'should not add if full' do
@@ -97,6 +112,15 @@ describe LogStash::Outputs::Loki do
       sleep(1)
       expect(loki.is_batch_expired).to be true
     end
+    it 'should expire if old different tenants' do
+      loki = LogStash::Outputs::Loki.new(simple_loki_config.merge!({'batch_wait'=>0.5}))
+      expect(loki.add_entry_to_batch(entry, "fake")).to eql true
+      sleep(1)
+      expect(loki.add_entry_to_batch(entry, "custom")).to eql true
+      expect(loki.is_batch_expired("fake")).to be true
+      expect(loki.is_batch_expired("custom")).to be false
+      expect(loki.is_batch_expired).to be false
+    end
   end
 
   context 'channel' do
@@ -174,6 +198,30 @@ describe LogStash::Outputs::Loki do
       loki.send(b, "custom")
       expect(post).to have_been_requested.times(1)
     end
+    it 'should send message tenant fake' do
+      conf = {
+        'url'=>'http://localhost:3100/loki/api/v1/push',
+        'username' => 'foo',
+        'password' => 'bar',
+        'tenant_id' => 'custom'
+      }
+      loki = LogStash::Outputs::Loki.new(conf)
+      loki.register
+      b = Batch.new(entry)
+      post = stub_request(:post, "http://localhost:3100/loki/api/v1/push").with(
+        basic_auth: ['foo', 'bar'],
+        body: b.to_json,
+        headers:{
+          'Content-Type' => 'application/json' ,
+          'User-Agent' => 'loki-logstash',
+          'X-Scope-OrgID'=>'fake',
+          'Accept'=>'*/*',
+          'Accept-Encoding'=>'gzip;q=1.0,deflate;q=0.6,identity;q=0.3',
+        }
+      )
+      loki.send(b, "fake")
+      expect(post).to have_been_requested.times(1)
+    end
     it 'should send credentials' do
       conf = {
           'url'=>'http://localhost:3100/loki/api/v1/push',

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: logstash-output-loki-tenants
 version: !ruby/object:Gem::Version
-  version: 1.0.3
+  version: 1.0.4
 platform: ruby
 authors:
 - Co.brick
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2021-05-25 00:00:00.000000000 Z
+date: 2021-07-02 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement