logstash-output-loki-tenants 1.0.3

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: 468a3f120f85bb99690179384555a9cc4d0b090f027fa6abdbc2e40200973964
+  data.tar.gz: 58532c550dff9474933c6df25b635586d0fd745014753babf8dfaf300e653939
+SHA512:
+  metadata.gz: 45db85447deeda78d894384604a1dd470b1b873d5ea854c6ab26390bdbfdc33640aaef7833af7aa66188ed96ad174414a85e0b90ad5ed186c0bbd7ac21ed4932
+  data.tar.gz: '005299b0c0541b42806f35ae6d0aa7f55967066f8851642c9ec791f58b2d5387065c4600e51bc91fb7830c082abddbf14de51c5c9a82e3ca9d806c6a896de5e1'

data/Gemfile

@@ -0,0 +1,14 @@
+source 'https://rubygems.org'
+
+gemspec
+
+logstash_path = ENV["LOGSTASH_PATH"] || "./logstash"
+
+if Dir.exist?(logstash_path)
+  gem 'logstash-core', :path => "#{logstash_path}/logstash-core"
+  gem 'logstash-core-plugin-api', :path => "#{logstash_path}/logstash-core-plugin-api"
+else
+  raise 'missing logstash vendoring'
+end
+
+gem "webmock", "~> 3.8"

data/README.md

@@ -0,0 +1,76 @@
+# Contributing to Loki Logstash Output Plugin
+
+For information about how to use this plugin see this [documentation](../../docs/sources/clients/logstash/_index.md).
+
+## Install dependencies
+
+First, make sure you have JDK version `8` or `11` installed and you have set the `JAVA_HOME` environment variable.
+
+You need to setup JRuby environment to build this plugin. Refer https://github.com/rbenv/rbenv for setting up your rbenv environment.
+
+After setting up `rbenv`. Install JRuby
+
+```bash
+rbenv install jruby-9.2.10.0
+rbenv local jruby-9.2.10.0
+```
+
+Check that the environment is configured
+
+```bash
+ruby --version
+jruby 9.2.10
+```
+
+You should make sure you are running `jruby` and not `ruby`. If the command `ruby --version` still shows `ruby` and not `jruby`, check that PATH contains `$HOME/.rbenv/shims` and `$HOME/.rbenv/bin`. Also verify that you have this in your bash profile:
+
+```bash
+export PATH="$HOME/.rbenv/bin:$PATH"
+eval "$(rbenv init -)"
+```
+
+Then install bundler
+`gem install bundler:2.1.4`
+
+Follow those instructions to [install logstash](https://www.elastic.co/guide/en/logstash/current/installing-logstash.html) before moving to the next section.
+
+## Build and test the plugin
+
+### Install required packages
+
+```bash
+git clone git@github.com:elastic/logstash.git
+cd logstash
+git checkout tags/v7.6.2
+export LOGSTASH_PATH=`pwd`
+export GEM_PATH=$LOGSTASH_PATH/vendor/bundle/jruby/2.5.0
+export GEM_HOME=$LOGSTASH_PATH/vendor/bundle/jruby/2.5.0
+./gradlew assemble
+cd ..
+ruby -S bundle install
+ruby -S bundle exec rake vendor
+```
+
+### Build the plugin
+
+`gem build logstash-output-loki.gemspec`
+
+### Test
+
+`ruby -S bundle exec rspec`
+
+Alternatively if you don't want to install JRuby. Enter inside logstash-loki container.
+
+```bash
+docker build -t logstash-loki ./
+docker run -v  `pwd`/spec:/home/logstash/spec -it --rm --entrypoint /bin/sh logstash-loki
+bundle exec rspec
+```
+
+## Install plugin to local logstash
+
+`bin/logstash-plugin install --no-verify --local logstash-output-loki-1.0.0.gem`
+
+## Send sample event and check plugin is working
+
+`bin/logstash -f loki.conf`

data/lib/logstash/outputs/loki.rb

@@ -0,0 +1,301 @@
+# encoding: utf-8
+require "logstash/outputs/base"
+require "logstash/outputs/loki/entry"
+require "logstash/outputs/loki/batch"
+require "logstash/namespace"
+require 'net/http'
+require 'time'
+require 'uri'
+require 'json'
+
+class LogStash::Outputs::Loki < LogStash::Outputs::Base
+  include Loki
+  config_name "loki"
+
+  ## 'A single instance of the Output will be shared among the pipeline worker threads'
+  concurrency :single
+
+  ## 'Loki URL'
+  config :url, :validate => :string, :required => true
+
+  ## 'BasicAuth credentials'
+  config :username, :validate => :string, :required => false
+  config :password, :validate => :string, secret: true, :required => false
+
+  ## 'Client certificate'
+  config :cert, :validate => :path, :required => false
+  config :key, :validate => :path, :required => false
+
+  ## 'TLS'
+  config :ca_cert, :validate => :path, :required => false
+
+  ## 'Disable server certificate verification'
+  config :insecure_skip_verify, :validate => :boolean, :default => false, :required => false
+
+  ## 'Loki Tenant ID'
+  config :tenant_id, :validate => :string, :required => false
+
+  ## 'Maximum batch size to accrue before pushing to loki. Defaults to 102400 bytes'
+  config :batch_size, :validate => :number, :default => 102400, :required => false
+
+  ## 'Interval in seconds to wait before pushing a batch of records to loki. Defaults to 1 second'
+  config :batch_wait, :validate => :number, :default => 1, :required => false
+
+  ## 'Log line field to pick from logstash. Defaults to "message"'
+  config :message_field, :validate => :string, :default => "message", :required => false
+
+  ## 'Backoff configuration. Initial backoff time between retries. Default 1s'
+  config :min_delay, :validate => :number, :default => 1, :required => false
+
+  ## 'Backoff configuration. Maximum backoff time between retries. Default 300s'
+  config :max_delay, :validate => :number, :default => 300, :required => false
+
+  ## 'Backoff configuration. Maximum number of retries to do'
+  config :retries, :validate => :number, :default => 10, :required => false
+
+  attr_reader :batches
+  public
+  def register
+
+    @uri = URI.parse(@url)
+    unless @uri.is_a?(URI::HTTP) || @uri.is_a?(URI::HTTPS)
+      raise LogStash::ConfigurationError, "url parameter must be valid HTTP, currently '#{@url}'"
+    end
+
+    if @min_delay > @max_delay
+      raise LogStash::ConfigurationError, "Min delay should be less than Max delay, currently 'Min delay is #{@min_delay} and Max delay is #{@max_delay}'"
+    end
+
+    @logger.info("Loki output plugin", :class => self.class.name)
+
+    # initialize Queue and Mutex
+    @entries = Queue.new
+    @mutex = Mutex.new
+    @stop = false
+
+    # create nil batch object.
+    @batches = Hash.new
+
+    # validate certs
+    if ssl_cert?
+      load_ssl
+      validate_ssl_key
+    end
+
+    # start batch_max_wait and batch_max_size threads
+    @batch_wait_thread = Thread.new{max_batch_wait()}
+    @batch_size_thread = Thread.new{max_batch_size()}
+
+  end
+
+  def batch(tenant = 'fake')
+    return nil if @batches.nil?
+    return @batches[tenant] if !tenant.nil? && !tenant.empty? && @batches.key?(tenant)
+    return @batches['fake'] if @batches.key?('fake')
+    return nil
+  end
+
+  def max_batch_size
+    loop do
+      @mutex.synchronize do
+        return if @stop
+      end
+
+      e = @entries.deq
+      return if e.nil?
+
+      tenant = nil
+      tenant = e.labels['tenant'] if !e.labels.nil? && e.labels.key?('tenant')
+      tenant = 'fake' if tenant.nil? or tenant.empty?
+
+      @mutex.synchronize do
+        if !add_entry_to_batch(e, tenant)
+          @logger.debug("Max batch_size is reached. Sending batch to loki. Tenant #{tenant}")
+          send_batch_for_tenant(tenant)
+          @batches[tenant] = Batch.new(e)
+        end
+      end
+    end
+  end
+
+  def max_batch_wait
+    # minimum wait frequency is 10 milliseconds
+	  min_wait_checkfrequency = 1/100
+	  max_wait_checkfrequency = @batch_wait
+	  if max_wait_checkfrequency < min_wait_checkfrequency
+		  max_wait_checkfrequency = min_wait_checkfrequency
+    end
+
+    loop do
+      @mutex.synchronize do
+        return if @stop
+      end
+
+      sleep(max_wait_checkfrequency)
+
+      @mutex.synchronize do
+        @batches.keys.clone.each { |tenant|
+          if is_batch_expired(tenant)
+            @logger.debug("Max batch_wait time is reached. Sending batch to loki. Tenant #{tenant}")
+            send_batch_for_tenant(tenant)
+            @batches.delete(tenant)
+          end
+        }
+      end
+    end
+  end
+
+  def ssl_cert?
+    !@key.nil? && !@cert.nil?
+  end
+
+  def load_ssl
+    @cert = OpenSSL::X509::Certificate.new(File.read(@cert)) if @cert
+    @key = OpenSSL::PKey.read(File.read(@key)) if @key
+  end
+
+  def validate_ssl_key
+    if !@key.is_a?(OpenSSL::PKey::RSA) && !@key.is_a?(OpenSSL::PKey::DSA)
+      raise LogStash::ConfigurationError, "Unsupported private key type '#{@key.class}''"
+    end
+  end
+
+  def ssl_opts(uri)
+    opts = {
+      use_ssl: uri.scheme == 'https'
+    }
+
+     # disable server certificate verification
+    if @insecure_skip_verify
+      opts = opts.merge(
+        verify_mode: OpenSSL::SSL::VERIFY_NONE
+      )
+    end
+
+    if !@cert.nil? && !@key.nil?
+      opts = opts.merge(
+        verify_mode: OpenSSL::SSL::VERIFY_PEER,
+        cert: @cert,
+        key: @key
+      )
+    end
+
+    unless @ca_cert.nil?
+      opts = opts.merge(
+        ca_file: @ca_cert
+      )
+    end
+    opts
+  end
+
+  # Add an entry to the current batch returns false if the batch is full
+  # and the entry can't be added.
+  def add_entry_to_batch(e, tenant = 'fake')
+    line = e.entry['line']
+    # we don't want to send empty lines.
+    return true if line.to_s.strip.empty?
+
+    tenant = 'fake' if tenant.nil? or tenant.empty?
+
+    if @batches.nil?
+      @batches = Hash.new
+    end
+
+    if !@batches.key?(tenant)
+      @batches[tenant] = Batch.new(e)
+      return true
+    end
+
+    if @batches[tenant].size_bytes_after(line) > @batch_size
+      return false
+    end
+
+    @batches[tenant].add(e)
+    return true
+  end
+
+  def is_batch_expired(tenant = 'fake')
+    tenant = 'fake' if tenant.nil? or tenant.empty?
+    return !@batches.nil? && @batches.key?(tenant) && @batches[tenant].age() >= @batch_wait
+  end
+
+  def send_batch_for_tenant(tenant)
+    send(batch(tenant), tenant)
+  end
+
+  ## Receives logstash events
+  public
+  def receive(event)
+    @entries << Entry.new(event, @message_field)
+  end
+
+  def close
+    @entries.close
+    @mutex.synchronize do
+      @stop = true
+    end
+    @batch_wait_thread.join
+    @batch_size_thread.join
+
+    # if by any chance we still have a forming batch, we need to send it.
+    @batches.keys.each { |tenant|
+      send_batch_for_tenant(tenant)
+    }
+    @batches.clear()
+    @batches = nil
+  end
+
+  def send(batch, tenant = 'fake')
+    payload = batch.to_json
+    res = loki_http_request(payload, tenant)
+    if res.is_a?(Net::HTTPSuccess)
+      @logger.debug("Successfully pushed data to loki")
+    else
+      @logger.debug("failed payload", :payload => payload)
+    end
+  end
+
+  def loki_http_request(payload, tenant = 'fake')
+    req = Net::HTTP::Post.new(
+      @uri.request_uri
+    )
+    req.add_field('Content-Type', 'application/json')
+    if !tenant.nil? && !tenant.empty? && !tenant.eql?('fake')
+      req.add_field('X-Scope-OrgID', tenant)
+    elsif !@tenant_id.nil? && !@tenant_id.empty?
+      req.add_field('X-Scope-OrgID', @tenant_id)
+    end
+
+    req['User-Agent']= 'loki-logstash'
+    req.basic_auth(@username, @password) if @username
+    req.body = payload
+
+    opts = ssl_opts(@uri)
+
+    @logger.debug("sending #{req.body.length} bytes to loki. tenant #{tenant}")
+    retry_count = 0
+    delay = @min_delay
+    begin
+      res = Net::HTTP.start(@uri.host, @uri.port, **opts) { |http|
+        http.request(req)
+      }
+      return res if !res.nil? && res.code.to_i != 429 && res.code.to_i.div(100) != 5
+      raise StandardError.new res
+    rescue StandardError => e
+      retry_count += 1
+      @logger.warn("Failed to send batch, attempt: #{retry_count}/#{@retries}", :error_inspect => e.inspect, :error => e)
+      if retry_count < @retries
+        sleep delay
+        if delay * 2 <= @max_delay
+          delay = delay * 2
+        else
+          delay = @max_delay
+        end
+        retry
+      else
+        @logger.error("Failed to send batch", :error_inspect => e.inspect, :error => e)
+        return res
+      end
+    end
+  end
+end

data/lib/logstash/outputs/loki/batch.rb

@@ -0,0 +1,63 @@
+require 'time'
+
+module Loki
+    class Batch
+        attr_reader :streams
+        def initialize(e)
+            @bytes = 0
+            @createdAt = Time.now
+            @streams = {}
+            add(e)
+        end
+
+        def size_bytes
+            return @bytes
+        end
+
+        def add(e)
+            @bytes = @bytes + e.entry['line'].length
+
+            # Append the entry to an already existing stream (if any)
+            labels = e.labels.sort.to_h
+            labelkey = labels.to_s
+            if @streams.has_key?(labelkey)
+                stream = @streams[labelkey]
+                stream['entries'].append(e.entry)
+                return
+            else
+                # Add the entry as a new stream
+                @streams[labelkey] = {
+                    "labels" => labels,
+                    "entries" => [e.entry],
+                }
+            end
+        end
+
+        def size_bytes_after(line)
+            return @bytes + line.length
+        end
+
+        def age()
+            return Time.now - @createdAt
+        end
+
+        def to_json
+            streams = []
+            @streams.each { |_ , stream|
+                streams.append(build_stream(stream))
+            }
+            return {"streams"=>streams}.to_json
+        end
+
+        def build_stream(stream)
+            values = []
+            stream['entries'].each { |entry|
+                values.append([entry['ts'].to_s, entry['line']])
+            }
+            return {
+                'stream'=>stream['labels'],
+                'values' => values
+            }
+        end
+    end
+end

data/lib/logstash/outputs/loki/entry.rb

@@ -0,0 +1,25 @@
+module Loki
+    def to_ns(s)
+        (s.to_f * (10**9)).to_i
+    end
+    class Entry
+        include Loki
+        attr_reader :labels, :entry
+        def initialize(event,message_field)
+            @entry = {
+                "ts" => to_ns(event.get("@timestamp")),
+                "line" => event.get(message_field).to_s
+            }
+            event = event.clone()
+            event.remove(message_field)
+            event.remove("@timestamp")
+
+            @labels = {}
+            event.to_hash.each { |key,value|
+                next if key.start_with?('@')
+                next if value.is_a?(Hash)
+                @labels[key] = value.to_s
+            }
+        end
+    end
+end

data/logstash-output-loki.gemspec

@@ -0,0 +1,26 @@
+Gem::Specification.new do |s|
+  s.name    = 'logstash-output-loki-tenants'
+  s.version = '1.0.3'
+  s.authors = ['Co.brick']
+  s.email   = ['aiops.dev@cobrick.com']
+
+  s.summary       = 'Output plugin to ship logs to a Grafana Loki server'
+  s.description   = 'Output plugin to ship logs to a Grafana Loki server'
+  s.homepage      = 'https://rubygems.org/gems/logstash-output-loki-tenants'
+  s.license       = 'Apache-2.0'
+  s.require_paths = ["lib"]
+
+  # Files
+  s.files = Dir['lib/**/*','spec/**/*','vendor/**/*','*.gemspec','*.md','CONTRIBUTORS','Gemfile']
+   # Tests
+  s.test_files = s.files.grep(%r{^(test|spec|features)/})
+
+  # Special flag to let us know this is actually a logstash plugin
+  s.metadata = { "logstash_plugin" => "true", "logstash_group" => "output" }
+
+  # Gem dependencies
+  #
+  s.add_runtime_dependency "logstash-core-plugin-api", ">= 1.60", "<= 2.99"
+  s.add_runtime_dependency "logstash-codec-plain", "3.0.6"
+  s.add_development_dependency 'logstash-devutils', "2.0.2"
+end

data/spec/outputs/loki/entry_spec.rb

@@ -0,0 +1,59 @@
+# encoding: utf-8
+require "logstash/devutils/rspec/spec_helper"
+require "logstash/outputs/loki"
+require "logstash/codecs/plain"
+require "logstash/event"
+require "net/http"
+include Loki
+
+describe Loki::Entry do
+  context 'test entry generation' do
+      let (:event) {
+        LogStash::Event.new(
+            {
+              'message' => 'hello',
+              '@metadata' => {'foo'=>'bar'},
+              '@version' => '1',
+              'foo' => 5,
+              'agent' => 'filebeat',
+              'log' => {
+                   'file' =>
+                        {'@path' => '/path/to/file.log'},
+                   },
+              'host' => '172.0.0.1',
+              '@timestamp' => Time.now
+            }
+        )
+      }
+
+      it 'labels extracted should not contains object and metadata or timestamp' do
+        entry = Entry.new(event,"message")
+        expect(entry.labels).to eql({ 'agent' => 'filebeat', 'host' => '172.0.0.1', 'foo'=>'5'})
+        expect(entry.entry['ts']).to eql to_ns(event.get("@timestamp"))
+        expect(entry.entry['line']).to eql 'hello'
+      end
+  end
+
+  context 'test batch generation with label order' do
+    let (:entries)  {[
+        Entry.new(LogStash::Event.new({"message"=>"foobuzz","buzz"=>"bar","cluster"=>"us-central1","@timestamp"=>Time.at(1)}),"message"),
+        Entry.new(LogStash::Event.new({"log"=>"foobar","bar"=>"bar","@timestamp"=>Time.at(2)}),"log"),
+        Entry.new(LogStash::Event.new({"cluster"=>"us-central1","message"=>"foobuzz","buzz"=>"bar","@timestamp"=>Time.at(3)}),"message"),
+
+    ]}
+    let (:expected) {
+      {"streams" => [
+          {"stream"=> {"buzz"=>"bar","cluster"=>"us-central1"}, "values" => [[to_ns(Time.at(1)).to_s,"foobuzz"],[to_ns(Time.at(3)).to_s,"foobuzz"]]},
+          {"stream"=> {"bar"=>"bar"}, "values"=>[[to_ns(Time.at(2)).to_s,"foobar"]]},
+      ] }
+    }
+
+    it 'to_json' do
+      @batch = Loki::Batch.new(entries.first)
+      entries.drop(1).each { |e| @batch.add(e)}
+      expect(JSON.parse(@batch.to_json)).to eql expected
+    end
+  end
+
+
+end

data/spec/outputs/loki_spec.rb

@@ -0,0 +1,281 @@
+# encoding: utf-8
+require "logstash/devutils/rspec/spec_helper"
+require "logstash/outputs/loki"
+require "logstash/codecs/plain"
+require "logstash/event"
+require "net/http"
+require 'webmock/rspec'
+include Loki
+
+describe LogStash::Outputs::Loki do
+
+  let (:simple_loki_config) { {'url' => 'http://localhost:3100'} }
+
+  context 'when initializing' do
+    it "should register" do
+      loki = LogStash::Plugin.lookup("output", "loki").new(simple_loki_config)
+      expect { loki.register }.to_not raise_error
+    end
+
+    it 'should populate loki config with default or initialized values' do
+      loki = LogStash::Outputs::Loki.new(simple_loki_config)
+      expect(loki.url).to eql 'http://localhost:3100'
+      expect(loki.tenant_id).to eql nil
+      expect(loki.batch_size).to eql 102400
+      expect(loki.batch_wait).to eql 1
+    end
+  end
+
+  context 'when adding en entry to the batch' do
+    let (:simple_loki_config) {{'url' => 'http://localhost:3100'}}
+    let (:entry) {Entry.new(LogStash::Event.new({"message"=>"foobuzz","buzz"=>"bar","cluster"=>"us-central1","@timestamp"=>Time.at(1)}),"message")}
+    let (:lbs) { {"buzz"=>"bar","cluster"=>"us-central1"}.sort.to_h}
+
+    it 'should add tenant batch' do
+      plugin = LogStash::Plugin.lookup("output", "loki").new(simple_loki_config)
+      expect(plugin.batch).to eql nil
+      expect(plugin.add_entry_to_batch(entry, "a")).to eql true
+      expect(plugin.add_entry_to_batch(entry, "b")).to eql true
+      expect(plugin.add_entry_to_batch(entry, nil)).to eql true
+      expect(plugin.add_entry_to_batch(entry, "")).to eql true
+      expect(plugin.batches.keys.length).to eq 3
+    end
+
+    it 'should not add empty line' do
+      plugin = LogStash::Plugin.lookup("output", "loki").new(simple_loki_config)
+      emptyEntry = Entry.new(LogStash::Event.new({"message"=>"foobuzz","buzz"=>"bar","cluster"=>"us-central1","@timestamp"=>Time.at(1)}),"foo")
+      expect(plugin.add_entry_to_batch(emptyEntry)).to eql true
+      expect(plugin.batch).to eql nil
+    end
+
+    it 'should add entry' do
+      plugin = LogStash::Plugin.lookup("output", "loki").new(simple_loki_config)
+      expect(plugin.batch).to eql nil
+      expect(plugin.add_entry_to_batch(entry)).to eql true
+      expect(plugin.add_entry_to_batch(entry)).to eql true
+      expect(plugin.batch).not_to be_nil
+      expect(plugin.batch.streams.length).to eq 1
+      expect(plugin.batch.streams[lbs.to_s]['entries'].length).to eq 2
+      expect(plugin.batch.streams[lbs.to_s]['labels']).to eq lbs
+      expect(plugin.batch.size_bytes).to eq 14
+    end
+
+    it 'should not add if full' do
+      plugin = LogStash::Plugin.lookup("output", "loki").new(simple_loki_config.merge!({'batch_size'=>10}))
+      expect(plugin.batch).to eql nil
+      expect(plugin.add_entry_to_batch(entry)).to eql true # first entry is fine.
+      expect(plugin.batch).not_to be_nil
+      expect(plugin.batch.streams.length).to eq 1
+      expect(plugin.batch.streams[lbs.to_s]['entries'].length).to eq 1
+      expect(plugin.batch.streams[lbs.to_s]['labels']).to eq lbs
+      expect(plugin.batch.size_bytes).to eq 7
+      expect(plugin.add_entry_to_batch(entry)).to eql false # second entry goes over the limit.
+      expect(plugin.batch).not_to be_nil
+      expect(plugin.batch.streams.length).to eq 1
+      expect(plugin.batch.streams[lbs.to_s]['entries'].length).to eq 1
+      expect(plugin.batch.streams[lbs.to_s]['labels']).to eq lbs
+      expect(plugin.batch.size_bytes).to eq 7
+    end
+  end
+
+  context 'batch expiration' do
+    let (:entry) {Entry.new(LogStash::Event.new({"message"=>"foobuzz","buzz"=>"bar","cluster"=>"us-central1","@timestamp"=>Time.at(1)}),"message")}
+
+    it 'should not expire if empty' do
+      loki = LogStash::Outputs::Loki.new(simple_loki_config.merge!({'batch_wait'=>0.5}))
+      sleep(1)
+      expect(loki.is_batch_expired).to be false
+    end
+    it 'should not expire batch if not old' do
+      loki = LogStash::Outputs::Loki.new(simple_loki_config.merge!({'batch_wait'=>0.5}))
+      expect(loki.add_entry_to_batch(entry)).to eql true
+      expect(loki.is_batch_expired).to be false
+    end
+    it 'should expire if old' do
+      loki = LogStash::Outputs::Loki.new(simple_loki_config.merge!({'batch_wait'=>0.5}))
+      expect(loki.add_entry_to_batch(entry)).to eql true
+      sleep(1)
+      expect(loki.is_batch_expired).to be true
+    end
+  end
+
+  context 'channel' do
+    let (:event) {LogStash::Event.new({"message"=>"foobuzz","buzz"=>"bar","cluster"=>"us-central1","@timestamp"=>Time.at(1)})}
+
+    it 'should send entry if batch size reached with no tenant' do
+      loki = LogStash::Outputs::Loki.new(simple_loki_config.merge!({'batch_wait'=>0.5,'batch_size'=>10}))
+      loki.register
+      sent = Queue.new
+      allow(loki).to receive(:send) do | batch|
+        Thread.new do
+          sent << batch
+        end
+      end
+      loki.receive(event)
+      loki.receive(event)
+      sent.deq
+      sent.deq
+      loki.close
+    end
+    it 'should send entry while closing' do
+      loki = LogStash::Outputs::Loki.new(simple_loki_config.merge!({'batch_wait'=>10,'batch_size'=>10}))
+      loki.register
+      sent = Queue.new
+      allow(loki).to receive(:send) do | batch|
+        Thread.new do
+          sent << batch
+        end
+      end
+      loki.receive(event)
+      loki.close
+      sent.deq
+    end
+    it 'should send entry when batch is expiring' do
+      loki = LogStash::Outputs::Loki.new(simple_loki_config.merge!({'batch_wait'=>0.5,'batch_size'=>10}))
+      loki.register
+      sent = Queue.new
+      allow(loki).to receive(:send) do | batch|
+        Thread.new  do
+          sent << batch
+        end
+      end
+      loki.receive(event)
+      sent.deq
+      sleep(0.01) # Adding a minimal sleep. In few cases @batch=nil might happen after evaluating for nil
+      expect(loki.batch).to be_nil
+      loki.close
+    end
+  end
+
+  context 'http requests' do
+    let (:entry) {Entry.new(LogStash::Event.new({"message"=>"foobuzz","buzz"=>"bar","cluster"=>"us-central1","@timestamp"=>Time.at(1)}),"message")}
+
+    it 'should send message tenant' do
+      conf = {
+        'url'=>'http://localhost:3100/loki/api/v1/push',
+        'username' => 'foo',
+        'password' => 'bar',
+        'tenant_id' => 't'
+      }
+      loki = LogStash::Outputs::Loki.new(conf)
+      loki.register
+      b = Batch.new(entry)
+      post = stub_request(:post, "http://localhost:3100/loki/api/v1/push").with(
+        basic_auth: ['foo', 'bar'],
+        body: b.to_json,
+        headers:{
+          'Content-Type' => 'application/json' ,
+          'User-Agent' => 'loki-logstash',
+          'X-Scope-OrgID'=>'custom',
+          'Accept'=>'*/*',
+          'Accept-Encoding'=>'gzip;q=1.0,deflate;q=0.6,identity;q=0.3',
+        }
+      )
+      loki.send(b, "custom")
+      expect(post).to have_been_requested.times(1)
+    end
+    it 'should send credentials' do
+      conf = {
+          'url'=>'http://localhost:3100/loki/api/v1/push',
+          'username' => 'foo',
+          'password' => 'bar',
+          'tenant_id' => 't'
+      }
+      loki = LogStash::Outputs::Loki.new(conf)
+      loki.register
+      b = Batch.new(entry)
+      post = stub_request(:post, "http://localhost:3100/loki/api/v1/push").with(
+          basic_auth: ['foo', 'bar'],
+          body: b.to_json,
+          headers:{
+              'Content-Type' => 'application/json' ,
+              'User-Agent' => 'loki-logstash',
+              'X-Scope-OrgID'=>'t',
+              'Accept'=>'*/*',
+              'Accept-Encoding'=>'gzip;q=1.0,deflate;q=0.6,identity;q=0.3',
+          }
+      )
+      loki.send(b)
+      expect(post).to have_been_requested.times(1)
+    end
+    it 'should not send credentials' do
+      conf = {
+          'url'=>'http://foo.com/loki/api/v1/push',
+      }
+      loki = LogStash::Outputs::Loki.new(conf)
+      loki.register
+      b = Batch.new(entry)
+      post = stub_request(:post, "http://foo.com/loki/api/v1/push").with(
+          body: b.to_json,
+          headers:{
+              'Content-Type' => 'application/json' ,
+              'User-Agent' => 'loki-logstash',
+              'Accept'=>'*/*',
+              'Accept-Encoding'=>'gzip;q=1.0,deflate;q=0.6,identity;q=0.3',
+          }
+      )
+      loki.send(b)
+      expect(post).to have_been_requested.times(1)
+    end
+    it 'should retry 500' do
+      conf = {
+          'url'=>'http://foo.com/loki/api/v1/push',
+          'retries' => 3,
+      }
+      loki = LogStash::Outputs::Loki.new(conf)
+      loki.register
+      b = Batch.new(entry)
+      post = stub_request(:post, "http://foo.com/loki/api/v1/push").with(
+          body: b.to_json,
+      ).to_return(status: [500, "Internal Server Error"])
+      loki.send(b)
+      loki.close
+      expect(post).to have_been_requested.times(3)
+    end
+    it 'should retry 429' do
+      conf = {
+          'url'=>'http://foo.com/loki/api/v1/push',
+          'retries' => 2,
+      }
+      loki = LogStash::Outputs::Loki.new(conf)
+      loki.register
+      b = Batch.new(entry)
+      post = stub_request(:post, "http://foo.com/loki/api/v1/push").with(
+          body: b.to_json,
+          ).to_return(status: [429, "stop spamming"])
+      loki.send(b)
+      loki.close
+      expect(post).to have_been_requested.times(2)
+    end
+    it 'should not retry 400' do
+      conf = {
+          'url'=>'http://foo.com/loki/api/v1/push',
+          'retries' => 11,
+      }
+      loki = LogStash::Outputs::Loki.new(conf)
+      loki.register
+      b = Batch.new(entry)
+      post = stub_request(:post, "http://foo.com/loki/api/v1/push").with(
+          body: b.to_json,
+          ).to_return(status: [400, "bad request"])
+      loki.send(b)
+      loki.close
+      expect(post).to have_been_requested.times(1)
+    end
+    it 'should retry exception' do
+      conf = {
+          'url'=>'http://foo.com/loki/api/v1/push',
+          'retries' => 11,
+      }
+      loki = LogStash::Outputs::Loki.new(conf)
+      loki.register
+      b = Batch.new(entry)
+      post = stub_request(:post, "http://foo.com/loki/api/v1/push").with(
+          body: b.to_json,
+          ).to_raise("some error").then.to_return(status: [200, "fine !"])
+      loki.send(b)
+      loki.close
+      expect(post).to have_been_requested.times(2)
+    end
+  end
+end

metadata

@@ -0,0 +1,103 @@
+--- !ruby/object:Gem::Specification
+name: logstash-output-loki-tenants
+version: !ruby/object:Gem::Version
+  version: 1.0.3
+platform: ruby
+authors:
+- Co.brick
+autorequire:
+bindir: bin
+cert_chain: []
+date: 2021-05-25 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '1.60'
+    - - "<="
+      - !ruby/object:Gem::Version
+        version: '2.99'
+  name: logstash-core-plugin-api
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '1.60'
+    - - "<="
+      - !ruby/object:Gem::Version
+        version: '2.99'
+- !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 3.0.6
+  name: logstash-codec-plain
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 3.0.6
+- !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 2.0.2
+  name: logstash-devutils
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 2.0.2
+description: Output plugin to ship logs to a Grafana Loki server
+email:
+- aiops.dev@cobrick.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- Gemfile
+- README.md
+- lib/logstash/outputs/loki.rb
+- lib/logstash/outputs/loki/batch.rb
+- lib/logstash/outputs/loki/entry.rb
+- logstash-output-loki.gemspec
+- spec/outputs/loki/entry_spec.rb
+- spec/outputs/loki_spec.rb
+homepage: https://rubygems.org/gems/logstash-output-loki-tenants
+licenses:
+- Apache-2.0
+metadata:
+  logstash_plugin: 'true'
+  logstash_group: output
+post_install_message:
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubygems_version: 3.0.6
+signing_key:
+specification_version: 4
+summary: Output plugin to ship logs to a Grafana Loki server
+test_files:
+- spec/outputs/loki/entry_spec.rb
+- spec/outputs/loki_spec.rb