logstash-output-loki 1.0.1 → 1.0.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 5d3ae7289992fc6a6b98db3f5f1d929041656f2cd1e7e53915d48df9a8bf216b
-  data.tar.gz: 21e0b37676d7f6d0bca071bd2644a8ce96370e1c62efbf26becb3bce1de63f53
+  metadata.gz: a76f3826de04d39e57f06c522125adfc150e1b9c44dee915cb5a220e829b5ae6
+  data.tar.gz: d4f881165097a6c04dcef15eca32f3570f0d35453a16713e5da1a50e022876d0
 SHA512:
-  metadata.gz: 74e562693367ffcae0a7c639185ca75a984a156a7b7d8c7c61626b17ea01e20470901a2847230aff5f89d7b800f14cdc35ed82be93dd865af91d54548332dbf2
-  data.tar.gz: 9caf43738978ad056df2661460452c7059bd8a4c646f6e9949b3c7ee492b57ec1d0fa9320df02748f928fe4b002404ec2cfa75c0e9f4d78465606d38088dc8d7
+  metadata.gz: 706a86a1852e2b2e97bbd340904b1a0a72751beca4ca4257bfbd66683e0c3a7fe5a54d273666c2da98947b84270b4e9544fe36b6e1742877ccffcbf67fe090bc
+  data.tar.gz: 5710b40ac392fa8e82c29343f22b444ce12284023497da3d0cb9c28049b81811ece3bcf431e5b948b7c60dd29c941dcd3c508289fc9a2d49faf5331eda893f69

data/README.md

@@ -1,10 +1,12 @@
 # Contributing to Loki Logstash Output Plugin
 
-For information about hwo to use this plugin see this [documentation](../../docs/clients/logstash/README.md).
+For information about how to use this plugin see this [documentation](../../docs/sources/clients/logstash/_index.md).
 
 ## Install dependencies
 
-First you need to setup JRuby environment to build this plugin. Refer https://github.com/rbenv/rbenv for setting up your rbenv environment.
+First, make sure you have JDK version `8` or `11` installed and you have set the `JAVA_HOME` environment variable.
+
+You need to setup JRuby environment to build this plugin. Refer https://github.com/rbenv/rbenv for setting up your rbenv environment.
 
 After setting up `rbenv`. Install JRuby
 
@@ -20,7 +22,7 @@ ruby --version
 jruby 9.2.10
 ```
 
-You should use make sure you are running jruby and not ruby. If the command below still shows ruby and not jruby, check that PATH contains `$HOME/.rbenv/shims` and `$HOME/.rbenv/bin`. Also verify that you have this in your bash profile:
+You should make sure you are running `jruby` and not `ruby`. If the command `ruby --version` still shows `ruby` and not `jruby`, check that PATH contains `$HOME/.rbenv/shims` and `$HOME/.rbenv/bin`. Also verify that you have this in your bash profile:
 
 ```bash
 export PATH="$HOME/.rbenv/bin:$PATH"
@@ -32,7 +34,7 @@ Then install bundler
 
 Follow those instructions to [install logstash](https://www.elastic.co/guide/en/logstash/current/installing-logstash.html) before moving to the next section.
 
-## Install dependencies and Build plugin
+## Build and test the plugin
 
 ### Install required packages
 

data/lib/logstash/outputs/loki.rb

@@ -4,7 +4,6 @@ require "logstash/outputs/loki/entry"
 require "logstash/outputs/loki/batch"
 require "logstash/namespace"
 require 'net/http'
-require 'concurrent-edge'
 require 'time'
 require 'uri'
 require 'json'
@@ -30,6 +29,9 @@ class LogStash::Outputs::Loki < LogStash::Outputs::Base
   ## 'TLS'
   config :ca_cert, :validate => :path, :required => false
 
+  ## 'Disable server certificate verification'
+  config :insecure_skip_verify, :validate => :boolean, :default => false, :required => false
+
   ## 'Loki Tenant ID'
   config :tenant_id, :validate => :string, :required => false
 
@@ -45,8 +47,8 @@ class LogStash::Outputs::Loki < LogStash::Outputs::Base
   ## 'Backoff configuration. Initial backoff time between retries. Default 1s'
   config :min_delay, :validate => :number, :default => 1, :required => false
 
-   ## 'Backoff configuration. Maximum backoff time between retries. Default 300s'
-   config :max_delay, :validate => :number, :default => 300, :required => false
+  ## 'Backoff configuration. Maximum backoff time between retries. Default 300s'
+  config :max_delay, :validate => :number, :default => 300, :required => false
 
   ## 'Backoff configuration. Maximum number of retries to do'
   config :retries, :validate => :number, :default => 10, :required => false
@@ -65,10 +67,10 @@ class LogStash::Outputs::Loki < LogStash::Outputs::Base
 
     @logger.info("Loki output plugin", :class => self.class.name)
 
-    # initialize channels
-    @Channel = Concurrent::Channel
-    @entries = @Channel.new
-    @stop = @Channel.new
+    # initialize Queue and Mutex
+    @entries = Queue.new 
+    @mutex = Mutex.new
+    @stop = false
 
     # create nil batch object.
     @batch = nil
@@ -79,7 +81,52 @@ class LogStash::Outputs::Loki < LogStash::Outputs::Base
       validate_ssl_key
     end
 
-    @Channel.go{run()}
+    # start batch_max_wait and batch_max_size threads
+    @batch_wait_thread = Thread.new{max_batch_wait()}
+    @batch_size_thread = Thread.new{max_batch_size()}
+  end
+
+  def max_batch_size
+    loop do
+      @mutex.synchronize do
+        return if @stop
+      end
+  
+      e = @entries.deq
+      return if e.nil?
+
+      @mutex.synchronize do
+        if !add_entry_to_batch(e)
+          @logger.debug("Max batch_size is reached. Sending batch to loki")
+          send(@batch)
+          @batch = Batch.new(e)
+        end
+      end
+    end
+  end
+
+  def max_batch_wait
+    # minimum wait frequency is 10 milliseconds
+	  min_wait_checkfrequency = 1/100
+	  max_wait_checkfrequency = @batch_wait
+	  if max_wait_checkfrequency < min_wait_checkfrequency
+		  max_wait_checkfrequency = min_wait_checkfrequency
+    end
+
+    loop do
+      @mutex.synchronize do
+        return if @stop
+      end
+
+      sleep(max_wait_checkfrequency)
+      if is_batch_expired
+        @mutex.synchronize do
+          @logger.debug("Max batch_wait time is reached. Sending batch to loki")
+          send(@batch)
+          @batch = nil
+        end
+      end
+    end
   end
 
   def ssl_cert?
@@ -102,6 +149,13 @@ class LogStash::Outputs::Loki < LogStash::Outputs::Base
       use_ssl: uri.scheme == 'https'
     }
 
+     # disable server certificate verification
+    if @insecure_skip_verify
+      opts = opts.merge(
+        verify_mode: OpenSSL::SSL::VERIFY_NONE
+      )
+    end
+
     if !@cert.nil? && !@key.nil?
       opts = opts.merge(
         verify_mode: OpenSSL::SSL::VERIFY_PEER,
@@ -118,39 +172,7 @@ class LogStash::Outputs::Loki < LogStash::Outputs::Base
     opts
   end
 
-  def run()
-	  min_wait_checkfrequency = 1/100 #1 millisecond
-	  max_wait_checkfrequency = @batch_wait / 10
-	  if max_wait_checkfrequency < min_wait_checkfrequency
-		  max_wait_checkfrequency = min_wait_checkfrequency
-    end
-
-    @max_wait_check = Concurrent::Channel.tick(max_wait_checkfrequency)
-    loop do
-      Concurrent::Channel.select do |s|
-        s.take(@stop) {
-          return
-        }
-        s.take(@entries) { |e|
-         if !add_entry_to_batch(e)
-           @logger.debug("Max batch_size is reached. Sending batch to loki")
-           send(@batch)
-           @batch = Batch.new(e)
-         end
-        }
-        s.take(@max_wait_check) {
-          # Send batch if max wait time has been reached
-          if is_batch_expired
-            @logger.debug("Max batch_wait time is reached. Sending batch to loki")
-            send(@batch)
-            @batch = nil
-          end
-        }
-      end
-    end
-  end
-
-  # add an entry to the current batch return false if the batch is full
+  # Add an entry to the current batch returns false if the batch is full
   # and the entry can't be added.
   def add_entry_to_batch(e)
     line = e.entry['line']
@@ -181,8 +203,11 @@ class LogStash::Outputs::Loki < LogStash::Outputs::Base
 
   def close
     @entries.close
-    @max_wait_check.close if !@max_wait_check.nil?
-    @stop << true # stop will block until it's accepted by the worker.
+    @mutex.synchronize do 
+      @stop = true 
+    end
+    @batch_wait_thread.join
+    @batch_size_thread.join
 
     # if by any chance we still have a forming batch, we need to send it.
     send(@batch) if !@batch.nil?
@@ -222,13 +247,13 @@ class LogStash::Outputs::Loki < LogStash::Outputs::Base
       raise StandardError.new res
     rescue StandardError => e
       retry_count += 1
-      @logger.warn("Failed to send batch attempt: #{retry_count}/#{@retries}", :error_inspect => e.inspect, :error => e)
+      @logger.warn("Failed to send batch, attempt: #{retry_count}/#{@retries}", :error_inspect => e.inspect, :error => e)
       if retry_count < @retries
         sleep delay
-        if (delay * 2 - delay) > @max_delay
-          delay = delay
-        else
+        if delay * 2 <= @max_delay
           delay = delay * 2
+        else
+          delay = @max_delay
         end
         retry
       else

data/logstash-output-loki.gemspec

@@ -1,6 +1,6 @@
 Gem::Specification.new do |s|
-  s.name = 'logstash-output-loki'
-  s.version         = '1.0.1'
+  s.name    = 'logstash-output-loki'
+  s.version = '1.0.3'
   s.authors = ['Aditya C S','Cyril Tovena']
   s.email   = ['aditya.gnu@gmail.com','cyril.tovena@grafana.com']
 
@@ -22,6 +22,5 @@ Gem::Specification.new do |s|
   #
   s.add_runtime_dependency "logstash-core-plugin-api", ">= 1.60", "<= 2.99"
   s.add_runtime_dependency "logstash-codec-plain", "3.0.6"
-  s.add_runtime_dependency "concurrent-ruby-edge", "0.6.0"
   s.add_development_dependency 'logstash-devutils', "2.0.2"
 end

data/spec/outputs/loki_spec.rb

@@ -95,7 +95,7 @@ describe LogStash::Outputs::Loki do
     it 'should send entry if batch size reached with no tenant' do
       loki = LogStash::Outputs::Loki.new(simple_loki_config.merge!({'batch_wait'=>0.5,'batch_size'=>10}))
       loki.register
-      sent = Concurrent::Channel.new(capacity: 3)
+      sent = Queue.new
       allow(loki).to receive(:send) do |batch|
         Thread.new do
           sent << batch
@@ -103,14 +103,14 @@ describe LogStash::Outputs::Loki do
       end
       loki.receive(event)
       loki.receive(event)
+      sent.deq
+      sent.deq
       loki.close
-      ~sent
-      ~sent
     end
     it 'should send entry while closing' do
       loki = LogStash::Outputs::Loki.new(simple_loki_config.merge!({'batch_wait'=>10,'batch_size'=>10}))
       loki.register
-      sent = Concurrent::Channel.new(capacity: 3)
+      sent = Queue.new
       allow(loki).to receive(:send) do | batch|
         Thread.new  do
           sent << batch
@@ -118,20 +118,21 @@ describe LogStash::Outputs::Loki do
       end
       loki.receive(event)
       loki.close
-      ~sent
+      sent.deq
     end
     it 'should send entry when batch is expiring' do
       loki = LogStash::Outputs::Loki.new(simple_loki_config.merge!({'batch_wait'=>0.5,'batch_size'=>10}))
       loki.register
-      sent = Concurrent::Channel.new(capacity: 3)
+      sent = Queue.new
       allow(loki).to receive(:send) do | batch|
         Thread.new  do
           sent << batch
         end
       end
       loki.receive(event)
-      ~sent
-      expect(loki.batch).to be_nil
+      sent.deq
+      sleep(0.01) # Adding a minimal sleep. In few cases @batch=nil might happen after evaluating for nil
+      expect(loki.batch).to be_nil 
       loki.close
     end
   end

metadata

@@ -1,17 +1,18 @@
 --- !ruby/object:Gem::Specification
 name: logstash-output-loki
 version: !ruby/object:Gem::Version
-  version: 1.0.1
+  version: 1.0.3
 platform: ruby
 authors:
 - Aditya C S
 - Cyril Tovena
-autorequire:
+autorequire: 
 bindir: bin
 cert_chain: []
-date: 2020-07-16 00:00:00.000000000 Z
+date: 2020-11-13 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
+  name: logstash-core-plugin-api
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
@@ -20,7 +21,6 @@ dependencies:
     - - "<="
       - !ruby/object:Gem::Version
         version: '2.99'
-  name: logstash-core-plugin-api
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
@@ -32,12 +32,12 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '2.99'
 - !ruby/object:Gem::Dependency
+  name: logstash-codec-plain
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - '='
       - !ruby/object:Gem::Version
         version: 3.0.6
-  name: logstash-codec-plain
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
@@ -46,26 +46,12 @@ dependencies:
       - !ruby/object:Gem::Version
         version: 3.0.6
 - !ruby/object:Gem::Dependency
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - '='
-      - !ruby/object:Gem::Version
-        version: 0.6.0
-  name: concurrent-ruby-edge
-  type: :runtime
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - '='
-      - !ruby/object:Gem::Version
-        version: 0.6.0
-- !ruby/object:Gem::Dependency
+  name: logstash-devutils
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - '='
       - !ruby/object:Gem::Version
         version: 2.0.2
-  name: logstash-devutils
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
@@ -95,7 +81,7 @@ licenses:
 metadata:
   logstash_plugin: 'true'
   logstash_group: output
-post_install_message:
+post_install_message: 
 rdoc_options: []
 require_paths:
 - lib
@@ -110,8 +96,8 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.0.6
-signing_key:
+rubygems_version: 3.0.3
+signing_key: 
 specification_version: 4
 summary: Output plugin to ship logs to a Grafana Loki server
 test_files: