logstash-output-edge_loki 1.0.10 → 1.0.11

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 459c97179f8e5871f7992ce20df5ba1ef2dc95ec0959eecbb1dbfda58c2dd81c
-  data.tar.gz: 672e5fc63bb488cd69fe6522cd210235fd6824d7677a713b45e16d8adc7864ce
+  metadata.gz: 5853b6587938a56259061c3bb90d770d233f3a204efe9867d5d35f761d85fb21
+  data.tar.gz: da0b9595f03cd605c6aa28fc66864bcf4375466879636385cef8816491be6494
 SHA512:
-  metadata.gz: 0727eb5e0abbf2cf0aeba08606b13a4081aa662f5b31afe1caebf7670d6ff62ed6059b1701d6662867feca4f1ab4a2b052eff678d02a7f2ee7849c3c7437b562
-  data.tar.gz: 2417817e5581a05067965c93a5e7e61a0f1403f4db0301c8f80d58eb03eca6e48eeacb9b583a046edb188e716793a15d2b3695ce15ad06eda6dfc4703bd1d2ac
+  metadata.gz: 7fb2656179619773e32aa46625819b2809c31da737e229033f483d5429d63f585a9b9dffd22a1fcd4daed1971137cc189badaee468c32a83a53f029caffcc99e
+  data.tar.gz: 19cb39bc4d5d9e6b79bf2014f3106615ea788e8abdb3d2abb2d90856234a766853ef859bc7ed0934cf2298c5ce8664d93ca2a7cd9d50725de7c56f8aa3cbc442

data/lib/logstash/outputs/edge_loki.rb

@@ -8,6 +8,7 @@ require 'time'
 require 'uri'
 require 'json'
 require 'oauth2'
+require "zlib"
 
 class LogStash::Outputs::EDGE_Loki < LogStash::Outputs::Base
   include Loki
@@ -51,6 +52,9 @@ class LogStash::Outputs::EDGE_Loki < LogStash::Outputs::Base
   ## 'Interval in seconds to wait before pushing a batch of records to loki. Defaults to 1 second'
   config :batch_wait, :validate => :number, :default => 1, :required => false
 
+  # Set this to true if you want to enable gzip compression for your http requests
+  config :http_compression, :validate => :boolean, :default => false
+
   ## 'Log line field to pick from logstash. Defaults to "message"'
   config :message_field, :validate => :string, :default => "message", :required => false
 
@@ -271,12 +275,22 @@ class LogStash::Outputs::EDGE_Loki < LogStash::Outputs::Base
 
   def edge_http_request(token, payload)
     #@logger.info("edge_http_request: started method")
+	  headers = {}
     cntLines = JSON.parse(payload)["streams"][0]["values"].size
     @logger.info("processing #{cntLines} lines to edge-loki")
     retry_count = 0
     delay = @min_delay
     begin
-      res = token.post(@uri, {:body => payload, :headers => {'Content-Type' => 'application/json'}})
+      puts payload
+	  headers["Content-Type"] = "application/json"
+      # Compress the payload and add appropriate header
+  	  if @http_compression == true
+	    headers["Content-Encoding"] = "gzip"
+	    payload1 = payload
+	    payload = gzip(payload1)
+	  end
+	  puts payload
+      res = token.post(@uri, {:body => payload, :headers => headers})
       status_code = "#{res.status}"
       return status_code if !status_code.nil? && status_code.to_i != 429 && status_code.to_i.div(100) != 5
       return res
@@ -299,4 +313,14 @@ class LogStash::Outputs::EDGE_Loki < LogStash::Outputs::Base
     end
   end
 
-end
+  # gzip data
+  def gzip(data)
+    gz = StringIO.new
+    gz.set_encoding("BINARY")
+    z = Zlib::GzipWriter.new(gz)
+    z.write(data)
+    z.close
+    gz.string
+  end
+
+end

data/lib/logstash/outputs/edge_loki_20240722.rb

@@ -0,0 +1,303 @@
+# encoding: utf-8
+require "logstash/outputs/base"
+require "logstash/outputs/loki/entry"
+require "logstash/outputs/loki/batch"
+require "logstash/namespace"
+require 'net/http'
+require 'time'
+require 'uri'
+require 'json'
+require 'oauth2'
+
+class LogStash::Outputs::EDGE_Loki < LogStash::Outputs::Base
+  include Loki
+  config_name "edge_loki"
+
+  ## 'A single instance of the Output will be shared among the pipeline worker threads'
+  concurrency :single
+  
+  ## 'Loki URL'
+  config :url, :validate => :string, :required => true
+
+  ## 'BasicAuth credentials'
+  config :client_id, :validate => :string, :required => true
+  config :client_secret, :validate => :string, secret: true, :required => true
+
+  ## 'Loki Token URL Domain'
+  config :tokenurl_domain, :validate => :string, :required => true
+
+  ## 'Loki Token URL Endpoint'
+  config :tokenurl_endpoint, :validate => :string, :default => "oauth2/v2.0/token", :required => false
+
+  ## 'Scopes'
+  config :scopes, :validate => :string, :required => true
+
+  ## 'Proxy URL'
+  config :proxy_url, :validate => :string, :required => false
+
+  ## 'Disable server certificate verification'
+  config :insecure_skip_verify, :validate => :boolean, :default => false, :required => false
+
+  ## 'Client certificate'
+  config :cert, :validate => :path, :required => false
+  config :key, :validate => :path, :required => false
+
+  ## 'TLS'
+  config :ca_cert, :validate => :path, :required => false
+
+  ## 'Maximum batch size to accrue before pushing to loki. Defaults to 102400 bytes'
+  config :batch_size, :validate => :number, :default => 102400, :required => false
+
+  ## 'Interval in seconds to wait before pushing a batch of records to loki. Defaults to 1 second'
+  config :batch_wait, :validate => :number, :default => 1, :required => false
+
+  ## 'Log line field to pick from logstash. Defaults to "message"'
+  config :message_field, :validate => :string, :default => "message", :required => false
+
+  ## 'Backoff configuration. Initial backoff time between retries. Default 1s'
+  config :min_delay, :validate => :number, :default => 1, :required => false
+
+  ## 'An array of fields to map to labels, if defined only fields in this list will be mapped.'
+  config :include_fields, :validate => :array, :default => [], :required => false
+
+  ## 'Backoff configuration. Maximum backoff time between retries. Default 300s'
+  config :max_delay, :validate => :number, :default => 300, :required => false
+
+  ## 'Backoff configuration. Maximum number of retries to do'
+  config :retries, :validate => :number, :default => 10, :required => false
+
+  attr_reader :batch
+  public
+  def register
+    @uri = URI.parse(@url)
+    unless @uri.is_a?(URI::HTTP) || @uri.is_a?(URI::HTTPS)
+      raise LogStash::ConfigurationError, "url parameter must be valid HTTP, currently '#{@url}'"
+    end
+
+    if @min_delay > @max_delay
+      raise LogStash::ConfigurationError, "Min delay should be less than Max delay, currently 'Min delay is #{@min_delay} and Max delay is #{@max_delay}'"
+    end
+
+    @logger.info("Loki output plugin", :class => self.class.name)
+
+    # initialize Queue and Mutex
+    @entries = Queue.new 
+    @mutex = Mutex.new
+    @stop = false
+
+    # create nil batch object.
+    @batch = nil
+
+    # validate certs
+    if ssl_cert?
+      load_ssl
+      validate_ssl_key
+    end
+
+    # start batch_max_wait and batch_max_size threads
+    @batch_wait_thread = Thread.new{max_batch_wait()}
+    @batch_size_thread = Thread.new{max_batch_size()}
+  end
+
+  def max_batch_size
+    #@logger.info("max_batch_size: started method")
+    loop do
+      @mutex.synchronize do
+        return if @stop
+      end
+  
+      e = @entries.deq
+      return if e.nil?
+
+      @mutex.synchronize do
+        if !add_entry_to_batch(e)
+          @logger.debug("Max batch_size is reached. Sending batch to edge-loki")
+          send(@batch)
+          @batch = Batch.new(e)
+        end
+      end
+    end
+  end
+
+  def max_batch_wait
+    # minimum wait frequency is 10 milliseconds
+	  min_wait_checkfrequency = 1/100
+	  max_wait_checkfrequency = @batch_wait
+	  if max_wait_checkfrequency < min_wait_checkfrequency
+		  max_wait_checkfrequency = min_wait_checkfrequency
+    end
+
+    loop do
+      @mutex.synchronize do
+        return if @stop
+      end
+
+      sleep(max_wait_checkfrequency)
+      if is_batch_expired
+        @mutex.synchronize do
+          @logger.debug("Max batch_wait time is reached. Sending batch to loki")
+          send(@batch)
+          @batch = nil
+        end
+      end
+    end
+  end
+
+  def ssl_cert?
+    !@key.nil? && !@cert.nil?
+  end
+
+  def load_ssl
+    @cert = OpenSSL::X509::Certificate.new(File.read(@cert)) if @cert
+    @key = OpenSSL::PKey.read(File.read(@key)) if @key
+  end
+
+  def validate_ssl_key
+    if !@key.is_a?(OpenSSL::PKey::RSA) && !@key.is_a?(OpenSSL::PKey::DSA)
+      raise LogStash::ConfigurationError, "Unsupported private key type '#{@key.class}''"
+    end
+  end
+
+  def ssl_opts(uri)
+    opts = {
+      use_ssl: uri.scheme == 'https'
+    }
+
+     # disable server certificate verification
+    if @insecure_skip_verify
+      opts = opts.merge(
+        verify_mode: OpenSSL::SSL::VERIFY_NONE
+      )
+    end
+
+    if !@cert.nil? && !@key.nil?
+      opts = opts.merge(
+        verify_mode: OpenSSL::SSL::VERIFY_PEER,
+        cert: @cert,
+        key: @key
+      )
+    end
+
+    unless @ca_cert.nil?
+      opts = opts.merge(
+        ca_file: @ca_cert
+      )
+    end
+    opts
+  end
+
+  # Add an entry to the current batch returns false if the batch is full and the entry can't be added.
+  def add_entry_to_batch(e)
+    line = e.entry['line']
+    # we don't want to send empty lines.
+    return true if line.to_s.strip.empty?
+
+    if @batch.nil?
+      @batch = Batch.new(e)
+      return true
+    end
+
+    if @batch.size_bytes_after(line) > @batch_size
+      return false
+    end
+    @batch.add(e)
+    return true
+  end
+
+  def is_batch_expired
+    return !@batch.nil? && @batch.age() >= @batch_wait
+  end
+
+  ## Receives logstash events
+  public
+  def receive(event)
+    @entries << Entry.new(event, @message_field, @include_fields)
+  end
+
+  def close
+    @entries.close
+    @mutex.synchronize do 
+      @stop = true 
+    end
+    @batch_wait_thread.join
+    @batch_size_thread.join
+
+    # if by any chance we still have a forming batch, we need to send it.
+    send(@batch) if !@batch.nil?
+    @batch = nil
+  end
+
+  def send(batch)
+    #@logger.info("send: started method")
+    payload = batch.to_json
+    if @proxy_url == ""
+      client = createClient()
+    else
+      client = createClientwithProxy()
+    end
+    access = getToken(client)
+    res = edge_http_request(access, payload)
+    if res == "200"
+      cntLines = JSON.parse(payload)["streams"][0]["values"].size
+      @logger.info("Successfully pushed data to edge_loki")
+      @logger.info("Sent #{cntLines} lines to edge-loki")
+    else
+      @logger.debug("failed payload", :payload => JSON.parse(payload)["streams"][0]["values"])
+    end
+  end
+
+  def createClient()
+    #@logger.info("createClient: started method")
+    client = OAuth2::Client.new(@client_id, @client_secret, auth_scheme: "basic_auth", site: @tokenurl_domain,:token_url => @tokenurl_endpoint)
+    return client
+  end
+
+  def createClientwithProxy()
+    #@logger.info("createClient: started method")
+	  conn_opts = {}
+	  conn_opts = conn_opts.merge('ssl' => {verify: false})
+	  conn_opts = conn_opts.merge('proxy' => @proxy_url)
+	  client = OAuth2::Client.new(@client_id, @client_secret, auth_scheme: "basic_auth", site: @tokenurl_domain,:token_url => @tokenurl_endpoint, :connection_opts => conn_opts)
+    return client
+  end
+
+  def getToken(client)
+    params = {}
+    opts = {}
+    params = params.merge('scope' => @scopes)
+    access = client.client_credentials.get_token(params, opts)
+    return access
+  end
+
+  def edge_http_request(token, payload)
+    #@logger.info("edge_http_request: started method")
+    cntLines = JSON.parse(payload)["streams"][0]["values"].size
+    @logger.info("processing #{cntLines} lines to edge-loki")
+    retry_count = 0
+    delay = @min_delay
+    begin
+      puts payload
+      res = token.post(@uri, {:body => payload, :headers => {'Content-Type' => 'application/json'}})
+      status_code = "#{res.status}"
+      return status_code if !status_code.nil? && status_code.to_i != 429 && status_code.to_i.div(100) != 5
+      return res
+      raise StandardError.new res
+    rescue StandardError => e
+      retry_count += 1
+      @logger.warn("Failed to send batch, attempt: #{retry_count}/#{@retries}", :error_inspect => e.inspect, :error => e)
+      if retry_count < @retries
+        sleep delay
+        if delay * 2 <= @max_delay
+          delay = delay * 2
+        else
+          delay = @max_delay
+        end
+        retry
+      else
+        @logger.error("Failed to send batch", :error_inspect => e.inspect, :error => e)
+        return res
+      end
+    end
+  end
+
+end

data/logstash-output-edge_loki.gemspec

@@ -1,11 +1,11 @@
 Gem::Specification.new do |s|
   s.name    = 'logstash-output-edge_loki'
-  s.version = '1.0.10'
+  s.version = '1.0.11'
   s.authors = ['Britto Prabhu']
   s.email   = ['britto.prabhu@apmterminals.com']
 
-  s.summary       = 'Output plugin to ship logs to Pensive Loki server'
-  s.description   = 'Output plugin to ship logs to Pensive Loki server'
+  s.summary       = 'Output plugin to ship logs to Loki server with oAuth2'
+  s.description   = 'Output plugin to ship logs to Loki server with oAuth2'
   s.homepage      = 'https://github.com/Maersk-Global/apmt-observability-deployment'
   s.license       = 'Apache-2.0'
   s.require_paths = ["lib"]

metadata

@@ -1,30 +1,31 @@
 --- !ruby/object:Gem::Specification
 name: logstash-output-edge_loki
 version: !ruby/object:Gem::Version
-  version: 1.0.10
+  version: 1.0.11
 platform: ruby
 authors:
 - Britto Prabhu
-autorequire:
+autorequire: 
 bindir: bin
 cert_chain: []
-date: 2023-09-08 00:00:00.000000000 Z
+date: 2024-08-06 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
+  name: oauth2
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
         version: 2.0.9
-  name: oauth2
-  prerelease: false
   type: :runtime
+  prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
         version: 2.0.9
 - !ruby/object:Gem::Dependency
+  name: logstash-core-plugin-api
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
@@ -33,9 +34,8 @@ dependencies:
     - - "<="
       - !ruby/object:Gem::Version
         version: '2.99'
-  name: logstash-core-plugin-api
-  prerelease: false
   type: :runtime
+  prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
@@ -45,34 +45,34 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '2.99'
 - !ruby/object:Gem::Dependency
+  name: logstash-codec-plain
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - '='
       - !ruby/object:Gem::Version
         version: 3.1.0
-  name: logstash-codec-plain
-  prerelease: false
   type: :runtime
+  prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - '='
       - !ruby/object:Gem::Version
         version: 3.1.0
 - !ruby/object:Gem::Dependency
+  name: logstash-devutils
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - '='
       - !ruby/object:Gem::Version
         version: 2.0.2
-  name: logstash-devutils
-  prerelease: false
   type: :development
+  prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - '='
       - !ruby/object:Gem::Version
         version: 2.0.2
-description: Output plugin to ship logs to Pensive Loki server
+description: Output plugin to ship logs to Loki server with oAuth2
 email:
 - britto.prabhu@apmterminals.com
 executables: []
@@ -82,6 +82,7 @@ files:
 - Gemfile
 - README.md
 - lib/logstash/outputs/edge_loki.rb
+- lib/logstash/outputs/edge_loki_20240722.rb
 - lib/logstash/outputs/loki/batch.rb
 - lib/logstash/outputs/loki/entry.rb
 - logstash-output-edge_loki.gemspec
@@ -93,7 +94,7 @@ licenses:
 metadata:
   logstash_plugin: 'true'
   logstash_group: output
-post_install_message:
+post_install_message: 
 rdoc_options: []
 require_paths:
 - lib
@@ -108,10 +109,10 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.2.33
-signing_key:
+rubygems_version: 3.0.3.1
+signing_key: 
 specification_version: 4
-summary: Output plugin to ship logs to Pensive Loki server
+summary: Output plugin to ship logs to Loki server with oAuth2
 test_files:
 - spec/outputs/loki/entry_spec.rb
 - spec/outputs/loki_spec.rb