logstash-output-edge_loki 1.0.7 → 1.0.10

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: b7815895543bbcb18b5a4b9383f083223f229713c537f336f48cc2cffd76fed7
-  data.tar.gz: d8d68d9291a738b47d4de30ec1d3c2a62894aae498dfb6869e14684d4cbebb37
+  metadata.gz: 459c97179f8e5871f7992ce20df5ba1ef2dc95ec0959eecbb1dbfda58c2dd81c
+  data.tar.gz: 672e5fc63bb488cd69fe6522cd210235fd6824d7677a713b45e16d8adc7864ce
 SHA512:
-  metadata.gz: 4baf39cedf6a80af43190b002452d7565e1e1277de335aa708fd0533f76a527250ec90c420b1a643f7adf4c61d9dfe3216d9df4c10217aa926b126e703b504b1
-  data.tar.gz: 7fc3f6ad11def007015b2180ed15c663919babce9f36d5e69d1533751f2ebbf78325f8fb949c7f92c8cf90aadc072e428f478cbc8f8dc8c6e37592072f93aebe
+  metadata.gz: 0727eb5e0abbf2cf0aeba08606b13a4081aa662f5b31afe1caebf7670d6ff62ed6059b1701d6662867feca4f1ab4a2b052eff678d02a7f2ee7849c3c7437b562
+  data.tar.gz: 2417817e5581a05067965c93a5e7e61a0f1403f4db0301c8f80d58eb03eca6e48eeacb9b583a046edb188e716793a15d2b3695ce15ad06eda6dfc4703bd1d2ac

data/lib/logstash/outputs/edge_loki.rb

@@ -15,29 +15,35 @@ class LogStash::Outputs::EDGE_Loki < LogStash::Outputs::Base
 
   ## 'A single instance of the Output will be shared among the pipeline worker threads'
   concurrency :single
-
+  
   ## 'Loki URL'
   config :url, :validate => :string, :required => true
 
   ## 'BasicAuth credentials'
-  config :client_id, :validate => :string, :required => false
-  config :client_secret, :validate => :string, secret: true, :required => false
+  config :client_id, :validate => :string, :required => true
+  config :client_secret, :validate => :string, secret: true, :required => true
 
-  ## 'Client certificate'
-  config :cert, :validate => :path, :required => false
-  config :key, :validate => :path, :required => false
+  ## 'Loki Token URL Domain'
+  config :tokenurl_domain, :validate => :string, :required => true
 
-  ## 'TLS'
-  config :ca_cert, :validate => :path, :required => false
+  ## 'Loki Token URL Endpoint'
+  config :tokenurl_endpoint, :validate => :string, :default => "oauth2/v2.0/token", :required => false
+
+  ## 'Scopes'
+  config :scopes, :validate => :string, :required => true
+
+  ## 'Proxy URL'
+  config :proxy_url, :validate => :string, :required => false
 
   ## 'Disable server certificate verification'
   config :insecure_skip_verify, :validate => :boolean, :default => false, :required => false
 
-  ## 'Loki Tenant ID'
-  config :tenant_id, :validate => :string, :required => false
+  ## 'Client certificate'
+  config :cert, :validate => :path, :required => false
+  config :key, :validate => :path, :required => false
 
-  ## 'Loki Token URL'
-  config :token_url, :validate => :string, :required => false
+  ## 'TLS'
+  config :ca_cert, :validate => :path, :required => false
 
   ## 'Maximum batch size to accrue before pushing to loki. Defaults to 102400 bytes'
   config :batch_size, :validate => :number, :default => 102400, :required => false
@@ -94,7 +100,7 @@ class LogStash::Outputs::EDGE_Loki < LogStash::Outputs::Base
   end
 
   def max_batch_size
-    @logger.info("max_batch_size: started method")
+    #@logger.info("max_batch_size: started method")
     loop do
       @mutex.synchronize do
         return if @stop
@@ -105,7 +111,7 @@ class LogStash::Outputs::EDGE_Loki < LogStash::Outputs::Base
 
       @mutex.synchronize do
         if !add_entry_to_batch(e)
-          @logger.debug("Max batch_size is reached. Sending batch to loki")
+          @logger.debug("Max batch_size is reached. Sending batch to edge-loki")
           send(@batch)
           @batch = Batch.new(e)
         end
@@ -180,8 +186,7 @@ class LogStash::Outputs::EDGE_Loki < LogStash::Outputs::Base
     opts
   end
 
-  # Add an entry to the current batch returns false if the batch is full
-  # and the entry can't be added.
+  # Add an entry to the current batch returns false if the batch is full and the entry can't be added.
   def add_entry_to_batch(e)
     line = e.entry['line']
     # we don't want to send empty lines.
@@ -223,38 +228,58 @@ class LogStash::Outputs::EDGE_Loki < LogStash::Outputs::Base
   end
 
   def send(batch)
-    @logger.info("send: started method")
-    client = createClient()
+    #@logger.info("send: started method")
     payload = batch.to_json
-    res = loki_http_request(client, payload)
-    if res.is_a?(Net::HTTPSuccess)
-      @logger.debug("Successfully pushed data to loki")
+    if @proxy_url == ""
+      client = createClient()
+    else
+      client = createClientwithProxy()
+    end
+    access = getToken(client)
+    res = edge_http_request(access, payload)
+    if res == "200"
+      cntLines = JSON.parse(payload)["streams"][0]["values"].size
+      @logger.info("Successfully pushed data to edge_loki")
+      @logger.info("Sent #{cntLines} lines to edge-loki")
     else
-      @logger.debug("failed payload", :payload => payload)
+      @logger.debug("failed payload", :payload => JSON.parse(payload)["streams"][0]["values"])
     end
   end
 
-  def loki_http_request(client, payload)
-    @logger.info("loki_http_request: started method")
-    req = Net::HTTP::Post.new(
-      @uri.request_uri
-    )
-    client.add_field('Content-Type', 'application/json')
-    client.add_field('X-Scope-OrgID', @tenant_id) if @tenant_id
-    client['User-Agent']= 'loki-logstash'
-    client.basic_auth(@client_id, @client_secret) if @client_id
-    client.body = payload
+  def createClient()
+    #@logger.info("createClient: started method")
+    client = OAuth2::Client.new(@client_id, @client_secret, auth_scheme: "basic_auth", site: @tokenurl_domain,:token_url => @tokenurl_endpoint)
+    return client
+  end
 
-    opts = ssl_opts(@uri)
+  def createClientwithProxy()
+    #@logger.info("createClient: started method")
+	  conn_opts = {}
+	  conn_opts = conn_opts.merge('ssl' => {verify: false})
+	  conn_opts = conn_opts.merge('proxy' => @proxy_url)
+	  client = OAuth2::Client.new(@client_id, @client_secret, auth_scheme: "basic_auth", site: @tokenurl_domain,:token_url => @tokenurl_endpoint, :connection_opts => conn_opts)
+    return client
+  end
 
-    @logger.debug("sending #{client.body.length} bytes to loki")
+  def getToken(client)
+    params = {}
+    opts = {}
+    params = params.merge('scope' => @scopes)
+    access = client.client_credentials.get_token(params, opts)
+    return access
+  end
+
+  def edge_http_request(token, payload)
+    #@logger.info("edge_http_request: started method")
+    cntLines = JSON.parse(payload)["streams"][0]["values"].size
+    @logger.info("processing #{cntLines} lines to edge-loki")
     retry_count = 0
     delay = @min_delay
     begin
-      res = Net::HTTP.start(@uri.host, @uri.port, **opts) { |http|
-        http.request(client)
-      }
-      return res if !res.nil? && res.code.to_i != 429 && res.code.to_i.div(100) != 5
+      res = token.post(@uri, {:body => payload, :headers => {'Content-Type' => 'application/json'}})
+      status_code = "#{res.status}"
+      return status_code if !status_code.nil? && status_code.to_i != 429 && status_code.to_i.div(100) != 5
+      return res
       raise StandardError.new res
     rescue StandardError => e
       retry_count += 1
@@ -274,13 +299,4 @@ class LogStash::Outputs::EDGE_Loki < LogStash::Outputs::Base
     end
   end
 
-  def createClient
-    @logger.info("createClient: started method")
-    client = OAuth2::Client.new(@client_id, @client_secret, site: @url)
-    client.auth_code.authorize_url(redirect_uri: @token_url)
-    @logger.info("createClient: started method", :client => client)
-    return client
-  end
-
 end
-

data/logstash-output-edge_loki.gemspec

@@ -1,6 +1,6 @@
 Gem::Specification.new do |s|
   s.name    = 'logstash-output-edge_loki'
-  s.version = '1.0.7'
+  s.version = '1.0.10'
   s.authors = ['Britto Prabhu']
   s.email   = ['britto.prabhu@apmterminals.com']
 

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: logstash-output-edge_loki
 version: !ruby/object:Gem::Version
-  version: 1.0.7
+  version: 1.0.10
 platform: ruby
 authors:
 - Britto Prabhu
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2023-09-04 00:00:00.000000000 Z
+date: 2023-09-08 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement