logstash-output-edge_loki 1.0.8 → 1.0.10

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: a5ec89dff9f872d7371462dc0e39118ac9e0fa9d662fa5eecd4e99e9565f29d7
-  data.tar.gz: b598d411be65be2e16c9ce26d6d954cc63496085b714e66bc6274af6c42ef765
+  metadata.gz: 459c97179f8e5871f7992ce20df5ba1ef2dc95ec0959eecbb1dbfda58c2dd81c
+  data.tar.gz: 672e5fc63bb488cd69fe6522cd210235fd6824d7677a713b45e16d8adc7864ce
 SHA512:
-  metadata.gz: 8eef5ab0b6d08ab8968575e33cb60b88ecc6b45775df95c7abde04229bcc80577f394a976d4041bdd954772f598b67b94ce9da028c4f4ce32cd67a03bd5739da
-  data.tar.gz: b7f7529cc7d32de3e6c47a579f5db49595f2671d156660add48091ff4be26a83621ecb1196b3769854d7eae22ade1e0dbf4ad9bbd3008327a9c6fe83b0b6ee9f
+  metadata.gz: 0727eb5e0abbf2cf0aeba08606b13a4081aa662f5b31afe1caebf7670d6ff62ed6059b1701d6662867feca4f1ab4a2b052eff678d02a7f2ee7849c3c7437b562
+  data.tar.gz: 2417817e5581a05067965c93a5e7e61a0f1403f4db0301c8f80d58eb03eca6e48eeacb9b583a046edb188e716793a15d2b3695ce15ad06eda6dfc4703bd1d2ac

data/lib/logstash/outputs/edge_loki.rb

@@ -16,31 +16,34 @@ class LogStash::Outputs::EDGE_Loki < LogStash::Outputs::Base
   ## 'A single instance of the Output will be shared among the pipeline worker threads'
   concurrency :single
   
-  ## 'Loki Domain'
-  config :domain, :validate => :string, :required => true
-
-  ## 'Loki Endpoint'
-  config :endpoint, :validate => :string, :required => true
+  ## 'Loki URL'
+  config :url, :validate => :string, :required => true
 
   ## 'BasicAuth credentials'
-  config :client_id, :validate => :string, :required => false
-  config :client_secret, :validate => :string, secret: true, :required => false
+  config :client_id, :validate => :string, :required => true
+  config :client_secret, :validate => :string, secret: true, :required => true
 
-  ## 'Client certificate'
-  config :cert, :validate => :path, :required => false
-  config :key, :validate => :path, :required => false
+  ## 'Loki Token URL Domain'
+  config :tokenurl_domain, :validate => :string, :required => true
 
-  ## 'TLS'
-  config :ca_cert, :validate => :path, :required => false
+  ## 'Loki Token URL Endpoint'
+  config :tokenurl_endpoint, :validate => :string, :default => "oauth2/v2.0/token", :required => false
+
+  ## 'Scopes'
+  config :scopes, :validate => :string, :required => true
+
+  ## 'Proxy URL'
+  config :proxy_url, :validate => :string, :required => false
 
   ## 'Disable server certificate verification'
   config :insecure_skip_verify, :validate => :boolean, :default => false, :required => false
 
-  ## 'Loki Tenant ID'
-  config :tenant_id, :validate => :string, :required => false
+  ## 'Client certificate'
+  config :cert, :validate => :path, :required => false
+  config :key, :validate => :path, :required => false
 
-  ## 'Loki Token URL'
-  config :token_url, :validate => :string, :required => false
+  ## 'TLS'
+  config :ca_cert, :validate => :path, :required => false
 
   ## 'Maximum batch size to accrue before pushing to loki. Defaults to 102400 bytes'
   config :batch_size, :validate => :number, :default => 102400, :required => false
@@ -97,7 +100,7 @@ class LogStash::Outputs::EDGE_Loki < LogStash::Outputs::Base
   end
 
   def max_batch_size
-    @logger.info("max_batch_size: started method")
+    #@logger.info("max_batch_size: started method")
     loop do
       @mutex.synchronize do
         return if @stop
@@ -108,7 +111,7 @@ class LogStash::Outputs::EDGE_Loki < LogStash::Outputs::Base
 
       @mutex.synchronize do
         if !add_entry_to_batch(e)
-          @logger.debug("Max batch_size is reached. Sending batch to loki")
+          @logger.debug("Max batch_size is reached. Sending batch to edge-loki")
           send(@batch)
           @batch = Batch.new(e)
         end
@@ -183,8 +186,7 @@ class LogStash::Outputs::EDGE_Loki < LogStash::Outputs::Base
     opts
   end
 
-  # Add an entry to the current batch returns false if the batch is full
-  # and the entry can't be added.
+  # Add an entry to the current batch returns false if the batch is full and the entry can't be added.
   def add_entry_to_batch(e)
     line = e.entry['line']
     # we don't want to send empty lines.
@@ -226,52 +228,58 @@ class LogStash::Outputs::EDGE_Loki < LogStash::Outputs::Base
   end
 
   def send(batch)
-    @logger.info("send: started method")
-    token = createClient()
+    #@logger.info("send: started method")
     payload = batch.to_json
-    res = pensive_http_request(token, payload)
-    @logger.info("send result: ", res => res)
-    if res.is_a?(Net::HTTPSuccess)
-      @logger.debug("Successfully pushed data to loki")
+    if @proxy_url == ""
+      client = createClient()
     else
-      @logger.debug("failed payload", :payload => payload)
+      client = createClientwithProxy()
     end
-  end
-
-  def send1(batch)
-    @logger.info("send: started method")
-    token = createClient()
-    payload = batch.to_json
-    res = loki_http_request(token, payload)
-    if res.is_a?(Net::HTTPSuccess)
-      @logger.debug("Successfully pushed data to loki")
+    access = getToken(client)
+    res = edge_http_request(access, payload)
+    if res == "200"
+      cntLines = JSON.parse(payload)["streams"][0]["values"].size
+      @logger.info("Successfully pushed data to edge_loki")
+      @logger.info("Sent #{cntLines} lines to edge-loki")
     else
-      @logger.debug("failed payload", :payload => payload)
+      @logger.debug("failed payload", :payload => JSON.parse(payload)["streams"][0]["values"])
     end
   end
 
-  def loki_http_request(token, payload)
-    @logger.info("loki_http_request: started method")
-    req = Net::HTTP::Post.new(
-      @uri.request_uri
-    )
-    
-    req.add_field('Content-Type', 'application/json')
-    req.add_field('X-Scope-OrgID', @tenant_id) if @tenant_id
-    req['User-Agent']= 'loki-logstash'
-    req.basic_auth(@client_id, @client_secret) if @client_id
-    req.body = payload
-
-    opts = ssl_opts(@uri)
-
-    @logger.debug("sending #{req.body.length} bytes to loki")
+  def createClient()
+    #@logger.info("createClient: started method")
+    client = OAuth2::Client.new(@client_id, @client_secret, auth_scheme: "basic_auth", site: @tokenurl_domain,:token_url => @tokenurl_endpoint)
+    return client
+  end
+
+  def createClientwithProxy()
+    #@logger.info("createClient: started method")
+	  conn_opts = {}
+	  conn_opts = conn_opts.merge('ssl' => {verify: false})
+	  conn_opts = conn_opts.merge('proxy' => @proxy_url)
+	  client = OAuth2::Client.new(@client_id, @client_secret, auth_scheme: "basic_auth", site: @tokenurl_domain,:token_url => @tokenurl_endpoint, :connection_opts => conn_opts)
+    return client
+  end
+
+  def getToken(client)
+    params = {}
+    opts = {}
+    params = params.merge('scope' => @scopes)
+    access = client.client_credentials.get_token(params, opts)
+    return access
+  end
+
+  def edge_http_request(token, payload)
+    #@logger.info("edge_http_request: started method")
+    cntLines = JSON.parse(payload)["streams"][0]["values"].size
+    @logger.info("processing #{cntLines} lines to edge-loki")
     retry_count = 0
     delay = @min_delay
     begin
-      res = Net::HTTP.start(@uri.host, @uri.port, **opts) { |http|
-        http.request(req)
-      }
-      return res if !res.nil? && res.code.to_i != 429 && res.code.to_i.div(100) != 5
+      res = token.post(@uri, {:body => payload, :headers => {'Content-Type' => 'application/json'}})
+      status_code = "#{res.status}"
+      return status_code if !status_code.nil? && status_code.to_i != 429 && status_code.to_i.div(100) != 5
+      return res
       raise StandardError.new res
     rescue StandardError => e
       retry_count += 1
@@ -291,21 +299,4 @@ class LogStash::Outputs::EDGE_Loki < LogStash::Outputs::Base
     end
   end
 
-  def createClient
-    @logger.info("createClient: started method")
-    client = OAuth2::Client.new(@client_id, @client_secret, site: @domain + @endpoint)
-    client.auth_code.authorize_url(redirect_uri: @token_url)
-    token = client.auth_code.get_token(code, :redirect_uri => @token_url)
-    @logger.info("createClient: started method", :client => client)
-    return token
-  end
-
-  def pensive_http_request(token, payload)
-    @logger.info("pensive_http_request: started method")
-    response = token.get(@endpoint, :headers => { 'Accept' => 'application/json', 'X-Scope-OrgID', @tenant_id }, :params => { page: 1 })
-    hash = JSON.parse(response.body)
-    @logger.info("pensive_http_request: ", :token_output => hash)
-    return hash
-  end
 end
-

data/logstash-output-edge_loki.gemspec

@@ -1,6 +1,6 @@
 Gem::Specification.new do |s|
   s.name    = 'logstash-output-edge_loki'
-  s.version = '1.0.8'
+  s.version = '1.0.10'
   s.authors = ['Britto Prabhu']
   s.email   = ['britto.prabhu@apmterminals.com']
 

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: logstash-output-edge_loki
 version: !ruby/object:Gem::Version
-  version: 1.0.8
+  version: 1.0.10
 platform: ruby
 authors:
 - Britto Prabhu
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2023-09-04 00:00:00.000000000 Z
+date: 2023-09-08 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement