fluent-plugin-grafana-loki 1.2.14 → 1.2.18

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 1fd9308b0afb758b4718b7ad46fc5a3c154400a93514ea79fe3453279abe5cdc
-  data.tar.gz: b67621f7aa29c775f6e81f546623dec74d326097071f2ce5bd04523ffd865769
+  metadata.gz: b4a8acb3f25f565acd7f4071ea961ab1d7a76b7a2f80068fbcc77c3e380f3f5d
+  data.tar.gz: 9b0c3124a9a12d2c5f2f5a1a96d885fe257040b0c58db62bd323d8799a7c62bc
 SHA512:
-  metadata.gz: 5e6fee1480cdd594f027cbbb22e503a7ecc0dd2384a6b5d46083eed2a6cdbb820d3c3bfe8f98f2cd197d89cbee2876d8a6659d57a070980b609e6b0cc99fec7a
-  data.tar.gz: 5452556eed1158c3ccaf5bcb5509a773ff89ad4f64e7b3e778603aef0d221cbd2d1ea5ff0e7bed47b1d68b51abae9bf58fccab5996b9ca67bcbfaf81a977cafe
+  metadata.gz: 65384975b8afc4d2fe377d7551caf461b14ea92dd9b21d254bf3cd001582807918671f724e902d012694fd7907601eb738c8d9f7adb14b2474163154a79eaa8b
+  data.tar.gz: f2fb6366e91d22068797f43cf7b439b06d788a4dac33936bd62fa7895056f1f03e5b439ba2e31e0523152af8c0b2a6c380cc15eb1b35502cefcd5a8daeaeb601

data/README.md

@@ -6,14 +6,17 @@ See [docs/client/fluentd/README.md](../../docs/sources/clients/fluentd/_index.md
 
 ## Development
 
-After checking out the repo, run `bin/setup` to install dependencies. Then, run `bundle exec rake spec` to run the tests. You can also run `bin/console` for an interactive prompt that will allow you to experiment.
+After checking out the repo, run `bin/setup` to install dependencies. Then, run `bin/test` to run the tests. You can also run `bin/console` for an interactive prompt that will allow you to experiment.
 
-To install this gem onto your local machine, run `bundle exec rake install`. To release a new version, update the version number in `fluent-plugin-grafana-loki.gemspec`, and then run `bundle exec rake release`, which will create a git tag for the version, push git commits and tags, and push the `.gem` file to [rubygems.org](https://rubygems.org).
+To install this gem onto your local machine, run `ruby -S bundle exec rake install`. To release a new version, update the version number in `fluent-plugin-grafana-loki.gemspec`, and then run `ruby -S bundle exec rake release`, which will create a git tag for the version, push git commits and tags, and push the `.gem` file to [rubygems.org](https://rubygems.org).
 
-To create the gem: `gem build fluent-plugin-grafana-loki.gemspec`
+To create the gem: `ruby -S gem build fluent-plugin-grafana-loki.gemspec`
 
 Useful additions:
-  `gem install rubocop`
+
+```bash
+ruby -S gem install rubocop
+```
 
 ## Testing
 

data/bin/setup

@@ -1,7 +1,9 @@
 #!/usr/bin/env bash
+
 set -euo pipefail
-IFS=$'\n\t'
-set -vx
 
-gem install bundler
-bundle install
+ruby --version
+echo ""
+ruby -S gem install bundler --version 2.3.4
+ruby -S bundle config set --local path $(pwd)/vendor/bundle
+ruby -S bundle install

data/bin/test

@@ -0,0 +1,5 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+
+ruby -S bundle exec rspec

data/lib/fluent/plugin/out_loki.rb

@@ -35,21 +35,24 @@ module Fluent
 
       DEFAULT_BUFFER_TYPE = 'memory'
 
-      desc 'url of loki server'
+      desc 'Loki API base URL'
       config_param :url, :string, default: 'https://logs-prod-us-central1.grafana.net'
 
-      desc 'BasicAuth credentials'
+      desc 'Authentication: basic auth credentials'
       config_param :username, :string, default: nil
       config_param :password, :string, default: nil, secret: true
 
-      desc 'Client certificate'
+      desc 'Authentication: Authorization header with Bearer token scheme'
+      config_param :bearer_token_file, :string, default: nil
+
+      desc 'TLS: parameters for presenting a client certificate'
       config_param :cert, :string, default: nil
       config_param :key, :string, default: nil
 
-      desc 'TLS'
+      desc 'TLS: CA certificate file for server certificate verification'
       config_param :ca_cert, :string, default: nil
 
-      desc 'Disable server certificate verification'
+      desc 'TLS: disable server certificate verification'
       config_param :insecure_tls, :bool, default: false
 
       desc 'Loki tenant id'
@@ -80,7 +83,7 @@ module Fluent
         super
         @uri = URI.parse(@url + '/loki/api/v1/push')
         unless @uri.is_a?(URI::HTTP) || @uri.is_a?(URI::HTTPS)
-          raise Fluent::ConfigError, 'url parameter must be valid HTTP'
+          raise Fluent::ConfigError, 'URL parameter must have HTTP/HTTPS scheme'
         end
 
         @record_accessors = {}
@@ -96,24 +99,42 @@ module Fluent
           @remove_keys_accessors.push(record_accessor_create(key))
         end
 
-        if ssl_cert?
-          load_ssl
-          validate_ssl_key
+        # If configured, load and validate client certificate (and corresponding key)
+        if client_cert_configured?
+          load_client_cert
+          validate_client_cert_key
+        end
+
+        raise "bearer_token_file #{@bearer_token_file} not found" if !@bearer_token_file.nil? && !File.exist?(@bearer_token_file)
+
+        @auth_token_bearer = nil
+        if !@bearer_token_file.nil?
+          if !File.exist?(@bearer_token_file)
+            raise "bearer_token_file #{@bearer_token_file} not found"
+          end
+
+          # Read the file once, assume long-lived authentication token.
+          @auth_token_bearer = File.read(@bearer_token_file)
+          if @auth_token_bearer.empty?
+            raise "bearer_token_file #{@bearer_token_file} is empty"
+          end
+          log.info "will use Bearer token from bearer_token_file #{@bearer_token_file} in Authorization header"
         end
 
+
         raise "CA certificate file #{@ca_cert} not found" if !@ca_cert.nil? && !File.exist?(@ca_cert)
       end
 
-      def ssl_cert?
+      def client_cert_configured?
         !@key.nil? && !@cert.nil?
       end
 
-      def load_ssl
+      def load_client_cert
         @cert = OpenSSL::X509::Certificate.new(File.read(@cert)) if @cert
         @key = OpenSSL::PKey.read(File.read(@key)) if @key
       end
 
-      def validate_ssl_key
+      def validate_client_cert_key
         if !@key.is_a?(OpenSSL::PKey::RSA) && !@key.is_a?(OpenSSL::PKey::DSA)
           raise "Unsupported private key type #{key.class}"
         end
@@ -123,13 +144,6 @@ module Fluent
         true
       end
 
-      def http_opts(uri)
-        opts = {
-          use_ssl: uri.scheme == 'https'
-        }
-        opts
-      end
-
       # flush a chunk to loki
       def write(chunk)
         # streams by label
@@ -141,7 +155,10 @@ module Fluent
         # add ingest path to loki url
         res = loki_http_request(body, tenant)
 
-        return if res.is_a?(Net::HTTPSuccess)
+        if res.is_a?(Net::HTTPSuccess)
+          log.debug "POST request was responded to with status code #{res.code}"
+          return
+        end
 
         res_summary = "#{res.code} #{res.message} #{res.body}"
         log.warn "failed to write post to #{@uri} (#{res_summary})"
@@ -151,19 +168,19 @@ module Fluent
         raise(LogPostError, res_summary) if res.is_a?(Net::HTTPTooManyRequests) || res.is_a?(Net::HTTPServerError)
       end
 
-      def ssl_opts(uri)
+      def http_request_opts(uri)
         opts = {
           use_ssl: uri.scheme == 'https'
         }
 
-        # Disable server TLS certificate verification
+        # Optionally disable server server certificate verification.
         if @insecure_tls
           opts = opts.merge(
             verify_mode: OpenSSL::SSL::VERIFY_NONE
           )
         end
 
-        # Verify client TLS certificate
+        # Optionally present client certificate
         if !@cert.nil? && !@key.nil?
           opts = opts.merge(
             cert: @cert,
@@ -171,7 +188,8 @@ module Fluent
           )
         end
 
-        # Specify custom certificate authority
+        # For server certificate verification: set custom CA bundle.
+        # Only takes effect when `insecure_tls` is not set.
         unless @ca_cert.nil?
           opts = opts.merge(
             ca_file: @ca_cert
@@ -194,11 +212,12 @@ module Fluent
           @uri.request_uri
         )
         req.add_field('Content-Type', 'application/json')
+        req.add_field('Authorization', "Bearer #{@auth_token_bearer}") if !@auth_token_bearer.nil?
         req.add_field('X-Scope-OrgID', tenant) if tenant
         req.body = Yajl.dump(body)
         req.basic_auth(@username, @password) if @username
 
-        opts = ssl_opts(@uri)
+        opts = http_request_opts(@uri)
 
         msg = "sending #{req.body.length} bytes to loki"
         msg += " (tenant: \"#{tenant}\")" if tenant
@@ -259,7 +278,17 @@ module Fluent
           when :key_value
             formatted_labels = []
             record.each do |k, v|
-              formatted_labels.push(%(#{k}="#{v}"))
+              # Remove non UTF-8 characters by force-encoding the string
+              if v.is_a?(String)
+                v = v.encode('utf-8', invalid: :replace)
+              end
+              # Escape double quotes and backslashes by prefixing them with a backslash
+              v = v.to_s.gsub(%r{(["\\])}, '\\\\\1')
+              if v.include?(' ') || v.include?('=')
+                formatted_labels.push(%(#{k}="#{v}"))
+              else
+                formatted_labels.push(%(#{k}=#{v}))
+              end
             end
             line = formatted_labels.join(' ')
           end
@@ -267,7 +296,6 @@ module Fluent
         line
       end
 
-      #
       # convert a line to loki line with labels
       def line_to_loki(record)
         chunk_labels = {}
@@ -281,9 +309,11 @@ module Fluent
 
           if @extract_kubernetes_labels && record.key?('kubernetes')
             kubernetes_labels = record['kubernetes']['labels']
-            kubernetes_labels.each_key do |l|
-              new_key = l.gsub(%r{[.\-\/]}, '_')
-              chunk_labels[new_key] = kubernetes_labels[l]
+            if !kubernetes_labels.nil?
+              kubernetes_labels.each_key do |l|
+                new_key = l.gsub(%r{[.\-\/]}, '_')
+                chunk_labels[new_key] = kubernetes_labels[l]
+              end
             end
           end
 
@@ -316,10 +346,8 @@ module Fluent
       # iterate through each chunk and create a loki stream entry
       def chunk_to_loki(chunk)
         streams = {}
-        last_time = nil
         chunk.each do |time, record|
           # each chunk has a unique set of labels
-          last_time = time if last_time.nil?
           result = line_to_loki(record)
           chunk_labels = result[:labels]
           # initialize a new stream with the chunk_labels if it does not exist

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-grafana-loki
 version: !ruby/object:Gem::Version
-  version: 1.2.14
+  version: 1.2.18
 platform: ruby
 authors:
 - woodsaj
@@ -10,7 +10,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2020-07-23 00:00:00.000000000 Z
+date: 2022-01-17 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fluentd
@@ -129,6 +129,7 @@ files:
 - README.md
 - bin/console
 - bin/setup
+- bin/test
 - lib/fluent/plugin/out_loki.rb
 homepage: https://github.com/grafana/loki/
 licenses:
@@ -149,7 +150,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.0.3
+rubygems_version: 3.0.3.1
 signing_key: 
 specification_version: 4
 summary: Output plugin to ship logs to a Grafana Loki server