fluent-plugin-grafana-loki 1.2.10 → 1.2.16

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: b9b80ced17e5aa680b13c310f15008e9fcbd219b046b9dd40eddfb43a6f454fe
-  data.tar.gz: 4e7721bf9e2bed189dd2ab9c7090aaac30621e8938658291ad6d622272cd0591
+  metadata.gz: d69446470f9b77ceaf43e03d15948de4afbf0573a0a1be6ca1825aaa43075a90
+  data.tar.gz: dbc6e1481ee9b55a236f5316f00f92631970fd5ed990c3f7e2c595d25e90bac1
 SHA512:
-  metadata.gz: 624c222606aac4e9725e10a69984457d01a0a28e8364ec50f073df570ba783235719b72117a00aacdc64f0252d7ad4f1e740add4b2a51d86b0ccabda3a04c0f6
-  data.tar.gz: 73ff3b3338cf03509890e26ca50449d8b0a956a8c7cc4a946faa030aa2efdabe34500a450dfc6869ceda06eadbb55fef6184df29fbce6b7edda1675568fbbce2
+  metadata.gz: 05165c783cf297c4e2cb64f5e421ea5b6c8aae41894269c06500177b722d82250cf8a78040c6e7389dab840e463a65f78e8ce4e07ae04dcd8fb7f66a4f90fe19
+  data.tar.gz: 0e8b7dab804916783771976c39eb3a9e1edbd3ed54438c72119ba9b0aef79e490af47e626c044084d80b0273d3eb22cfc71b345a0c01a22299da6b83c73d697f

data/README.md

@@ -1,10 +1,12 @@
-# fluent-plugin-grafana-loki
+# Fluentd output plugin
 
-[Fluentd](https://fluentd.org/) output plugin to ship logs to a Loki server. See [docs/client/fluentd/README.md](../../docs/clients/fluentd/README.md) for detailed information.
+[Fluentd](https://fluentd.org/) is a data collector for unified logging layer, it can be configured with the Loki output plugin, provided in this folder, to ship logs to Loki.
+
+See [docs/client/fluentd/README.md](../../docs/sources/clients/fluentd/_index.md) for detailed information.
 
 ## Development
 
-After checking out the repo, run `bin/setup` to install dependencies. Then, run `rake spec` to run the tests. You can also run `bin/console` for an interactive prompt that will allow you to experiment.
+After checking out the repo, run `bin/setup` to install dependencies. Then, run `bundle exec rake spec` to run the tests. You can also run `bin/console` for an interactive prompt that will allow you to experiment.
 
 To install this gem onto your local machine, run `bundle exec rake install`. To release a new version, update the version number in `fluent-plugin-grafana-loki.gemspec`, and then run `bundle exec rake release`, which will create a git tag for the version, push git commits and tags, and push the `.gem` file to [rubygems.org](https://rubygems.org).
 
@@ -13,6 +15,70 @@ To create the gem: `gem build fluent-plugin-grafana-loki.gemspec`
 Useful additions:
   `gem install rubocop`
 
+## Testing
+
+Start Loki using:
+
+```bash
+docker run -it -p 3100:3100 grafana/loki:latest
+```
+
+Verify that Loki accept and stores logs:
+
+```bash
+curl -H "Content-Type: application/json" -XPOST -s "http://localhost:3100/loki/api/v1/push" --data-raw "{\"streams\": [{\"stream\": {\"job\": \"test\"}, \"values\": [[\"$(date +%s)000000000\", \"fizzbuzz\"]]}]}"
+curl "http://localhost:3100/loki/api/v1/query_range" --data-urlencode 'query={job="test"}' --data-urlencode 'step=300' | jq .data.result
+```
+
+The expected output is:
+
+```json
+[
+  {
+    "stream": {
+      "job": "test"
+    },
+    "values": [
+      [
+        "1588337198000000000",
+        "fizzbuzz"
+      ]
+    ]
+  }
+]
+```
+
+Start and send test logs with Fluentd using:
+
+```bash
+LOKI_URL=http://{{ IP }}:3100 make fluentd-test
+```
+
+Verify that syslogs are being feeded into Loki:
+
+```bash
+curl "http://localhost:3100/loki/api/v1/query_range" --data-urlencode 'query={job="fluentd"}' --data-urlencode 'step=300' | jq .data.result
+```
+
+The expected output is:
+
+```json
+[
+  {
+    "stream": {
+      "job": "fluentd"
+    },
+    "values": [
+      [
+        "1588336950379591919",
+        "log=\"May  1 14:42:30 ibuprofen avahi-daemon[859]: New relevant interface vethb503225.IPv6 for mDNS.\""
+      ],
+      ...
+    ]
+  }
+]
+```
+
 ## Copyright
 
 * Copyright(c) 2018- Grafana Labs

data/lib/fluent/plugin/out_loki.rb

@@ -15,6 +15,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+require 'fluent/env'
 require 'fluent/plugin/output'
 require 'net/http'
 require 'yajl'
@@ -26,26 +27,34 @@ module Fluent
     class LokiOutput < Fluent::Plugin::Output # rubocop:disable Metrics/ClassLength
       Fluent::Plugin.register_output('loki', self)
 
+      class LogPostError < StandardError; end
+
       helpers :compat_parameters, :record_accessor
 
       attr_accessor :record_accessors
 
       DEFAULT_BUFFER_TYPE = 'memory'
 
-      desc 'url of loki server'
-      config_param :url, :string, default: 'https://logs-us-west1.grafana.net'
+      desc 'Loki API base URL'
+      config_param :url, :string, default: 'https://logs-prod-us-central1.grafana.net'
 
-      desc 'BasicAuth credentials'
+      desc 'Authentication: basic auth credentials'
       config_param :username, :string, default: nil
       config_param :password, :string, default: nil, secret: true
 
-      desc 'Client certificate'
+      desc 'Authentication: Authorization header with Bearer token scheme'
+      config_param :bearer_token_file, :string, default: nil
+
+      desc 'TLS: parameters for presenting a client certificate'
       config_param :cert, :string, default: nil
       config_param :key, :string, default: nil
 
-      desc 'TLS'
+      desc 'TLS: CA certificate file for server certificate verification'
       config_param :ca_cert, :string, default: nil
 
+      desc 'TLS: disable server certificate verification'
+      config_param :insecure_tls, :bool, default: false
+
       desc 'Loki tenant id'
       config_param :tenant, :string, default: nil
 
@@ -74,7 +83,7 @@ module Fluent
         super
         @uri = URI.parse(@url + '/loki/api/v1/push')
         unless @uri.is_a?(URI::HTTP) || @uri.is_a?(URI::HTTPS)
-          raise Fluent::ConfigError, 'url parameter must be valid HTTP'
+          raise Fluent::ConfigError, 'URL parameter must have HTTP/HTTPS scheme'
         end
 
         @record_accessors = {}
@@ -90,24 +99,42 @@ module Fluent
           @remove_keys_accessors.push(record_accessor_create(key))
         end
 
-        if ssl_cert?
-          load_ssl
-          validate_ssl_key
+        # If configured, load and validate client certificate (and corresponding key)
+        if client_cert_configured?
+          load_client_cert
+          validate_client_cert_key
         end
 
+        raise "bearer_token_file #{@bearer_token_file} not found" if !@bearer_token_file.nil? && !File.exist?(@bearer_token_file)
+
+        @auth_token_bearer = nil
+        if !@bearer_token_file.nil?
+          if !File.exist?(@bearer_token_file)
+            raise "bearer_token_file #{@bearer_token_file} not found"
+          end
+
+          # Read the file once, assume long-lived authentication token.
+          @auth_token_bearer = File.read(@bearer_token_file)
+          if @auth_token_bearer.empty?
+            raise "bearer_token_file #{@bearer_token_file} is empty"
+          end
+          log.info "will use Bearer token from bearer_token_file #{@bearer_token_file} in Authorization header"
+        end
+
+
         raise "CA certificate file #{@ca_cert} not found" if !@ca_cert.nil? && !File.exist?(@ca_cert)
       end
 
-      def ssl_cert?
+      def client_cert_configured?
         !@key.nil? && !@cert.nil?
       end
 
-      def load_ssl
+      def load_client_cert
         @cert = OpenSSL::X509::Certificate.new(File.read(@cert)) if @cert
         @key = OpenSSL::PKey.read(File.read(@key)) if @key
       end
 
-      def validate_ssl_key
+      def validate_client_cert_key
         if !@key.is_a?(OpenSSL::PKey::RSA) && !@key.is_a?(OpenSSL::PKey::DSA)
           raise "Unsupported private key type #{key.class}"
         end
@@ -117,58 +144,52 @@ module Fluent
         true
       end
 
-      def http_opts(uri)
-        opts = {
-          use_ssl: uri.scheme == 'https'
-        }
-        opts
-      end
-
       # flush a chunk to loki
       def write(chunk)
         # streams by label
         payload = generic_to_loki(chunk)
         body = { 'streams' => payload }
 
-        # add ingest path to loki url
+        tenant = extract_placeholders(@tenant, chunk) if @tenant
 
-        req = Net::HTTP::Post.new(
-          @uri.request_uri
-        )
-        req.add_field('Content-Type', 'application/json')
-        req.add_field('X-Scope-OrgID', @tenant) if @tenant
-        req.body = Yajl.dump(body)
-        req.basic_auth(@username, @password) if @username
+        # add ingest path to loki url
+        res = loki_http_request(body, tenant)
 
-        opts = ssl_opts(@uri)
+        if res.is_a?(Net::HTTPSuccess)
+          log.debug "POST request was responded to with status code #{res.code}"
+          return
+        end
 
-        log.debug "sending #{req.body.length} bytes to loki"
-        res = Net::HTTP.start(@uri.host, @uri.port, **opts) { |http| http.request(req) }
-        unless res&.is_a?(Net::HTTPSuccess)
-          res_summary = if res
-                          "#{res.code} #{res.message} #{res.body}"
-                        else
-                          'res=nil'
-                        end
-          log.warn "failed to #{req.method} #{@uri} (#{res_summary})"
-          log.debug Yajl.dump(body)
+        res_summary = "#{res.code} #{res.message} #{res.body}"
+        log.warn "failed to write post to #{@uri} (#{res_summary})"
+        log.debug Yajl.dump(body)
 
-        end
+        # Only retry 429 and 500s
+        raise(LogPostError, res_summary) if res.is_a?(Net::HTTPTooManyRequests) || res.is_a?(Net::HTTPServerError)
       end
 
-      def ssl_opts(uri)
+      def http_request_opts(uri)
         opts = {
           use_ssl: uri.scheme == 'https'
         }
 
+        # Optionally disable server server certificate verification.
+        if @insecure_tls
+          opts = opts.merge(
+            verify_mode: OpenSSL::SSL::VERIFY_NONE
+          )
+        end
+
+        # Optionally present client certificate
         if !@cert.nil? && !@key.nil?
           opts = opts.merge(
-            verify_mode: OpenSSL::SSL::VERIFY_PEER,
             cert: @cert,
             key: @key
           )
         end
 
+        # For server certificate verification: set custom CA bundle.
+        # Only takes effect when `insecure_tls` is not set.
         unless @ca_cert.nil?
           opts = opts.merge(
             ca_file: @ca_cert
@@ -186,6 +207,25 @@ module Fluent
 
       private
 
+      def loki_http_request(body, tenant)
+        req = Net::HTTP::Post.new(
+          @uri.request_uri
+        )
+        req.add_field('Content-Type', 'application/json')
+        req.add_field('Authorization', "Bearer #{@auth_token_bearer}") if !@auth_token_bearer.nil?
+        req.add_field('X-Scope-OrgID', tenant) if tenant
+        req.body = Yajl.dump(body)
+        req.basic_auth(@username, @password) if @username
+
+        opts = http_request_opts(@uri)
+
+        msg = "sending #{req.body.length} bytes to loki"
+        msg += " (tenant: \"#{tenant}\")" if tenant
+        log.debug msg
+
+        Net::HTTP.start(@uri.host, @uri.port, **opts) { |http| http.request(req) }
+      end
+
       def numeric?(val)
         !Float(val).nil?
       rescue StandardError
@@ -238,7 +278,13 @@ module Fluent
           when :key_value
             formatted_labels = []
             record.each do |k, v|
-              formatted_labels.push(%(#{k}="#{v}"))
+              # Escape double quotes and backslashes by prefixing them with a backslash
+              v = v.to_s.gsub(%r{(["\\])}, '\\\\\1')
+              if v.include?(' ') || v.include?('=')
+                formatted_labels.push(%(#{k}="#{v}"))
+              else
+                formatted_labels.push(%(#{k}=#{v}))
+              end
             end
             line = formatted_labels.join(' ')
           end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-grafana-loki
 version: !ruby/object:Gem::Version
-  version: 1.2.10
+  version: 1.2.16
 platform: ruby
 authors:
 - woodsaj
@@ -10,8 +10,28 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2020-03-17 00:00:00.000000000 Z
+date: 2020-10-27 00:00:00.000000000 Z
 dependencies:
+- !ruby/object:Gem::Dependency
+  name: fluentd
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 1.9.3
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '2'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 1.9.3
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '2'
 - !ruby/object:Gem::Dependency
   name: bundler
   requirement: !ruby/object:Gem::Requirement
@@ -55,25 +75,47 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '3.0'
 - !ruby/object:Gem::Dependency
-  name: fluentd
+  name: rubocop-rspec
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 1.9.0
-    - - "<"
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
       - !ruby/object:Gem::Version
-        version: '2'
-  type: :runtime
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: simplecov
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 1.9.0
-    - - "<"
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: test-unit
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
       - !ruby/object:Gem::Version
-        version: '2'
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 description: Output plugin to ship logs to a Grafana Loki server
 email:
 - awoods@grafana.com