fluent-plugin-grafana-loki 1.2.9 → 1.2.15

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 33ae09dd97c7f753b7bc507a701df03e4630c9561681c88fcdfb02446157b920
-  data.tar.gz: 1eb0e85e3136bcb251a861cca28693394c90615e403cc02306650607ec9ca98b
+  metadata.gz: 9da0220850d940db558feefbf825cd3e1b5a73c73f0a2e35516c98a9bc86f4f8
+  data.tar.gz: 50f3c8cfda6747c98cf0335f6cf60705b828e34e44c5372ac9ff5c13f59caf96
 SHA512:
-  metadata.gz: 3e52e7c93fcd20ea3ce687a0c74b0273de4367ede9addd14e5c59bbba477f769db1d1a8bd8bb84dc5e464dfa095fb7e9ce3214504535c4fbd903744e1b410348
-  data.tar.gz: 2d7bbca865bb13f2a2b8dc3a9e62ae84a9740fd1a09413569a8b892a4453ebada71205d678e6824885145b4d2bfdd694423d0db9ec3360d32a14423fc1852716
+  metadata.gz: 64ee54f30a6cb6da3c5b2725eebe9f3a8b25e41ba51bcf8027d876d33080ffbd831371252ba05f39a3dced2fad6d94a9b35f918d4be060e25f3d690223005a06
+  data.tar.gz: f6c507d56f16056306e8f53688a5a509cb1425947b57dba7e143aa4b2496dcb438adc38be2a1b9bde4051ef8c522214f514149888bb93fcd38760f08db143968

data/README.md

@@ -1,10 +1,12 @@
-# fluent-plugin-grafana-loki
+# Fluentd output plugin
 
-[Fluentd](https://fluentd.org/) output plugin to ship logs to a Loki server. See [docs/client/fluentd/README.md](../../docs/clients/fluentd/README.md) for detailed information.
+[Fluentd](https://fluentd.org/) is a data collector for unified logging layer, it can be configured with the Loki output plugin, provided in this folder, to ship logs to Loki.
+
+See [docs/client/fluentd/README.md](../../docs/sources/clients/fluentd/_index.md) for detailed information.
 
 ## Development
 
-After checking out the repo, run `bin/setup` to install dependencies. Then, run `rake spec` to run the tests. You can also run `bin/console` for an interactive prompt that will allow you to experiment.
+After checking out the repo, run `bin/setup` to install dependencies. Then, run `bundle exec rake spec` to run the tests. You can also run `bin/console` for an interactive prompt that will allow you to experiment.
 
 To install this gem onto your local machine, run `bundle exec rake install`. To release a new version, update the version number in `fluent-plugin-grafana-loki.gemspec`, and then run `bundle exec rake release`, which will create a git tag for the version, push git commits and tags, and push the `.gem` file to [rubygems.org](https://rubygems.org).
 
@@ -13,6 +15,70 @@ To create the gem: `gem build fluent-plugin-grafana-loki.gemspec`
 Useful additions:
   `gem install rubocop`
 
+## Testing
+
+Start Loki using:
+
+```bash
+docker run -it -p 3100:3100 grafana/loki:latest
+```
+
+Verify that Loki accept and stores logs:
+
+```bash
+curl -H "Content-Type: application/json" -XPOST -s "http://localhost:3100/loki/api/v1/push" --data-raw "{\"streams\": [{\"stream\": {\"job\": \"test\"}, \"values\": [[\"$(date +%s)000000000\", \"fizzbuzz\"]]}]}"
+curl "http://localhost:3100/loki/api/v1/query_range" --data-urlencode 'query={job="test"}' --data-urlencode 'step=300' | jq .data.result
+```
+
+The expected output is:
+
+```json
+[
+  {
+    "stream": {
+      "job": "test"
+    },
+    "values": [
+      [
+        "1588337198000000000",
+        "fizzbuzz"
+      ]
+    ]
+  }
+]
+```
+
+Start and send test logs with Fluentd using:
+
+```bash
+LOKI_URL=http://{{ IP }}:3100 make fluentd-test
+```
+
+Verify that syslogs are being feeded into Loki:
+
+```bash
+curl "http://localhost:3100/loki/api/v1/query_range" --data-urlencode 'query={job="fluentd"}' --data-urlencode 'step=300' | jq .data.result
+```
+
+The expected output is:
+
+```json
+[
+  {
+    "stream": {
+      "job": "fluentd"
+    },
+    "values": [
+      [
+        "1588336950379591919",
+        "log=\"May  1 14:42:30 ibuprofen avahi-daemon[859]: New relevant interface vethb503225.IPv6 for mDNS.\""
+      ],
+      ...
+    ]
+  }
+]
+```
+
 ## Copyright
 
 * Copyright(c) 2018- Grafana Labs

data/lib/fluent/plugin/out_loki.rb

@@ -15,6 +15,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+require 'fluent/env'
 require 'fluent/plugin/output'
 require 'net/http'
 require 'yajl'
@@ -26,26 +27,34 @@ module Fluent
     class LokiOutput < Fluent::Plugin::Output # rubocop:disable Metrics/ClassLength
       Fluent::Plugin.register_output('loki', self)
 
+      class LogPostError < StandardError; end
+
       helpers :compat_parameters, :record_accessor
 
       attr_accessor :record_accessors
 
       DEFAULT_BUFFER_TYPE = 'memory'
 
-      desc 'url of loki server'
-      config_param :url, :string, default: 'https://logs-us-west1.grafana.net'
+      desc 'Loki API base URL'
+      config_param :url, :string, default: 'https://logs-prod-us-central1.grafana.net'
 
-      desc 'BasicAuth credentials'
+      desc 'Authentication: basic auth credentials'
       config_param :username, :string, default: nil
       config_param :password, :string, default: nil, secret: true
 
-      desc 'Client certificate'
+      desc 'Authentication: Authorization header with Bearer token scheme'
+      config_param :bearer_token_file, :string, default: nil
+
+      desc 'TLS: parameters for presenting a client certificate'
       config_param :cert, :string, default: nil
       config_param :key, :string, default: nil
 
-      desc 'TLS'
+      desc 'TLS: CA certificate file for server certificate verification'
       config_param :ca_cert, :string, default: nil
 
+      desc 'TLS: disable server certificate verification'
+      config_param :insecure_tls, :bool, default: false
+
       desc 'Loki tenant id'
       config_param :tenant, :string, default: nil
 
@@ -74,7 +83,7 @@ module Fluent
         super
         @uri = URI.parse(@url + '/loki/api/v1/push')
         unless @uri.is_a?(URI::HTTP) || @uri.is_a?(URI::HTTPS)
-          raise Fluent::ConfigError, 'url parameter must be valid HTTP'
+          raise Fluent::ConfigError, 'URL parameter must have HTTP/HTTPS scheme'
         end
 
         @record_accessors = {}
@@ -90,24 +99,42 @@ module Fluent
           @remove_keys_accessors.push(record_accessor_create(key))
         end
 
-        if ssl_cert?
-          load_ssl
-          validate_ssl_key
+        # If configured, load and validate client certificate (and corresponding key)
+        if client_cert_configured?
+          load_client_cert
+          validate_client_cert_key
         end
 
+        raise "bearer_token_file #{@bearer_token_file} not found" if !@bearer_token_file.nil? && !File.exist?(@bearer_token_file)
+
+        @auth_token_bearer = nil
+        if !@bearer_token_file.nil?
+          if !File.exist?(@bearer_token_file)
+            raise "bearer_token_file #{@bearer_token_file} not found"
+          end
+
+          # Read the file once, assume long-lived authentication token.
+          @auth_token_bearer = File.read(@bearer_token_file)
+          if @auth_token_bearer.empty?
+            raise "bearer_token_file #{@bearer_token_file} is empty"
+          end
+          log.info "will use Bearer token from bearer_token_file #{@bearer_token_file} in Authorization header"
+        end
+
+
         raise "CA certificate file #{@ca_cert} not found" if !@ca_cert.nil? && !File.exist?(@ca_cert)
       end
 
-      def ssl_cert?
+      def client_cert_configured?
         !@key.nil? && !@cert.nil?
       end
 
-      def load_ssl
+      def load_client_cert
         @cert = OpenSSL::X509::Certificate.new(File.read(@cert)) if @cert
         @key = OpenSSL::PKey.read(File.read(@key)) if @key
       end
 
-      def validate_ssl_key
+      def validate_client_cert_key
         if !@key.is_a?(OpenSSL::PKey::RSA) && !@key.is_a?(OpenSSL::PKey::DSA)
           raise "Unsupported private key type #{key.class}"
         end
@@ -117,58 +144,52 @@ module Fluent
         true
       end
 
-      def http_opts(uri)
-        opts = {
-          use_ssl: uri.scheme == 'https'
-        }
-        opts
-      end
-
       # flush a chunk to loki
       def write(chunk)
         # streams by label
         payload = generic_to_loki(chunk)
         body = { 'streams' => payload }
 
-        # add ingest path to loki url
+        tenant = extract_placeholders(@tenant, chunk) if @tenant
 
-        req = Net::HTTP::Post.new(
-          @uri.request_uri
-        )
-        req.add_field('Content-Type', 'application/json')
-        req.add_field('X-Scope-OrgID', @tenant) if @tenant
-        req.body = Yajl.dump(body)
-        req.basic_auth(@username, @password) if @username
+        # add ingest path to loki url
+        res = loki_http_request(body, tenant)
 
-        opts = ssl_opts(@uri)
+        if res.is_a?(Net::HTTPSuccess)
+          log.debug "POST request was responded to with status code #{res.code}"
+          return
+        end
 
-        log.debug "sending #{req.body.length} bytes to loki"
-        res = Net::HTTP.start(@uri.host, @uri.port, **opts) { |http| http.request(req) }
-        unless res&.is_a?(Net::HTTPSuccess)
-          res_summary = if res
-                          "#{res.code} #{res.message} #{res.body}"
-                        else
-                          'res=nil'
-                        end
-          log.warn "failed to #{req.method} #{@uri} (#{res_summary})"
-          log.warn Yajl.dump(body)
+        res_summary = "#{res.code} #{res.message} #{res.body}"
+        log.warn "failed to write post to #{@uri} (#{res_summary})"
+        log.debug Yajl.dump(body)
 
-        end
+        # Only retry 429 and 500s
+        raise(LogPostError, res_summary) if res.is_a?(Net::HTTPTooManyRequests) || res.is_a?(Net::HTTPServerError)
       end
 
-      def ssl_opts(uri)
+      def http_request_opts(uri)
         opts = {
           use_ssl: uri.scheme == 'https'
         }
 
+        # Optionally disable server server certificate verification.
+        if @insecure_tls
+          opts = opts.merge(
+            verify_mode: OpenSSL::SSL::VERIFY_NONE
+          )
+        end
+
+        # Optionally present client certificate
         if !@cert.nil? && !@key.nil?
           opts = opts.merge(
-            verify_mode: OpenSSL::SSL::VERIFY_PEER,
             cert: @cert,
             key: @key
           )
         end
 
+        # For server certificate verification: set custom CA bundle.
+        # Only takes effect when `insecure_tls` is not set.
         unless @ca_cert.nil?
           opts = opts.merge(
             ca_file: @ca_cert
@@ -186,6 +207,25 @@ module Fluent
 
       private
 
+      def loki_http_request(body, tenant)
+        req = Net::HTTP::Post.new(
+          @uri.request_uri
+        )
+        req.add_field('Content-Type', 'application/json')
+        req.add_field('Authorization', "Bearer #{@auth_token_bearer}") if !@auth_token_bearer.nil?
+        req.add_field('X-Scope-OrgID', tenant) if tenant
+        req.body = Yajl.dump(body)
+        req.basic_auth(@username, @password) if @username
+
+        opts = http_request_opts(@uri)
+
+        msg = "sending #{req.body.length} bytes to loki"
+        msg += " (tenant: \"#{tenant}\")" if tenant
+        log.debug msg
+
+        Net::HTTP.start(@uri.host, @uri.port, **opts) { |http| http.request(req) }
+      end
+
       def numeric?(val)
         !Float(val).nil?
       rescue StandardError

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-grafana-loki
 version: !ruby/object:Gem::Version
-  version: 1.2.9
+  version: 1.2.15
 platform: ruby
 authors:
 - woodsaj
@@ -10,8 +10,28 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2020-02-18 00:00:00.000000000 Z
+date: 2020-10-16 00:00:00.000000000 Z
 dependencies:
+- !ruby/object:Gem::Dependency
+  name: fluentd
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 1.9.3
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '2'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 1.9.3
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '2'
 - !ruby/object:Gem::Dependency
   name: bundler
   requirement: !ruby/object:Gem::Requirement
@@ -55,25 +75,47 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '3.0'
 - !ruby/object:Gem::Dependency
-  name: fluentd
+  name: rubocop-rspec
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 1.9.0
-    - - "<"
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
       - !ruby/object:Gem::Version
-        version: '2'
-  type: :runtime
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: simplecov
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 1.9.0
-    - - "<"
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: test-unit
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
       - !ruby/object:Gem::Version
-        version: '2'
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 description: Output plugin to ship logs to a Grafana Loki server
 email:
 - awoods@grafana.com