fluent-plugin-grafana-loki 1.0.2 → 1.1.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: adedfda5ed9ce22ed7a9c46d5497b0b2aa608631df08dc68273349567cadc2e6
-  data.tar.gz: 8c74cf88dc52bccdb51b5efc72e69a543a2e53988be8ce92402303f3054bbc41
+  metadata.gz: 3d29e69ac3bede62dac1fa9aba38b5b96bd322150f13748a85ef91e5a044c5d8
+  data.tar.gz: dc521c086b7a88e1163e1257b31dab968b26082655b764b89685e0945e72cc25
 SHA512:
-  metadata.gz: 17fba6b641ec604837afe7003fb1332f1b28b83a80f67abfc39cf240ea5b296dc2317f1e7315c6cc08f86b367c3797ee42d42ecbabb416afde05973145d5fed0
-  data.tar.gz: ceeb3e8d842bfb67d9f95f0c8c6000bb0e1895bf91f4306642a8ec2d8ce3c56d499b70adc42888b9a7cfd8b51132deb9dfdfe253cc631bfa87dc7c84cce51bd1
+  metadata.gz: b8fdacb9474dba6bcf8a7395a9126945bc3c123b8cdb318fc95f7658943b5465819e226a8ac65a5c0b1dc0a3d66c9fd8b551170d5b9416601d693484c10f87d4
+  data.tar.gz: 025c5f3e296af6e3e19daf132ad0411795386073c1573ba2766984812c2801f5cc985a90ddeb7bba4c736b70538d55047070bb98b6d1dfe6e669ab7437204a89

data/README.md

@@ -6,7 +6,7 @@ This plugin offers two line formats and uses protobuf to send compressed data to
 
 Key features:
   * extra_labels - labels to be added to every line of a logfile, useful for designating environments
-  * label_keys - customizable list of keys for stream labels
+  * label - This section allows you to specify labels from your log fields
 
 ## Installation
 
@@ -29,6 +29,64 @@ In your Fluentd configuration, use `@type loki`. Additional configuration is opt
 </match>
 ```
 
+### Using labels
+
+Simple label from top level attribute
+```
+<match mytag>
+  @type loki
+  # ...
+  <label>
+    fluentd_worker 
+  </label>
+  # ...
+</match>
+```
+
+You can rewrite the label keys as well as the following
+
+```
+<match mytag>
+  @type loki
+  # ...
+  <label>
+    worker fluentd_worker
+  </label>
+  # ...
+</match>
+```
+
+You can use record accessor syntax for nested field. https://docs.fluentd.org/plugin-helper-overview/api-plugin-helper-record_accessor#syntax
+
+```
+<match mytag>
+  @type loki
+  # ...
+  <label>
+    container $.kubernetes.container
+  </label>
+  # ...
+</match>
+```
+
+### Extracting Kubernetes labels
+
+As Kubernetes labels are a list of nested key-value pairs there is a separate option to extract them.
+Note that special characters like "`. - /`" will be overwritten with `_`.
+Use with the `remove_keys kubernetes` option to eliminate metadata from the log.
+``` 
+<match mytag>
+  @type loki
+  # ...
+  extract_kubernetes_labels true
+  remove_keys kubernetes
+  <label>
+    container $.kubernetes.container
+  </label>
+  # ...
+</match>
+```
+
 ### Multi-worker usage
 
 Loki doesn't currently support out-of-order inserts - if you try to insert a log entry an earlier timestamp after a log entry with with identical labels but a later timestamp, the insert will fail with `HTTP status code: 500, message: rpc error: code = Unknown desc = Entry out of order`. Therefore, in order to use this plugin in a multi worker Fluentd setup, you'll need to include the worker ID in the labels.
@@ -45,7 +103,9 @@ For example, using [fluent-plugin-record-modifier](https://github.com/repeatedly
 <match mytag>
   @type loki
   # ...
-  label_keys "fluentd_worker"
+  <label>
+    fluentd_worker
+  </label>
   # ...
 </match>
 ```
@@ -112,8 +172,7 @@ Loki is intended to index and group log streams using only a small set of labels
 
 There are few configurations settings to control the output format.
  - extra_labels: (default: nil) set of labels to include with every Loki stream. eg `{"env":"dev", "datacenter": "dc1"}`
- - remove_keys: (default: nil) comma separated list of needless record keys to remove. All other keys will be placed into the log line
- - label_keys: (default: "job,instance") comma separated list of keys to use as stream labels. All other keys will be placed into the log line
+ - remove_keys: (default: nil) comma separated list of needless record keys to remove. All other keys will be placed into the log line. You can use [record_accessor syntax](https://docs.fluentd.org/plugin-helper-overview/api-plugin-helper-record_accessor#syntax).
  - line_format: format to use when flattening the record to a log line. Valid values are "json" or "key_value". If set to "json" the log line sent to Loki will be the fluentd record (excluding any keys extracted out as labels) dumped as json. If set to "key_value", the log line will be each item in the record concatenated together (separated by a single space) in the format `<key>=<value>`.
  - drop_single_key: if set to true and after extracting label_keys a record only has a single key remaining, the log line sent to Loki will just be the value of the record key.
 

data/lib/fluent/plugin/out_loki.rb

@@ -24,36 +24,38 @@ require 'time'
 module Fluent
   module Plugin
     # Subclass of Fluent Plugin Output
-    class LokiOutput < Fluent::Plugin::Output
+    class LokiOutput < Fluent::Plugin::Output # rubocop:disable Metrics/ClassLength
       Fluent::Plugin.register_output('loki', self)
 
-      helpers :compat_parameters
+      helpers :compat_parameters, :record_accessor
+
+      attr_accessor :record_accessors
 
       DEFAULT_BUFFER_TYPE = 'memory'
 
-      # url of loki server
+      desc 'url of loki server'
       config_param :url, :string, default: 'https://logs-us-west1.grafana.net'
 
-      # BasicAuth credentials
+      desc 'BasicAuth credentials'
       config_param :username, :string, default: nil
       config_param :password, :string, default: nil, secret: true
 
-      # Loki tenant id
+      desc 'Loki tenant id'
       config_param :tenant, :string, default: nil
 
-      # extra labels to add to all log streams
+      desc 'extra labels to add to all log streams'
       config_param :extra_labels, :hash, default: {}
 
-      # format to use when flattening the record to a log line
+      desc 'format to use when flattening the record to a log line'
       config_param :line_format, :enum, list: %i[json key_value], default: :key_value
 
-      # comma separated list of keys to use as stream lables.  All other keys will be placed into the log line
-      config_param :label_keys, :string, default: 'job,instance'
+      desc 'extract kubernetes labels as loki labels'
+      config_param :extract_kubernetes_labels, :bool, default: false
 
-      # comma separated list of needless record keys to remove
-      config_param :remove_keys, :string, default: nil
+      desc 'comma separated list of needless record keys to remove'
+      config_param :remove_keys, :array, default: %w[], value_type: :string
 
-      # if a record only has 1 key, then just set the log line to the value and discard the key.
+      desc 'if a record only has 1 key, then just set the log line to the value and discard the key.'
       config_param :drop_single_key, :bool, default: false
 
       config_section :buffer do
@@ -64,9 +66,18 @@ module Fluent
       def configure(conf)
         compat_parameters_convert(conf, :buffer)
         super
-
-        @label_keys = @label_keys.split(/\s*,\s*/) if @label_keys
-        @remove_keys = @remove_keys.split(',').map(&:strip) if @remove_keys
+        @record_accessors = {}
+        conf.elements.select { |element| element.name == 'label' }.each do |element|
+          element.each_pair do |k, v|
+            element.key?(k) # to suppress unread configuration warning
+            v = k if v.empty?
+            @record_accessors[k] = record_accessor_create(v)
+          end
+        end
+        @remove_keys_accessors = []
+        @remove_keys.each do |key|
+          @remove_keys_accessors.push(record_accessor_create(key))
+        end
       end
 
       def multi_workers_ready?
@@ -84,7 +95,7 @@ module Fluent
       def write(chunk)
         # streams by label
         payload = generic_to_loki(chunk)
-        body = { 'streams': payload }
+        body = { 'streams' => payload }
 
         # add ingest path to loki url
         uri = URI.parse(url + '/api/prom/push')
@@ -101,7 +112,7 @@ module Fluent
         }
         log.debug "sending #{req.body.length} bytes to loki"
         res = Net::HTTP.start(uri.hostname, uri.port, **opts) { |http| http.request(req) }
-        unless res && res.is_a?(Net::HTTPSuccess)
+        unless res&.is_a?(Net::HTTPSuccess)
           res_summary = if res
                           "#{res.code} #{res.message} #{res.body}"
                         else
@@ -135,10 +146,8 @@ module Fluent
         data_labels = {} if data_labels.nil?
         data_labels = data_labels.merge(@extra_labels)
 
-        unless data_labels.nil?
-          data_labels.each do |k, v|
-            formatted_labels.push("#{k}=\"#{v}\"")
-          end
+        data_labels.each do |k, v|
+          formatted_labels.push(%(#{k}="#{v.gsub('"', '\\"')}")) if v
         end
         '{' + formatted_labels.join(',') + '}'
       end
@@ -147,7 +156,7 @@ module Fluent
         payload = []
         streams.each do |k, v|
           # create a stream for each label set.
-          # Additionally sort the entries by timestamp just incase we
+          # Additionally sort the entries by timestamp just in case we
           # got them out of order.
           # 'labels' => '{worker="0"}',
           payload.push(
@@ -169,7 +178,7 @@ module Fluent
           when :key_value
             formatted_labels = []
             record.each do |k, v|
-              formatted_labels.push("#{k}=\"#{v}\"")
+              formatted_labels.push(%(#{k}="#{v}"))
             end
             line = formatted_labels.join(' ')
           end
@@ -177,22 +186,31 @@ module Fluent
         line
       end
 
+      #
       # convert a line to loki line with labels
       def line_to_loki(record)
         chunk_labels = {}
         line = ''
         if record.is_a?(Hash)
-          # remove needless keys.
-          @remove_keys.each { |v|
-            record.delete(v)
-          } if @remove_keys
-          # extract white listed record keys into labels.
-          @label_keys.each do |k|
-            if record.key?(k)
-              chunk_labels[k] = record[k]
-              record.delete(k)
+          @record_accessors&.each do |name, accessor|
+            new_key = name.gsub(%r{[.\-\/]}, '_')
+            chunk_labels[new_key] = accessor.call(record)
+            accessor.delete(record)
+          end
+
+          if @extract_kubernetes_labels && record.key?('kubernetes')
+            kubernetes_labels = record['kubernetes']['labels']
+            kubernetes_labels.each_key do |l|
+              new_key = l.gsub(%r{[.\-\/]}, '_')
+              chunk_labels[new_key] = kubernetes_labels[l]
             end
-          end if @label_keys
+          end
+
+          # remove needless keys.
+          @remove_keys_accessors&.each do |deleter|
+            deleter.delete(record)
+          end
+
           line = record_to_line(record)
         else
           line = record.to_s

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-grafana-loki
 version: !ruby/object:Gem::Version
-  version: 1.0.2
+  version: 1.1.0
 platform: ruby
 authors:
 - woodsaj
@@ -9,22 +9,22 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2019-07-26 00:00:00.000000000 Z
+date: 2019-10-25 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
-        version: '1.15'
+        version: '0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
-        version: '1.15'
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: rake
   requirement: !ruby/object:Gem::Requirement
@@ -105,7 +105,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.0.4
+rubygems_version: 3.0.3
 signing_key: 
 specification_version: 4
 summary: Output plugin to ship logs to a Grafana Loki server