fluent-plugin-grafana-loki 1.2.7 → 1.2.9

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 6272020eefeef15ea9b65b49ede71e60c08c298c7c56d2ca366a95130fbf3f3a
-  data.tar.gz: b3699f4d8dcc778bfc63d6d2a0fe456ecfd95fb55dfbf6048af0846fdd741259
+  metadata.gz: 33ae09dd97c7f753b7bc507a701df03e4630c9561681c88fcdfb02446157b920
+  data.tar.gz: 1eb0e85e3136bcb251a861cca28693394c90615e403cc02306650607ec9ca98b
 SHA512:
-  metadata.gz: 15cd081338b9da5c2b8ea7212d4ebd0cb455782cb02722992bdd671e6715d960c6a1bfb828d14a763c60175cc82d156f912a624dde81072eaeb85c726b8a06c5
-  data.tar.gz: dbec238d8c2e2b52c5508cc1cebe9a55677dfbb6de26de33af2b45399742fd57b8a5a727b6e763560b4f775eafa5172c4fed86e4cd9162ad06b90e2233708851
+  metadata.gz: 3e52e7c93fcd20ea3ce687a0c74b0273de4367ede9addd14e5c59bbba477f769db1d1a8bd8bb84dc5e464dfa095fb7e9ce3214504535c4fbd903744e1b410348
+  data.tar.gz: 2d7bbca865bb13f2a2b8dc3a9e62ae84a9740fd1a09413569a8b892a4453ebada71205d678e6824885145b4d2bfdd694423d0db9ec3360d32a14423fc1852716

data/README.md

@@ -1,216 +1,6 @@
 # fluent-plugin-grafana-loki
 
-[Fluentd](https://fluentd.org/) output plugin to ship logs to a Loki server.
-
-This plugin offers two line formats and uses protobuf to send compressed data to Loki.
-
-Key features:
-  * extra_labels - labels to be added to every line of a logfile, useful for designating environments
-  * label - This section allows you to specify labels from your log fields
-
-## Installation
-
-```
-$ gem install fluent-plugin-grafana-loki
-```
-
-## Usage
-In your Fluentd configuration, use `@type loki`. Additional configuration is optional, default values would look like this:
-```
-<match **>
-  @type loki
-  url "https://logs-us-west1.grafana.net"
-  username "#{ENV['LOKI_USERNAME']}"
-  password "#{ENV['LOKI_PASSWORD']}"
-  extra_labels {"env":"dev"}
-  flush_interval 10s
-  flush_at_shutdown true
-  buffer_chunk_limit 1m
-</match>
-```
-
-### Using labels
-
-Simple label from top level attribute
-```
-<match mytag>
-  @type loki
-  # ...
-  <label>
-    fluentd_worker 
-  </label>
-  # ...
-</match>
-```
-
-You can rewrite the label keys as well as the following
-
-```
-<match mytag>
-  @type loki
-  # ...
-  <label>
-    worker fluentd_worker
-  </label>
-  # ...
-</match>
-```
-
-You can use record accessor syntax for nested field. https://docs.fluentd.org/plugin-helper-overview/api-plugin-helper-record_accessor#syntax
-
-```
-<match mytag>
-  @type loki
-  # ...
-  <label>
-    container $.kubernetes.container
-  </label>
-  # ...
-</match>
-```
-
-### Extracting Kubernetes labels
-
-As Kubernetes labels are a list of nested key-value pairs there is a separate option to extract them.
-Note that special characters like "`. - /`" will be overwritten with `_`.
-Use with the `remove_keys kubernetes` option to eliminate metadata from the log.
-``` 
-<match mytag>
-  @type loki
-  # ...
-  extract_kubernetes_labels true
-  remove_keys kubernetes
-  <label>
-    container $.kubernetes.container
-  </label>
-  # ...
-</match>
-```
-
-### Multi-worker usage
-
-Loki doesn't currently support out-of-order inserts - if you try to insert a log entry an earlier timestamp after a log entry with with identical labels but a later timestamp, the insert will fail with `HTTP status code: 500, message: rpc error: code = Unknown desc = Entry out of order`. Therefore, in order to use this plugin in a multi worker Fluentd setup, you'll need to include the worker ID in the labels.
-
-For example, using [fluent-plugin-record-modifier](https://github.com/repeatedly/fluent-plugin-record-modifier):
-```
-<filter mytag>
-    @type record_modifier
-    <record>
-        fluentd_worker "#{worker_id}"
-    </record>
-</filter>
-
-<match mytag>
-  @type loki
-  # ...
-  <label>
-    fluentd_worker
-  </label>
-  # ...
-</match>
-```
-
-### Using multiple buffer flush threads
-
-Similarly, when using `flush_thread_count` > 1 in the [`buffer`](https://docs.fluentd.org/configuration/buffer-section#flushing-parameters)
-section, a thread identifier must be added as a label to ensure that log chunks flushed in parallel to loki by fluentd always have increasing
-times for their unique label sets.
-
-This plugin automatically adds a `fluentd_thread` label with the name of the buffer flush thread when `flush_thread_count` > 1.
-
-## Docker Image
-
-There is a Docker image `grafana/fluent-plugin-grafana-loki:master` which contains default configuration files to git log information
-a host's `/var/log` dir, and from the host's Journald. To use it, you can set the `LOKI_URL`, `LOKI_USERNAME`, and `LOKI_PASSWORD` environment variables (you can leave the USERNAME and PASSWORD blank if they're not used.)
-
-
-A Docker Swarm Compose configuration that will work looks like:
-
-```
-services:
-  fluentd:
-    image: grafana/fluent-plugin-grafana-loki:master
-    command:
-      - "fluentd"
-      - "-v"
-      - "-p"
-      - "/fluentd/plugins"
-    environment:
-      LOKI_URL: http://loki:3100
-      LOKI_USERNAME:
-      LOKI_PASSWORD:
-    deploy:
-      mode: global
-    configs:
-      - source: loki_config
-        target: /fluentd/etc/loki/loki.conf
-    networks:
-      - loki
-    volumes:
-      - host_logs:/var/log
-      # Needed for journald log ingestion:
-      - /etc/machine-id:/etc/machine-id
-      - /dev/log:/dev/log
-      - /var/run/systemd/journal/:/var/run/systemd/journal/
-    logging:
-      options:
-         tag: infra.monitoring
-```
-
-## Configuration
-
-### url
-The url of the Loki server to send logs to.  When sending data the publish path (`/api/prom/push`) will automatically be appended.
-By default the url is set to `https://logs-us-west1.grafana.net`, the url of the Grafana Labs preview (hosted Loki)[https://grafana.com/loki] service.
-
-#### Proxy Support
-
-Starting with version 0.8.0, this gem uses excon, which supports proxy with environment variables - https://github.com/excon/excon#proxy-support
-
-### username / password
-Specify a username and password if the Loki server requires authentication.
-If using the GrafanaLab's hosted Loki, the username needs to be set to your instanceId and the password should be a Grafana.com api key.
-
-### tenant
-Loki is a multi-tenant log storage platform and all requests sent must include a tenant.  For some installations the tenant will be set automatically by an authenticating proxy.  Otherwise you can define a tenant to be passed through.  The tenant can be any string value.
-
-### client certificate verification
-Specify a pair of client certificate and private key with `cert` and `key` if a reverse proxy with client certificate verification is configured in front of Loki. `ca_cert` can also be specified if the server uses custom certificate authority.
-
-```
-<match **>
-  @type loki
-
-  url "https://loki"
-
-  cert /path/to/certificate.pem
-  key /path/to/key.key
-  ca_cert /path/to/ca.pem
-
-  ...
-</match>
-```
-
-### output format
-Loki is intended to index and group log streams using only a small set of labels.  It is not intended for full-text indexing.  When sending logs to Loki the majority of log message will be sent as a single log "line".
-
-There are few configurations settings to control the output format.
- - extra_labels: (default: nil) set of labels to include with every Loki stream. eg `{"env":"dev", "datacenter": "dc1"}`
- - remove_keys: (default: nil) comma separated list of needless record keys to remove. All other keys will be placed into the log line. You can use [record_accessor syntax](https://docs.fluentd.org/plugin-helper-overview/api-plugin-helper-record_accessor#syntax).
- - line_format: format to use when flattening the record to a log line. Valid values are "json" or "key_value". If set to "json" the log line sent to Loki will be the fluentd record (excluding any keys extracted out as labels) dumped as json. If set to "key_value", the log line will be each item in the record concatenated together (separated by a single space) in the format `<key>=<value>`.
- - drop_single_key: if set to true and after extracting label_keys a record only has a single key remaining, the log line sent to Loki will just be the value of the record key.
-
-### Buffer options
-
-`fluentd-plugin-loki` extends [Fluentd's builtin Output plugin](https://docs.fluentd.org/v1.0/articles/output-plugin-overview) and use `compat_parameters` plugin helper. It adds the following options:
-
-```
-buffer_type memory
-flush_interval 10s
-retry_limit 17
-retry_wait 1.0
-num_threads 1
-```
+[Fluentd](https://fluentd.org/) output plugin to ship logs to a Loki server. See [docs/client/fluentd/README.md](../../docs/clients/fluentd/README.md) for detailed information.
 
 ## Development
 

data/bin/setup

@@ -3,5 +3,5 @@ set -euo pipefail
 IFS=$'\n\t'
 set -vx
 
-sudo gem install bundle
+gem install bundler
 bundle install

data/lib/fluent/plugin/out_loki.rb

@@ -17,7 +17,6 @@
 
 require 'fluent/plugin/output'
 require 'net/http'
-require 'uri'
 require 'yajl'
 require 'time'
 
@@ -70,9 +69,14 @@ module Fluent
         config_set_default :chunk_keys, []
       end
 
-      def configure(conf)
+      def configure(conf) # rubocop:disable Metrics/CyclomaticComplexity
         compat_parameters_convert(conf, :buffer)
         super
+        @uri = URI.parse(@url + '/loki/api/v1/push')
+        unless @uri.is_a?(URI::HTTP) || @uri.is_a?(URI::HTTPS)
+          raise Fluent::ConfigError, 'url parameter must be valid HTTP'
+        end
+
         @record_accessors = {}
         conf.elements.select { |element| element.name == 'label' }.each do |element|
           element.each_pair do |k, v|
@@ -127,27 +131,26 @@ module Fluent
         body = { 'streams' => payload }
 
         # add ingest path to loki url
-        uri = URI.parse(url + '/loki/api/v1/push')
 
         req = Net::HTTP::Post.new(
-          uri.request_uri
+          @uri.request_uri
         )
         req.add_field('Content-Type', 'application/json')
         req.add_field('X-Scope-OrgID', @tenant) if @tenant
         req.body = Yajl.dump(body)
         req.basic_auth(@username, @password) if @username
 
-        opts = ssl_opts(uri)
+        opts = ssl_opts(@uri)
 
         log.debug "sending #{req.body.length} bytes to loki"
-        res = Net::HTTP.start(uri.hostname, uri.port, **opts) { |http| http.request(req) }
+        res = Net::HTTP.start(@uri.host, @uri.port, **opts) { |http| http.request(req) }
         unless res&.is_a?(Net::HTTPSuccess)
           res_summary = if res
                           "#{res.code} #{res.message} #{res.body}"
                         else
                           'res=nil'
                         end
-          log.warn "failed to #{req.method} #{uri} (#{res_summary})"
+          log.warn "failed to #{req.method} #{@uri} (#{res_summary})"
           log.warn Yajl.dump(body)
 
         end
@@ -215,7 +218,13 @@ module Fluent
       end
 
       def to_nano(time)
-        time.to_i * (10**9) + time.nsec
+        # time is a Fluent::EventTime object, or an Integer which represents unix timestamp (seconds from Epoch)
+        # https://docs.fluentd.org/plugin-development/api-plugin-output#chunk-each-and-block
+        if time.is_a?(Fluent::EventTime)
+          time.to_i * (10**9) + time.nsec
+        else
+          time.to_i * (10**9)
+        end
       end
 
       def record_to_line(record)

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-grafana-loki
 version: !ruby/object:Gem::Version
-  version: 1.2.7
+  version: 1.2.9
 platform: ruby
 authors:
 - woodsaj
@@ -10,7 +10,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2020-01-17 00:00:00.000000000 Z
+date: 2020-02-18 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -60,7 +60,7 @@ dependencies:
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 0.14.10
+        version: 1.9.0
     - - "<"
       - !ruby/object:Gem::Version
         version: '2'
@@ -70,7 +70,7 @@ dependencies:
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 0.14.10
+        version: 1.9.0
     - - "<"
       - !ruby/object:Gem::Version
         version: '2'