fluent-plugin-deis-kafka 0.1.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 52fe5165a1eafbb97ae6af21ce01e4ac2c7aaaa0
+  data.tar.gz: 8acc20b11e2e5c4d3d85b2ff528637259dc13925
+SHA512:
+  metadata.gz: 171b8ceea5f2fd90e48550d9a9f72daadb6b59dc3b770035c4841d3fe6cc678b559f3489f6b442a6d2e9f5cae4c24535bc57a620827b3e605be4244fa9cea830
+  data.tar.gz: d26124dca9d98030be9359460dbbb9c4eb7b593665cc59807a44b7cc0a6a9faab66766dc4f3f8fac0b25dca0573bb8fa37eaa0e8762c42dd401decedffbd7592

data/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2017 Top Free Games
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

data/README.md

@@ -0,0 +1 @@
+# fluentd-plugin-deis-kafka

data/lib/fluent/mixin/deis.rb

@@ -0,0 +1,67 @@
+# frozen_string_literal: true
+
+module Fluent
+  module Mixin
+    module Deis
+      def kubernetes?(message)
+        !message['kubernetes'].nil?
+      end
+
+      def from_router?(message)
+        from_container?(message, 'deis-router')
+      end
+
+      def from_container?(message, regex)
+        if kubernetes? message
+          return true unless Regexp.new(regex).match(message['kubernetes']['container_name']).nil?
+        end
+        false
+      end
+
+      def build_series(message)
+        metric = parse_router_log(message['log'], message['kubernetes']['host'])
+        if metric
+          tags = { app: metric['app'], status_code: metric['status_code'], host: metric['host'] }
+          data = [
+            {
+              series: 'deis_router_request_time_ms',
+              values: { value: metric['request_time'] },
+              tags: tags
+            },
+            {
+              series: 'deis_router_response_time_ms',
+              values: { value: metric['response_time'] },
+              tags: tags
+            },
+            {
+              series: 'deis_router_bytes_sent',
+              values: { value: metric['bytes_sent'] },
+              tags: tags
+            }
+          ]
+          return data
+        end
+        nil
+      end
+
+      # {"log"=>"[2016-05-31T16:56:12+00:00] - foo - 10.164.1.1 - - - 200 - \"GET / HTTP/1.0\" - 211 - \"-\" - \"ApacheBench/2.3\" - \"~^foo\\x5C.(?<domain>.+)$\" - 10.167.243.4:80 - foo.devart.io - 0.002 - 0.046\n"}
+      # request_time - request processing time in seconds with a milliseconds resolution; time elapsed between the first bytes were read from the client and the log write after the last bytes were sent to the client
+      # response_time - keeps time spent on receiving the response from the upstream server; the time is kept in seconds with millisecond resolution.
+      def parse_router_log(message, host)
+        split_message = message.split(' - ')
+        return nil if split_message.length < 14
+        metric = {}
+        metric['app'] = split_message[1].strip
+        metric['status_code'] = split_message[4].strip
+        metric['bytes_sent'] = split_message[6].strip.to_f
+        metric['response_time'] = split_message[12].strip.to_f
+        metric['request_time'] = split_message[13].strip.to_f
+        metric['host'] = host
+        return metric
+      rescue Exception => e # rubocop:disable RescueException
+        puts "Error:#{e.message}"
+        return nil
+      end
+    end
+  end
+end

data/lib/fluent/plugin/kafka_producer_ext.rb

@@ -0,0 +1,28 @@
+# frozen_string_literal: true
+
+require 'kafka/pending_message'
+require 'kafka/compressor'
+require 'kafka/producer'
+
+# for out_kafka_buffered
+module Kafka
+  class Producer
+    def produce2(value, topic:)
+      create_time = Time.now
+
+      message = PendingMessage.new(
+        value,
+        nil,
+        topic,
+        nil,
+        nil,
+        create_time
+      )
+
+      @target_topics.add(topic)
+      @pending_message_queue.write(message)
+
+      nil
+    end
+  end
+end

data/lib/fluent/plugin/out_deis.rb

@@ -0,0 +1,214 @@
+# frozen_string_literal: true
+
+require 'fluent/mixin/config_placeholders'
+require 'fluent/mixin/plaintextformatter'
+require 'fluent/mixin/rewrite_tag_name'
+require 'fluent/mixin/deis'
+require 'fluent/output'
+require 'influxdb'
+
+module Fluent
+  class DeisOutput < Output
+    Fluent::Plugin.register_output('deis', self)
+
+    include Fluent::Mixin::PlainTextFormatter
+    include Fluent::Mixin::ConfigPlaceholders
+    include Fluent::HandleTagNameMixin
+    include Fluent::Mixin::RewriteTagName
+    include Fluent::Mixin::Deis
+    config_param :brokers, :string, default: 'localhost:9092',
+                                    desc: <<-DESC
+                                     Set brokers directly:
+                                     <broker1_host>:<broker1_port>,<broker2_host>:<broker2_port>,..
+                                    DESC
+    config_param :client_id, :string, default: 'fluentd'
+    config_param :metrics_topic, :string, default: 'metrics'
+    config_param :discard_kafka_delivery_failed, :bool, default: false
+
+    # ruby-kafka producer options
+    config_param :max_send_retries, :integer, default: 2,
+                                              desc: 'Number of times to retry '\
+                                              'sending of messages to a leader.'
+    config_param :required_acks, :integer, default: 1,
+                                           desc: 'The number of acks required per request.'
+    config_param :ack_timeout, :time, default: nil,
+                                      desc: 'How long the producer waits for acks.'
+    config_param :compression_codec, :string, default: nil,
+                                              desc: <<~DESC
+                                                The codec the producer uses to compress messages.
+                                                Supported codecs: (gzip|snappy)
+                                              DESC
+
+    config_param :max_send_limit_bytes, :size, default: nil
+    config_param :kafka_agg_max_bytes, :size, default: 4 * 1024 # 4k
+    config_param :kafka_agg_max_messages, :integer, default: nil
+
+    define_method('log') { $log } unless method_defined?(:log) # rubocop:disable GlobalVars
+
+    def initialize
+      super
+      require 'kafka'
+      require 'fluent/plugin/kafka_producer_ext'
+
+      @kafka = nil
+      @producers = {}
+      @producers_mutex = Mutex.new
+    end
+
+    def start
+      super
+      refresh_client
+    end
+
+    def shutdown
+      super
+      shutdown_producers
+      @kafka = nil
+    end
+
+    def format_stream(_tag, es)
+      es.to_msgpack_stream
+    end
+
+    def shutdown_producers
+      @producers_mutex.synchronize do
+        @producers.each_value(&:shutdown)
+        @producers = {}
+      end
+    end
+
+    def get_producer # rubocop:disable AccessorMethodName
+      @producers_mutex.synchronize do
+        producer = @producers[Thread.current.object_id]
+        unless producer
+          producer = @kafka.producer(@producer_opts)
+          @producers[Thread.current.object_id] = producer
+        end
+        producer
+      end
+    end
+
+    def deliver_messages(producer, tag)
+      if @discard_kafka_delivery_failed
+        begin
+          producer.deliver_messages
+        rescue Kafka::DeliveryFailed => e
+          log.warn 'DeliveryFailed occurred. Discard broken event:',
+                   error: e.to_s, error_class: e.class.to_s, tag: tag
+          producer.clear_buffer
+        end
+      else
+        producer.deliver_messages
+      end
+    end
+
+    def refresh_client(raise_error = true)
+      @kafka = Kafka.new(seed_brokers: @brokers.split(','), client_id: @client_id)
+      log.info "initialized kafka producer: #{@client_id}"
+    rescue Exception => e # rubocop:disable RescueException
+      raise e if raise_error
+      log.error e
+    end
+
+    def configure(conf)
+      super
+
+      @producer_opts = { max_retries: @max_send_retries, required_acks: @required_acks }
+      @producer_opts[:ack_timeout] = @ack_timeout if @ack_timeout
+      @producer_opts[:compression_codec] = @compression_codec.to_sym if @compression_codec
+
+      return unless @discard_kafka_delivery_failed
+      log.warn "'discard_kafka_delivery_failed' option discards events which "\
+               'cause delivery failure, e.g. invalid topic or something.'
+      log.warn 'If this is unexpected, you need to check your configuration or data.'
+    end
+
+    # def emit(tag, es, chain)
+    #   super(tag, es, chain, tag)
+    # end
+
+    def filter_record(record)
+      return unless from_router?(record)
+      data = build_series(record)
+      return unless data
+      return data.map do |point|
+        InfluxDB::PointValue.new(point).dump
+      end.join("\n")
+    rescue Exception => e # rubocop:disable RescueException
+      puts "Error:#{e.backtrace}"
+    end
+
+    def write(chunk)
+      tag = chunk.metadata.tag
+      producer = get_producer
+
+      records_by_topic = {}
+      bytes_by_topic = {}
+      messages = 0
+      messages_bytes = 0
+      record_buf = nil
+      record_buf_bytes = nil
+      begin
+        Fluent::Engine.msgpack_factory.unpacker(chunk.open).each do |time, record|
+          begin
+            topic = @metrics_topic
+            records_by_topic[topic] ||= 0
+            bytes_by_topic[topic] ||= 0
+            line = filter_record(record)
+
+            next unless line
+            record_buf_bytes = line.bytesize
+            if @max_send_limit_bytes && record_buf_bytes > @max_send_limit_bytes
+              log.warn 'record size exceeds max_send_limit_bytes. Skip event:',
+                       time: time, record: record
+              next
+            end
+          rescue StandardError => e
+            log.warn 'unexpected error during format record. Skip broken event:',
+                     error: e.to_s, error_class: e.class.to_s, time: time, record: record
+            next
+          end
+
+          if messages.positive? &&
+             (messages_bytes + record_buf_bytes > @kafka_agg_max_bytes) ||
+             (@kafka_agg_max_messages && messages >= @kafka_agg_max_messages)
+            log.debug do
+              "#{messages} messages send because reaches the limit of batch transmission."
+            end
+            deliver_messages(producer, tag)
+            messages = 0
+            messages_bytes = 0
+          end
+
+          log.trace do
+            "message will send to #{topic} with partition_key: #{partition_key},"\
+            "partition: #{partition}, message_key: #{message_key} and value: #{record_buf}."
+          end
+
+          messages += 1
+          producer.produce2(
+            line,
+            topic: topic
+          )
+          messages_bytes += record_buf_bytes
+
+          records_by_topic[topic] += 1
+          bytes_by_topic[topic] += record_buf_bytes
+        end
+        if messages.positive?
+          log.debug { "#{messages} messages send." }
+          deliver_messages(producer, tag)
+        end
+        log.debug { "(records|bytes) (#{records_by_topic}|#{bytes_by_topic})" }
+      end
+    rescue Exception => e # rubocop:disable RescueException
+      log.warn "Send exception occurred: #{e}"
+      log.warn "Exception Backtrace : #{e.backtrace.join("\n")}"
+      # For safety, refresh client and its producers
+      shutdown_producers
+      refresh_client(false)
+      # Raise exception to retry sendind messages
+      raise e
+    end
+  end
+end

metadata

@@ -0,0 +1,218 @@
+--- !ruby/object:Gem::Specification
+name: fluent-plugin-deis-kafka
+version: !ruby/object:Gem::Version
+  version: 0.1.0
+platform: ruby
+authors:
+- Guilherme Souza
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2017-09-19 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: fluentd
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.14'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.14'
+- !ruby/object:Gem::Dependency
+  name: fluent-mixin-plaintextformatter
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: fluent-mixin-config-placeholders
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: fluent-mixin-rewrite-tag-name
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: influxdb
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.4'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.4'
+- !ruby/object:Gem::Dependency
+  name: ruby-kafka
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.4.2
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.4.2
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.3'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.3'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '10.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '10.0'
+- !ruby/object:Gem::Dependency
+  name: test-unit
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 3.1.7
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 3.1.7
+- !ruby/object:Gem::Dependency
+  name: rubocop
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: pry
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: simplecov
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+description: Fluentd plugin to send deis-router metricsto influxdb through kafka
+email:
+- backend@tfgco.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- LICENSE
+- README.md
+- lib/fluent/mixin/deis.rb
+- lib/fluent/plugin/kafka_producer_ext.rb
+- lib/fluent/plugin/out_deis.rb
+homepage: https://github.com/topfreegames/fluent-plugin-deis-kafka
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: 2.0.0
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.6.13
+signing_key: 
+specification_version: 4
+summary: 'Fluentd plugin to send deis-router metricsto influxdb through kafkabased
+  on: https://github.com/deis/fluentd and https://github.com/fluent/fluent-plugin-kafka'
+test_files: []