fluent-plugin-datadog 0.9.2.beta

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: b93c02f502b84f29f21972b23951089eac51b38a
+  data.tar.gz: 7dd0ef40fb8ac4d90ff724a3b91c081a4fb443dc
+SHA512:
+  metadata.gz: b6e9aa3bca5010c1b2bbc347450174359d79710754a2175c45cc3c17094ab6651ffe256f310e0c63ef60824d4ae75e2cfde2f4369f26d87dfb1d9a61590b1024
+  data.tar.gz: 4921013a7da5cd1ab3400d920fe8b51dee1b8b58f1063c0ba7a060add48d67b9603583de8585908a38543e71b13fa56dfd96a093396bf3f4c0bb8b89e56f8132

data/.gitignore

@@ -0,0 +1,41 @@
+*.gem
+*.rbc
+/.config
+/coverage/
+/InstalledFiles
+/pkg/
+/spec/reports/
+/spec/examples.txt
+/test/tmp/
+/test/version_tmp/
+/tmp/
+
+## Specific to RubyMotion:
+.dat*
+.repl_history
+build/
+
+## Documentation cache and generated files:
+/.yardoc/
+/_yardoc/
+/doc/
+/rdoc/
+
+## Environment normalization:
+/.bundle/
+/vendor/bundle
+/lib/bundler/man/
+
+# for a library or gem, you might want to ignore these files since the code is
+# intended to run in multiple environments; otherwise, check them in:
+# Gemfile.lock
+# .ruby-version
+# .ruby-gemset
+
+# unless supporting rvm < 1.11.0 or doing something fancy, ignore this:
+.rvmrc
+
+# Misc
+foo/
+*.iml
+.idea/

data/Gemfile

@@ -0,0 +1,4 @@
+source 'https://rubygems.org'
+
+# Specify your gem's dependencies in fluent-plugin-datadog.gemspec
+gemspec

data/LICENSE

@@ -0,0 +1,21 @@
+The MIT License (MIT)
+
+Copyright (c) 2017 Datadog
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

data/README.md

@@ -0,0 +1,69 @@
+# Fluentd output plugin for Datadog
+
+It mainly contains a proper JSON formatter and a socket handler that
+streams logs directly to Datadog - so no need to use a log shipper
+if you don't wan't to.
+
+## Pre-requirements
+
+To add the plugin to your fluentd agent, use the following command:
+
+    gem install fluent-plugin-datadog
+
+If you installed the td-agent instead
+
+    /usr/sbin/td-agent-gem install fluent-plugin-datadog
+
+## Usage
+### Configure the output plugin
+
+To match events and send them to Datadog, simply add the following code to your configuration file.
+
+TCP example
+```xml
+# Match events tagged with "datadog.**" and
+# send them to Datadog
+<match datadog.**>
+
+  @type datadog
+  @id awesome_agent
+  api_key <your_api_key>
+
+  # Optional
+  include_tag_key true
+  tag_key 'tag'
+
+</match>
+```
+
+After a restart of FluentD, any child events tagged with `datadog` are shipped to your platform.
+
+### Validation
+Let's make a simple test.
+
+```bash
+ curl -X POST -d 'json={"message":"hello Datadog from fluentd"}' http://localhost:8888/datadog.test
+```
+
+Produces the following event:
+
+```javascript
+{
+    "message": "hello Datadog from fluentd"
+}
+```
+
+### fluent-plugin-datadog properties
+Let's go deeper on the plugin configuration.
+
+As fluent-plugin-datadog is an output_buffer, you can set all output_buffer properties like it's describe in the [fluentd documentation](http://docs.fluentd.org/articles/output-plugin-overview#buffered-output-parameters "documentation").
+
+
+|  Property   |  Description                                                             |  Default value |
+|-------------|--------------------------------------------------------------------------|----------------|
+| **api_key** | This parameter is required in order to authenticate your fluent agent.   | nil            |
+| **use_json**| Event format, if true, the event is sent in json format. Othwerwise, in plain text. | true      |
+| **include_tag_key**| Automatically include tags in the record. | false      |
+| **tag_key**| Name of the tag attribute, if they are included. | "tag"      |
+| **use_ssl** | If true, the agent initializes a secure connection to Datadog. In clear TCP otherwise. | true |
+|**max_retries**| The number of retries before the output plugin stops. Set to -1 for unlimited retries | -1 |

data/Rakefile

@@ -0,0 +1 @@
+require "bundler/gem_tasks"

data/fluent-plugin-datadog.gemspec

@@ -0,0 +1,22 @@
+# coding: utf-8
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+
+Gem::Specification.new do |spec|
+  spec.name          = "fluent-plugin-datadog"
+  spec.version       = "0.9.2.beta"
+  spec.authors       = ["Datadog support team"]
+  spec.email         = ["support@datadoghq.com"]
+  spec.summary       = "Datadog output plugin for Fluent event collector"
+  spec.homepage      = "http://datadoghq.com"
+  spec.license       = "MIT"
+
+  spec.files         = [".gitignore", "Gemfile", "LICENSE", "README.md", "Rakefile", "fluent-plugin-datadog.gemspec", "lib/fluent/plugin/out_datadog.rb"]
+  spec.executables   = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
+  spec.test_files    = spec.files.grep(%r{^(test|spec|features)/})
+  spec.require_paths = ["lib"]
+
+  spec.add_development_dependency "bundler", "~> 1.5"
+  spec.add_development_dependency "rake"
+  spec.add_development_dependency "yajl-ruby", "~> 1.2"
+end

data/lib/fluent/plugin/out_datadog.rb

@@ -0,0 +1,151 @@
+require 'socket'
+require 'openssl'
+require 'yajl'
+
+class Fluent::DatadogOutput < Fluent::BufferedOutput
+  class ConnectionFailure < StandardError; end
+
+  # Register the plugin
+  Fluent::Plugin.register_output('datadog', self)
+  # Output settings
+  config_param :use_json,       :bool,    :default => true
+  config_param :include_tag_key,:bool,    :default => false
+  config_param :tag_key,        :string,  :default => 'tag'
+
+  # Connection settings
+  config_param :host,           :string,  :default => 'intake.logs.datadoghq.com'
+  config_param :use_ssl,        :bool,    :default => false
+  config_param :port,           :integer, :default => 10514
+  config_param :ssl_port,       :integer, :default => 10516
+  config_param :max_retries,    :integer, :default => -1
+  config_param :tcp_ping_rate,  :integer, :default => 10
+
+  # API Settings
+  config_param :api_key,  :string
+
+  def initialize
+    super
+  end
+
+  # Define `log` method for v0.10.42 or earlier
+  unless method_defined?(:log)
+    define_method("log") { $log }
+  end
+
+  def configure(conf)
+    super
+  end
+
+  def client
+    @_socket ||= if @use_ssl
+      context    = OpenSSL::SSL::SSLContext.new
+      socket     = TCPSocket.new @host, @ssl_port
+      ssl_client = OpenSSL::SSL::SSLSocket.new socket, context
+      ssl_client.connect
+    else
+      socket = TCPSocket.new @host, @port
+    end
+
+    return @_socket
+
+  end
+
+  #not used for now...
+  def init_socket(socket)
+    socket.setsockopt(Socket::SOL_SOCKET, Socket::SO_KEEPALIVE, true)
+
+    begin
+      socket.setsockopt(Socket::SOL_TCP, Socket::TCP_KEEPINTVL, 3)
+      socket.setsockopt(Socket::SOL_TCP, Socket::TCP_KEEPCNT, 3)
+      socket.setsockopt(Socket::SOL_TCP, Socket::TCP_KEEPIDLE, 10)
+    rescue
+      log.info "DatadogOutput: Fallback on socket options during initialization"
+    end
+
+    return socket
+  end
+
+  def start
+    super
+    @my_mutex = Mutex.new
+    @running = true
+
+    if @tcp_ping_rate > 0
+      @timer = Thread.new do
+        while @running do
+          messages = Array.new
+          messages.push("fp\n")
+          send_to_datadog(messages)
+          sleep(15)
+        end
+      end
+    end
+
+  end
+
+  def shutdown
+    super
+    @running = false
+    if @_socket
+      @_socket.close()
+    end
+  end
+
+  # This method is called when an event reaches Fluentd.
+  def format(tag, time, record)
+    return [tag, record].to_msgpack
+  end
+
+  # NOTE! This method is called by internal thread, not Fluentd's main thread.
+  # 'chunk' is a buffer chunk that includes multiple formatted events.
+  def write(chunk)
+    messages = Array.new
+    chunk.msgpack_each do |tag, record|
+      next unless record.is_a? Hash
+      next unless record.has_key? "message"
+
+      if @include_tag_key
+        record[@tag_key] = tag
+      end
+      if @use_json
+        messages.push "#{api_key} " + Yajl.dump(record) + "\n"
+      else
+        messages.push "#{api_key} " + record["message"].rstrip() + "\n"
+      end
+    end
+    send_to_datadog(messages)
+
+  end
+
+  def send_to_datadog(data)
+    @my_mutex.synchronize do
+      retries = 0
+      begin
+        log.trace "Send nb_event=#{data.size} events to Datadog"
+
+        # Check the connectivity and write messages
+        log.info "New attempt to Datadog attempt=#{retries}" if retries > 0
+
+        retries = retries + 1
+        data.each do |event|
+          client.write(event)
+        end
+
+        # Handle some failures
+      rescue Errno::EHOSTUNREACH, Errno::ECONNREFUSED, Errno::ETIMEDOUT, Errno::EPIPE => e
+
+        if retries < @max_retries || max_retries == -1
+          @_socket = nil
+          a_couple_of_seconds = retries ** 2
+          a_couple_of_seconds = 30 unless a_couple_of_seconds < 30
+          retries += 1
+          log.warn "Could not push logs to Datadog, attempt=#{retries} max_attempts=#{max_retries} wait=#{a_couple_of_seconds}s error=#{e.message}"
+          sleep a_couple_of_seconds
+          retry
+        end
+        raise ConnectionFailure, "Could not push logs to Datadog after #{retries} retries, #{e.message}"
+      end
+    end
+  end
+
+end

metadata

@@ -0,0 +1,93 @@
+--- !ruby/object:Gem::Specification
+name: fluent-plugin-datadog
+version: !ruby/object:Gem::Version
+  version: 0.9.2.beta
+platform: ruby
+authors:
+- Datadog support team
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2017-10-05 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.5'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.5'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: yajl-ruby
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.2'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.2'
+description: 
+email:
+- support@datadoghq.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- ".gitignore"
+- Gemfile
+- LICENSE
+- README.md
+- Rakefile
+- fluent-plugin-datadog.gemspec
+- lib/fluent/plugin/out_datadog.rb
+homepage: http://datadoghq.com
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">"
+    - !ruby/object:Gem::Version
+      version: 1.3.1
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.6.13
+signing_key: 
+specification_version: 4
+summary: Datadog output plugin for Fluent event collector
+test_files: []