fluent-plugin-splunk-ex 1.0.0

data/.travis.yml

@@ -0,0 +1,7 @@
+rvm:
+  - 1.9.3
+  - 2.0.0
+  - 2.1.*
+gemfile:
+  - Gemfile
+

data/Gemfile

@@ -0,0 +1,4 @@
+source "http://rubygems.org"
+
+gemspec
+

data/README.md

@@ -0,0 +1,56 @@
+[![Build Status](https://travis-ci.org/gtrevg/fluent-plugin-splunk-ex.svg?branch=master)](https://travis-ci.org/gtrevg/fluent-plugin-splunk-ex)
+
+## Overview
+
+This plugin will send your fluentd logs to a splunk server.  It can send the data in either
+key/value (k1=v1 k2=v2) or json format for easy splunk parsing.
+
+
+## Installation
+
+    gem install fluent-plugin-splunk-ex
+
+## Configuration
+
+### Plugin
+
+    <match pattern>
+      type splunk_ex
+      host <splunk_host>          # default: localhost
+      port <splunk_port>          # default: 9997
+      output_format json|kv       # default: json
+    </match>
+
+### Splunk
+
+You may need to open up a special TCP port just for the fluentd logs.  To do that, go to
+`Manager -> Data Inputs -> TCP -> New`.  Then decide the following:
+
+* Port
+* Source Name
+* Source Type
+* Index ( default works well )
+
+After enabling these settings, you'll be able to see your fluentd logs appear in your Splunk search interface.
+The JSON format will automagically be parsed and indexed based on the keys passed in.
+
+Because the plugin batch send data to Splunk, you'll want to update your `apps/search/local/props.conf`
+file to specify that Splunk should split on newlines. If you do not update this setting, you find that
+all logs from a similar time slice will be stacked upon each other.  Because the kv & json formats do
+not contain any newline characters, splitting on the newline will solve this problem.  The values to
+add to this file are:
+
+    [<source_type_here>]
+    SHOULD_LINEMERGE = false
+    
+This will make sure that the new source type you just set up for fluentd will always split on the newline character.
+
+## Copyright
+
+Copyright (c) 2014 Trevor Gattis
+
+## License
+
+Apache License, Version 2.0
+
+

data/Rakefile

@@ -0,0 +1,16 @@
+# encoding: utf-8
+require "bundler/gem_tasks"
+
+require 'rspec/core'
+require 'rspec/core/rake_task'
+RSpec::Core::RakeTask.new(:spec) do |spec|
+  spec.pattern = FileList['spec/**/*_spec.rb']
+end
+task :default => :spec
+
+desc 'Open an irb session preloaded with the gem library'
+task :console do
+    sh 'irb -rubygems -I lib'
+end
+task :c => :console
+

data/fluent-plugin-splunk-ex.gemspec

@@ -0,0 +1,28 @@
+# encoding: utf-8
+
+Gem::Specification.new do |gem|
+  gem.name         = "fluent-plugin-splunk-ex"
+  gem.version      = "1.0.0"
+
+  gem.authors      = ["Trevor Gattis"]
+  gem.email        = "github@trevorgattis.com"
+  gem.description  = "Splunk output plugin for Fluent event collector"
+  gem.homepage     = "https://github.com/ggatti000/fluent-plugin-splunk-ex"
+  gem.summary      = gem.description
+  gem.license      = "APLv2"
+  gem.has_rdoc = false
+
+  gem.files = `git ls-files`.split("\n")
+  gem.test_files = `git ls-files -- {test,spec,features}/*`.split("\n")
+  gem.executables = `git ls-files -- bin/*`.split("\n").map{ |f| File.basename(f) }
+  gem.require_paths = ['lib']
+
+  gem.add_dependency "fluentd", "~> 0.10.17"
+  gem.add_runtime_dependency "json"
+
+  gem.add_development_dependency "rake"
+  gem.add_development_dependency "rspec"
+  gem.add_development_dependency "pry"
+  gem.add_development_dependency "pry-nav"
+end
+

data/lib/fluent/plugin/out_splunk_ex.rb

@@ -0,0 +1,124 @@
+require 'open-uri'
+require 'json'
+
+class Fluent::SplunkExOutput < Fluent::Output
+
+  SOCKET_TRY_MAX = 3
+
+  Fluent::Plugin.register_output('splunk_ex', self)
+
+  include Fluent::SetTagKeyMixin
+  config_set_default :include_tag_key, false
+
+  include Fluent::SetTimeKeyMixin
+  config_set_default :include_time_key, false
+
+  config_param :host, :string, :default => 'localhost'
+  config_param :port, :string, :default => 9997
+  config_param :output_format, :string, :default => 'json'
+
+  config_param :test_mode, :bool, :default => false
+
+  # To support log_level option implemented by Fluentd v0.10.43
+  unless method_defined?(:log)
+    define_method(:log) { $log }
+  end
+
+
+  def configure(conf)
+    super
+  end
+
+  def start
+    super
+
+    if @output_format == 'kv'
+      @format_fn = self.class.method(:format_kv)
+    else
+      @format_fn = self.class.method(:format_json)
+    end
+
+    if @test_mode
+      @send_data = proc { |text| log.trace("test mode text: #{text}") }
+    else
+      begin
+        @splunk_connection = TCPSocket.open(@host, @port)
+      rescue Errno::ECONNREFUSED
+        # we'll try again when data is sent
+      end
+      @send_data = self.method(:splunk_send)
+    end
+
+  end
+
+
+  def shutdown
+    super
+    if !@test_mode && @splunk_connection.respond_to?(:close)
+      @splunk_connection.close
+    end
+  end
+
+
+  def emit(tag, es, chain)
+    chain.next
+    es.each {|time,record|
+      @send_data.call( @format_fn.call(record) )
+    }
+  end
+
+  # =================================================================
+
+  protected
+
+  def self.format_kv(record)
+    kv_out_str = ''
+    record.each { |k, v|
+      kv_out_str << sprintf('%s=%s ', URI::encode(k), URI::encode(v.to_s))
+    }
+    kv_out_str
+  end
+
+  def self.format_json(record)
+    json_str = record.to_json
+  end
+
+  def splunk_send(text, try_count=0)
+    log.debug("splunk_send: #{text}")
+
+    successful_send = false
+    try_count = 0
+
+    while (!successful_send && try_count < SOCKET_TRY_MAX)
+      begin
+        @splunk_connection.puts(text)
+        successful_send = true
+
+      rescue NoMethodError, Errno::ECONNRESET, Errno::ECONNREFUSED, Errno::EPIPE => se
+        log.error("splunk_send - socket send retry (#{try_count}) failed: #{se}")
+        try_count = try_count + 1
+
+        successful_reopen = false
+        while (!successful_reopen && try_count < SOCKET_TRY_MAX)
+          begin
+            # Try reopening
+            @splunk_connection = TCPSocket.open(@host, @port)
+            successful_reopen = true
+          rescue Errno::ECONNRESET, Errno::ECONNREFUSED, Errno::EPIPE => se
+            log.error("splunk_send - socket open retry (#{try_count}) failed: #{se}")
+            try_count = try_count + 1
+          end
+        end
+      end
+    end
+
+    if !successful_send
+      log.fatal("splunk_send - retry of sending data failed after #{SOCKET_TRY_MAX} chances.")
+      log.warn(text)
+    end
+    
+  end
+
+
+end
+

data/spec/out_splunk_ex_spec.rb

@@ -0,0 +1,74 @@
+# encoding: UTF-8
+require_relative 'spec_helper'
+require 'benchmark'
+Fluent::Test.setup
+
+def create_driver(config, tag = 'foo.bar')
+  Fluent::Test::OutputTestDriver.new(Fluent::SplunkExOutput, tag).configure(config)
+end
+
+# setup
+single_key_message = {
+  'msg' => 'testing some data'
+}
+
+multi_key_message = {
+  'msg'            => 'testing some data',
+  'chars'          => 'let"s put !@#$%^&*()-= some weird :\'?><,./ characters',
+  'dt'             => '2014/04/03T07:02:11.124202',
+  'debug_line'     => 24,
+  'debug_file'     => '/some/path/to/myFile.py',
+  'statsd_key'     => 'fluent_plugin_splunk_ex',
+  'statsd_timing'  => 0.234,
+  'statsd_type'    => 'timing',
+  'tx'             => '280c3e80-bb6c-11e3-a5e2-0800200c9a66',
+  'host'           => 'my01.cool.server.com'
+}
+
+time = Time.now.to_i
+
+driver_kv = create_driver(%[
+  log_level     fatal
+  test_mode     true
+  output_format kv
+])
+
+driver_json = create_driver(%[
+  log_level     fatal
+  test_mode     true
+  output_format json
+])
+
+driver_kv_time = create_driver(%[
+  log_level fatal
+  test_mode true
+  output_format kv
+  time_key myKey
+  include_time_key true
+])
+
+driver_json_time = create_driver(%[
+  log_level fatal
+  test_mode true
+  output_format json
+  time_key myKey
+  include_time_key true
+])
+
+
+
+# bench
+n = 10000
+Benchmark.bm(7) do |x|
+  x.report("single_kv       ") { driver_kv.run        { n.times { driver_kv.emit(       single_key_message, time) } } }
+  x.report("single_kv_time  ") { driver_kv_time.run   { n.times { driver_kv_time.emit(  single_key_message, time) } } }
+  x.report("single_json     ") { driver_json.run      { n.times { driver_json.emit(     single_key_message, time) } } }
+  x.report("single_json_time") { driver_json_time.run { n.times { driver_json_time.emit(single_key_message, time) } } }
+
+  x.report("multi_kv        ") { driver_kv.run        { n.times { driver_kv.emit(       multi_key_message, time ) } } }
+  x.report("multi_kv_time   ") { driver_kv_time.run   { n.times { driver_kv_time.emit(  multi_key_message, time ) } } }
+  x.report("multi_json      ") { driver_json.run      { n.times { driver_json.emit(     multi_key_message, time ) } } }
+  x.report("multi_json_time ") { driver_json_time.run { n.times { driver_json_time.emit(multi_key_message, time ) } } }
+
+end
+

data/spec/spec_helper.rb

@@ -0,0 +1,22 @@
+# This file was generated by the `rspec --init` command. Conventionally, all
+# specs live under a `spec` directory, which RSpec adds to the `$LOAD_PATH`.
+# Require this file using `require "spec_helper"` to ensure that it is only
+# loaded once.
+#
+# See http://rubydoc.info/gems/rspec-core/RSpec/Core/Configuration
+RSpec.configure do |config|
+  config.treat_symbols_as_metadata_keys_with_true_values = true
+  config.run_all_when_everything_filtered = true
+  config.filter_run :focus
+
+  # Run specs in random order to surface order dependencies. If you find an
+  # order dependency and want to debug it, you can fix the order by providing
+  # the seed, which is printed after each run.
+  #     --seed 1234
+  config.order = 'random'
+end
+
+require 'fluent/test'
+
+require 'fluent/plugin/out_splunk_ex'
+

metadata

@@ -0,0 +1,151 @@
+--- !ruby/object:Gem::Specification
+name: fluent-plugin-splunk-ex
+version: !ruby/object:Gem::Version
+  version: 1.0.0
+  prerelease: 
+platform: ruby
+authors:
+- Trevor Gattis
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2014-04-14 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: fluentd
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 0.10.17
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 0.10.17
+- !ruby/object:Gem::Dependency
+  name: json
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: rspec
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: pry
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: pry-nav
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+description: Splunk output plugin for Fluent event collector
+email: github@trevorgattis.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- .travis.yml
+- Gemfile
+- README.md
+- Rakefile
+- fluent-plugin-splunk-ex.gemspec
+- lib/fluent/plugin/out_splunk_ex.rb
+- spec/out_splunk_ex_spec.rb
+- spec/spec_helper.rb
+homepage: https://github.com/ggatti000/fluent-plugin-splunk-ex
+licenses:
+- APLv2
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 1.8.23
+signing_key: 
+specification_version: 3
+summary: Splunk output plugin for Fluent event collector
+test_files:
+- spec/out_splunk_ex_spec.rb
+- spec/spec_helper.rb