fluent-plugin-splunkapi 0.1.5 → 0.2.0

data/README.md

@@ -12,6 +12,55 @@ Splunk Storm API:
 
   http://docs.splunk.com/Documentation/Storm/latest/User/UseStormsRESTAPI
 
+## Notes
+
+Although this plugin is capable of sending Fluent events directly to
+Splunk servers or Splunk Storm, it is not recommended to do so.
+Please use "Universal Forwarder" as a gateway, as described below.
+
+It is known that this plugin has several issues of performance and
+error handling in dealing with large data sets.  With a local/reliable
+forwarder, you can aggregate a number of events locally and send them
+to a server in bulk.
+
+In short, I'd recommend to install a forwarder in each host, and use
+this plugin to deliver events to the local forwarder:
+
+    <match **>
+      # Deliver events to the local forwarder.
+      type splunkapi
+      protocol rest
+      server 127.0.0.1:8089
+      verify false
+      auth admin:changeme
+
+      # Convert fluent tags to Splunk sources.
+      # If you set an index, "check_index false" is required.
+      host YOUR-HOSTNAME
+      index SOME-INDEX
+      check_index false
+      source {TAG}
+      sourcetype fluent
+
+      # TIMESTAMP: key1="value1" key2="value2" ...
+      time_format unixtime
+      format kvp
+
+      # Memory buffer with a short flush internal.
+      buffer_type memory
+      buffer_queue_limit 16
+      buffer_chunk_limit 8m
+      flush_interval 2s
+    </match>
+
+## Additional Notes
+
+Splunk 5 has a new feature called "Modular Inputs":
+
+http://blogs.splunk.com/2013/04/16/modular-inputs-tools/
+
+My plan is switching to Modular Inputs rather than staying with APIs.
+
 ## Installation
 
 Add this line to your application's Gemfile:

data/fluent-plugin-splunkapi.gemspec

@@ -3,7 +3,7 @@ $:.push File.expand_path("../lib", __FILE__)
 
 Gem::Specification.new do |gem|
   gem.name          = "fluent-plugin-splunkapi"
-  gem.version       = "0.1.5"
+  gem.version       = "0.2.0"
   gem.authors       = ["Keisuke Nishida"]
   gem.email         = ["knishida@bizmobile.co.jp"]
   gem.summary       = %q{Splunk output plugin (REST API / Storm API) for Fluent event collector}

data/lib/fluent/plugin/out_splunkapi.rb

@@ -166,12 +166,23 @@ class SplunkAPIOutput < BufferedOutput
       # retry up to :post_retry_max times
       1.upto(@post_retry_max) do |c|
         response = @http.request uri, post
-        break if response.code != "503"
         $log.debug "=> #{response.code} (#{response.message})"
-        sleep @post_retry_interval
-        # fluentd will retry processing on exception
-        # FIXME: this may duplicate logs with multiple buffers
-        raise "#{uri}: #{response.message}" if c == @post_retry_max
+        if response.code == "200"
+          # success
+          break
+        elsif response.code.match(/^40/)
+          # user error
+          $log.error "#{uri}: #{response.code} (#{response.message})\n#{response.body}"
+          break
+        elsif c < @post_retry_max
+          # retry
+          sleep @post_retry_interval
+          next
+        else
+          # other errors. fluentd will retry processing on exception
+          # FIXME: this may duplicate logs when using multiple buffers
+          raise "#{uri}: #{response.message}"
+        end
       end
     end
   end

metadata

@@ -4,9 +4,9 @@ version: !ruby/object:Gem::Version
   prerelease: false
   segments: 
   - 0
-  - 1
-  - 5
-  version: 0.1.5
+  - 2
+  - 0
+  version: 0.2.0
 platform: ruby
 authors: 
 - Keisuke Nishida
@@ -14,7 +14,7 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2013-07-28 00:00:00 +09:00
+date: 2013-08-11 00:00:00 +09:00
 default_executable: 
 dependencies: 
 - !ruby/object:Gem::Dependency