fluent-plugin-splunk-hec 1.0.0 → 1.0.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 891d6573230e5e79e740d8a1d583f5f8b205313169e9c7ae809fcac2e7bf2367
-  data.tar.gz: 5d6e4fbf0e6f80eb9eaf7ea1ba9b1b0e1c0ffb231e8682be1ecdb0f353a7a8cb
+  metadata.gz: 1a5e33a0b115b4d515ce94f1bf904bddf192994ff1b54fb624f5ea9d470bb4cd
+  data.tar.gz: 306d2647c27a63e98d3f530eacf2abd0fd16cd5f7f23213291ff061a11d4fb92
 SHA512:
-  metadata.gz: 8d728f7565cb240a50759acf353bfc5ef1de88223d79c81f5a9ba4f1c848414df8448d1642652ba42e0bc82983b0ffbc9918e6e6a8576aa3051f9eea18f44ac2
-  data.tar.gz: a5a0e99b20197fca605bc03f094657cb06367b61e69fd5562f0ad35c5cfe733b2ee66de62019d8576467d357f2e7de8eb8594497d9e30e651cafd15eff6ff526
+  metadata.gz: a813d1d0ef4e157c95c49a0f75641e51af2a18d56c02020ec702a06a4d48fad02cb7e60f044637adfc261d8ec6fce21775ef38aa6215e8dea5cdb850c2faa74c
+  data.tar.gz: 584ffa55837e358d705780cdd01f86bcd3245fe3ebcc4f3f0f999ae03fa5634443db531cb1a51ac806402399cac7d1f6d9ed43f612b4c331de445a9df6376172

data/VERSION

@@ -1 +1 @@
-1.0.0
+1.0.1

data/lib/fluent/plugin/out_splunk_hec.rb

@@ -121,7 +121,6 @@ module Fluent::Plugin
     def initialize
       super
       @default_host = Socket.gethostname
-      @chunk_queue = SizedQueue.new 1
       @extra_fields = nil
     end
 
@@ -143,21 +142,17 @@ module Fluent::Plugin
 
     def start
       super
-      start_worker_threads
+
+      @hec_conn = new_connection
     end
 
     def format(tag, time, record)
       # this method will be replaced in `configure`
     end
 
-    def try_write(chunk)
+    def write(chunk)
       log.debug { "Received new chunk, size=#{chunk.read.bytesize}" }
-      @chunk_queue << chunk
-    end
-
-    def close
-      @chunk_queue.close
-      super
+      send_to_hec chunk
     end
 
     def multi_workers_ready?
@@ -239,7 +234,11 @@ module Fluent::Plugin
     def format_event(tag, time, record)
       MultiJson.dump({
 	host: @host ? @host.(tag, record) : @default_host,
-	time: time.to_i
+	# From the API reference
+	# http://docs.splunk.com/Documentation/Splunk/latest/RESTREF/RESTinput#services.2Fcollector
+	# `time` should be a string or unsigned integer.
+	# That's why we use `to_s` here.
+	time: time.to_f.to_s
       }.tap { |payload|
 	payload[:index] = @index.(tag, record) if @index
 	payload[:source] = @source.(tag, record) if @source
@@ -264,7 +263,11 @@ module Fluent::Plugin
     def format_metric(tag, time, record)
       payload = {
 	host: @host ? @host.(tag, record) : @default_host,
-	time: time.to_i,
+	# From the API reference
+	# http://docs.splunk.com/Documentation/Splunk/latest/RESTREF/RESTinput#services.2Fcollector
+	# `time` should be a string or unsigned integer.
+	# That's why we use `to_s` here.
+	time: time.to_f.to_s,
 	event: 'metric'
       }
       payload[:index] = @index.(tag, record) if @index
@@ -304,51 +307,35 @@ module Fluent::Plugin
       raise Fluent::ConfigError, "hec_host (#{@hec_host}) and/or hec_port (#{@hec_port}) are invalid."
     end
 
-    def start_worker_threads
-      thread_create :"hec_worker_#{@hec_api}" do
-	http = new_connection
-	while chunk = get_next_chunk
-	  send_to_hec http, chunk
-	end
-      end
-    end
-
-    def get_next_chunk
-      @chunk_queue.pop @chunk_queue.closed?
-    rescue ThreadError # see SizedQueue#pop doc
-      nil
-    end
-
     def new_connection
       Net::HTTP::Persistent.new.tap do |c|
-	c.verify_mode = @insecure_ssl ? OpenSSL::SSL::VERIFY_NONE : OpenSSL::SSL::VERIFY_PEER
-	c.cert = OpenSSL::X509::Certificate.new File.read(@client_cert) if @client_cert
-	c.key = OpenSSL::PKey::RSA.new File.read(@client_key) if @client_key
-	c.ca_file = @ca_file
-	c.ca_path = @ca_path
-	c.ciphers = @ssl_ciphers
-
-	c.override_headers['Content-Type'] = 'application/json'
-	c.override_headers['User-Agent'] = "fluent-plugin-splunk_hec_out/#{VERSION}"
-	c.override_headers['Authorization'] = "Splunk #{@hec_token}"
+        c.verify_mode = @insecure_ssl ? OpenSSL::SSL::VERIFY_NONE : OpenSSL::SSL::VERIFY_PEER
+        c.cert = OpenSSL::X509::Certificate.new File.read(@client_cert) if @client_cert
+        c.key = OpenSSL::PKey::RSA.new File.read(@client_key) if @client_key
+        c.ca_file = @ca_file
+        c.ca_path = @ca_path
+        c.ciphers = @ssl_ciphers
+
+        c.override_headers['Content-Type'] = 'application/json'
+        c.override_headers['User-Agent'] = "fluent-plugin-splunk_hec_out/#{VERSION}"
+        c.override_headers['Authorization'] = "Splunk #{@hec_token}"
       end
     end
 
-    def send_to_hec(http, chunk)
+    def send_to_hec(chunk)
       post = Net::HTTP::Post.new @hec_api.request_uri
       post.body = chunk.read
       log.debug { "Sending #{post.body.bytesize} bytes to Splunk." }
 
       log.trace { "POST #{@hec_api} body=#{post.body}" }
-      response = http.request @hec_api, post
+      response = @hec_conn.request @hec_api, post
       log.debug { "[Response] POST #{@hec_api}: #{response.inspect}" }
 
       # raise Exception to utilize Fluentd output plugin retry machanism
-      raise "Server error for POST #{@hec_api}, response: #{response.body}" if response.code.start_with?('5')
+      raise "Server error (#{response.code}) for POST #{@hec_api}, response: #{response.body}" if response.code.start_with?('5')
 
       # For both success response (2xx) and client errors (4xx), we will consume the chunk.
       # Because there probably a bug in the code if we get 4xx errors, retry won't do any good.
-      commit_write(chunk.unique_id)
       if not response.code.start_with?('2')
 	log.error "Failed POST to #{@hec_api}, response: #{response.body}"
 	log.debug { "Failed request body: #{post.body}" }

data/test/fluent/plugin/out_splunk_hec_test.rb

@@ -31,6 +31,15 @@ describe Fluent::Plugin::SplunkHecOutput do
     expect(req).must_be_requested times: 1
   end
 
+  it "should use string for event time, and the value of the string should be a float" do
+    verify_sent_events { |batch|
+      batch.each do |item|
+	expect(item['time']).must_be_instance_of String
+	expect(item['time']).must_match /^\d+\.\d+$/
+      end
+    }
+  end
+
   it "should use host machine's hostname for event host by default" do
     verify_sent_events { |batch|
       batch.each do |item|

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-splunk-hec
 version: !ruby/object:Gem::Version
-  version: 1.0.0
+  version: 1.0.1
 platform: ruby
 authors:
 - Zhimin (Gimi) Liang
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2018-05-15 00:00:00.000000000 Z
+date: 2018-06-27 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fluentd