logstash-input-gelf 2.0.2 → 2.0.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: c300317ea03aabab7bd1c7f99882e91a0b6c245a
-  data.tar.gz: b4c6ceb2eec3fe01f5e13a38067a384db0b5acc7
+  metadata.gz: 943a77f8602e7e8cfa73b87113150d15fdb5ba7b
+  data.tar.gz: 1698c9d2177fa3b5f1fdd2df9c2a06e0ca871633
 SHA512:
-  metadata.gz: 7580a06ea44838c481241d069ce18d8e2e0a75b0b6dd592720adf069b3e5b0f4f75986ea3c5c02e6928c847b105526f526f6162756c50688ee1b7e7a0fd699e5
-  data.tar.gz: ec105386f002250bf4cbd58ef8d9addb3b923fc51d89c9df1d1a4468bd90ce71648829b39d5dcf9a8f4104758b6d3ebb2f2837646d3f01541e7ff3f664c5e5fd
+  metadata.gz: ec422f290472e01dd2a5e2d1482f00d82d0d51d000686d6b1fb5d5ed44e47f3927484b1687e2477c6625eaffc00aecad5559da433eeb5fc0a36b8baa53e30a74
+  data.tar.gz: d36417132925a905b0b91012a92c65428a2d3443bb1252300982361585d9d35f73883b2939bfc3fc67a177b542faa3c27e347117e4d36c29c6ca5f0fa8b8fb2e

data/CHANGELOG.md

@@ -1,5 +1,8 @@
+## 2.0.3
+ - Fix Timestamp coercion to preserve upto microsecond precision, https://github.com/logstash-plugins/logstash-input-gelf/pull/35
+
 ## 2.0.0
- - Plugins were updated to follow the new shutdown semantic, this mainly allows Logstash to instruct input plugins to terminate gracefully, 
+ - Plugins were updated to follow the new shutdown semantic, this mainly allows Logstash to instruct input plugins to terminate gracefully,
    instead of using Thread.raise on the plugins' threads. Ref: https://github.com/elastic/logstash/pull/3895
  - Dependency on logstash-core update to 2.0
 

data/README.md

@@ -1,5 +1,8 @@
 # Logstash Plugin
 
+[![Build
+Status](http://build-eu-00.elastic.co/view/LS%20Plugins/view/LS%20Inputs/job/logstash-plugin-input-gelf-unit/badge/icon)](http://build-eu-00.elastic.co/view/LS%20Plugins/view/LS%20Inputs/job/logstash-plugin-input-gelf-unit/)
+
 This is a plugin for [Logstash](https://github.com/elastic/logstash).
 
 It is fully free and fully open source. The license is Apache 2.0, meaning you are pretty much free to use it however you want in whatever way.

data/lib/logstash/inputs/gelf.rb

@@ -11,7 +11,15 @@ require "socket"
 # making it a good choice if you already use Graylog2 today.
 #
 # The main use case for this input is to leverage existing GELF
-# logging libraries such as the GELF log4j appender.
+# logging libraries such as the GELF log4j appender. A library used
+# by this plugin has a bug which prevents it parsing uncompressed data.
+# If you use the log4j appender you need to configure it like this to force
+# gzip even for small messages:
+#
+#   <Socket name="logstash" protocol="udp" host="logstash.example.com" port="5001">
+#      <GelfLayout compressionType="GZIP" compressionThreshold="1" />
+#   </Socket>
+#
 #
 class LogStash::Inputs::Gelf < LogStash::Inputs::Base
   config_name "gelf"
@@ -44,6 +52,8 @@ class LogStash::Inputs::Gelf < LogStash::Inputs::Base
   config :strip_leading_underscore, :validate => :boolean, :default => true
 
   RECONNECT_BACKOFF_SLEEP = 5
+  TIMESTAMP_GELF_FIELD = "timestamp".freeze
+  SOURCE_HOST_FIELD = "source_host".freeze
 
   public
   def initialize(params)
@@ -96,13 +106,7 @@ class LogStash::Inputs::Gelf < LogStash::Inputs::Base
       # Gelfd parser outputs null if it received and cached a non-final chunk
       next if data.nil?
 
-      event = LogStash::Event.new(LogStash::Json.load(data))
-
-      event["source_host"] = client[3]
-      if event["timestamp"].is_a?(Numeric)
-        event.timestamp = LogStash::Timestamp.at(event["timestamp"])
-        event.remove("timestamp")
-      end
+      event = self.class.new_event(data, client[3])
 
       remap_gelf(event) if @remap
       strip_leading_underscore(event) if @strip_leading_underscore
@@ -112,15 +116,41 @@ class LogStash::Inputs::Gelf < LogStash::Inputs::Base
     end
   end # def udp_listener
 
+  # generate a new LogStash::Event from json input and assign host to source_host event field.
+  # @param json_gelf [String] GELF json data
+  # @param host [String] source host of GELF data
+  # @return [LogStash::Event] new event with parsed json gelf, assigned source host and coerced timestamp
+  def self.new_event(json_gelf, host)
+    event = LogStash::Event.new(LogStash::Json.load(json_gelf))
+
+    event[SOURCE_HOST_FIELD] = host
+
+    if (gelf_timestamp = event[TIMESTAMP_GELF_FIELD]).is_a?(Numeric)
+      event.timestamp = self.coerce_timestamp(gelf_timestamp)
+      event.remove(TIMESTAMP_GELF_FIELD)
+    end
+
+    event
+  end
+
+  # transform a given timestamp value into a proper LogStash::Timestamp, preserving microsecond precision
+  # and work around a JRuby issue with Time.at loosing fractional part with BigDecimal.
+  # @param timestamp [Numeric] a Numeric (integer, float or bigdecimal) timestampo representation
+  # @return [LogStash::Timestamp] the proper LogStash::Timestamp representation
+  def self.coerce_timestamp(timestamp)
+    # bug in JRuby prevents correcly parsing a BigDecimal fractional part, see https://github.com/elastic/logstash/issues/4565
+    timestamp.is_a?(BigDecimal) ? LogStash::Timestamp.at(timestamp.to_i, timestamp.frac * 1000000) : LogStash::Timestamp.at(timestamp)
+  end
+
   private
   def remap_gelf(event)
-    if event["full_message"]
+    if event["full_message"] && !event["full_message"].empty?
       event["message"] = event["full_message"].dup
       event.remove("full_message")
       if event["short_message"] == event["message"]
         event.remove("short_message")
       end
-    elsif event["short_message"]
+    elsif event["short_message"]  && !event["short_message"].empty?
       event["message"] = event["short_message"].dup
       event.remove("short_message")
     end

data/logstash-input-gelf.gemspec

@@ -1,7 +1,7 @@
 Gem::Specification.new do |s|
 
   s.name            = 'logstash-input-gelf'
-  s.version         = '2.0.2'
+  s.version         = '2.0.3'
   s.licenses        = ['Apache License (2.0)']
   s.summary         = "This input will read GELF messages as events over the network, making it a good choice if you already use Graylog2 today."
   s.description     = "This gem is a logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/plugin install gemname. This gem is not a stand-alone program"

data/spec/inputs/gelf_spec.rb

@@ -66,6 +66,84 @@ describe LogStash::Inputs::Gelf do
 
     events.each_with_index do |e, i|
       insist { e["message"] } == messages[i]
+      insist { e["host"] } == Socket.gethostname
+    end
+  end
+
+  context "timestamp coercion" do
+    # these test private methods, this is advisable for now until we roll out this coercion in the Timestamp class
+    # and remove this
+
+    context "integer numeric values" do
+      it "should coerce" do
+        expect(LogStash::Inputs::Gelf.coerce_timestamp(946702800).to_iso8601).to eq("2000-01-01T05:00:00.000Z")
+        expect(LogStash::Inputs::Gelf.coerce_timestamp(946702800).usec).to eq(0)
+      end
+    end
+
+    context "float numeric values" do
+      # using explicit and certainly useless to_f here just to leave no doubt about the numeric type involved
+
+      it "should coerce and preserve millisec precision in iso8601" do
+        expect(LogStash::Inputs::Gelf.coerce_timestamp(946702800.1.to_f).to_iso8601).to eq("2000-01-01T05:00:00.100Z")
+        expect(LogStash::Inputs::Gelf.coerce_timestamp(946702800.12.to_f).to_iso8601).to eq("2000-01-01T05:00:00.120Z")
+        expect(LogStash::Inputs::Gelf.coerce_timestamp(946702800.123.to_f).to_iso8601).to eq("2000-01-01T05:00:00.123Z")
+      end
+
+      it "should coerce and preserve usec precision" do
+        expect(LogStash::Inputs::Gelf.coerce_timestamp(946702800.1.to_f).usec).to eq(100000)
+        expect(LogStash::Inputs::Gelf.coerce_timestamp(946702800.12.to_f).usec).to eq(120000)
+        expect(LogStash::Inputs::Gelf.coerce_timestamp(946702800.123.to_f).usec).to eq(123000)
+
+        # since Java Timestamp in 2.3+ relies on JodaTime which supports only millisec precision
+        # the usec method will only be precise up to millisec.
+        expect(LogStash::Inputs::Gelf.coerce_timestamp(946702800.1234.to_f).usec).to be_within(1000).of(123400)
+        expect(LogStash::Inputs::Gelf.coerce_timestamp(946702800.12345.to_f).usec).to be_within(1000).of(123450)
+        expect(LogStash::Inputs::Gelf.coerce_timestamp(946702800.123456.to_f).usec).to be_within(1000).of(123456)
+      end
+    end
+
+    context "BigDecimal numeric values" do
+      it "should coerce and preserve millisec precision in iso8601" do
+        expect(LogStash::Inputs::Gelf.coerce_timestamp(BigDecimal.new("946702800.1")).to_iso8601).to eq("2000-01-01T05:00:00.100Z")
+        expect(LogStash::Inputs::Gelf.coerce_timestamp(BigDecimal.new("946702800.12")).to_iso8601).to eq("2000-01-01T05:00:00.120Z")
+        expect(LogStash::Inputs::Gelf.coerce_timestamp(BigDecimal.new("946702800.123")).to_iso8601).to eq("2000-01-01T05:00:00.123Z")
+      end
+
+      it "should coerce and preserve usec precision" do
+        expect(LogStash::Inputs::Gelf.coerce_timestamp(BigDecimal.new("946702800.1")).usec).to eq(100000)
+        expect(LogStash::Inputs::Gelf.coerce_timestamp(BigDecimal.new("946702800.12")).usec).to eq(120000)
+        expect(LogStash::Inputs::Gelf.coerce_timestamp(BigDecimal.new("946702800.123")).usec).to eq(123000)
+
+        # since Java Timestamp in 2.3+ relies on JodaTime which supports only millisec precision
+        # the usec method will only be precise up to millisec.
+        expect(LogStash::Inputs::Gelf.coerce_timestamp(BigDecimal.new("946702800.1234")).usec).to be_within(1000).of(123400)
+        expect(LogStash::Inputs::Gelf.coerce_timestamp(BigDecimal.new("946702800.12345")).usec).to be_within(1000).of(123450)
+        expect(LogStash::Inputs::Gelf.coerce_timestamp(BigDecimal.new("946702800.123456")).usec).to be_within(1000).of(123456)
+      end
+    end
+  end
+
+  context "json timestamp coercion" do
+    # these test private methods, this is advisable for now until we roll out this coercion in the Timestamp class
+    # and remove this
+
+    it "should coerce integer numeric json timestamp input" do
+      event = LogStash::Inputs::Gelf.new_event("{\"timestamp\":946702800}", "dummy")
+      expect(event.timestamp.to_iso8601).to eq("2000-01-01T05:00:00.000Z")
+    end
+
+    it "should coerce float numeric value and preserve milliseconds precision in iso8601" do
+      event = LogStash::Inputs::Gelf.new_event("{\"timestamp\":946702800.123}", "dummy")
+      expect(event.timestamp.to_iso8601).to eq("2000-01-01T05:00:00.123Z")
+    end
+
+    it "should coerce float numeric value and preserve usec precision" do
+      # since Java Timestamp in 2.3+ relies on JodaTime which supports only millisec precision
+      # the usec method will only be precise up to millisec.
+
+      event = LogStash::Inputs::Gelf.new_event("{\"timestamp\":946702800.123456}", "dummy")
+      expect(event.timestamp.usec).to be_within(1000).of(123456)
     end
   end
 end

metadata

@@ -1,17 +1,18 @@
 --- !ruby/object:Gem::Specification
 name: logstash-input-gelf
 version: !ruby/object:Gem::Version
-  version: 2.0.2
+  version: 2.0.3
 platform: ruby
 authors:
 - Elastic
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2015-10-14 00:00:00.000000000 Z
+date: 2016-03-18 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
-  requirement: !ruby/object:Gem::Requirement
+  name: logstash-core
+  version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - '>='
       - !ruby/object:Gem::Version
@@ -19,10 +20,7 @@ dependencies:
     - - <
       - !ruby/object:Gem::Version
         version: 3.0.0
-  name: logstash-core
-  prerelease: false
-  type: :runtime
-  version_requirements: !ruby/object:Gem::Requirement
+  requirement: !ruby/object:Gem::Requirement
     requirements:
     - - '>='
       - !ruby/object:Gem::Version
@@ -30,90 +28,92 @@ dependencies:
     - - <
       - !ruby/object:Gem::Version
         version: 3.0.0
+  prerelease: false
+  type: :runtime
 - !ruby/object:Gem::Dependency
+  name: gelfd
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 0.2.0
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - '='
       - !ruby/object:Gem::Version
         version: 0.2.0
-  name: gelfd
   prerelease: false
   type: :runtime
+- !ruby/object:Gem::Dependency
+  name: logstash-codec-plain
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - '='
+    - - '>='
       - !ruby/object:Gem::Version
-        version: 0.2.0
-- !ruby/object:Gem::Dependency
+        version: '0'
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - '>='
       - !ruby/object:Gem::Version
         version: '0'
-  name: logstash-codec-plain
   prerelease: false
   type: :runtime
+- !ruby/object:Gem::Dependency
+  name: stud
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ~>
       - !ruby/object:Gem::Version
-        version: '0'
-- !ruby/object:Gem::Dependency
+        version: 0.0.22
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
         version: 0.0.22
-  name: stud
   prerelease: false
   type: :runtime
+- !ruby/object:Gem::Dependency
+  name: logstash-devutils
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - '>='
       - !ruby/object:Gem::Version
-        version: 0.0.22
-- !ruby/object:Gem::Dependency
+        version: '0'
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - '>='
       - !ruby/object:Gem::Version
         version: '0'
-  name: logstash-devutils
   prerelease: false
   type: :development
+- !ruby/object:Gem::Dependency
+  name: gelf
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - '='
       - !ruby/object:Gem::Version
-        version: '0'
-- !ruby/object:Gem::Dependency
+        version: 1.3.2
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - '='
       - !ruby/object:Gem::Version
         version: 1.3.2
-  name: gelf
   prerelease: false
   type: :development
+- !ruby/object:Gem::Dependency
+  name: flores
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - '='
+    - - '>='
       - !ruby/object:Gem::Version
-        version: 1.3.2
-- !ruby/object:Gem::Dependency
+        version: '0'
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - '>='
       - !ruby/object:Gem::Version
         version: '0'
-  name: flores
   prerelease: false
   type: :development
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - '>='
-      - !ruby/object:Gem::Version
-        version: '0'
 description: This gem is a logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/plugin install gemname. This gem is not a stand-alone program
 email: info@elastic.co
 executables: []