logstash-output-newrelic 1.0.2 → 1.0.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 998bef6edbc8fb754b22ab4cca0f25b3e27182e3ac22b5992840ebeab1086764
-  data.tar.gz: 9d2517ffcc0b573a5c2321dea3c4ca8ced58df4563346f93a1925983169c1a6d
+  metadata.gz: 6fb3ab8cf5881043a05b318e3a5158a15f9e39f71d35bf1b98a2b0526ec59df8
+  data.tar.gz: b78ec9ccaddeb88589815019ef3a6b8336b9b12aef7d51b03ccb07df21e7c679
 SHA512:
-  metadata.gz: 5ebeed8b92b33be3ff45d7148ff402d0128fb0a29fa8028380a046b32819ea5c76ee2d388386ebc801a0840371ccc991a2253c7cf5806d8bc9a0cf050420f0d6
-  data.tar.gz: '02922a85561259f830f3c142dbabac0bb59867e8bf1b87c2fc0810304c51f0d9333251768e32715014c0d1c60bc071ad0b5f0f142bf5dbab54677b0dc9fea477'
+  metadata.gz: d5d13fbb5791e8f7ecf98c6aaebae14596f94581246691cbc9b92a8129b97207405e7af396de356cf1a3bd8fbc882e77700be6353356f54c806a631b1052cd06
+  data.tar.gz: f57941893058a0fdd95c2ab7d3b307051275f428591314d5c2ae93afa71ad18cc29e7ec64bc17db17073670c031f1b0293525c675d00e8db52a1fddda6f85cce

data/README.md

@@ -6,13 +6,8 @@ This is a plugin for [Logstash](https://github.com/elastic/logstash) that output
 Install the New Relic Logstash plugin using the following command:</br>
 `logstash-plugin install logstash-output-newrelic`
 
-(Optional) If you are interested in installing the gem directly, run the following command. If you want a specific version, specify it by appending the `-v <VERSION>` option.<br/>
-`gem install logstash-output-newrelic`
-
-```
-Old version: 0.9.1 (unmaintained)
-Current: 1.0.0
-```
+### Versions
+Versions of this plugin less than 1.0.0 are unsupported.
 
 ## Configuration
 
@@ -50,7 +45,8 @@ output {
 
 ## Testing
 
-An easy way to test the plugin is to make sure Logstash is getting input from a log file you can write to. Something like this in your logstash.conf:
+An easy way to test the plugin is to make sure Logstash is getting input from a log file you 
+can write to. Something like this in your logstash.conf:
 ```
 input {
   file {
@@ -62,32 +58,29 @@ input {
 * Append a test log message to your log file: `echo "test message" >> /path/to/your/log/file`
 * Search New Relic Logs for `"test message"`
 
-## Notes
+## JSON message parsing
 
-This plugin will attempt to parse any 'message' attribute as JSON -- if it is JSON, its JSON attributes will be added to the event.
+This plugin will attempt to parse any 'message' attribute as JSON -- if it is JSON, it will be parsed and 
+the JSON attributes will be added to the event.
 
-For example, the events:
+For example, the event:
 ```
-[{
-  "message": "some message",
-  "timestamp": 1531414060739
-},
 {
-  {"message":"some_message","timestamp":"12897439", "compound" :"{\"a\":111, \"b\":222}"},
-}]
+  "timestamp": 1562767499238,
+  "message": "{\"service-name\": \"login-service\", \"user\": {\"id\": 123, \"name\": \"alice\"}}"
+}
+
 ```
 
-Will be output as:
+Will be treated as:
 ```
-[{  "message": "{\"key\": \"value1\", \"compound\": {\"sub_key\": \"value2\"}}",
-  "key": "value1",
-  "compound": {
-    "sub_key": "value2"
-  },
-  "other": "other value"
-}]
+{
+  "timestamp": 1562767499238,
+  "message": "{\"service-name\": \"my-service\", \"user\": {\"id\": 123, \"name\": \"alice\"}}",
+  "service-name": "login-service",
+  "user": {
+    "id": 123,
+    "name": "alice"
+  }
+}
 ```
-
-## Development
-
-See [DEVELOPER.md](DEVELOPER.md)

data/lib/logstash/outputs/newrelic.rb

@@ -38,13 +38,32 @@ class LogStash::Outputs::NewRelic < LogStash::Outputs::Base
     @executor&.shutdown
   end
 
+  def time_to_milliseconds(time)
+    timestamp = time_to_logstash_timestamp(time)
+    if timestamp.nil?
+      return nil
+    end
+
+    (timestamp.to_f * 1000).to_i # to_f gives seconds with a fractional portion
+  end
+
+
+  def time_to_logstash_timestamp(time)
+    begin
+      LogStash::Timestamp.coerce(time)
+    rescue
+      nil
+    end
+  end
+
+
   def encode(event_hash)
     log_message_hash = {
-      'attributes' => {}
+        :attributes => {}
     }
 
     if event_hash.has_key?('message')
-      log_message_hash['attributes'] = maybe_parse_json(event_hash['message'])
+      log_message_hash[:attributes] = maybe_parse_json(event_hash['message'])
     end
 
     event_hash.each do |key, value|
@@ -52,22 +71,17 @@ class LogStash::Outputs::NewRelic < LogStash::Outputs::Base
         # intrinsic attributes go at the top level
         log_message_hash[key] = value
       elsif key == '@timestamp'
-        # Do not add @timestamp
+        milliseconds = time_to_milliseconds(value)
+        if !milliseconds.nil?
+          log_message_hash[:timestamp] = milliseconds
+        end
       else
         # non-intrinsic attributes get put into 'attributes'
-        log_message_hash['attributes'][key] = value
+        log_message_hash[:attributes][key] = value
       end
     end
-    
-    log_message_hash
-  end
 
-  def maybe_parse_message_json(event_hash)
-    if event_hash.has_key?('message')
-      message = event_hash['message']
-      event_hash = event_hash.merge(maybe_parse_json(message))
-    end
-    event_hash
+    log_message_hash
   end
 
   def maybe_parse_json(message)
@@ -77,8 +91,9 @@ class LogStash::Outputs::NewRelic < LogStash::Outputs::Base
         return parsed
       end
     rescue JSON::ParserError
+      # Not JSON
     end
-    return {}
+    {}
   end
 
   def multi_receive(events)
@@ -87,15 +102,15 @@ class LogStash::Outputs::NewRelic < LogStash::Outputs::Base
       payload.push(encode(event.to_hash))
     end
     payload = {
-      'common' => {
-        'attributes' => {
-          'plugin' =>  {
-            'type' => 'logstash',
-            'version' => LogStash::Outputs::NewRelicVersion::VERSION,
-          }
-        }
-      },
-      'logs' => payload
+        :common => {
+            :attributes => {
+                :plugin => {
+                    :type => 'logstash',
+                    :version => LogStash::Outputs::NewRelicVersion::VERSION,
+                }
+            }
+        },
+        :logs => payload
     }
     @semaphor.acquire()
     execute = @executor.java_method :submit, [java.lang.Runnable]

data/lib/logstash/outputs/newrelic_version/version.rb

@@ -1,7 +1,7 @@
 module LogStash
   module Outputs
     module NewRelicVersion
-      VERSION = "1.0.2"
+      VERSION = "1.0.3"
     end
   end
 end

data/spec/outputs/newrelic_spec.rb

@@ -24,6 +24,13 @@ describe LogStash::Outputs::NewRelic do
     }
   }
 
+  # An arbitrary time to use in these tests, with different representations
+  class FixedTime
+    MILLISECONDS = 1562888528123
+    ISO_8601_STRING_TIME = '2019-07-11T23:42:08.123Z'
+    LOGSTASH_TIMESTAMP = LogStash::Timestamp.coerce(ISO_8601_STRING_TIME)
+  end
+
   def gunzip(bytes)
     gz = Zlib::GzipReader.new(StringIO.new(bytes))
     gz.read
@@ -39,6 +46,16 @@ describe LogStash::Outputs::NewRelic do
     JSON.parse(gunzip(body))
   end
 
+  def now_in_milliseconds()
+    (Time.now.to_f * 1000).to_i # to_f gives seconds with a fractional portion
+  end
+
+  def within_five_seconds_of(time_in_millis, expected_in_millis)
+    five_seconds_in_millis = 5 * 1000
+    (time_in_millis - expected_in_millis).abs < five_seconds_in_millis
+  end
+
+
   before(:each) do
     @newrelic_output = LogStash::Plugin.lookup("output", "newrelic").new(simple_config)
     @newrelic_output.register
@@ -61,7 +78,7 @@ describe LogStash::Outputs::NewRelic do
     it "all present" do
       stub_request(:any, base_uri).to_return(status: 200)
 
-      event = LogStash::Event.new({ "message" => "Test message" })
+      event = LogStash::Event.new({:message => "Test message" })
       @newrelic_output.multi_receive([event])
 
       wait_for(a_request(:post, base_uri)
@@ -72,6 +89,7 @@ describe LogStash::Outputs::NewRelic do
               })).to have_been_made
     end
   end
+
   context "request body" do
 
     it "message contains plugin information" do
@@ -88,16 +106,47 @@ describe LogStash::Outputs::NewRelic do
       .to have_been_made
     end
 
-    # TODO: why is this field always removed?
-    it "'@timestamp' field is removed" do
-      stub_request(:any, base_uri).to_return(status: 200)
 
-      event = LogStash::Event.new({ :message => "Test message", :@timestamp => '123' })
-      @newrelic_output.multi_receive([event])
+    context "@timestamp field" do
 
-      wait_for(a_request(:post, base_uri)
-        .with { |request| single_gzipped_message(request.body)['@timestamp'] == nil })
-        .to have_been_made
+      def test_timestamp(timestamp, message_matcher)
+        stub_request(:any, base_uri).to_return(status: 200)
+
+        event = LogStash::Event.new({ :message => "Test message", :@timestamp => timestamp })
+        @newrelic_output.multi_receive([event])
+
+        wait_for(a_request(:post, base_uri)
+                     .with { |request|
+                       message = single_gzipped_message(request.body)
+                       message_matcher.call(message)
+                     }).to have_been_made
+      end
+
+      it "LogStash::Timestamp '@timestamp' field is added as 'timestamp' as milliseconds since epoch" do
+        def message_matcher(message)
+          message['timestamp'] == FixedTime::MILLISECONDS
+        end
+
+        test_timestamp(FixedTime::LOGSTASH_TIMESTAMP, method(:message_matcher))
+      end
+
+      it "String ISO 8601 '@timestamp' field is added as 'timestamp' as milliseconds since epoch" do
+        def message_matcher(message)
+          message['timestamp'] == FixedTime::MILLISECONDS
+        end
+
+        test_timestamp(FixedTime::ISO_8601_STRING_TIME, method(:message_matcher))
+      end
+
+      # Ideally we might not even set a timestamp if parsing fails. However, the time parsing library doesn't tell us if
+      # parsing fails -- it just returns the current time.
+      it "Unparseable '@timestamp' field is parsed as current time and put into 'timestamp'" do
+        def message_matcher(message)
+          within_five_seconds_of(message['timestamp'], now_in_milliseconds)
+        end
+
+        test_timestamp("Not an ISO 8601 string", method(:message_matcher))
+      end
     end
 
     it "all other fields passed through as is" do
@@ -109,7 +158,6 @@ describe LogStash::Outputs::NewRelic do
       wait_for(a_request(:post, base_uri)
         .with { |request|
           message = single_gzipped_message(request.body)
-          puts message
           message['message'] == 'Test message' &&
           message['attributes']['other'] == 'Other value' })
         .to have_been_made

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: logstash-output-newrelic
 version: !ruby/object:Gem::Version
-  version: 1.0.2
+  version: 1.0.3
 platform: ruby
 authors:
 - New Relic Logging Team
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2019-07-09 00:00:00.000000000 Z
+date: 2019-07-12 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement