logstash-output-loggly 4.0.0 → 5.0.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 1e15e34b7a8077fdb9c36ebd61ddad7d3e89adf138381cb339849f7dfe55b2ea
-  data.tar.gz: 509a2c487b8faf7821e8e83b32db90096dc46c23af35bfc7757dc8a698643901
+  metadata.gz: c5ef3911c30709e44be9b65e5be43f192b716bc00b40c9d373fcc78677f7b060
+  data.tar.gz: 1be2aa1769cf66c2bf19d431738a6995217b8ee9007533a1848647c0b358b2dd
 SHA512:
-  metadata.gz: aef1bc454004b3978558d783ae2fc2194f715405cc3e60bc9f06b635fe722eab60a6b9b088c8c0a61fd0ac92cc4c609510e2c17198d9980db006505b66905050
-  data.tar.gz: 36dd4857187cefd3bca6fed7c65badd69b3a0adb8b0f220af085758b99f4d23a82a6a598bab479dee55d3b2179bb74d065e2af0ab5f51e8984e6038f83f2fe91
+  metadata.gz: 85b87a48f7c044a25fb8739c21761ced70f6d131614890fb39a7ae7edd03be025e0f0cc0bbc4b4ab16ca19d398ff389c95f9aea759d31e63cb294ded666e659f
+  data.tar.gz: 6bfa8d3d9ef0dbbceb2332b30b13fc794dc75935ff31c45ba7494ad4de243db5f433642b78538dccbaf30593f476702307d01913788e5c684a6e56d0734b1c41

data/CHANGELOG.md

@@ -1,3 +1,17 @@
+## 5.0.0
+  - This version introduces "breaking" changes for users who never copied/renamed
+    their `@timestamp` field to `timestamp`: their events will suddenly appear
+    in Loggly with a `timestamp` based on Logstash's value of `@timestamp`.
+    This would especially be noticed at times where processing is behind, and
+    events need to be "backfilled".
+    - The plugin now sets field `timestamp` so that Loggly will recognize the
+      correct timestamp.
+      - The event's timestamps will however not be touched at all if `timestamp`
+        is already set on the event or if `@timestamp` is missing.
+    - This version introduces attribute `convert_timestamp` (defaults to true), which
+      triggers the timestamp mingling.
+  - Now log a debug message with all of the plugin's configuration upon initialization.
+
 ## 4.0.0
   - The plugin now uses the Loggly bulk API.
   - If you need to modify event batch sizes and max delay between flushes,

data/docs/index.asciidoc

@@ -38,6 +38,7 @@ This plugin supports the following configuration options plus the <<plugins-{typ
 |=======================================================================
 |Setting |Input type|Required
 | <<plugins-{type}s-{plugin}-can_retry>> |<<boolean,boolean>>|No
+| <<plugins-{type}s-{plugin}-convert_timestamp>> |<<boolean,boolean>>|No
 | <<plugins-{type}s-{plugin}-host>> |<<string,string>>|No
 | <<plugins-{type}s-{plugin}-key>> |<<string,string>>|Yes
 | <<plugins-{type}s-{plugin}-max_event_size>> |<<bytes,bytes>>|Yes
@@ -57,7 +58,7 @@ output plugins.
 &nbsp;
 
 [id="plugins-{type}s-{plugin}-can_retry"]
-===== `can_retry` 
+===== `can_retry`
 
   * Value type is <<boolean,boolean>>
   * Default value is `true`
@@ -65,6 +66,21 @@ output plugins.
 Can Retry.
 Setting this value true helps user to send multiple retry attempts if the first request fails
 
+[id="plugins-{type}s-{plugin}-convert_timestamp"]
+===== `convert_timestamp`
+
+  * Value type is <<boolean,boolean>>
+  * Default value is `true`
+
+The plugin renames Logstash's '@timestamp' field to 'timestamp' before sending,
+so that Loggly recognizes it automatically.
+
+This will do nothing if your event doesn't have a '@timestamp' field or if
+your event already has a 'timestamp' field.
+
+Note that the actual Logstash event is not modified by the output. This modification
+only happens on a copy of the event, prior to sending.
+
 [id="plugins-{type}s-{plugin}-host"]
 ===== `host`
 

data/lib/logstash/outputs/loggly.rb

@@ -1,6 +1,7 @@
 # encoding: utf-8
 require "logstash/outputs/base"
 require "logstash/namespace"
+require "json"
 require "timeout"
 require "uri"
 # TODO(sissel): Move to something that performs better than net/http
@@ -30,6 +31,16 @@ class LogStash::Outputs::Loggly < LogStash::Outputs::Base
 
   config_name "loggly"
 
+  # Rename Logstash's '@timestamp' field to 'timestamp' before sending,
+  # so that Loggly recognizes it automatically.
+  #
+  # This will do nothing if your event doesn't have a '@timestamp' field or if
+  # your event already has a 'timestamp' field.
+  #
+  # Note that the actual Logstash event is not modified by the output. This
+  # modification only happens on a copy of the event, prior to sending.
+  config :convert_timestamp, :validate => :boolean, :default => true
+
   # The hostname to send logs to. This should target the loggly http input
   # server which is usually "logs-01.loggly.com" (Gen2 account).
   # See Loggly HTTP endpoint documentation at
@@ -105,6 +116,7 @@ class LogStash::Outputs::Loggly < LogStash::Outputs::Base
 
   public
   def register
+    @logger.debug "Initializing Loggly Output", @config
   end
 
   public
@@ -133,7 +145,12 @@ class LogStash::Outputs::Loggly < LogStash::Outputs::Base
     # we should ship logs with the default tag value.
     tag = DEFAULT_LOGGLY_TAG if /%{\w+}/.match(tag)
 
-    meta_event = {  key: key, tag: tag, event: event }
+    event_hash = event.to_hash # Don't want to modify the event in an output
+    if @convert_timestamp && event_hash['@timestamp'] && !event_hash['timestamp']
+      event_hash['timestamp'] = event_hash.delete('@timestamp')
+    end
+
+    meta_event = {  key: key, tag: tag, event: event_hash }
   end # prepare_meta
 
   public

data/logstash-output-loggly.gemspec

@@ -1,6 +1,6 @@
 Gem::Specification.new do |s|
   s.name            = 'logstash-output-loggly'
-  s.version         = '4.0.0'
+  s.version         = '5.0.0'
   s.licenses        = ['Apache License (2.0)']
   s.summary         = "Ships logs to Loggly"
   s.description     = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program"
@@ -23,5 +23,6 @@ Gem::Specification.new do |s|
 
   s.add_development_dependency 'logstash-devutils'
   s.add_development_dependency 'logstash-codec-plain'
+  s.add_development_dependency 'rake', '~> 12.2.1' # for JRuby 1.7, Ruby 1.9
 end
 

data/spec/outputs/loggly_spec.rb

@@ -7,7 +7,7 @@ def logger_for(plugin)
 end
 
 describe 'outputs/loggly' do
-  let(:config) { { 'key' => 'abcdef123456' } }
+  let(:config) { { 'key' => 'abcdef123456', 'convert_timestamp' => false } }
 
   let(:output) do
     LogStash::Outputs::Loggly.new(config).tap do |output|
@@ -41,7 +41,7 @@ describe 'outputs/loggly' do
 
   context 'when sending events' do
     it 'should set the default tag to logstash' do
-      expect(output).to receive(:send_batch).with([{event: event, key: 'abcdef123456', tag: 'logstash'}])
+      expect(output).to receive(:send_batch).with([{event: event.to_hash, key: 'abcdef123456', tag: 'logstash'}])
       output.receive(event)
     end
 
@@ -50,19 +50,19 @@ describe 'outputs/loggly' do
       event.set('token', 'xxxxxxx1234567')
       config['key'] = '%{token}'
 
-      expect(output).to receive(:send_batch).with([{event: event, key: 'xxxxxxx1234567', tag: 'logstash'}])
+      expect(output).to receive(:send_batch).with([{event: event.to_hash, key: 'xxxxxxx1234567', tag: 'logstash'}])
       output.receive(event)
     end
 
     it 'should support field interpolation for tag' do
       config['tag'] = '%{source}'
-      expect(output).to receive(:send_batch).with([{event: event, key: 'abcdef123456', tag: 'someapp'}])
+      expect(output).to receive(:send_batch).with([{event: event.to_hash, key: 'abcdef123456', tag: 'someapp'}])
       output.receive(event)
     end
 
     it 'should default tag to logstash if interpolated field for tag does not exist' do
       config['tag'] = '%{foobar}'
-      expect(output).to receive(:send_batch).with([{event: event, key: 'abcdef123456', tag: 'logstash'}])
+      expect(output).to receive(:send_batch).with([{event: event.to_hash, key: 'abcdef123456', tag: 'logstash'}])
       output.receive(event)
     end
 
@@ -72,7 +72,7 @@ describe 'outputs/loggly' do
       event2 = event.clone
       event2.remove('custom_key')
 
-      expect(output).to receive(:send_batch).once.with([{event: event, key: 'a_key', tag: 'logstash'}, nil])
+      expect(output).to receive(:send_batch).once.with([{event: event.to_hash, key: 'a_key', tag: 'logstash'}, nil])
       logger = logger_for(output)
       expect(logger).to receive(:warn).with(/No key provided/)
       expect(logger).to receive(:debug).with(/Dropped message/, kind_of(Hash))
@@ -109,6 +109,48 @@ describe 'outputs/loggly' do
         output.multi_receive([event1, event2, event3, event4])
       end
     end
+
+    context 'timestamp mingling' do
+      context 'when convert_timestamp is false' do
+        let(:config) { super.merge('convert_timestamp' => false) }
+
+        it 'should not create a timestamp field nor delete @timestamp' do
+          expected_time = event.get('@timestamp')
+          meta_event = output.send :prepare_meta, event
+
+          expect(meta_event[:event]['@timestamp']).to eq(expected_time)
+          expect(meta_event[:event]['timestamp']).to be_nil
+        end
+      end
+
+      context 'when convert_timestamp is true' do
+        let(:config) { super.merge('convert_timestamp' => true) }
+
+        it 'should rename @timestamp to timestamp in the normal case' do
+          expected_time = event.get('@timestamp')
+          meta_event = output.send :prepare_meta, event
+
+          expect(meta_event[:event]['timestamp']).to eq(expected_time)
+          expect(meta_event[:event]['@timestamp']).to be_nil
+        end
+
+        it 'should not overwrite an existing timestamp field' do
+          event.set('timestamp', "no tocar")
+          meta_event = output.send :prepare_meta, event
+
+          expect(meta_event[:event]['timestamp']).to  eq('no tocar')
+          expect(meta_event[:event]['@timestamp']).to eq(event.get('@timestamp'))
+        end
+
+        it 'should not attempt to set timestamp if the event has no @timestamp' do
+          event.remove('@timestamp')
+          meta_event = output.send :prepare_meta, event
+
+          expect(meta_event[:event]['timestamp']).to  be_nil
+          expect(meta_event[:event]['@timestamp']).to be_nil
+        end
+      end
+    end
   end
 
   context 'splitting batches of events' do

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: logstash-output-loggly
 version: !ruby/object:Gem::Version
-  version: 4.0.0
+  version: 5.0.0
 platform: ruby
 authors:
 - Elastic
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2018-05-15 00:00:00.000000000 Z
+date: 2018-05-21 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
@@ -58,6 +58,20 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 12.2.1
+  name: rake
+  prerelease: false
+  type: :development
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 12.2.1
 description: This gem is a Logstash plugin required to be installed on top of the
   Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This
   gem is not a stand-alone program