logstash-output-loggly 5.0.0 → 6.0.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: c5ef3911c30709e44be9b65e5be43f192b716bc00b40c9d373fcc78677f7b060
-  data.tar.gz: 1be2aa1769cf66c2bf19d431738a6995217b8ee9007533a1848647c0b358b2dd
+  metadata.gz: bccd01fccea37ecf90a43b572b94db5e05dad7e415b8d2b8c97afb9c835eb8f6
+  data.tar.gz: fab97721a923024d364c1a36cdfad1be3d2b1b59c78b370867ebb744457c83f6
 SHA512:
-  metadata.gz: 85b87a48f7c044a25fb8739c21761ced70f6d131614890fb39a7ae7edd03be025e0f0cc0bbc4b4ab16ca19d398ff389c95f9aea759d31e63cb294ded666e659f
-  data.tar.gz: 6bfa8d3d9ef0dbbceb2332b30b13fc794dc75935ff31c45ba7494ad4de243db5f433642b78538dccbaf30593f476702307d01913788e5c684a6e56d0734b1c41
+  metadata.gz: c3da1ce06b6dbf4ba04e2baad8082e753cc7cd7759b1a8d17e164565e8c02fa685599899053641f91789f17dce231b8fb5e9669db02be6711cdf867066d55f6a
+  data.tar.gz: 6fb1b4d3ff313299f29863208a4882f0c554cf5541eba386135250510cd6bcfb7aab980715c4d43a695c23e22cc732b84a836f18cc973f9b9161777039cda62a

data/CHANGELOG.md

@@ -1,5 +1,13 @@
+## 6.0.0
+  - Fixed bug when interpolation fails in a list of tag. The list of tags no
+    longer gets completely replaced with a default value.
+    [#31](https://github.com/logstash-plugins/logstash-output-loggly/pull/31)
+  - [BREAKING] Changed tagging to no longer set a tag by default.
+    The tag is optional with Loggly.
+    [#31](https://github.com/logstash-plugins/logstash-output-loggly/pull/31)
+
 ## 5.0.0
-  - This version introduces "breaking" changes for users who never copied/renamed
+  - [BREAKING] This version introduces "breaking" changes for users who never copied/renamed
     their `@timestamp` field to `timestamp`: their events will suddenly appear
     in Loggly with a `timestamp` based on Logstash's value of `@timestamp`.
     This would especially be noticed at times where processing is behind, and
@@ -20,9 +28,9 @@
   - New settings: `max_event_size` and `max_payload_size`.
     Both are currently set according to Loggly's [published API limits](https://www.loggly.com/docs/http-bulk-endpoint/).
     They only need to be changed if Loggly changes these limits.
-  - The plugin now skips events bigger than the API limit for single event size.
+  - [BREAKING] The plugin now skips events bigger than the API limit for single event size.
     A proper warning is logged when this happens.
-  - When interpolating `key` field, drop messages where interpolation doesn't
+  - [BREAKING] When interpolating `key` field, drop messages where interpolation doesn't
     resolve (meaning we don't have the API key for the event).
   - When interpolating `tag` field, revert to default of 'logstash' if interpolation doesn't resolve.
   - Beef up unit tests significantly.

data/docs/index.asciidoc

@@ -189,16 +189,17 @@ It will try to submit request until retry_count and then halt
 ===== `tag` 
 
   * Value type is <<string,string>>
-  * Default value is `"logstash"`
 
 Loggly Tags help you to find your logs in the Loggly dashboard easily.
-You can search for a tag in Loggly using `"tag:logstash"`.
+You can search for a tag in Loggly, using `"tag:your_tag"`.
 
 If you need to specify multiple tags here on your events,
 specify them as outlined in https://www.loggly.com/docs/tags/[the tag documentation].
 E.g. `"tag" => "foo,bar,myApp"`.
 
-You can also use `"tag" => "%{somefield}"` to take your tag value from `somefield` on your event.
+You can also use `"tag" => "%{somefield},%{another_field}"` to take your tag values
+from `somefield` and `another_field` on your event. If the field doesn't exist,
+no tag will be created.
 Helpful for leveraging https://www.loggly.com/docs/source-groups/[Loggly source groups].
 
 

data/lib/logstash/outputs/loggly.rb

@@ -59,17 +59,18 @@ class LogStash::Outputs::Loggly < LogStash::Outputs::Base
   config :proto, :validate => :string, :default => "http"
 
   # Loggly Tags help you to find your logs in the Loggly dashboard easily.
-  # You can search for a tag in Loggly using `"tag:logstash"`.
+  # You can search for a tag in Loggly using `"tag:your_tag"`.
   #
   # If you need to specify multiple tags here on your events,
   # specify them as outlined in the tag documentation (https://www.loggly.com/docs/tags/).
   # E.g. `"tag" => "foo,bar,myApp"`.
   #
-  # You can also use `"tag" => "%{somefield}"` to take your tag value from `somefield` on your event.
+  # You can also use `"tag" => "%{somefield},%{another_field}"` to take your tag values
+  # from `somefield` and `another_field` on your event. If the field doesn't exist,
+  # no tag will be created.
   # Helpful for leveraging Loggly source groups (https://www.loggly.com/docs/source-groups/).
 
-  DEFAULT_LOGGLY_TAG = 'logstash'
-  config :tag, :validate => :string, :default => DEFAULT_LOGGLY_TAG
+  config :tag, :validate => :string, :default => ''
 
   # Retry count.
   # It may be possible that the request may timeout due to slow Internet connection
@@ -81,6 +82,8 @@ class LogStash::Outputs::Loggly < LogStash::Outputs::Base
   # Setting this value true helps user to send multiple retry attempts if the first request fails
   config :can_retry, :validate => :boolean, :default => true
 
+  config :mime_type, :validate => :string, :default => 'application/json'
+	
   # Proxy Host
   config :proxy_host, :validate => :string
 
@@ -133,7 +136,16 @@ class LogStash::Outputs::Loggly < LogStash::Outputs::Base
   # or returns nil, if event's key doesn't resolve.
   def prepare_meta(event)
     key = event.sprintf(@key)
-    tag = event.sprintf(@tag)
+    tags = @tag.split(",")
+    tag_array = []
+
+    tags.each do |t|
+      t = event.sprintf(t)
+      # For those cases where %{somefield} doesn't exist we don't include it
+      unless /%{\w+}/.match(t) || t.blank?
+        tag_array.push(t)
+      end
+    end
 
     if expected_field = key[/%{(.*)}/, 1]
       @logger.warn "Skipping sending message to Loggly. No key provided (key='#{key}'). Make sure to set field '#{expected_field}'."
@@ -141,9 +153,9 @@ class LogStash::Outputs::Loggly < LogStash::Outputs::Base
       return nil
     end
 
-    # For those cases where %{somefield} doesn't exist
-    # we should ship logs with the default tag value.
-    tag = DEFAULT_LOGGLY_TAG if /%{\w+}/.match(tag)
+    unless tag_array.empty?
+      tag = tag_array.uniq.join(",")
+    end
 
     event_hash = event.to_hash # Don't want to modify the event in an output
     if @convert_timestamp && event_hash['@timestamp'] && !event_hash['timestamp']
@@ -163,7 +175,12 @@ class LogStash::Outputs::Loggly < LogStash::Outputs::Base
   def send_batch(meta_events)
     split_batches(meta_events.compact).each_pair do |k, batch|
       key, tag = *k
-      url = "#{@proto}://#{@host}/bulk/#{key}/tag/#{tag}"
+      if tag.nil?
+        url = "#{@proto}://#{@host}/bulk/#{key}"
+      else
+        url = "#{@proto}://#{@host}/bulk/#{key}/tag/#{tag}"
+      end
+
 
       build_message_bodies(batch) do |body|
         perform_api_call url, body
@@ -241,7 +258,7 @@ class LogStash::Outputs::Loggly < LogStash::Outputs::Base
       http.verify_mode = OpenSSL::SSL::VERIFY_NONE
     end
 
-    request = Net::HTTP::Post.new(url.path, {'Content-Type' =>'application/json'})
+    request = Net::HTTP::Post.new(url.path, {'Content-Type' => @mime_type})
     request.body = message
 
     # Variable for count total retries

data/logstash-output-loggly.gemspec

@@ -1,7 +1,7 @@
 Gem::Specification.new do |s|
   s.name            = 'logstash-output-loggly'
-  s.version         = '5.0.0'
-  s.licenses        = ['Apache License (2.0)']
+  s.version         = '6.0.0'
+  s.licenses        = ['Apache-2.0']
   s.summary         = "Ships logs to Loggly"
   s.description     = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program"
   s.authors         = ["Elastic"]

data/spec/outputs/loggly_spec.rb

@@ -33,15 +33,15 @@ describe 'outputs/loggly' do
     it 'should have default config values' do
       expect(subject.proto).to eq('http')
       expect(subject.host).to eq('logs-01.loggly.com')
-      expect(subject.tag).to eq('logstash')
+      expect(subject.tag).to eq('')
       expect(subject.max_event_size).to eq(1_048_576)
       expect(subject.max_payload_size).to eq(5_242_880)
     end
   end
 
   context 'when sending events' do
-    it 'should set the default tag to logstash' do
-      expect(output).to receive(:send_batch).with([{event: event.to_hash, key: 'abcdef123456', tag: 'logstash'}])
+    it 'should set the default tag to nil' do
+      expect(output).to receive(:send_batch).with([{event: event.to_hash, key: 'abcdef123456', tag: nil}])
       output.receive(event)
     end
 
@@ -50,7 +50,7 @@ describe 'outputs/loggly' do
       event.set('token', 'xxxxxxx1234567')
       config['key'] = '%{token}'
 
-      expect(output).to receive(:send_batch).with([{event: event.to_hash, key: 'xxxxxxx1234567', tag: 'logstash'}])
+      expect(output).to receive(:send_batch).with([{event: event.to_hash, key: 'xxxxxxx1234567', tag: nil}])
       output.receive(event)
     end
 
@@ -60,9 +60,9 @@ describe 'outputs/loggly' do
       output.receive(event)
     end
 
-    it 'should default tag to logstash if interpolated field for tag does not exist' do
+    it 'should have no tag if the interpolated field for the tag does not exist' do
       config['tag'] = '%{foobar}'
-      expect(output).to receive(:send_batch).with([{event: event.to_hash, key: 'abcdef123456', tag: 'logstash'}])
+      expect(output).to receive(:send_batch).with([{event: event.to_hash, key: 'abcdef123456', tag: nil}])
       output.receive(event)
     end
 
@@ -72,7 +72,7 @@ describe 'outputs/loggly' do
       event2 = event.clone
       event2.remove('custom_key')
 
-      expect(output).to receive(:send_batch).once.with([{event: event.to_hash, key: 'a_key', tag: 'logstash'}, nil])
+      expect(output).to receive(:send_batch).once.with([{event: event.to_hash, key: 'a_key', tag: nil}, nil])
       logger = logger_for(output)
       expect(logger).to receive(:warn).with(/No key provided/)
       expect(logger).to receive(:debug).with(/Dropped message/, kind_of(Hash))
@@ -94,11 +94,11 @@ describe 'outputs/loggly' do
         expect(output).to receive(:perform_api_call) { |url, body|
           expect(body).to match /"event1"/
           expect(body).to match /"event4"/
-          expect(url).to eq('http://logs-01.loggly.com/bulk/generally_used_key/tag/logstash')
+          expect(url).to eq('http://logs-01.loggly.com/bulk/generally_used_key')
         }
         expect(output).to receive(:perform_api_call) { |url, body|
           expect(body).to match /"event2"/
-          expect(url).to eq('http://logs-01.loggly.com/bulk/other_key/tag/logstash')
+          expect(url).to eq('http://logs-01.loggly.com/bulk/other_key')
         }
         expect(output).to receive(:perform_api_call) { |url, body|
           expect(body).to match /"event3"/
@@ -110,6 +110,54 @@ describe 'outputs/loggly' do
       end
     end
 
+    context 'when computing the tags' do
+      it 'should not leave a failed interpolation in the tag list' do
+        config['tag']  =  '%{field1},%{field2}'
+        event.set('field1', 'some_value1')
+
+        meta_event = output.send :prepare_meta, event
+        expect(meta_event[:tag]).to eq('some_value1')
+      end
+
+      it 'should support no interpolation' do
+        config['tag']  =  'some_tag'
+
+        meta_event = output.send :prepare_meta, event
+        expect(meta_event[:tag]).to eq('some_tag')
+      end
+
+      it 'should support interpolation of one tag' do
+        config['tag']  =  '%{field1}'
+        event.set('field1', 'some_value1')
+
+        meta_event = output.send :prepare_meta, event
+        expect(meta_event[:tag]).to eq('some_value1')
+      end
+
+      it 'should support interpolation of one tag in a list of tags' do
+        config['tag']  =  '%{field1},some_value2'
+        event.set('field1', 'some_value1')
+
+        meta_event = output.send :prepare_meta, event
+        expect(meta_event[:tag]).to eq('some_value1,some_value2')
+      end
+
+      it 'should remove duplicate tags' do
+        config['tag']  =  'some_value,some_value'
+
+        meta_event = output.send :prepare_meta, event
+        expect(meta_event[:tag]).to eq('some_value')
+      end
+
+      it 'should remove tag if field interpolated to empty string' do
+        config['tag']  =  '%{field1}'
+        event.set('field1', '')
+
+        meta_event = output.send :prepare_meta, event
+        expect(meta_event[:tag]).to eq(nil)
+      end
+    end
+
     context 'timestamp mingling' do
       context 'when convert_timestamp is false' do
         let(:config) { super.merge('convert_timestamp' => false) }
@@ -173,14 +221,21 @@ describe 'outputs/loggly' do
           {event: :event5, key: 'key2', tag: 'tag1'},
           {event: :event6, key: 'key1', tag: 'tag1'},
           {event: :event7, key: 'key1', tag: 'tag1'},
+          {event: :event8, key: 'key1', tag: 'tag1,tag2'},
+          {event: :event9, key: 'key2', tag: 'tag1,tag2'},
         ])
-        expect(batches.size).to eq(3)
+        expect(batches.size).to eq(5)
         expect(batches).to eq(
           { ['key1', 'tag1'] => [:event1, :event4, :event6, :event7],
             ['key2', 'tag1'] => [:event2, :event5],
             ['key2', 'tag2'] => [:event3],
+            ['key1', 'tag1,tag2'] => [:event8],
+            ['key2', 'tag1,tag2'] => [:event9],
           })
       end
+
+
+
     end
   end
 

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: logstash-output-loggly
 version: !ruby/object:Gem::Version
-  version: 5.0.0
+  version: 6.0.0
 platform: ruby
 authors:
 - Elastic
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2018-05-21 00:00:00.000000000 Z
+date: 2018-07-03 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
@@ -92,7 +92,7 @@ files:
 - spec/outputs/loggly_spec.rb
 homepage: http://www.elastic.co/guide/en/logstash/current/index.html
 licenses:
-- Apache License (2.0)
+- Apache-2.0
 metadata:
   logstash_plugin: 'true'
   logstash_group: output