logstash-filter-de_dot 0.1.0 → 0.1.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 40cd7fa6cd2b0e8d4bdc3c40dfda787d1f2cdf9e
-  data.tar.gz: 267194e821436b230dbb9d9e15be5aabd7a9e4bd
+  metadata.gz: 28e3ce95451aea24574ed2b0f86e699140537a3e
+  data.tar.gz: 7bb70232fea7b261a9a92ce57c386c13b2deae42
 SHA512:
-  metadata.gz: 750bf7f0bd52fcf52652afb34603115906b2a5ec7f333991400fe7b8145014f18e5073ce092a10c0442076818db3cecc2dcababf109529419dd1ae95b9b026da
-  data.tar.gz: ab574e0eafe500541678d6e5d4c2f9e7286c741193b8742e748d6d84e9e4bd2402f5a3db6255a07462a2172815fb96a81e4b44a170fad1cc3c60a5ecc6eded4c
+  metadata.gz: ff44ec509d4582fc128dba358b20f4446a9bce0b778cbe2da6f2b365a92307bf580a1ac2fff5138ef6a1002b75ec1046a3df8ef22e57b3748255a5c158e6bf80
+  data.tar.gz: 083bbed40f2322060aade113b65d06bed09c7052c6189272877c82f77afbefd07e6a3011a732e8b6a99bcfdd438da56ca70753ffe21854a825364059f481db48

data/CHANGELOG.md

@@ -1,2 +1,9 @@
+## 0.1.1
+ - Fix issue #4, where fields specified by `fields => ` were added to events if
+   they were not present.
+ - Fixed a bug where objects (nested fields) were not properly deleted if there
+   were multiple levels and each had a dotted field name.
+ - Fix issue #3, where dynamic fields are not recognized properly because of the
+   instance variable.
 ## 0.1.0
  - Initial release

data/lib/logstash/filters/de_dot.rb

@@ -28,6 +28,10 @@ class LogStash::Filters::De_dot < LogStash::Filters::Base
   #
   config :fields, :validate => :array
 
+  public
+  def has_dot?(fieldref)
+    fieldref =~ /\./
+  end
 
   public
   def register
@@ -35,6 +39,26 @@ class LogStash::Filters::De_dot < LogStash::Filters::Base
     # Add instance variables here, if any
   end # def register
 
+  private
+  def find_fieldref_for_delete(source)
+    # In cases where fieldref may look like [a.b][c.d][e.f], we only want to delete
+    # the first level at which the dotted field appears.
+    fieldref = ''
+    @logger.debug? && @logger.debug("de_dot: source fieldref for delete", :source => source)
+    # Iterate over each level of source
+    source.delete('[').split(']').each do |ref|
+      fieldref = fieldref + '['
+      if has_dot?(ref)
+        # return when we find the first ref with a '.'
+        @logger.debug? && @logger.debug("de_dot: fieldref for delete", :fieldref => fieldref + ref + ']')
+        return fieldref + ref + ']'
+      else
+        fieldref = fieldref + ref + ']'
+        @logger.debug? && @logger.debug("de_dot: fieldref still building", :fieldref => fieldref)
+      end
+    end
+  end
+
   private
   def rename_field(event, fieldref)
     @logger.debug? && @logger.debug("de_dot: preprocess", :event => event.to_hash.to_s)
@@ -49,7 +73,7 @@ class LogStash::Filters::De_dot < LogStash::Filters::Base
     @logger.debug? && @logger.debug("de_dot: replacement field reference", :newref => newref)
     event[newref] = event[fieldref]
     @logger.debug? && @logger.debug("de_dot: event with both new and old field references", :event => event.to_hash.to_s)
-    event.remove(fieldref)
+    event.remove(find_fieldref_for_delete(fieldref))
     @logger.debug? && @logger.debug("de_dot: postprocess", :event => event.to_hash.to_s)
   end
 
@@ -57,9 +81,17 @@ class LogStash::Filters::De_dot < LogStash::Filters::Base
   def filter(event)
     @separator = '][' if @nested
     @logger.debug? && @logger.debug("de_dot: Replace dots with separator", :separator => @separator)
-    @fields = event.to_hash.keys if @fields.nil?
-    @logger.debug? && @logger.debug("de_dot: Act on these fields", :fields => @fields)
-    @fields.each { |ref| rename_field(event, ref) if !(ref =~ /\./).nil? }
+    if @fields.nil?
+      fields = event.to_hash.keys
+    else
+      fields = @fields
+    end
+    @logger.debug? && @logger.debug("de_dot: Act on these fields", :fields => fields)
+    fields.each do |ref|
+      if event[ref]
+        rename_field(event, ref) if has_dot?(ref)
+      end
+    end
     filter_matched(event)
   end # def filter
 end # class LogStash::Filters::De_dot

data/logstash-filter-de_dot.gemspec

@@ -1,6 +1,6 @@
 Gem::Specification.new do |s|
   s.name = 'logstash-filter-de_dot'
-  s.version         = '0.1.0'
+  s.version         = '0.1.1'
   s.licenses = ['Apache License (2.0)']
   s.summary = "This filter removes dots from field names and replaces them with a different separator."
   s.description = "This gem is a logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/plugin install gemname. This gem is not a stand-alone program"

data/spec/filters/de_dot_spec.rb

@@ -128,11 +128,40 @@ describe LogStash::Filters::De_dot do
       expect(event['[acme][roller][skates]']).to eq('coyote')
       expect(event.to_hash.keys).not_to include('foo.bar')
       expect(event['[foo][bar]']).to eq('nochange')
-      expect(event.to_hash.keys).not_to include('a.a')
+      expect(event.to_hash.keys).not_to include('a.b')
       expect(event['[a][b][c][d][e][f]']).to eq('finally')
     end
   end
 
+  describe "Multiple specific nested fields with some not present" do
+    let(:config) {
+      {
+        "nested" => true,
+        "fields" => [ "[acme][roller.skates]", "foo.bar", "[a.b][c.d][e.f]" ]
+      }
+    }
+    let(:attrs) {
+      {
+        "acme" => { "roller.skates" => "coyote" },
+        "a.b" => { "c.d" => { "e.f" => "finally"} }
+      }
+    }
+
+    it "should replace all dots with underscores within specified fields" do
+      subject.filter(event)
+      expect(event['acme']).not_to include('roller.skates')
+      expect(event['[acme][roller][skates]']).to eq('coyote')
+      expect(event.to_hash.keys).not_to include('a.b')
+      expect(event['[a][b][c][d][e][f]']).to eq('finally')
+    end
+
+    it "should not add [foo][bar]" do
+      subject.filter(event)
+      expect(event.to_hash.keys).not_to include('foo.bar')
+      expect(event.to_hash.keys).not_to include('foo')
+    end
+  end
+
   describe "Specific nested fields with underscores already" do
     let(:config) {
       {

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: logstash-filter-de_dot
 version: !ruby/object:Gem::Version
-  version: 0.1.0
+  version: 0.1.1
 platform: ruby
 authors:
 - Elastic
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2015-11-09 00:00:00.000000000 Z
+date: 2015-11-17 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement