logstash-filter-fingerprint 3.3.0 → 3.4.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 0a14dcbc1585cf2f6afea7bed7a0e056d2f098bc896dc6d7571caf3c5cc3c51d
-  data.tar.gz: 2c7714afaaee24b9a8995d541ffa1cab5d16dc907dd3d1e98213d7b218fb0ebf
+  metadata.gz: 25ac9ded3114af17962469d77481c6992d65778a9dcf67012aaa3c7fb63c6ff7
+  data.tar.gz: d42a1fc7ff16d940f5634cb5f14ee2b4f2216176bdf7b90a8e7e6ed23cc95909
 SHA512:
-  metadata.gz: cbaccf02918c11b39f2f1f7bc78d7a16fbbac76d7bef9edebcb907f89cba98126708c07c05497d1e1cd5862e1846bb37b5779aaa51937382d8f416264f111328
-  data.tar.gz: 2df22689f778872a0cd85e000ff8ef940e7a16dfec4c469c4a846e7457e599e887768d3d26bd630af0af45ae0b6740cd800d3b2d89cd08bdb08f76124cad1d2b
+  metadata.gz: dc8e8df09287df6f5f0649397f0b6b326396508afb101b73b7b51ca95802c314719a74440657059114ddab7b5d47cd65b7a48babea76e2b341cbdf46753f4da6
+  data.tar.gz: 605eec24d6a4587e565acfe35c405beb6ff782410ed23c955e3d7e938e9a8fb3f633f40bc282f7ccaec77be07dfe233843ce2564dceb6cbe87dbe7fec0759668

data/CHANGELOG.md

@@ -1,3 +1,12 @@
+## 3.4.0
+  - Added support for 128bit murmur variant [#66](https://github.com/logstash-plugins/logstash-filter-fingerprint/pull/66).
+
+## 3.3.2
+  - [DOC] Clarify behavior when key is set [#65](https://github.com/logstash-plugins/logstash-filter-fingerprint/pull/65). 
+
+## 3.3.1
+  - Force encoding to UTF-8 when concatenating sources to generate fingerprint [#64](https://github.com/logstash-plugins/logstash-filter-fingerprint/pull/64)
+
 ## 3.3.0
   - Add ECS compatibility [#62](https://github.com/logstash-plugins/logstash-filter-fingerprint/pull/62)
 

data/docs/index.asciidoc

@@ -23,7 +23,7 @@ include::{include_path}/plugin_header.asciidoc[]
 Create consistent hashes (fingerprints) of one or more fields and store
 the result in a new field.
 
-You can use this plugin to create consistent document ids when  events are
+You can use this plugin to create consistent document ids when events are
 inserted into Elasticsearch. This approach means that existing documents can be
 updated instead of creating new documents.
 
@@ -60,7 +60,7 @@ This plugin supports the following configuration options plus the <<plugins-{typ
 | <<plugins-{type}s-{plugin}-concatenate_all_fields>> |<<boolean,boolean>>|No
 | <<plugins-{type}s-{plugin}-ecs_compatibility>> | <<string,string>>|No
 | <<plugins-{type}s-{plugin}-key>> |<<string,string>>|No
-| <<plugins-{type}s-{plugin}-method>> |<<string,string>>, one of `["SHA1", "SHA256", "SHA384", "SHA512", "MD5", "MURMUR3", "IPV4_NETWORK", "UUID", "PUNCTUATION"]`|Yes
+| <<plugins-{type}s-{plugin}-method>> |<<string,string>>, one of `["SHA1", "SHA256", "SHA384", "SHA512", "MD5", "MURMUR3", "MURMUR3_128", IPV4_NETWORK", "UUID", "PUNCTUATION"]`|Yes
 | <<plugins-{type}s-{plugin}-source>> |<<array,array>>|No
 | <<plugins-{type}s-{plugin}-target>> |<<string,string>>|No
 |=======================================================================
@@ -76,7 +76,7 @@ filter plugins.
   * Value type is <<boolean,boolean>>
   * Default value is `false`
 
-When set to `true`, the `SHA1`, `SHA256`, `SHA384`, `SHA512` and `MD5` fingerprint methods will produce
+When set to `true`, the `SHA1`, `SHA256`, `SHA384`, `SHA512`, `MD5` and `MURMUR3_128` fingerprint methods will produce
 base64 encoded rather than hex encoded strings.
 
 [id="plugins-{type}s-{plugin}-concatenate_sources"]
@@ -174,17 +174,16 @@ With other methods, optionally fill in the HMAC key.
 ===== `method` 
 
   * This is a required setting.
-  * Value can be any of: `SHA1`, `SHA256`, `SHA384`, `SHA512`, `MD5`, `MURMUR3`, `IPV4_NETWORK`, `UUID`, `PUNCTUATION`
+  * Value can be any of: `SHA1`, `SHA256`, `SHA384`, `SHA512`, `MD5`, `MURMUR3`, `MURMUR3_128`, `IPV4_NETWORK`, `UUID`, `PUNCTUATION`
   * Default value is `"SHA1"`
 
 The fingerprint method to use.
 
-If set to `SHA1`, `SHA256`, `SHA384`, `SHA512`, or `MD5` and a key is set,
-the cryptographic hash function with the same name will be used to generate
-the fingerprint. When a key set, the keyed-hash (HMAC) digest function will
-be used.
+If set to `SHA1`, `SHA256`, `SHA384`, `SHA512`, or `MD5` and a key is set, the
+corresponding cryptographic hash function and the keyed-hash (HMAC) digest function
+are used to generate the fingerprint.
 
-If set to `MURMUR3` the non-cryptographic 64 bit MurmurHash function will be used.
+If set to `MURMUR3` or `MURMUR3_128` the non-cryptographic MurmurHash function (either the 32-bit or 128-bit implementation, respectively) will be used.
 
 If set to `IPV4_NETWORK` the input data needs to be a IPv4 address and
 the hash value will be the masked-out address using the number of bits

data/lib/logstash/filters/fingerprint.rb

@@ -23,6 +23,10 @@ require "logstash/plugin_mixins/ecs_compatibility_support"
 # https://en.wikipedia.org/wiki/Universally_unique_identifier[UUID].
 # To generate UUIDs, prefer the <<plugins-filters-uuid,uuid filter>>.
 class LogStash::Filters::Fingerprint < LogStash::Filters::Base
+
+  INTEGER_MAX_32BIT = (1 << 31) - 1
+  INTEGER_MIN_32BIT = -(1 << 31)
+
   include LogStash::PluginMixins::ECSCompatibilitySupport(:disabled, :v1, :v8 => :v1)
 
   config_name "fingerprint"
@@ -40,8 +44,8 @@ class LogStash::Filters::Fingerprint < LogStash::Filters::Base
   # With other methods, optionally fill in the HMAC key.
   config :key, :validate => :string
 
-  # When set to `true`, the `SHA1`, `SHA256`, `SHA384`, `SHA512` and `MD5` fingerprint methods will produce
-  # base64 encoded rather than hex encoded strings.
+  # When set to `true`, the `SHA1`, `SHA256`, `SHA384`, `SHA512`, `MD5` and `MURMUR3_128` fingerprint
+  # methods will produce base64 encoded rather than hex encoded strings.
   config :base64encode, :validate => :boolean, :default => false
 
   # The fingerprint method to use.
@@ -51,7 +55,9 @@ class LogStash::Filters::Fingerprint < LogStash::Filters::Base
   # the fingerprint. When a key set, the keyed-hash (HMAC) digest function will
   # be used.
   #
-  # If set to `MURMUR3` the non-cryptographic MurmurHash function will be used.
+  # If set to `MURMUR3` or `MURMUR3_128` the non-cryptographic MurmurHash
+  # function (either the 32-bit or 128-bit implementation, respectively)
+  # will be used.
   #
   # If set to `IPV4_NETWORK` the input data needs to be a IPv4 address and
   # the hash value will be the masked-out address using the number of bits
@@ -64,7 +70,7 @@ class LogStash::Filters::Fingerprint < LogStash::Filters::Base
   # If set to `UUID`, a
   # https://en.wikipedia.org/wiki/Universally_unique_identifier[UUID] will
   # be generated. The result will be random and thus not a consistent hash.
-  config :method, :validate => ['SHA1', 'SHA256', 'SHA384', 'SHA512', 'MD5', "MURMUR3", "IPV4_NETWORK", "UUID", "PUNCTUATION"], :required => true, :default => 'SHA1'
+  config :method, :validate => ['SHA1', 'SHA256', 'SHA384', 'SHA512', 'MD5', "MURMUR3", "MURMUR3_128", "IPV4_NETWORK", "UUID", "PUNCTUATION"], :required => true, :default => 'SHA1'
 
   # When set to `true` and `method` isn't `UUID` or `PUNCTUATION`, the
   # plugin concatenates the names and values of all fields given in the
@@ -102,6 +108,8 @@ class LogStash::Filters::Fingerprint < LogStash::Filters::Base
       class << self; alias_method :fingerprint, :fingerprint_ipv4_network; end
     when :MURMUR3
       class << self; alias_method :fingerprint, :fingerprint_murmur3; end
+    when :MURMUR3_128
+      class << self; alias_method :fingerprint, :fingerprint_murmur3_128; end
     when :UUID
       # nothing
     when :PUNCTUATION
@@ -128,11 +136,13 @@ class LogStash::Filters::Fingerprint < LogStash::Filters::Base
         to_string = ""
         if @concatenate_all_fields
           deep_sort_hashes(event.to_hash).each do |k,v|
-            to_string << "|#{k}|#{v}"
+            # Force encoding to UTF-8 to get around https://github.com/jruby/jruby/issues/6748
+            to_string << "|#{k}|#{v}".force_encoding("UTF-8")
           end
         else
           @source.sort.each do |k|
-            to_string << "|#{k}|#{deep_sort_hashes(event.get(k))}"
+            # Force encoding to UTF-8 to get around https://github.com/jruby/jruby/issues/6748
+            to_string << "|#{k}|#{deep_sort_hashes(event.get(k))}".force_encoding("UTF-8")
           end
         end
         to_string << "|"
@@ -208,6 +218,30 @@ class LogStash::Filters::Fingerprint < LogStash::Filters::Base
     end
   end
 
+  def fingerprint_murmur3_128(value)
+    if value.is_a?(Integer)
+      if (INTEGER_MIN_32BIT <= value) && (value <= INTEGER_MAX_32BIT)
+        if @base64encode
+          [MurmurHash3::V128.int32_hash(value, 2).pack("L*")].pack("m").chomp!
+        else
+          MurmurHash3::V128.int32_hash(value, 2).pack("L*").unpack("H*")[0]
+        end
+      else
+        if @base64encode
+          [MurmurHash3::V128.int64_hash(value, 2).pack("L*")].pack("m").chomp!
+        else
+          MurmurHash3::V128.int64_hash(value, 2).pack("L*").unpack("H*")[0]
+        end
+      end
+    else
+      if @base64encode
+        MurmurHash3::V128.str_base64digest(value.to_s, 2)
+      else
+        MurmurHash3::V128.str_hexdigest(value.to_s, 2)
+      end
+    end
+  end
+
   def select_digest(method)
     case method
     when :SHA1

data/logstash-filter-fingerprint.gemspec

@@ -1,7 +1,7 @@
 Gem::Specification.new do |s|
 
   s.name            = 'logstash-filter-fingerprint'
-  s.version         = '3.3.0'
+  s.version         = '3.4.0'
   s.licenses        = ['Apache-2.0']
   s.summary         = "Fingerprints fields by replacing values with a consistent hash"
   s.description     = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program"
@@ -21,8 +21,7 @@ Gem::Specification.new do |s|
 
   # Gem dependencies
   s.add_runtime_dependency "logstash-core-plugin-api", ">= 1.60", "<= 2.99"
-  s.add_runtime_dependency "murmurhash3"                      #(MIT license)
+  s.add_runtime_dependency "murmurhash3"                    #(MIT license)
   s.add_development_dependency 'logstash-devutils'
   s.add_runtime_dependency 'logstash-mixin-ecs_compatibility_support', '~>1.2'
 end
-

data/spec/filters/fingerprint_spec.rb

@@ -50,6 +50,59 @@ describe LogStash::Filters::Fingerprint, :ecs_compatibility_support, :aggregate_
         end
       end
 
+      describe "the MURMUR3_128 method" do
+        let(:fingerprint_method) { "MURMUR3_128" }
+
+        context "string hex encoding" do
+          it "fingerprints the value" do
+            expect(fingerprint).to eq("41cbc4056eed401d091dfbeabf7ea9e0")
+          end
+        end
+
+        context "string base64 encoding" do
+          let(:config) { super().merge("base64encode" => true) }
+          it "fingerprints the value" do
+            expect(fingerprint).to eq("QcvEBW7tQB0JHfvqv36p4A==")
+          end
+        end
+
+        context "int32 hex encoding" do
+          let(:config) { super().merge("base64encode" => false) }
+          let(:data) { {"clientip" => 123 } }
+
+          it "fingerprints the value" do
+            expect(fingerprint).to eq("286816c693ac410ed63e1430dcd6f6fe")
+          end
+        end
+
+        context "int32 base64 encoding" do
+          let(:config) { super().merge("base64encode" => true) }
+          let(:data) { {"clientip" => 123 } }
+
+          it "fingerprints the value" do
+            expect(fingerprint).to eq("KGgWxpOsQQ7WPhQw3Nb2/g==")
+          end
+        end
+
+        context "int64 hex encoding" do
+          let(:config) { super().merge("base64encode" => false) }
+          let(:data) { {"clientip" => 2148483647 } }
+
+          it "fingerprints the value" do
+            expect(fingerprint).to eq("fdc7699a82556c8c584131f0133ee989")
+          end
+        end
+
+        context "int64 base64 encoding" do
+          let(:config) { super().merge("base64encode" => true) }
+          let(:data) { {"clientip" => 2148483647 } }
+
+          it "fingerprints the value" do
+            expect(fingerprint).to eq("/cdpmoJVbIxYQTHwEz7piQ==")
+          end
+        end
+      end
+
       describe "the SHA1 method" do
         let(:fingerprint_method) { "SHA1" }
 
@@ -192,6 +245,23 @@ describe LogStash::Filters::Fingerprint, :ecs_compatibility_support, :aggregate_
       end
     end
 
+    context "when utf-8 chars used" do
+      let(:config) { super().merge("source" => ['field1', 'field2']) }
+      let(:data) { {"field1"=>[{"inner_key"=>"🂡"}, {"1"=>"2"}], "field2"=>"🂡"} }
+      it "fingerprints the value of the last value" do
+        # SHA1 of "|field1|inner_key|🂡|1|2|field2|🂡|"
+        expect(fingerprint).to eq("58fa9e0e60c9f0d24b51d84cddb26732a39eeb3d")
+      end
+
+      describe "with concatenate_sources" do
+        let(:config) { super().merge("concatenate_sources" => true) }
+        it "fingerprints the value of concatenated key/pairs" do
+          # SHA1 of "|field1|inner_key|🂡|1|2|field2|🂡|"
+          expect(fingerprint).to eq("d74f41841c7cdc793a97c218d2ff18064a5f1950")
+        end
+      end
+    end
+
     describe "PUNCTUATION method" do
       let(:fingerprint_method) { 'PUNCTUATION' }
       let(:config) { super().merge("source" => 'field1') }
@@ -222,6 +292,14 @@ describe LogStash::Filters::Fingerprint, :ecs_compatibility_support, :aggregate_
           expect(fingerprint).to eq(743372282)
         end
       end
+
+      describe 'MURMUR3_128 Fingerprinting' do
+        let(:fingerprint_method) { "MURMUR3_128" }
+        let(:data) { { "@timestamp" => epoch_time } }
+        it "fingerprints the timestamp correctly" do
+          expect(fingerprint).to eq("37785b62a8cae473acc315d39b66d86e")
+        end
+      end
     end
 
     describe "post fingerprint execution triggers" do

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: logstash-filter-fingerprint
 version: !ruby/object:Gem::Version
-  version: 3.3.0
+  version: 3.4.0
 platform: ruby
 authors:
 - Elastic
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2021-06-08 00:00:00.000000000 Z
+date: 2022-05-03 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
@@ -111,8 +111,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubyforge_project:
-rubygems_version: 2.6.13
+rubygems_version: 3.1.6
 signing_key:
 specification_version: 4
 summary: Fingerprints fields by replacing values with a consistent hash