logstash-filter-fingerprint 3.3.2 → 3.4.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: f6c1e99ae34e1681eba14af022fa9fc79909bfaf306542326f0ae88881810370
-  data.tar.gz: ffe46fdd11dc94d0fdcdb35d889ec41f3560f205c78e872985ea426fdd776cef
+  metadata.gz: 25ac9ded3114af17962469d77481c6992d65778a9dcf67012aaa3c7fb63c6ff7
+  data.tar.gz: d42a1fc7ff16d940f5634cb5f14ee2b4f2216176bdf7b90a8e7e6ed23cc95909
 SHA512:
-  metadata.gz: c343f48b3f479e5a38c4de0510fc9a51b6785c4437441206baf491c1373df9abae24d30b5efc4f761295857e7d376c7c276e69378ed305e53432fe55e6e5477a
-  data.tar.gz: 964ad01b6482e60069421ec1c26c1ad422c44e88372ff63305d752210fd31755752409af9dd2bee96adf8fafa9b9065ca1ce40b53f18da05b5a76b86a54fffb0
+  metadata.gz: dc8e8df09287df6f5f0649397f0b6b326396508afb101b73b7b51ca95802c314719a74440657059114ddab7b5d47cd65b7a48babea76e2b341cbdf46753f4da6
+  data.tar.gz: 605eec24d6a4587e565acfe35c405beb6ff782410ed23c955e3d7e938e9a8fb3f633f40bc282f7ccaec77be07dfe233843ce2564dceb6cbe87dbe7fec0759668

data/CHANGELOG.md

@@ -1,3 +1,6 @@
+## 3.4.0
+  - Added support for 128bit murmur variant [#66](https://github.com/logstash-plugins/logstash-filter-fingerprint/pull/66).
+
 ## 3.3.2
   - [DOC] Clarify behavior when key is set [#65](https://github.com/logstash-plugins/logstash-filter-fingerprint/pull/65). 
 

data/docs/index.asciidoc

@@ -60,7 +60,7 @@ This plugin supports the following configuration options plus the <<plugins-{typ
 | <<plugins-{type}s-{plugin}-concatenate_all_fields>> |<<boolean,boolean>>|No
 | <<plugins-{type}s-{plugin}-ecs_compatibility>> | <<string,string>>|No
 | <<plugins-{type}s-{plugin}-key>> |<<string,string>>|No
-| <<plugins-{type}s-{plugin}-method>> |<<string,string>>, one of `["SHA1", "SHA256", "SHA384", "SHA512", "MD5", "MURMUR3", "IPV4_NETWORK", "UUID", "PUNCTUATION"]`|Yes
+| <<plugins-{type}s-{plugin}-method>> |<<string,string>>, one of `["SHA1", "SHA256", "SHA384", "SHA512", "MD5", "MURMUR3", "MURMUR3_128", IPV4_NETWORK", "UUID", "PUNCTUATION"]`|Yes
 | <<plugins-{type}s-{plugin}-source>> |<<array,array>>|No
 | <<plugins-{type}s-{plugin}-target>> |<<string,string>>|No
 |=======================================================================
@@ -76,7 +76,7 @@ filter plugins.
   * Value type is <<boolean,boolean>>
   * Default value is `false`
 
-When set to `true`, the `SHA1`, `SHA256`, `SHA384`, `SHA512` and `MD5` fingerprint methods will produce
+When set to `true`, the `SHA1`, `SHA256`, `SHA384`, `SHA512`, `MD5` and `MURMUR3_128` fingerprint methods will produce
 base64 encoded rather than hex encoded strings.
 
 [id="plugins-{type}s-{plugin}-concatenate_sources"]
@@ -174,7 +174,7 @@ With other methods, optionally fill in the HMAC key.
 ===== `method` 
 
   * This is a required setting.
-  * Value can be any of: `SHA1`, `SHA256`, `SHA384`, `SHA512`, `MD5`, `MURMUR3`, `IPV4_NETWORK`, `UUID`, `PUNCTUATION`
+  * Value can be any of: `SHA1`, `SHA256`, `SHA384`, `SHA512`, `MD5`, `MURMUR3`, `MURMUR3_128`, `IPV4_NETWORK`, `UUID`, `PUNCTUATION`
   * Default value is `"SHA1"`
 
 The fingerprint method to use.
@@ -183,7 +183,7 @@ If set to `SHA1`, `SHA256`, `SHA384`, `SHA512`, or `MD5` and a key is set, the
 corresponding cryptographic hash function and the keyed-hash (HMAC) digest function
 are used to generate the fingerprint.
 
-If set to `MURMUR3` the non-cryptographic 64 bit MurmurHash function will be used.
+If set to `MURMUR3` or `MURMUR3_128` the non-cryptographic MurmurHash function (either the 32-bit or 128-bit implementation, respectively) will be used.
 
 If set to `IPV4_NETWORK` the input data needs to be a IPv4 address and
 the hash value will be the masked-out address using the number of bits

data/lib/logstash/filters/fingerprint.rb

@@ -23,6 +23,10 @@ require "logstash/plugin_mixins/ecs_compatibility_support"
 # https://en.wikipedia.org/wiki/Universally_unique_identifier[UUID].
 # To generate UUIDs, prefer the <<plugins-filters-uuid,uuid filter>>.
 class LogStash::Filters::Fingerprint < LogStash::Filters::Base
+
+  INTEGER_MAX_32BIT = (1 << 31) - 1
+  INTEGER_MIN_32BIT = -(1 << 31)
+
   include LogStash::PluginMixins::ECSCompatibilitySupport(:disabled, :v1, :v8 => :v1)
 
   config_name "fingerprint"
@@ -40,8 +44,8 @@ class LogStash::Filters::Fingerprint < LogStash::Filters::Base
   # With other methods, optionally fill in the HMAC key.
   config :key, :validate => :string
 
-  # When set to `true`, the `SHA1`, `SHA256`, `SHA384`, `SHA512` and `MD5` fingerprint methods will produce
-  # base64 encoded rather than hex encoded strings.
+  # When set to `true`, the `SHA1`, `SHA256`, `SHA384`, `SHA512`, `MD5` and `MURMUR3_128` fingerprint
+  # methods will produce base64 encoded rather than hex encoded strings.
   config :base64encode, :validate => :boolean, :default => false
 
   # The fingerprint method to use.
@@ -51,7 +55,9 @@ class LogStash::Filters::Fingerprint < LogStash::Filters::Base
   # the fingerprint. When a key set, the keyed-hash (HMAC) digest function will
   # be used.
   #
-  # If set to `MURMUR3` the non-cryptographic MurmurHash function will be used.
+  # If set to `MURMUR3` or `MURMUR3_128` the non-cryptographic MurmurHash
+  # function (either the 32-bit or 128-bit implementation, respectively)
+  # will be used.
   #
   # If set to `IPV4_NETWORK` the input data needs to be a IPv4 address and
   # the hash value will be the masked-out address using the number of bits
@@ -64,7 +70,7 @@ class LogStash::Filters::Fingerprint < LogStash::Filters::Base
   # If set to `UUID`, a
   # https://en.wikipedia.org/wiki/Universally_unique_identifier[UUID] will
   # be generated. The result will be random and thus not a consistent hash.
-  config :method, :validate => ['SHA1', 'SHA256', 'SHA384', 'SHA512', 'MD5', "MURMUR3", "IPV4_NETWORK", "UUID", "PUNCTUATION"], :required => true, :default => 'SHA1'
+  config :method, :validate => ['SHA1', 'SHA256', 'SHA384', 'SHA512', 'MD5', "MURMUR3", "MURMUR3_128", "IPV4_NETWORK", "UUID", "PUNCTUATION"], :required => true, :default => 'SHA1'
 
   # When set to `true` and `method` isn't `UUID` or `PUNCTUATION`, the
   # plugin concatenates the names and values of all fields given in the
@@ -102,6 +108,8 @@ class LogStash::Filters::Fingerprint < LogStash::Filters::Base
       class << self; alias_method :fingerprint, :fingerprint_ipv4_network; end
     when :MURMUR3
       class << self; alias_method :fingerprint, :fingerprint_murmur3; end
+    when :MURMUR3_128
+      class << self; alias_method :fingerprint, :fingerprint_murmur3_128; end
     when :UUID
       # nothing
     when :PUNCTUATION
@@ -210,6 +218,30 @@ class LogStash::Filters::Fingerprint < LogStash::Filters::Base
     end
   end
 
+  def fingerprint_murmur3_128(value)
+    if value.is_a?(Integer)
+      if (INTEGER_MIN_32BIT <= value) && (value <= INTEGER_MAX_32BIT)
+        if @base64encode
+          [MurmurHash3::V128.int32_hash(value, 2).pack("L*")].pack("m").chomp!
+        else
+          MurmurHash3::V128.int32_hash(value, 2).pack("L*").unpack("H*")[0]
+        end
+      else
+        if @base64encode
+          [MurmurHash3::V128.int64_hash(value, 2).pack("L*")].pack("m").chomp!
+        else
+          MurmurHash3::V128.int64_hash(value, 2).pack("L*").unpack("H*")[0]
+        end
+      end
+    else
+      if @base64encode
+        MurmurHash3::V128.str_base64digest(value.to_s, 2)
+      else
+        MurmurHash3::V128.str_hexdigest(value.to_s, 2)
+      end
+    end
+  end
+
   def select_digest(method)
     case method
     when :SHA1

data/logstash-filter-fingerprint.gemspec

@@ -1,7 +1,7 @@
 Gem::Specification.new do |s|
 
   s.name            = 'logstash-filter-fingerprint'
-  s.version         = '3.3.2'
+  s.version         = '3.4.0'
   s.licenses        = ['Apache-2.0']
   s.summary         = "Fingerprints fields by replacing values with a consistent hash"
   s.description     = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program"
@@ -21,8 +21,7 @@ Gem::Specification.new do |s|
 
   # Gem dependencies
   s.add_runtime_dependency "logstash-core-plugin-api", ">= 1.60", "<= 2.99"
-  s.add_runtime_dependency "murmurhash3"                      #(MIT license)
+  s.add_runtime_dependency "murmurhash3"                    #(MIT license)
   s.add_development_dependency 'logstash-devutils'
   s.add_runtime_dependency 'logstash-mixin-ecs_compatibility_support', '~>1.2'
 end
-

data/spec/filters/fingerprint_spec.rb

@@ -50,6 +50,59 @@ describe LogStash::Filters::Fingerprint, :ecs_compatibility_support, :aggregate_
         end
       end
 
+      describe "the MURMUR3_128 method" do
+        let(:fingerprint_method) { "MURMUR3_128" }
+
+        context "string hex encoding" do
+          it "fingerprints the value" do
+            expect(fingerprint).to eq("41cbc4056eed401d091dfbeabf7ea9e0")
+          end
+        end
+
+        context "string base64 encoding" do
+          let(:config) { super().merge("base64encode" => true) }
+          it "fingerprints the value" do
+            expect(fingerprint).to eq("QcvEBW7tQB0JHfvqv36p4A==")
+          end
+        end
+
+        context "int32 hex encoding" do
+          let(:config) { super().merge("base64encode" => false) }
+          let(:data) { {"clientip" => 123 } }
+
+          it "fingerprints the value" do
+            expect(fingerprint).to eq("286816c693ac410ed63e1430dcd6f6fe")
+          end
+        end
+
+        context "int32 base64 encoding" do
+          let(:config) { super().merge("base64encode" => true) }
+          let(:data) { {"clientip" => 123 } }
+
+          it "fingerprints the value" do
+            expect(fingerprint).to eq("KGgWxpOsQQ7WPhQw3Nb2/g==")
+          end
+        end
+
+        context "int64 hex encoding" do
+          let(:config) { super().merge("base64encode" => false) }
+          let(:data) { {"clientip" => 2148483647 } }
+
+          it "fingerprints the value" do
+            expect(fingerprint).to eq("fdc7699a82556c8c584131f0133ee989")
+          end
+        end
+
+        context "int64 base64 encoding" do
+          let(:config) { super().merge("base64encode" => true) }
+          let(:data) { {"clientip" => 2148483647 } }
+
+          it "fingerprints the value" do
+            expect(fingerprint).to eq("/cdpmoJVbIxYQTHwEz7piQ==")
+          end
+        end
+      end
+
       describe "the SHA1 method" do
         let(:fingerprint_method) { "SHA1" }
 
@@ -239,6 +292,14 @@ describe LogStash::Filters::Fingerprint, :ecs_compatibility_support, :aggregate_
           expect(fingerprint).to eq(743372282)
         end
       end
+
+      describe 'MURMUR3_128 Fingerprinting' do
+        let(:fingerprint_method) { "MURMUR3_128" }
+        let(:data) { { "@timestamp" => epoch_time } }
+        it "fingerprints the timestamp correctly" do
+          expect(fingerprint).to eq("37785b62a8cae473acc315d39b66d86e")
+        end
+      end
     end
 
     describe "post fingerprint execution triggers" do

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: logstash-filter-fingerprint
 version: !ruby/object:Gem::Version
-  version: 3.3.2
+  version: 3.4.0
 platform: ruby
 authors:
 - Elastic
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2021-07-13 00:00:00.000000000 Z
+date: 2022-05-03 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
@@ -111,8 +111,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubyforge_project:
-rubygems_version: 2.6.13
+rubygems_version: 3.1.6
 signing_key:
 specification_version: 4
 summary: Fingerprints fields by replacing values with a consistent hash