logstash-filter-fingerprint 3.3.1 → 3.4.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 595333133cc076fc879955fa5a665387dfd4d33d4af757d56d163a6a3eeb16d8
-  data.tar.gz: 912e104bd727b7b3f427d24f038288ca5d08087242267d55b4fc6899403ba378
+  metadata.gz: 314340e6e62bfa1dc34740e1f44af4313668c797dc99cb15b8fc37519f601c75
+  data.tar.gz: bba012a2b02065879152ac51fe926e8ba8beef8ed96d577b3aecb4555a255dce
 SHA512:
-  metadata.gz: d9f4631d499c888a5681c63be1d254d45392fc00c6e281d6bb5ddcd24028d0af43e4e0c6b9f99a0a082693e5559a44fb4d2ce5a3894e2c5081d9937e94ccb15e
-  data.tar.gz: c280e2fc1b976d11e9e57ccb444f8153e953f9f0ff7f74b393e1b1c670d9ed4c6932d8af14a6883806f0079419292df8aae9130fb1334ff4013e193a0c989925
+  metadata.gz: bb420df3e75d292c83cf6e7f6488ff6bad9b194d45787ee43c1e2b2fadb9fc038b42b6ebb8d2403b852d111799ebc139819109a559abd59692693ca8d7fed42b
+  data.tar.gz: 451164868428c63860da386b856c4ea0adff10ce290ae24dc4a5d45a7b8ab12a6c90c74bf352bcab0af702cec820d9cba1fcf04889ed02bac63a45b93037d03a

data/CHANGELOG.md

@@ -1,3 +1,12 @@
+## 3.4.1
+  - Added backward compatibility of timestamp format to provide consistent fingerprint [#67](https://github.com/logstash-plugins/logstash-filter-fingerprint/pull/67)
+
+## 3.4.0
+  - Added support for 128bit murmur variant [#66](https://github.com/logstash-plugins/logstash-filter-fingerprint/pull/66).
+
+## 3.3.2
+  - [DOC] Clarify behavior when key is set [#65](https://github.com/logstash-plugins/logstash-filter-fingerprint/pull/65). 
+
 ## 3.3.1
   - Force encoding to UTF-8 when concatenating sources to generate fingerprint [#64](https://github.com/logstash-plugins/logstash-filter-fingerprint/pull/64)
 

data/docs/index.asciidoc

@@ -23,7 +23,7 @@ include::{include_path}/plugin_header.asciidoc[]
 Create consistent hashes (fingerprints) of one or more fields and store
 the result in a new field.
 
-You can use this plugin to create consistent document ids when  events are
+You can use this plugin to create consistent document ids when events are
 inserted into Elasticsearch. This approach means that existing documents can be
 updated instead of creating new documents.
 
@@ -60,7 +60,7 @@ This plugin supports the following configuration options plus the <<plugins-{typ
 | <<plugins-{type}s-{plugin}-concatenate_all_fields>> |<<boolean,boolean>>|No
 | <<plugins-{type}s-{plugin}-ecs_compatibility>> | <<string,string>>|No
 | <<plugins-{type}s-{plugin}-key>> |<<string,string>>|No
-| <<plugins-{type}s-{plugin}-method>> |<<string,string>>, one of `["SHA1", "SHA256", "SHA384", "SHA512", "MD5", "MURMUR3", "IPV4_NETWORK", "UUID", "PUNCTUATION"]`|Yes
+| <<plugins-{type}s-{plugin}-method>> |<<string,string>>, one of `["SHA1", "SHA256", "SHA384", "SHA512", "MD5", "MURMUR3", "MURMUR3_128", IPV4_NETWORK", "UUID", "PUNCTUATION"]`|Yes
 | <<plugins-{type}s-{plugin}-source>> |<<array,array>>|No
 | <<plugins-{type}s-{plugin}-target>> |<<string,string>>|No
 |=======================================================================
@@ -76,7 +76,7 @@ filter plugins.
   * Value type is <<boolean,boolean>>
   * Default value is `false`
 
-When set to `true`, the `SHA1`, `SHA256`, `SHA384`, `SHA512` and `MD5` fingerprint methods will produce
+When set to `true`, the `SHA1`, `SHA256`, `SHA384`, `SHA512`, `MD5` and `MURMUR3_128` fingerprint methods will produce
 base64 encoded rather than hex encoded strings.
 
 [id="plugins-{type}s-{plugin}-concatenate_sources"]
@@ -174,17 +174,16 @@ With other methods, optionally fill in the HMAC key.
 ===== `method` 
 
   * This is a required setting.
-  * Value can be any of: `SHA1`, `SHA256`, `SHA384`, `SHA512`, `MD5`, `MURMUR3`, `IPV4_NETWORK`, `UUID`, `PUNCTUATION`
+  * Value can be any of: `SHA1`, `SHA256`, `SHA384`, `SHA512`, `MD5`, `MURMUR3`, `MURMUR3_128`, `IPV4_NETWORK`, `UUID`, `PUNCTUATION`
   * Default value is `"SHA1"`
 
 The fingerprint method to use.
 
-If set to `SHA1`, `SHA256`, `SHA384`, `SHA512`, or `MD5` and a key is set,
-the cryptographic hash function with the same name will be used to generate
-the fingerprint. When a key set, the keyed-hash (HMAC) digest function will
-be used.
+If set to `SHA1`, `SHA256`, `SHA384`, `SHA512`, or `MD5` and a key is set, the
+corresponding cryptographic hash function and the keyed-hash (HMAC) digest function
+are used to generate the fingerprint.
 
-If set to `MURMUR3` the non-cryptographic 64 bit MurmurHash function will be used.
+If set to `MURMUR3` or `MURMUR3_128` the non-cryptographic MurmurHash function (either the 32-bit or 128-bit implementation, respectively) will be used.
 
 If set to `IPV4_NETWORK` the input data needs to be a IPv4 address and
 the hash value will be the masked-out address using the number of bits

data/lib/logstash/filters/fingerprint.rb

@@ -23,6 +23,29 @@ require "logstash/plugin_mixins/ecs_compatibility_support"
 # https://en.wikipedia.org/wiki/Universally_unique_identifier[UUID].
 # To generate UUIDs, prefer the <<plugins-filters-uuid,uuid filter>>.
 class LogStash::Filters::Fingerprint < LogStash::Filters::Base
+
+  ##
+  # Logstash 8+ has variable-length serialization of timestamps
+  # that do not include subsecond info for whole-second timestamps.
+  # For backward-compatibility we refine the implementation to use
+  # our own three-decimal-place formatter for whole-second
+  # timestamps.
+  if LOGSTASH_VERSION.split('.').first.to_i >= 8
+    module MinimumSerializationLengthTimestamp
+      THREE_DECIMAL_INSTANT_FORMATTER = java.time.format.DateTimeFormatterBuilder.new.appendInstant(3).toFormatter
+      refine LogStash::Timestamp do
+        def to_s
+          return super unless nsec == 0
+          THREE_DECIMAL_INSTANT_FORMATTER.format(to_java.toInstant)
+        end
+      end
+    end
+    using MinimumSerializationLengthTimestamp
+  end
+
+  INTEGER_MAX_32BIT = (1 << 31) - 1
+  INTEGER_MIN_32BIT = -(1 << 31)
+
   include LogStash::PluginMixins::ECSCompatibilitySupport(:disabled, :v1, :v8 => :v1)
 
   config_name "fingerprint"
@@ -40,8 +63,8 @@ class LogStash::Filters::Fingerprint < LogStash::Filters::Base
   # With other methods, optionally fill in the HMAC key.
   config :key, :validate => :string
 
-  # When set to `true`, the `SHA1`, `SHA256`, `SHA384`, `SHA512` and `MD5` fingerprint methods will produce
-  # base64 encoded rather than hex encoded strings.
+  # When set to `true`, the `SHA1`, `SHA256`, `SHA384`, `SHA512`, `MD5` and `MURMUR3_128` fingerprint
+  # methods will produce base64 encoded rather than hex encoded strings.
   config :base64encode, :validate => :boolean, :default => false
 
   # The fingerprint method to use.
@@ -51,7 +74,9 @@ class LogStash::Filters::Fingerprint < LogStash::Filters::Base
   # the fingerprint. When a key set, the keyed-hash (HMAC) digest function will
   # be used.
   #
-  # If set to `MURMUR3` the non-cryptographic MurmurHash function will be used.
+  # If set to `MURMUR3` or `MURMUR3_128` the non-cryptographic MurmurHash
+  # function (either the 32-bit or 128-bit implementation, respectively)
+  # will be used.
   #
   # If set to `IPV4_NETWORK` the input data needs to be a IPv4 address and
   # the hash value will be the masked-out address using the number of bits
@@ -64,7 +89,7 @@ class LogStash::Filters::Fingerprint < LogStash::Filters::Base
   # If set to `UUID`, a
   # https://en.wikipedia.org/wiki/Universally_unique_identifier[UUID] will
   # be generated. The result will be random and thus not a consistent hash.
-  config :method, :validate => ['SHA1', 'SHA256', 'SHA384', 'SHA512', 'MD5', "MURMUR3", "IPV4_NETWORK", "UUID", "PUNCTUATION"], :required => true, :default => 'SHA1'
+  config :method, :validate => ['SHA1', 'SHA256', 'SHA384', 'SHA512', 'MD5', "MURMUR3", "MURMUR3_128", "IPV4_NETWORK", "UUID", "PUNCTUATION"], :required => true, :default => 'SHA1'
 
   # When set to `true` and `method` isn't `UUID` or `PUNCTUATION`, the
   # plugin concatenates the names and values of all fields given in the
@@ -102,6 +127,8 @@ class LogStash::Filters::Fingerprint < LogStash::Filters::Base
       class << self; alias_method :fingerprint, :fingerprint_ipv4_network; end
     when :MURMUR3
       class << self; alias_method :fingerprint, :fingerprint_murmur3; end
+    when :MURMUR3_128
+      class << self; alias_method :fingerprint, :fingerprint_murmur3_128; end
     when :UUID
       # nothing
     when :PUNCTUATION
@@ -210,6 +237,30 @@ class LogStash::Filters::Fingerprint < LogStash::Filters::Base
     end
   end
 
+  def fingerprint_murmur3_128(value)
+    if value.is_a?(Integer)
+      if (INTEGER_MIN_32BIT <= value) && (value <= INTEGER_MAX_32BIT)
+        if @base64encode
+          [MurmurHash3::V128.int32_hash(value, 2).pack("L*")].pack("m").chomp!
+        else
+          MurmurHash3::V128.int32_hash(value, 2).pack("L*").unpack("H*")[0]
+        end
+      else
+        if @base64encode
+          [MurmurHash3::V128.int64_hash(value, 2).pack("L*")].pack("m").chomp!
+        else
+          MurmurHash3::V128.int64_hash(value, 2).pack("L*").unpack("H*")[0]
+        end
+      end
+    else
+      if @base64encode
+        MurmurHash3::V128.str_base64digest(value.to_s, 2)
+      else
+        MurmurHash3::V128.str_hexdigest(value.to_s, 2)
+      end
+    end
+  end
+
   def select_digest(method)
     case method
     when :SHA1

data/logstash-filter-fingerprint.gemspec

@@ -1,7 +1,7 @@
 Gem::Specification.new do |s|
 
   s.name            = 'logstash-filter-fingerprint'
-  s.version         = '3.3.1'
+  s.version         = '3.4.1'
   s.licenses        = ['Apache-2.0']
   s.summary         = "Fingerprints fields by replacing values with a consistent hash"
   s.description     = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program"
@@ -21,8 +21,7 @@ Gem::Specification.new do |s|
 
   # Gem dependencies
   s.add_runtime_dependency "logstash-core-plugin-api", ">= 1.60", "<= 2.99"
-  s.add_runtime_dependency "murmurhash3"                      #(MIT license)
+  s.add_runtime_dependency "murmurhash3"                    #(MIT license)
   s.add_development_dependency 'logstash-devutils'
   s.add_runtime_dependency 'logstash-mixin-ecs_compatibility_support', '~>1.2'
 end
-

data/spec/filters/fingerprint_spec.rb

@@ -19,6 +19,10 @@ describe LogStash::Filters::Fingerprint, :ecs_compatibility_support, :aggregate_
       plugin.filter(event)
     end
 
+    def ge_version_8
+      LOGSTASH_VERSION.split('.').first.to_i >= 8
+    end
+
     context "with a string field" do
       let(:data) { {"clientip" => "123.123.123.123" } }
       let(:config) { super().merge("source" => ["clientip" ]) }
@@ -50,6 +54,59 @@ describe LogStash::Filters::Fingerprint, :ecs_compatibility_support, :aggregate_
         end
       end
 
+      describe "the MURMUR3_128 method" do
+        let(:fingerprint_method) { "MURMUR3_128" }
+
+        context "string hex encoding" do
+          it "fingerprints the value" do
+            expect(fingerprint).to eq("41cbc4056eed401d091dfbeabf7ea9e0")
+          end
+        end
+
+        context "string base64 encoding" do
+          let(:config) { super().merge("base64encode" => true) }
+          it "fingerprints the value" do
+            expect(fingerprint).to eq("QcvEBW7tQB0JHfvqv36p4A==")
+          end
+        end
+
+        context "int32 hex encoding" do
+          let(:config) { super().merge("base64encode" => false) }
+          let(:data) { {"clientip" => 123 } }
+
+          it "fingerprints the value" do
+            expect(fingerprint).to eq("286816c693ac410ed63e1430dcd6f6fe")
+          end
+        end
+
+        context "int32 base64 encoding" do
+          let(:config) { super().merge("base64encode" => true) }
+          let(:data) { {"clientip" => 123 } }
+
+          it "fingerprints the value" do
+            expect(fingerprint).to eq("KGgWxpOsQQ7WPhQw3Nb2/g==")
+          end
+        end
+
+        context "int64 hex encoding" do
+          let(:config) { super().merge("base64encode" => false) }
+          let(:data) { {"clientip" => 2148483647 } }
+
+          it "fingerprints the value" do
+            expect(fingerprint).to eq("fdc7699a82556c8c584131f0133ee989")
+          end
+        end
+
+        context "int64 base64 encoding" do
+          let(:config) { super().merge("base64encode" => true) }
+          let(:data) { {"clientip" => 2148483647 } }
+
+          it "fingerprints the value" do
+            expect(fingerprint).to eq("/cdpmoJVbIxYQTHwEz7piQ==")
+          end
+        end
+      end
+
       describe "the SHA1 method" do
         let(:fingerprint_method) { "SHA1" }
 
@@ -220,7 +277,7 @@ describe LogStash::Filters::Fingerprint, :ecs_compatibility_support, :aggregate_
     end
 
     context 'Timestamps' do
-      epoch_time = Time.at(0).gmtime
+      let(:epoch_time) { Time.at(0).gmtime }
       let(:config) { super().merge("source" => ['@timestamp']) }
 
       describe 'OpenSSL Fingerprinting' do
@@ -239,6 +296,40 @@ describe LogStash::Filters::Fingerprint, :ecs_compatibility_support, :aggregate_
           expect(fingerprint).to eq(743372282)
         end
       end
+
+      describe 'MURMUR3_128 Fingerprinting' do
+        let(:fingerprint_method) { "MURMUR3_128" }
+        let(:data) { { "@timestamp" => epoch_time } }
+        it "fingerprints the timestamp correctly" do
+          expect(fingerprint).to eq('37785b62a8cae473acc315d39b66d86e')
+        end
+      end
+
+      describe "fractional seconds" do
+        let(:fingerprint_method) { "MURMUR3" }
+        let(:data) { { "@timestamp" => epoch_time } }
+
+        describe "millisecond" do
+          let(:epoch_time) { LogStash::Timestamp.new('2000-01-01T05:00:00.12Z') }
+          it "fingerprints the timestamp correctly" do
+            expect(fingerprint).to eq(4263087275)
+          end
+        end
+
+        describe "microsecond" do
+          let(:epoch_time) { LogStash::Timestamp.new('2000-01-01T05:00:00.123456Z') }
+          it "fingerprints the timestamp correctly" do
+            expect(fingerprint).to eq(4188855160)
+          end
+        end if ge_version_8
+
+        describe "nanosecond" do
+          let(:epoch_time) { LogStash::Timestamp.new('2000-01-01T05:00:00.123456789Z') }
+          it "fingerprints the timestamp correctly" do
+            expect(fingerprint).to eq(3520111535)
+          end
+        end if ge_version_8
+      end
     end
 
     describe "post fingerprint execution triggers" do

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: logstash-filter-fingerprint
 version: !ruby/object:Gem::Version
-  version: 3.3.1
+  version: 3.4.1
 platform: ruby
 authors:
 - Elastic
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2021-07-08 00:00:00.000000000 Z
+date: 2022-06-16 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
@@ -111,8 +111,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubyforge_project:
-rubygems_version: 2.6.13
+rubygems_version: 3.1.6
 signing_key:
 specification_version: 4
 summary: Fingerprints fields by replacing values with a consistent hash