logstash-filter-fingerprint 3.4.0 → 3.4.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 25ac9ded3114af17962469d77481c6992d65778a9dcf67012aaa3c7fb63c6ff7
-  data.tar.gz: d42a1fc7ff16d940f5634cb5f14ee2b4f2216176bdf7b90a8e7e6ed23cc95909
+  metadata.gz: 5f2a16bce946eb4f6e8397822a4abf882c600dcce7dee9fb10d4a94c186921b7
+  data.tar.gz: 74a90fce053b44de8fcd0991f24f84fe2cfe6eedf7971ef4dd170f467d7b04c5
 SHA512:
-  metadata.gz: dc8e8df09287df6f5f0649397f0b6b326396508afb101b73b7b51ca95802c314719a74440657059114ddab7b5d47cd65b7a48babea76e2b341cbdf46753f4da6
-  data.tar.gz: 605eec24d6a4587e565acfe35c405beb6ff782410ed23c955e3d7e938e9a8fb3f633f40bc282f7ccaec77be07dfe233843ce2564dceb6cbe87dbe7fec0759668
+  metadata.gz: e086c37f7f9a7321741febefb51a3ab35ffdad9ca635da178aec9378131842cce321df124c545b7c10149ec5c588b965ee13b2a2857fe034fd31654e3aebb1b2
+  data.tar.gz: 4fef51f2ecf2a468006d8aca31ea225aac49ceca8826b8f4e4048bb2ad2d6052d22d388fa50b468ed5618c130c5e94d6a6b4f846c5cc86be4a4b45a446e85e13

data/CHANGELOG.md

@@ -1,3 +1,9 @@
+## 3.4.2
+  - Key config type changed to `Password` type for better protection from leaks. [#71](https://github.com/logstash-plugins/logstash-filter-fingerprint/pull/71)
+
+## 3.4.1
+  - Added backward compatibility of timestamp format to provide consistent fingerprint [#67](https://github.com/logstash-plugins/logstash-filter-fingerprint/pull/67)
+
 ## 3.4.0
   - Added support for 128bit murmur variant [#66](https://github.com/logstash-plugins/logstash-filter-fingerprint/pull/66).
 

data/docs/index.asciidoc

@@ -59,7 +59,7 @@ This plugin supports the following configuration options plus the <<plugins-{typ
 | <<plugins-{type}s-{plugin}-concatenate_sources>> |<<boolean,boolean>>|No
 | <<plugins-{type}s-{plugin}-concatenate_all_fields>> |<<boolean,boolean>>|No
 | <<plugins-{type}s-{plugin}-ecs_compatibility>> | <<string,string>>|No
-| <<plugins-{type}s-{plugin}-key>> |<<string,string>>|No
+| <<plugins-{type}s-{plugin}-key>> |<<password,password>>|No
 | <<plugins-{type}s-{plugin}-method>> |<<string,string>>, one of `["SHA1", "SHA256", "SHA384", "SHA512", "MD5", "MURMUR3", "MURMUR3_128", IPV4_NETWORK", "UUID", "PUNCTUATION"]`|Yes
 | <<plugins-{type}s-{plugin}-source>> |<<array,array>>|No
 | <<plugins-{type}s-{plugin}-target>> |<<string,string>>|No
@@ -164,7 +164,7 @@ See <<plugins-{type}s-{plugin}-ecs_metadata>> for detailed information.
 [id="plugins-{type}s-{plugin}-key"]
 ===== `key` 
 
-  * Value type is <<string,string>>
+  * Value type is <<password,password>>
   * There is no default value for this setting.
 
 When used with the `IPV4_NETWORK` method fill in the subnet prefix length.

data/lib/logstash/filters/fingerprint.rb

@@ -24,6 +24,25 @@ require "logstash/plugin_mixins/ecs_compatibility_support"
 # To generate UUIDs, prefer the <<plugins-filters-uuid,uuid filter>>.
 class LogStash::Filters::Fingerprint < LogStash::Filters::Base
 
+  ##
+  # Logstash 8+ has variable-length serialization of timestamps
+  # that do not include subsecond info for whole-second timestamps.
+  # For backward-compatibility we refine the implementation to use
+  # our own three-decimal-place formatter for whole-second
+  # timestamps.
+  if LOGSTASH_VERSION.split('.').first.to_i >= 8
+    module MinimumSerializationLengthTimestamp
+      THREE_DECIMAL_INSTANT_FORMATTER = java.time.format.DateTimeFormatterBuilder.new.appendInstant(3).toFormatter
+      refine LogStash::Timestamp do
+        def to_s
+          return super unless nsec == 0
+          THREE_DECIMAL_INSTANT_FORMATTER.format(to_java.toInstant)
+        end
+      end
+    end
+    using MinimumSerializationLengthTimestamp
+  end
+
   INTEGER_MAX_32BIT = (1 << 31) - 1
   INTEGER_MIN_32BIT = -(1 << 31)
 
@@ -42,7 +61,7 @@ class LogStash::Filters::Fingerprint < LogStash::Filters::Base
 
   # When used with the `IPV4_NETWORK` method fill in the subnet prefix length.
   # With other methods, optionally fill in the HMAC key.
-  config :key, :validate => :string
+  config :key, :validate => :password
 
   # When set to `true`, the `SHA1`, `SHA256`, `SHA384`, `SHA512`, `MD5` and `MURMUR3_128` fingerprint
   # methods will produce base64 encoded rather than hex encoded strings.
@@ -180,7 +199,7 @@ class LogStash::Filters::Fingerprint < LogStash::Filters::Base
 
   def fingerprint_ipv4_network(ip_string)
     # in JRuby 1.7.11 outputs as US-ASCII
-    IPAddr.new(ip_string).mask(@key.to_i).to_s.force_encoding(Encoding::UTF_8)
+    IPAddr.new(ip_string).mask(@key.value.to_i).to_s.force_encoding(Encoding::UTF_8)
   end
 
   def fingerprint_openssl(data)
@@ -201,10 +220,10 @@ class LogStash::Filters::Fingerprint < LogStash::Filters::Base
       end
     else
       if @base64encode
-        hash = OpenSSL::HMAC.digest(digest, @key, data.to_s)
+        hash = OpenSSL::HMAC.digest(digest, @key.value, data.to_s)
         Base64.strict_encode64(hash).force_encoding(Encoding::UTF_8)
       else
-        OpenSSL::HMAC.hexdigest(digest, @key, data.to_s).force_encoding(Encoding::UTF_8)
+        OpenSSL::HMAC.hexdigest(digest, @key.value, data.to_s).force_encoding(Encoding::UTF_8)
       end
     end
   end

data/logstash-filter-fingerprint.gemspec

@@ -1,7 +1,7 @@
 Gem::Specification.new do |s|
 
   s.name            = 'logstash-filter-fingerprint'
-  s.version         = '3.4.0'
+  s.version         = '3.4.2'
   s.licenses        = ['Apache-2.0']
   s.summary         = "Fingerprints fields by replacing values with a consistent hash"
   s.description     = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program"

data/spec/filters/fingerprint_spec.rb

@@ -19,13 +19,17 @@ describe LogStash::Filters::Fingerprint, :ecs_compatibility_support, :aggregate_
       plugin.filter(event)
     end
 
+    def ge_version_8
+      LOGSTASH_VERSION.split('.').first.to_i >= 8
+    end
+
     context "with a string field" do
       let(:data) { {"clientip" => "123.123.123.123" } }
       let(:config) { super().merge("source" => ["clientip" ]) }
 
       describe "the IPV4_NETWORK method" do
         let(:fingerprint_method) { "IPV4_NETWORK" }
-        let(:config) { super().merge("key" => 24) }
+        let(:config) { super().merge("key" => ::LogStash::Util::Password.new("24")) }
 
         it "fingerprints the ip as the network" do
           expect(fingerprint).to eq("123.123.123.0")
@@ -111,7 +115,7 @@ describe LogStash::Filters::Fingerprint, :ecs_compatibility_support, :aggregate_
         end
 
         context "with HMAC" do
-          let(:config) { super().merge("key" => "longencryptionkey") }
+          let(:config) { super().merge("key" => ::LogStash::Util::Password.new("longencryptionkey")) }
 
           it "fingerprints the value" do
             expect(fingerprint).to eq("fdc60acc4773dc5ac569ffb78fcb93c9630797f4")
@@ -137,7 +141,7 @@ describe LogStash::Filters::Fingerprint, :ecs_compatibility_support, :aggregate_
           expect(fingerprint).to eq("4dabcab210766e35f03e77120e6986d6e6d4752b2a9ff22980b9253d026080d8")
         end
         context "with HMAC" do
-          let(:config) { super().merge("key" => "longencryptionkey") }
+          let(:config) { super().merge("key" => ::LogStash::Util::Password.new("longencryptionkey")) }
           it "fingerprints the value" do
             expect(fingerprint).to eq("345bec3eff242d53b568916c2610b3e393d885d6b96d643f38494fd74bf4a9ca")
           end
@@ -156,7 +160,7 @@ describe LogStash::Filters::Fingerprint, :ecs_compatibility_support, :aggregate_
           expect(fingerprint).to eq("fd605b0a3af3e04ce0d7a0b0d9c48d67a12dab811f60072e6eae84e35d567793ffb68a1807536f11c90874065c2a4392")
         end
         context "with HMAC" do
-          let(:config) { super().merge("key" => "longencryptionkey") }
+          let(:config) { super().merge("key" => ::LogStash::Util::Password.new("longencryptionkey")) }
           it "fingerprints the value" do
             expect(fingerprint).to eq("22d4c0e8c4fbcdc4887d2038fca7650f0e2e0e2457ff41c06eb2a980dded6749561c814fe182aff93e2538d18593947a")
           end
@@ -174,7 +178,7 @@ describe LogStash::Filters::Fingerprint, :ecs_compatibility_support, :aggregate_
           expect(fingerprint).to eq("5468e2dc64ea92b617782aae884b35af60041ac9e168a283615b6a462c54c13d42fa9542cce9b7d76a8124ac6616818905e3e5dd35d6e519f77c3b517558639a")
         end
         context "with HMAC" do
-          let(:config) { super().merge("key" => "longencryptionkey") }
+          let(:config) { super().merge("key" => ::LogStash::Util::Password.new("longencryptionkey")) }
           it "fingerprints the value" do
             expect(fingerprint).to eq("11c19b326936c08d6c50a3c847d883e5a1362e6a64dd55201a25f2c1ac1b673f7d8bf15b8f112a4978276d573275e3b14166e17246f670c2a539401c5bfdace8")
           end
@@ -192,7 +196,7 @@ describe LogStash::Filters::Fingerprint, :ecs_compatibility_support, :aggregate_
           expect(fingerprint).to eq("ccdd8d3d940a01b2fb3258c059924c0d")
         end
         context "with HMAC" do
-          let(:config) { super().merge("key" => "longencryptionkey") }
+          let(:config) { super().merge("key" => ::LogStash::Util::Password.new("longencryptionkey")) }
           it "fingerprints the value" do
             expect(fingerprint).to eq("9336c879e305c9604a3843fc3e75948f")
           end
@@ -273,11 +277,11 @@ describe LogStash::Filters::Fingerprint, :ecs_compatibility_support, :aggregate_
     end
 
     context 'Timestamps' do
-      epoch_time = Time.at(0).gmtime
+      let(:epoch_time) { Time.at(0).gmtime }
       let(:config) { super().merge("source" => ['@timestamp']) }
 
       describe 'OpenSSL Fingerprinting' do
-        let(:config) { super().merge("key" => '0123') }
+        let(:config) { super().merge("key" => ::LogStash::Util::Password.new("0123")) }
         let(:fingerprint_method) { "SHA1" }
         let(:data) { { "@timestamp" => epoch_time } }
         it "fingerprints the timestamp correctly" do
@@ -297,9 +301,35 @@ describe LogStash::Filters::Fingerprint, :ecs_compatibility_support, :aggregate_
         let(:fingerprint_method) { "MURMUR3_128" }
         let(:data) { { "@timestamp" => epoch_time } }
         it "fingerprints the timestamp correctly" do
-          expect(fingerprint).to eq("37785b62a8cae473acc315d39b66d86e")
+          expect(fingerprint).to eq('37785b62a8cae473acc315d39b66d86e')
         end
       end
+
+      describe "fractional seconds" do
+        let(:fingerprint_method) { "MURMUR3" }
+        let(:data) { { "@timestamp" => epoch_time } }
+
+        describe "millisecond" do
+          let(:epoch_time) { LogStash::Timestamp.new('2000-01-01T05:00:00.12Z') }
+          it "fingerprints the timestamp correctly" do
+            expect(fingerprint).to eq(4263087275)
+          end
+        end
+
+        describe "microsecond" do
+          let(:epoch_time) { LogStash::Timestamp.new('2000-01-01T05:00:00.123456Z') }
+          it "fingerprints the timestamp correctly" do
+            expect(fingerprint).to eq(4188855160)
+          end
+        end if ge_version_8
+
+        describe "nanosecond" do
+          let(:epoch_time) { LogStash::Timestamp.new('2000-01-01T05:00:00.123456789Z') }
+          it "fingerprints the timestamp correctly" do
+            expect(fingerprint).to eq(3520111535)
+          end
+        end if ge_version_8
+      end
     end
 
     describe "post fingerprint execution triggers" do

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: logstash-filter-fingerprint
 version: !ruby/object:Gem::Version
-  version: 3.4.0
+  version: 3.4.2
 platform: ruby
 authors:
 - Elastic
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2022-05-03 00:00:00.000000000 Z
+date: 2023-01-13 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement