logstash-filter-fingerprint 3.2.1 → 3.2.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 3a80216d6dfbd2b4724fe593ce35e2c57ea7501f3c9fb78efbed7122c953b875
-  data.tar.gz: 3ad6b211481f4b7eb7233055e2665b13d5329a2c41af220b77bad02566581e0e
+  metadata.gz: 13e031f1122fff7d28e213e702e7d4ffbae573c76daf65445bb8e1059187f7db
+  data.tar.gz: 20046dd2014ef538c96de60c2c7ac7c5ec62542f14f3dbb65f5987ad79e4bf81
 SHA512:
-  metadata.gz: 4315fa59ed1276927c5af00e033652eed383e375956a632ddd572866d8e702daef3406040ccbd649f15885bd74e1ad68be70e93b812d1af9a0a9b15d3198d18d
-  data.tar.gz: e9137e8dd254c15b6a27ed4eddd2dd785c09410e84978148adf424449aefeadfc6f7af48dd2431c1032bab4d3dc0500d2ecfd4411f00009d578f529c020b6c26
+  metadata.gz: 4213b3572ebbae2559d5873fc0b2e2322c0c2b5d7df1dd2e8361c55b5d684560862b1b26f3d0ad54a2e0e827a51d31c393e466ed1d0a979694567aff7e7a9289
+  data.tar.gz: 4c4acdab141a7159c8eabd411f23e2d66925c3baecda1f2b75b0a57e000e18883c55edf81a1e1601d2ced5b3ecbda9d9884c12fc0f129ecab7987d35d72627a2

data/CHANGELOG.md

@@ -1,3 +1,6 @@
+## 3.2.2
+  - Fixed lack of consistent fingerprints on Hash/Map objects [#55](https://github.com/logstash-plugins/logstash-filter-fingerprint/pull/55)
+
 ## 3.2.1
   - Fixed concurrent SHA fingerprinting by making the instances thread local
 

data/LICENSE

@@ -1,13 +1,202 @@
-Copyright (c) 2012-2018 Elasticsearch <http://www.elastic.co>
 
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
+                                 Apache License
+                           Version 2.0, January 2004
+                        http://www.apache.org/licenses/
 
-    http://www.apache.org/licenses/LICENSE-2.0
+   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
 
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
+   1. Definitions.
+
+      "License" shall mean the terms and conditions for use, reproduction,
+      and distribution as defined by Sections 1 through 9 of this document.
+
+      "Licensor" shall mean the copyright owner or entity authorized by
+      the copyright owner that is granting the License.
+
+      "Legal Entity" shall mean the union of the acting entity and all
+      other entities that control, are controlled by, or are under common
+      control with that entity. For the purposes of this definition,
+      "control" means (i) the power, direct or indirect, to cause the
+      direction or management of such entity, whether by contract or
+      otherwise, or (ii) ownership of fifty percent (50%) or more of the
+      outstanding shares, or (iii) beneficial ownership of such entity.
+
+      "You" (or "Your") shall mean an individual or Legal Entity
+      exercising permissions granted by this License.
+
+      "Source" form shall mean the preferred form for making modifications,
+      including but not limited to software source code, documentation
+      source, and configuration files.
+
+      "Object" form shall mean any form resulting from mechanical
+      transformation or translation of a Source form, including but
+      not limited to compiled object code, generated documentation,
+      and conversions to other media types.
+
+      "Work" shall mean the work of authorship, whether in Source or
+      Object form, made available under the License, as indicated by a
+      copyright notice that is included in or attached to the work
+      (an example is provided in the Appendix below).
+
+      "Derivative Works" shall mean any work, whether in Source or Object
+      form, that is based on (or derived from) the Work and for which the
+      editorial revisions, annotations, elaborations, or other modifications
+      represent, as a whole, an original work of authorship. For the purposes
+      of this License, Derivative Works shall not include works that remain
+      separable from, or merely link (or bind by name) to the interfaces of,
+      the Work and Derivative Works thereof.
+
+      "Contribution" shall mean any work of authorship, including
+      the original version of the Work and any modifications or additions
+      to that Work or Derivative Works thereof, that is intentionally
+      submitted to Licensor for inclusion in the Work by the copyright owner
+      or by an individual or Legal Entity authorized to submit on behalf of
+      the copyright owner. For the purposes of this definition, "submitted"
+      means any form of electronic, verbal, or written communication sent
+      to the Licensor or its representatives, including but not limited to
+      communication on electronic mailing lists, source code control systems,
+      and issue tracking systems that are managed by, or on behalf of, the
+      Licensor for the purpose of discussing and improving the Work, but
+      excluding communication that is conspicuously marked or otherwise
+      designated in writing by the copyright owner as "Not a Contribution."
+
+      "Contributor" shall mean Licensor and any individual or Legal Entity
+      on behalf of whom a Contribution has been received by Licensor and
+      subsequently incorporated within the Work.
+
+   2. Grant of Copyright License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      copyright license to reproduce, prepare Derivative Works of,
+      publicly display, publicly perform, sublicense, and distribute the
+      Work and such Derivative Works in Source or Object form.
+
+   3. Grant of Patent License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      (except as stated in this section) patent license to make, have made,
+      use, offer to sell, sell, import, and otherwise transfer the Work,
+      where such license applies only to those patent claims licensable
+      by such Contributor that are necessarily infringed by their
+      Contribution(s) alone or by combination of their Contribution(s)
+      with the Work to which such Contribution(s) was submitted. If You
+      institute patent litigation against any entity (including a
+      cross-claim or counterclaim in a lawsuit) alleging that the Work
+      or a Contribution incorporated within the Work constitutes direct
+      or contributory patent infringement, then any patent licenses
+      granted to You under this License for that Work shall terminate
+      as of the date such litigation is filed.
+
+   4. Redistribution. You may reproduce and distribute copies of the
+      Work or Derivative Works thereof in any medium, with or without
+      modifications, and in Source or Object form, provided that You
+      meet the following conditions:
+
+      (a) You must give any other recipients of the Work or
+          Derivative Works a copy of this License; and
+
+      (b) You must cause any modified files to carry prominent notices
+          stating that You changed the files; and
+
+      (c) You must retain, in the Source form of any Derivative Works
+          that You distribute, all copyright, patent, trademark, and
+          attribution notices from the Source form of the Work,
+          excluding those notices that do not pertain to any part of
+          the Derivative Works; and
+
+      (d) If the Work includes a "NOTICE" text file as part of its
+          distribution, then any Derivative Works that You distribute must
+          include a readable copy of the attribution notices contained
+          within such NOTICE file, excluding those notices that do not
+          pertain to any part of the Derivative Works, in at least one
+          of the following places: within a NOTICE text file distributed
+          as part of the Derivative Works; within the Source form or
+          documentation, if provided along with the Derivative Works; or,
+          within a display generated by the Derivative Works, if and
+          wherever such third-party notices normally appear. The contents
+          of the NOTICE file are for informational purposes only and
+          do not modify the License. You may add Your own attribution
+          notices within Derivative Works that You distribute, alongside
+          or as an addendum to the NOTICE text from the Work, provided
+          that such additional attribution notices cannot be construed
+          as modifying the License.
+
+      You may add Your own copyright statement to Your modifications and
+      may provide additional or different license terms and conditions
+      for use, reproduction, or distribution of Your modifications, or
+      for any such Derivative Works as a whole, provided Your use,
+      reproduction, and distribution of the Work otherwise complies with
+      the conditions stated in this License.
+
+   5. Submission of Contributions. Unless You explicitly state otherwise,
+      any Contribution intentionally submitted for inclusion in the Work
+      by You to the Licensor shall be under the terms and conditions of
+      this License, without any additional terms or conditions.
+      Notwithstanding the above, nothing herein shall supersede or modify
+      the terms of any separate license agreement you may have executed
+      with Licensor regarding such Contributions.
+
+   6. Trademarks. This License does not grant permission to use the trade
+      names, trademarks, service marks, or product names of the Licensor,
+      except as required for reasonable and customary use in describing the
+      origin of the Work and reproducing the content of the NOTICE file.
+
+   7. Disclaimer of Warranty. Unless required by applicable law or
+      agreed to in writing, Licensor provides the Work (and each
+      Contributor provides its Contributions) on an "AS IS" BASIS,
+      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+      implied, including, without limitation, any warranties or conditions
+      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+      PARTICULAR PURPOSE. You are solely responsible for determining the
+      appropriateness of using or redistributing the Work and assume any
+      risks associated with Your exercise of permissions under this License.
+
+   8. Limitation of Liability. In no event and under no legal theory,
+      whether in tort (including negligence), contract, or otherwise,
+      unless required by applicable law (such as deliberate and grossly
+      negligent acts) or agreed to in writing, shall any Contributor be
+      liable to You for damages, including any direct, indirect, special,
+      incidental, or consequential damages of any character arising as a
+      result of this License or out of the use or inability to use the
+      Work (including but not limited to damages for loss of goodwill,
+      work stoppage, computer failure or malfunction, or any and all
+      other commercial damages or losses), even if such Contributor
+      has been advised of the possibility of such damages.
+
+   9. Accepting Warranty or Additional Liability. While redistributing
+      the Work or Derivative Works thereof, You may choose to offer,
+      and charge a fee for, acceptance of support, warranty, indemnity,
+      or other liability obligations and/or rights consistent with this
+      License. However, in accepting such obligations, You may act only
+      on Your own behalf and on Your sole responsibility, not on behalf
+      of any other Contributor, and only if You agree to indemnify,
+      defend, and hold each Contributor harmless for any liability
+      incurred by, or claims asserted against, such Contributor by reason
+      of your accepting any such warranty or additional liability.
+
+   END OF TERMS AND CONDITIONS
+
+   APPENDIX: How to apply the Apache License to your work.
+
+      To apply the Apache License to your work, attach the following
+      boilerplate notice, with the fields enclosed by brackets "[]"
+      replaced with your own identifying information. (Don't include
+      the brackets!)  The text should be enclosed in the appropriate
+      comment syntax for the file format. We also recommend that a
+      file or class name and description of purpose be included on the
+      same "printed page" as the copyright notice for easier
+      identification within third-party archives.
+
+   Copyright 2020 Elastic and contributors
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.

data/lib/logstash/filters/fingerprint.rb

@@ -119,12 +119,12 @@ class LogStash::Filters::Fingerprint < LogStash::Filters::Base
       if @concatenate_sources || @concatenate_all_fields
         to_string = ""
         if @concatenate_all_fields
-          event.to_hash.sort.map do |k,v|
+          deep_sort_hashes(event.to_hash).each do |k,v|
             to_string << "|#{k}|#{v}"
           end
         else
           @source.sort.each do |k|
-            to_string << "|#{k}|#{event.get(k)}"
+            to_string << "|#{k}|#{deep_sort_hashes(event.get(k))}"
           end
         end
         to_string << "|"
@@ -134,9 +134,9 @@ class LogStash::Filters::Fingerprint < LogStash::Filters::Base
         @source.each do |field|
           next unless event.include?(field)
           if event.get(field).is_a?(Array)
-            event.set(@target, event.get(field).collect { |v| fingerprint(v) })
+            event.set(@target, event.get(field).collect { |v| fingerprint(deep_sort_hashes(v)) })
           else
-            event.set(@target, fingerprint(event.get(field)))
+            event.set(@target, fingerprint(deep_sort_hashes(event.get(field))))
           end
         end
       end
@@ -145,6 +145,20 @@ class LogStash::Filters::Fingerprint < LogStash::Filters::Base
   end
 
   private
+  def deep_sort_hashes(object)
+    case object
+    when Hash
+      sorted_hash = Hash.new
+      object.sort.each do |sorted_key, value|
+        sorted_hash[sorted_key] = deep_sort_hashes(value)
+      end
+      sorted_hash
+    when Array
+      object.map {|element| deep_sort_hashes(element) }
+    else
+      object
+    end
+  end
 
   def fingerprint_ipv4_network(ip_string)
     # in JRuby 1.7.11 outputs as US-ASCII

data/logstash-filter-fingerprint.gemspec

@@ -1,7 +1,7 @@
 Gem::Specification.new do |s|
 
   s.name            = 'logstash-filter-fingerprint'
-  s.version         = '3.2.1'
+  s.version         = '3.2.2'
   s.licenses        = ['Apache-2.0']
   s.summary         = "Fingerprints fields by replacing values with a consistent hash"
   s.description     = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program"

data/spec/filters/fingerprint_spec.rb

@@ -4,412 +4,284 @@ require "logstash/filters/fingerprint"
 
 describe LogStash::Filters::Fingerprint do
 
-  describe "fingerprint ipaddress with IPV4_NETWORK method" do
-    config <<-CONFIG
-      filter {
-        fingerprint {
-          source => ["clientip"]
-          method => "IPV4_NETWORK"
-          key => 24
-        }
-      }
-    CONFIG
-
-    sample("clientip" => "233.255.13.44") do
-      insist { subject.get("fingerprint") } == "233.255.13.0"
-    end
+  let(:plugin) { described_class.new(config) }
+  let(:config) { { "method" => fingerprint_method } }
+  let(:fingerprint_method) { "SHA1" } # default
+  let(:data) { {} }
+  let(:event) { LogStash::Event.new(data) }
+  let(:fingerprint) { event.get("fingerprint") }
+
+  before(:each) do
+    plugin.register
+    plugin.filter(event)
   end
 
-  describe "fingerprint string with MURMUR3 method" do
-    config <<-CONFIG
-      filter {
-        fingerprint {
-          source => ["clientip"]
-          method => "MURMUR3"
-        }
-      }
-    CONFIG
-
-    sample("clientip" => "123.52.122.33") do
-      insist { subject.get("fingerprint") } == 1541804874
-    end
-  end
+  context "with a string field" do
+    let(:data) { {"clientip" => "123.123.123.123" } }
+    let(:config) { super.merge("source" => ["clientip" ]) }
 
-  describe "fingerprint string with SHA1 algorithm" do
-    config <<-CONFIG
-      filter {
-        fingerprint {
-          source => ["clientip"]
-          method => 'SHA1'
-        }
-      }
-    CONFIG
+    describe "the IPV4_NETWORK method" do
+      let(:fingerprint_method) { "IPV4_NETWORK" }
+      let(:config) { super.merge("key" => 24) }
 
-    sample("clientip" => "123.123.123.123") do
-      insist { subject.get("fingerprint") } == "3a5076c520b4b463f43806896ea0b3978d09dcae"
+      it "fingerprints the ip as the network" do
+        expect(fingerprint).to eq("123.123.123.0")
+      end
     end
-  end
 
-  describe "fingerprint string with SHA1 HMAC algorithm" do
-    config <<-CONFIG
-      filter {
-        fingerprint {
-          source => ["clientip"]
-          key => "longencryptionkey"
-          method => 'SHA1'
-        }
-      }
-    CONFIG
+    describe "the MURMUR3 method" do
+      let(:fingerprint_method) { "MURMUR3" }
 
-    sample("clientip" => "123.123.123.123") do
-      insist { subject.get("fingerprint") } == "fdc60acc4773dc5ac569ffb78fcb93c9630797f4"
+      it "fingerprints the value" do
+        expect(fingerprint).to eq(4013733395)
+      end
     end
-  end
-
-  describe "fingerprint string with SHA1 HMAC algorithm on all event fields" do
-    config <<-CONFIG
-      filter {
-        fingerprint {
-          concatenate_all_fields => true
-          key => "longencryptionkey"
-          method => 'SHA1'
-        }
-      }
-    CONFIG
 
-    # The @timestamp field is specified in this sample event as we need the event contents to be constant for the tests
-    sample("@timestamp" => "2017-07-26T14:44:27.064Z", "clientip" => "123.123.123.123", "message" => "This is a test message", "log_level" => "INFO", "offset" => 123456789, "type" => "test") do
-      insist { subject.get("fingerprint") } == "d7c617f4d40b2cb677a7003af68a41c415f58031"
-    end
-  end
+    describe "the SHA1 method" do
+      let(:fingerprint_method) { "SHA1" }
 
-  describe "fingerprint string with SHA1 algorithm and base64 encoding" do
-    config <<-CONFIG
-      filter {
-        fingerprint {
-          source => ["clientip"]
-          method => 'SHA1'
-          base64encode => true
-        }
-      }
-    CONFIG
+      it "fingerprints the value" do
+        expect(fingerprint).to eq("3a5076c520b4b463f43806896ea0b3978d09dcae")
+      end
 
-    sample("clientip" => "123.123.123.123") do
-      insist { subject.get("fingerprint") } == "OlB2xSC0tGP0OAaJbqCzl40J3K4="
+      context "with HMAC" do
+        let(:config) { super.merge("key" => "longencryptionkey") }
+
+        it "fingerprints the value" do
+          expect(fingerprint).to eq("fdc60acc4773dc5ac569ffb78fcb93c9630797f4")
+        end
+        context "with HMAC and base64 encoding" do
+          let(:config) { super.merge("base64encode" => true) }
+          it "fingerprints the value" do
+            expect(fingerprint).to eq("/cYKzEdz3FrFaf+3j8uTyWMHl/Q=")
+          end
+        end
+      end
+      context "and base64 encoding" do
+        let(:config) { super.merge("base64encode" => true) }
+        it "fingerprints the value" do
+          expect(fingerprint).to eq("OlB2xSC0tGP0OAaJbqCzl40J3K4=")
+        end
+      end
     end
-  end
-
-  describe "fingerprint string with SHA1 HMAC algorithm and base64 encoding" do
-    config <<-CONFIG
-      filter {
-        fingerprint {
-          source => ["clientip"]
-          key => "longencryptionkey"
-          method => 'SHA1'
-          base64encode => true
-        }
-      }
-    CONFIG
 
-    sample("clientip" => "123.123.123.123") do
-      insist { subject.get("fingerprint") } == "/cYKzEdz3FrFaf+3j8uTyWMHl/Q="
+    context "the SHA256 algorithm" do
+      let(:fingerprint_method) { "SHA256" }
+      it "fingerprints the value" do
+        expect(fingerprint).to eq("4dabcab210766e35f03e77120e6986d6e6d4752b2a9ff22980b9253d026080d8")
+      end
+      context "with HMAC" do
+        let(:config) { super.merge("key" => "longencryptionkey") }
+        it "fingerprints the value" do
+          expect(fingerprint).to eq("345bec3eff242d53b568916c2610b3e393d885d6b96d643f38494fd74bf4a9ca")
+        end
+        context "and base64 encoding" do
+          let(:config) { super.merge("base64encode" => true) }
+          it "fingerprints the value" do
+            expect(fingerprint).to eq("NFvsPv8kLVO1aJFsJhCz45PYhda5bWQ/OElP10v0qco=")
+          end
+        end
+      end
     end
-  end
-
-  describe "fingerprint string with SHA256 algorithm" do
-    config <<-CONFIG
-      filter {
-        fingerprint {
-          source => ["clientip"]
-          method => 'SHA256'
-        }
-      }
-    CONFIG
 
-    sample("clientip" => "123.123.123.123") do
-      insist { subject.get("fingerprint") } == "4dabcab210766e35f03e77120e6986d6e6d4752b2a9ff22980b9253d026080d8"
+    context "the SHA384 algorithm" do
+      let(:fingerprint_method) { "SHA384" }
+      it "fingerprints the value" do
+        expect(fingerprint).to eq("fd605b0a3af3e04ce0d7a0b0d9c48d67a12dab811f60072e6eae84e35d567793ffb68a1807536f11c90874065c2a4392")
+      end
+      context "with HMAC" do
+        let(:config) { super.merge("key" => "longencryptionkey") }
+        it "fingerprints the value" do
+          expect(fingerprint).to eq("22d4c0e8c4fbcdc4887d2038fca7650f0e2e0e2457ff41c06eb2a980dded6749561c814fe182aff93e2538d18593947a")
+        end
+        context "and base64 encoding" do
+          let(:config) { super.merge("base64encode" => true) }
+          it "fingerprints the value" do
+            expect(fingerprint).to eq("ItTA6MT7zcSIfSA4/KdlDw4uDiRX/0HAbrKpgN3tZ0lWHIFP4YKv+T4lONGFk5R6")
+          end
+        end
+      end
     end
-  end
-
-  describe "fingerprint string with SHA256 HMAC algorithm" do
-    config <<-CONFIG
-      filter {
-        fingerprint {
-          source => ["clientip"]
-          key => "longencryptionkey"
-          method => 'SHA256'
-        }
-      }
-    CONFIG
-
-    sample("clientip" => "123.123.123.123") do
-      insist { subject.get("fingerprint") } == "345bec3eff242d53b568916c2610b3e393d885d6b96d643f38494fd74bf4a9ca"
+    context "the SHA512 algorithm" do
+      let(:fingerprint_method) { "SHA512" }
+      it "fingerprints the value" do
+        expect(fingerprint).to eq("5468e2dc64ea92b617782aae884b35af60041ac9e168a283615b6a462c54c13d42fa9542cce9b7d76a8124ac6616818905e3e5dd35d6e519f77c3b517558639a")
+      end
+      context "with HMAC" do
+        let(:config) { super.merge("key" => "longencryptionkey") }
+        it "fingerprints the value" do
+          expect(fingerprint).to eq("11c19b326936c08d6c50a3c847d883e5a1362e6a64dd55201a25f2c1ac1b673f7d8bf15b8f112a4978276d573275e3b14166e17246f670c2a539401c5bfdace8")
+        end
+        context "and base64 encoding" do
+          let(:config) { super.merge("base64encode" => true) }
+          it "fingerprints the value" do
+            expect(fingerprint).to eq("EcGbMmk2wI1sUKPIR9iD5aE2Lmpk3VUgGiXywawbZz99i/FbjxEqSXgnbVcydeOxQWbhckb2cMKlOUAcW/2s6A==")
+          end
+        end
+      end
     end
-  end
-
-  describe "fingerprint string with SHA256 HMAC algorithm and base64 encoding" do
-    config <<-CONFIG
-      filter {
-        fingerprint {
-          source => ["clientip"]
-          key => "longencryptionkey"
-          method => 'SHA256'
-          base64encode => true
-        }
-      }
-    CONFIG
-
-    sample("clientip" => "123.123.123.123") do
-      insist { subject.get("fingerprint") } == "NFvsPv8kLVO1aJFsJhCz45PYhda5bWQ/OElP10v0qco="
+    context "the MD5 algorithm" do
+      let(:fingerprint_method) { "MD5" }
+      it "fingerprints the value" do
+        expect(fingerprint).to eq("ccdd8d3d940a01b2fb3258c059924c0d")
+      end
+      context "with HMAC" do
+        let(:config) { super.merge("key" => "longencryptionkey") }
+        it "fingerprints the value" do
+          expect(fingerprint).to eq("9336c879e305c9604a3843fc3e75948f")
+        end
+        context "and base64 encoding" do
+          let(:config) { super.merge("base64encode" => true) }
+          it "fingerprints the value" do
+            expect(fingerprint).to eq("kzbIeeMFyWBKOEP8PnWUjw==")
+          end
+        end
+      end
     end
   end
 
-  describe "fingerprint string with SHA384 algorithm" do
-    config <<-CONFIG
-      filter {
-        fingerprint {
-          source => ["clientip"]
-          method => 'SHA384'
-        }
-      }
-    CONFIG
+  context "multiple values in the source field" do
+    let(:config) { super.merge("source" => ["clientip" ]) }
+    let(:data) { { "clientip" => [ "123.123.123.123", "223.223.223.223" ] } }
 
-    sample("clientip" => "123.123.123.123") do
-      insist { subject.get("fingerprint") } == "fd605b0a3af3e04ce0d7a0b0d9c48d67a12dab811f60072e6eae84e35d567793ffb68a1807536f11c90874065c2a4392"
+    it "produces a fingerprint array" do
+      expect(fingerprint).to eq(["3a5076c520b4b463f43806896ea0b3978d09dcae", "47bbc4e06edebbace047fed35abeceec64968b81"])
     end
   end
 
-  describe "fingerprint string with SHA384 HMAC algorithm" do
-    config <<-CONFIG
-      filter {
-        fingerprint {
-          source => ["clientip"]
-          key => "longencryptionkey"
-          method => 'SHA384'
-        }
-      }
-    CONFIG
-
-    sample("clientip" => "123.123.123.123") do
-      insist { subject.get("fingerprint") } == "22d4c0e8c4fbcdc4887d2038fca7650f0e2e0e2457ff41c06eb2a980dded6749561c814fe182aff93e2538d18593947a"
+  describe "concatenate_all_fields" do
+    let(:config) { { "concatenate_all_fields" => true } }
+    # The @timestamp field is specified in this sample event as we need the event contents to be constant for the tests
+    let(:data) do
+      { "@timestamp" => "2017-07-26T14:44:27.064Z", "clientip" => "123.123.123.123", "message" => "This is a test message", "log_level" => "INFO", "offset" => 123456789, "type" => "test" }
     end
-  end
-
-  describe "fingerprint string with SHA384 HMAC algorithm and base64 encoding" do
-    config <<-CONFIG
-      filter {
-        fingerprint {
-          source => ["clientip"]
-          key => "longencryptionkey"
-          method => 'SHA384'
-          base64encode => true
-        }
-      }
-    CONFIG
 
-    sample("clientip" => "123.123.123.123") do
-      insist { subject.get("fingerprint") } == "ItTA6MT7zcSIfSA4/KdlDw4uDiRX/0HAbrKpgN3tZ0lWHIFP4YKv+T4lONGFk5R6"
+    it "fingerprints the concatenated values" do
+      expect(fingerprint).to eq("cbf022518e97860403160ed8a41847c0db104e63")
     end
   end
 
-  describe "fingerprint string with SHA512 algorithm" do
-    config <<-CONFIG
-      filter {
-        fingerprint {
-          source => ["clientip"]
-          method => 'SHA512'
-        }
-      }
-    CONFIG
+  context "when multiple fields are used" do
+    let(:config) { super.merge("source" => ['field1', 'field2']) }
+    let(:data) { { "field1" => "test1", "field2" => "test2" } }
 
-    sample("clientip" => "123.123.123.123") do
-      insist { subject.get("fingerprint") } == "5468e2dc64ea92b617782aae884b35af60041ac9e168a283615b6a462c54c13d42fa9542cce9b7d76a8124ac6616818905e3e5dd35d6e519f77c3b517558639a"
+    it "fingerprints the value of the last value" do
+      # SHA1 of "test2"
+      expect(fingerprint).to eq("109f4b3c50d7b0df729d299bc6f8e9ef9066971f")
     end
-  end
-
-  describe "fingerprint string with SHA512 HMAC algorithm" do
-    config <<-CONFIG
-      filter {
-        fingerprint {
-          source => ["clientip"]
-          key => "longencryptionkey"
-          method => 'SHA512'
-        }
-      }
-    CONFIG
 
-    sample("clientip" => "123.123.123.123") do
-      insist { subject.get("fingerprint") } == "11c19b326936c08d6c50a3c847d883e5a1362e6a64dd55201a25f2c1ac1b673f7d8bf15b8f112a4978276d573275e3b14166e17246f670c2a539401c5bfdace8"
+    describe "with concatenate_sources" do
+      let(:config) { super.merge("concatenate_sources" => true) }
+      it "fingerprints the value of concatenated key/pairs" do
+        # SHA1 of "|field1|test1|field2|test2|"
+        expect(fingerprint).to eq("e3b6b71eedc656f1d29408264e8a75535db985cb")
+      end
     end
   end
 
-  describe "fingerprint string with SHA512 HMAC algorithm and base64 encoding" do
-    config <<-CONFIG
-      filter {
-        fingerprint {
-          source => ["clientip"]
-          key => "longencryptionkey"
-          method => 'SHA512'
-          base64encode => true
-        }
-      }
-    CONFIG
+  describe "PUNCTUATION method" do
+    let(:fingerprint_method) { 'PUNCTUATION' }
+    let(:config) { super.merge("source" => 'field1') }
+    let(:data) { { "field1" =>  "PHP Warning:  json_encode() [<a href='function.json-encode'>function.json-encode</a>]: Invalid UTF-8 sequence in argument in /var/www/htdocs/test.php on line 233" } }
 
-    sample("clientip" => "123.123.123.123") do
-      insist { subject.get("fingerprint") } == "EcGbMmk2wI1sUKPIR9iD5aE2Lmpk3VUgGiXywawbZz99i/FbjxEqSXgnbVcydeOxQWbhckb2cMKlOUAcW/2s6A=="
+    it "extracts punctiation as the fingerprint" do
+      expect(fingerprint).to eq(":_()[<='.-'>.-</>]:-////.")
     end
   end
 
-  describe "fingerprint string with MD5 algorithm" do
-    config <<-CONFIG
-      filter {
-        fingerprint {
-          source => ["clientip"]
-          method => 'MD5'
-        }
-      }
-    CONFIG
+  context 'Timestamps' do
+    epoch_time = Time.at(0).gmtime
+    let(:config) { super.merge("source" => ['@timestamp']) }
 
-    sample("clientip" => "123.123.123.123") do
-      insist { subject.get("fingerprint") } == "ccdd8d3d940a01b2fb3258c059924c0d"
+    describe 'OpenSSL Fingerprinting' do
+      let(:config) { super.merge("key" => '0123') }
+      let(:fingerprint_method) { "SHA1" }
+      let(:data) { { "@timestamp" => epoch_time } }
+      it "fingerprints the timestamp correctly" do
+        expect(fingerprint).to eq('1d5379ec92d86a67cfc642d55aa050ca312d3b9a')
+      end
     end
-  end
-
-  describe "fingerprint string with MD5 HMAC algorithm" do
-    config <<-CONFIG
-      filter {
-        fingerprint {
-          source => ["clientip"]
-          key => "longencryptionkey"
-          method => 'MD5'
-        }
-      }
-    CONFIG
 
-    sample("clientip" => "123.123.123.123") do
-      insist { subject.get("fingerprint") } == "9336c879e305c9604a3843fc3e75948f"
+    describe 'MURMUR3 Fingerprinting' do
+      let(:fingerprint_method) { "MURMUR3" }
+      let(:data) { { "@timestamp" => epoch_time } }
+      it "fingerprints the timestamp correctly" do
+        expect(fingerprint).to eq(743372282)
+      end
     end
   end
 
-  describe "fingerprint string with MD5 HMAC algorithm and base64 encoding" do
-    config <<-CONFIG
-      filter {
-        fingerprint {
-          source => ["clientip"]
-          key => "longencryptionkey"
-          method => 'MD5'
-          base64encode => true
-        }
+  describe "post fingerprint execution triggers" do
+    let(:fingerprint_method) { "PUNCTUATION" }
+    let(:config) do
+      {
+        "source" => 'field1',
+        "add_field" => { 'myfield' => 'myvalue' },
+        "add_tag" => ['mytag']
       }
-    CONFIG
-
-    sample("clientip" => "123.123.123.123") do
-      insist { subject.get("fingerprint") } == "kzbIeeMFyWBKOEP8PnWUjw=="
     end
-  end
-
-  describe "Test field with multiple values" do
-    config <<-CONFIG
-      filter {
-        fingerprint {
-          source => ["clientip"]
-          key => "longencryptionkey"
-          method => 'MD5'
-        }
-      }
-    CONFIG
+    let(:data) { { "field1" => "Hello, World!" } }
 
-    sample("clientip" => [ "123.123.123.123", "223.223.223.223" ]) do
-      insist { subject.get("fingerprint")} == [ "9336c879e305c9604a3843fc3e75948f", "7a6c66b8d3f42a7d650e3354af508df3" ]
+    it "adds the new field" do
+      expect(event.get("myfield")).to eq("myvalue")
     end
-  end
-
-  describe "Concatenate multiple values into 1" do
-    config <<-CONFIG
-      filter {
-        fingerprint {
-          source => ['field1', 'field2']
-          key => "longencryptionkey"
-          method => 'MD5'
-        }
-      }
-    CONFIG
-
-    sample("field1" => "test1", "field2" => "test2") do
-      insist { subject.get("fingerprint")} == "872da745e45192c2a1d4bf7c1ff8a370"
+    it "adds the new tag" do
+      expect(event.get("tags")).to include("mytag")
     end
   end
 
-  describe "PUNCTUATION method" do
-    config <<-CONFIG
-      filter {
-        fingerprint {
-          source => 'field1'
-          method => 'PUNCTUATION'
-        }
-      }
-    CONFIG
-
-    sample("field1" =>  "PHP Warning:  json_encode() [<a href='function.json-encode'>function.json-encode</a>]: Invalid UTF-8 sequence in argument in /var/www/htdocs/test.php on line 233") do
-      insist { subject.get("fingerprint") } == ":_()[<='.-'>.-</>]:-////."
+  describe "tolerance to hash order" do
+    # insertion order can influence the result of to_hash's keys
+    let(:data1) { {
+        "a" => {"a0" => 0, "a1" => 1},
+        "b" => {"b0" => 0, "b1" => 1},
+    } }
+    let(:event1) { LogStash::Event.new(data1) }
+    let(:data2) { {
+        "b" => {"b1" => 1, "b0" => 0},
+        "a" => {"a1" => 1, "a0" => 0},
+    } }
+    let(:event2) { LogStash::Event.new(data2) }
+    let(:config) { { "source" => [ "a" ] } }
+
+    before(:each) do
+      # for testing purposes we want to ensure the hash order is different.
+      # since we can't easily control the order on the underlying Map,
+      # we're mocking the order here:
+      allow(event1).to receive(:to_hash).and_return(data1)
+      allow(event2).to receive(:to_hash).and_return(data2)
+      # by default event.get(key) fetches data from the event.
+      # mocking the default value has to be done first, and only
+      # then we can mock the getting "a" and "b"
+      allow(event1).to receive(:get).and_call_original
+      allow(event2).to receive(:get).and_call_original
+      # mock event.get("a") and event.get("b") for both events
+      # so we can inject an inconsistent order for the tests
+      allow(event1).to receive(:get).with("a") {|arg| data1["a"] }
+      allow(event1).to receive(:get).with("b") {|arg| data1["b"] }
+      allow(event2).to receive(:get).with("a") {|arg| data2["a"] }
+      allow(event2).to receive(:get).with("b") {|arg| data2["b"] }
+      plugin.filter(event1)
+      plugin.filter(event2)
     end
-
-    sample("field1" => "Warning: Ruby(ルビ) is an awesome language.") do
-      insist { subject.get("fingerprint") } == ":()."
+    it "computes the same hash" do
+      # confirm the order of the keys in the nested hash is different
+      # (of course it is, we just mocked the to_hash return)
+      expect(event1.to_hash["a"].keys).to_not eq(event2.to_hash["a"].keys)
+      # let's check that the fingerprint doesn't care about the insertion order
+      expect(event1.get("fingerprint")).to eq(event2.get("fingerprint"))
     end
-  end
-
-  context 'Timestamps' do
-    epoch_time = Time.at(0).gmtime
-
-    describe 'OpenSSL Fingerprinting' do
-      config <<-CONFIG
-        filter {
-          fingerprint {
-            source => ['@timestamp']
-            key    => '0123'
-            method => 'SHA1'
-          }
-        }
-      CONFIG
-
-      sample("@timestamp" => epoch_time) do
-        insist { subject.get("fingerprint") } == '1d5379ec92d86a67cfc642d55aa050ca312d3b9a'
+    context "concatenate_sources" do
+      let("config") { { "source" => [ "a", "b"], "concatenate_sources" => true } }
+      it "computes the same hash" do
+        expect(event1.get("fingerprint")).to eq(event2.get("fingerprint"))
       end
     end
-
-    describe 'MURMUR3 Fingerprinting' do
-      config <<-CONFIG
-        filter {
-          fingerprint {
-            source => ['@timestamp']
-            method => 'MURMUR3'
-          }
-        }
-      CONFIG
-
-      sample("@timestamp" => epoch_time) do
-        insist { subject.get("fingerprint") } == 743372282
+    context "concatenate_all_fields => true" do
+      let(:config) { { "concatenate_all_fields" => true } }
+      it "computes the same hash" do
+        expect(event1.get("fingerprint")).to eq(event2.get("fingerprint"))
       end
     end
   end
-
-  describe "execution triggers addition of fields and tags" do
-    config <<-CONFIG
-      filter {
-        fingerprint {
-          source => 'field1'
-          method => 'PUNCTUATION'
-          add_field => { 'myfield' => 'myvalue' }
-          add_tag => ['mytag']
-        }
-      }
-    CONFIG
-
-    sample("field1" => "Hello, World!") do
-      insist { subject.get("myfield") } == "myvalue"
-      insist { subject.get("tags") } == ["mytag"]
-    end
-  end
-
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: logstash-filter-fingerprint
 version: !ruby/object:Gem::Version
-  version: 3.2.1
+  version: 3.2.2
 platform: ruby
 authors:
 - Elastic
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2018-11-23 00:00:00.000000000 Z
+date: 2020-08-27 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement