logstash-filter-fingerprint 3.1.2 → 3.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: b88d55df3b200279ae29824d36676c7da08dd4f06d34fef026c8cd7e584d578b
-  data.tar.gz: 9324a50bb0ad60b767acff100362fd282a180fcdf22fdce48d7db6b30c16ee1d
+  metadata.gz: 3e5af8a524f7184afb61b5918806aedb56bb8c3e78de31b4eda34bec96166f6c
+  data.tar.gz: beddca5042e663506a3d5310bd82f0968ebf1c6d95fda038711e57339231e674
 SHA512:
-  metadata.gz: 6cee3dfe92acf4397cacd95c7ffff68de1bb4583cd6483de3ed7c374f7ad3a8f696ca8f69b1beb3b96754ddd5a6e44f101bdaacb7bf46483db3a24b88f3ed10a
-  data.tar.gz: f88a06056b0e4f215cd03224d226af33737a33b94e80fa21c9b3b501e5e3820ff75d9f4757dc3349f8773ab318605981b8f092d812a0673f1f54d7d3e8efa7e1
+  metadata.gz: 4e4058c40531a6e5f33475c7d847b52c64399e884e3dd7feb4585233fe1c91932bc9f0caea3b047b853351e97c83cb8b9edab0af59ff2e94b92cdc64c8ff3e99
+  data.tar.gz: ef3f51852929411c81cd655455611d0dbdd409618ba90df676c2ff2dc65f47803f315046734a4803f858e3bfc6130c0b246c024ed152d8d6c2b2bb205419ce42

data/CHANGELOG.md

@@ -1,3 +1,6 @@
+## 3.2.0
+  - Added support for non-keyed, regular hash functions [#18](https://github.com/logstash-plugins/logstash-filter-fingerprint/issues/18)
+
 ## 3.1.2
   - Update gemspec summary
 

data/CONTRIBUTORS

@@ -8,6 +8,8 @@ Contributors:
 * Richard Pijnenburg (electrical)
 * Suyog Rao (suyograo)
 * Tray (torrancew)
+* praseodym
+
 
 Note: If you've sent us patches, bug reports, or otherwise contributed to
 Logstash, and you aren't on the list above and want to be, please let us know

data/LICENSE

@@ -1,4 +1,4 @@
-Copyright (c) 2012–2016 Elasticsearch <http://www.elastic.co>
+Copyright (c) 2012-2018 Elasticsearch <http://www.elastic.co>
 
 Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.

data/docs/index.asciidoc

@@ -27,13 +27,10 @@ This can e.g. be used to create consistent document ids when inserting
 events into Elasticsearch, allowing events in Logstash to cause existing
 documents to be updated rather than new documents to be created.
 
-NOTE: When using any method other than 'UUID', 'PUNCTUATION' or 'MURMUR3'
-you must set the key, otherwise the plugin will raise an exception
-
 NOTE: When the `target` option is set to `UUID` the result won't be
 a consistent hash but a random
 https://en.wikipedia.org/wiki/Universally_unique_identifier[UUID].
-To generate UUIDs, prefer the <<plugins-filters-uuid,uuid filter>>.
+To generate UUIDs, prefer the {logstash-ref}/plugins-filters-uuid.html[uuid filter].
 
 [id="plugins-{type}s-{plugin}-options"]
 ==== Fingerprint Filter Configuration Options
@@ -80,7 +77,7 @@ fields are given, the target field will be an array with fingerprints
 of the source fields given.
 
 [id="plugins-{type}s-{plugin}-concatenate_all_fields"]
-===== `concatenate_sources` 
+===== `concatenate_all_fields` 
 
   * Value type is <<boolean,boolean>>
   * Default value is `false`
@@ -99,8 +96,7 @@ source fields given.
   * There is no default value for this setting.
 
 When used with the `IPV4_NETWORK` method fill in the subnet prefix length.
-Key is required with all methods except  `MURMUR3`, `PUNCTUATION` or `UUID`.
-With other methods fill in the HMAC key.
+With other methods, optionally fill in the HMAC key.
 
 [id="plugins-{type}s-{plugin}-method"]
 ===== `method` 
@@ -111,10 +107,12 @@ With other methods fill in the HMAC key.
 
 The fingerprint method to use.
 
-If set to `SHA1`, `SHA256`, `SHA384`, `SHA512`, or `MD5` the
-cryptographic keyed-hash function with the same name will be used to
-generate the fingerprint. If set to `MURMUR3` the non-cryptographic
-MurmurHash function will be used.
+If set to `SHA1`, `SHA256`, `SHA384`, `SHA512`, or `MD5` and a key is set,
+the cryptographic hash function with the same name will be used to generate
+the fingerprint. When a key set, the keyed-hash (HMAC) digest function will
+be used.
+
+If set to `MURMUR3` the non-cryptographic MurmurHash function will be used.
 
 If set to `IPV4_NETWORK` the input data needs to be a IPv4 address and
 the hash value will be the masked-out address using the number of bits
@@ -150,4 +148,4 @@ Any current contents of that field will be overwritten.
 
 
 [id="plugins-{type}s-{plugin}-common-options"]
-include::{include_path}/{type}.asciidoc[]
+include::{include_path}/{type}.asciidoc[]

data/lib/logstash/filters/fingerprint.rb

@@ -34,8 +34,7 @@ class LogStash::Filters::Fingerprint < LogStash::Filters::Base
   config :target, :validate => :string, :default => 'fingerprint'
 
   # When used with the `IPV4_NETWORK` method fill in the subnet prefix length.
-  # Key is required with all methods except  `MURMUR3`, `PUNCTUATION` or `UUID`.
-  # With other methods fill in the HMAC key.
+  # With other methods, optionally fill in the HMAC key.
   config :key, :validate => :string
 
   # When set to `true`, the `SHA1`, `SHA256`, `SHA384`, `SHA512` and `MD5` fingerprint methods will produce
@@ -44,10 +43,12 @@ class LogStash::Filters::Fingerprint < LogStash::Filters::Base
 
   # The fingerprint method to use.
   #
-  # If set to `SHA1`, `SHA256`, `SHA384`, `SHA512`, or `MD5` the
-  # cryptographic keyed-hash function with the same name will be used to
-  # generate the fingerprint. If set to `MURMUR3` the non-cryptographic
-  # MurmurHash function will be used.
+  # If set to `SHA1`, `SHA256`, `SHA384`, `SHA512`, or `MD5` and a key is set,
+  # the cryptographic hash function with the same name will be used to generate
+  # the fingerprint. When a key set, the keyed-hash (HMAC) digest function will
+  # be used.
+  #
+  # If set to `MURMUR3` the non-cryptographic MurmurHash function will be used.
   #
   # If set to `IPV4_NETWORK` the input data needs to be a IPv4 address and
   # the hash value will be the masked-out address using the number of bits
@@ -79,7 +80,7 @@ class LogStash::Filters::Fingerprint < LogStash::Filters::Base
     # convert to symbol for faster comparisons
     @method = @method.to_sym
 
-    # require any library and set the anonymize function
+    # require any library and set the fingerprint function
     case @method
     when :IPV4_NETWORK
       if @key.nil?
@@ -90,23 +91,15 @@ class LogStash::Filters::Fingerprint < LogStash::Filters::Base
           :error => "Key value is empty. please fill in a subnet prefix length"
         )
       end
-      class << self; alias_method :anonymize, :anonymize_ipv4_network; end
+      class << self; alias_method :fingerprint, :fingerprint_ipv4_network; end
     when :MURMUR3
-      class << self; alias_method :anonymize, :anonymize_murmur3; end
+      class << self; alias_method :fingerprint, :fingerprint_murmur3; end
     when :UUID
       # nothing
     when :PUNCTUATION
       # nothing
     else
-      if @key.nil?
-        raise LogStash::ConfigurationError, I18n.t(
-          "logstash.runner.configuration.invalid_plugin_register",
-          :plugin => "filter",
-          :type => "fingerprint",
-          :error => "Key value is empty. Please fill in an encryption key"
-        )
-      end
-      class << self; alias_method :anonymize, :anonymize_openssl; end
+      class << self; alias_method :fingerprint, :fingerprint_openssl; end
       @digest = select_digest(@method)
     end
   end
@@ -137,14 +130,14 @@ class LogStash::Filters::Fingerprint < LogStash::Filters::Base
         end
         to_string << "|"
         @logger.debug? && @logger.debug("String built", :to_checksum => to_string)
-        event.set(@target, anonymize(to_string))
+        event.set(@target, fingerprint(to_string))
       else
         @source.each do |field|
           next unless event.include?(field)
           if event.get(field).is_a?(Array)
-            event.set(@target, event.get(field).collect { |v| anonymize(v) })
+            event.set(@target, event.get(field).collect { |v| fingerprint(v) })
           else
-            event.set(@target, anonymize(event.get(field)))
+            event.set(@target, fingerprint(event.get(field)))
           end
         end
       end
@@ -154,22 +147,30 @@ class LogStash::Filters::Fingerprint < LogStash::Filters::Base
 
   private
 
-  def anonymize_ipv4_network(ip_string)
+  def fingerprint_ipv4_network(ip_string)
     # in JRuby 1.7.11 outputs as US-ASCII
     IPAddr.new(ip_string).mask(@key.to_i).to_s.force_encoding(Encoding::UTF_8)
   end
 
-  def anonymize_openssl(data)
+  def fingerprint_openssl(data)
     # in JRuby 1.7.11 outputs as ASCII-8BIT
-    if @base64encode
-      hash  = OpenSSL::HMAC.digest(@digest, @key, data.to_s)
-      Base64.strict_encode64(hash).force_encoding(Encoding::UTF_8)
+    if @key.nil?
+      if @base64encode
+        @digest.base64digest(data.to_s).force_encoding(Encoding::UTF_8)
+      else
+        @digest.hexdigest(data.to_s).force_encoding(Encoding::UTF_8)
+      end
     else
-      OpenSSL::HMAC.hexdigest(@digest, @key, data.to_s).force_encoding(Encoding::UTF_8)
+      if @base64encode
+        hash = OpenSSL::HMAC.digest(@digest, @key, data.to_s)
+        Base64.strict_encode64(hash).force_encoding(Encoding::UTF_8)
+      else
+        OpenSSL::HMAC.hexdigest(@digest, @key, data.to_s).force_encoding(Encoding::UTF_8)
+      end
     end
   end
 
-  def anonymize_murmur3(value)
+  def fingerprint_murmur3(value)
     case value
     when Fixnum
       MurmurHash3::V32.int_hash(value)

data/logstash-filter-fingerprint.gemspec

@@ -1,8 +1,8 @@
 Gem::Specification.new do |s|
 
   s.name            = 'logstash-filter-fingerprint'
-  s.version         = '3.1.2'
-  s.licenses        = ['Apache License (2.0)']
+  s.version         = '3.2.0'
+  s.licenses        = ['Apache-2.0']
   s.summary         = "Fingerprints fields by replacing values with a consistent hash"
   s.description     = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program"
   s.authors         = ["Elastic"]

data/spec/filters/fingerprint_spec.rb

@@ -35,7 +35,22 @@ describe LogStash::Filters::Fingerprint do
     end
   end
 
-  describe "fingerprint string with SHA1 alogrithm" do
+  describe "fingerprint string with SHA1 algorithm" do
+    config <<-CONFIG
+      filter {
+        fingerprint {
+          source => ["clientip"]
+          method => 'SHA1'
+        }
+      }
+    CONFIG
+
+    sample("clientip" => "123.123.123.123") do
+      insist { subject.get("fingerprint") } == "3a5076c520b4b463f43806896ea0b3978d09dcae"
+    end
+  end
+
+  describe "fingerprint string with SHA1 HMAC algorithm" do
     config <<-CONFIG
       filter {
         fingerprint {
@@ -51,7 +66,7 @@ describe LogStash::Filters::Fingerprint do
     end
   end
 
-  describe "fingerprint string with SHA1 alogrithm on all event fields" do
+  describe "fingerprint string with SHA1 HMAC algorithm on all event fields" do
     config <<-CONFIG
       filter {
         fingerprint {
@@ -68,7 +83,23 @@ describe LogStash::Filters::Fingerprint do
     end
   end
 
-  describe "fingerprint string with SHA1 alogrithm and base64 encoding" do
+  describe "fingerprint string with SHA1 algorithm and base64 encoding" do
+    config <<-CONFIG
+      filter {
+        fingerprint {
+          source => ["clientip"]
+          method => 'SHA1'
+          base64encode => true
+        }
+      }
+    CONFIG
+
+    sample("clientip" => "123.123.123.123") do
+      insist { subject.get("fingerprint") } == "OlB2xSC0tGP0OAaJbqCzl40J3K4="
+    end
+  end
+
+  describe "fingerprint string with SHA1 HMAC algorithm and base64 encoding" do
     config <<-CONFIG
       filter {
         fingerprint {
@@ -85,7 +116,22 @@ describe LogStash::Filters::Fingerprint do
     end
   end
 
-  describe "fingerprint string with SHA256 alogrithm" do
+  describe "fingerprint string with SHA256 algorithm" do
+    config <<-CONFIG
+      filter {
+        fingerprint {
+          source => ["clientip"]
+          method => 'SHA256'
+        }
+      }
+    CONFIG
+
+    sample("clientip" => "123.123.123.123") do
+      insist { subject.get("fingerprint") } == "4dabcab210766e35f03e77120e6986d6e6d4752b2a9ff22980b9253d026080d8"
+    end
+  end
+
+  describe "fingerprint string with SHA256 HMAC algorithm" do
     config <<-CONFIG
       filter {
         fingerprint {
@@ -101,7 +147,7 @@ describe LogStash::Filters::Fingerprint do
     end
   end
 
-  describe "fingerprint string with SHA256 alogrithm and base64 encoding" do
+  describe "fingerprint string with SHA256 HMAC algorithm and base64 encoding" do
     config <<-CONFIG
       filter {
         fingerprint {
@@ -118,7 +164,22 @@ describe LogStash::Filters::Fingerprint do
     end
   end
 
-  describe "fingerprint string with SHA384 alogrithm" do
+  describe "fingerprint string with SHA384 algorithm" do
+    config <<-CONFIG
+      filter {
+        fingerprint {
+          source => ["clientip"]
+          method => 'SHA384'
+        }
+      }
+    CONFIG
+
+    sample("clientip" => "123.123.123.123") do
+      insist { subject.get("fingerprint") } == "fd605b0a3af3e04ce0d7a0b0d9c48d67a12dab811f60072e6eae84e35d567793ffb68a1807536f11c90874065c2a4392"
+    end
+  end
+
+  describe "fingerprint string with SHA384 HMAC algorithm" do
     config <<-CONFIG
       filter {
         fingerprint {
@@ -134,7 +195,7 @@ describe LogStash::Filters::Fingerprint do
     end
   end
 
-  describe "fingerprint string with SHA384 alogrithm and base64 encoding" do
+  describe "fingerprint string with SHA384 HMAC algorithm and base64 encoding" do
     config <<-CONFIG
       filter {
         fingerprint {
@@ -151,7 +212,22 @@ describe LogStash::Filters::Fingerprint do
     end
   end
 
-  describe "fingerprint string with SHA512 alogrithm" do
+  describe "fingerprint string with SHA512 algorithm" do
+    config <<-CONFIG
+      filter {
+        fingerprint {
+          source => ["clientip"]
+          method => 'SHA512'
+        }
+      }
+    CONFIG
+
+    sample("clientip" => "123.123.123.123") do
+      insist { subject.get("fingerprint") } == "5468e2dc64ea92b617782aae884b35af60041ac9e168a283615b6a462c54c13d42fa9542cce9b7d76a8124ac6616818905e3e5dd35d6e519f77c3b517558639a"
+    end
+  end
+
+  describe "fingerprint string with SHA512 HMAC algorithm" do
     config <<-CONFIG
       filter {
         fingerprint {
@@ -167,7 +243,7 @@ describe LogStash::Filters::Fingerprint do
     end
   end
 
-  describe "fingerprint string with SHA512 alogrithm and base64 encoding" do
+  describe "fingerprint string with SHA512 HMAC algorithm and base64 encoding" do
     config <<-CONFIG
       filter {
         fingerprint {
@@ -184,7 +260,22 @@ describe LogStash::Filters::Fingerprint do
     end
   end
 
-  describe "fingerprint string with MD5 alogrithm" do
+  describe "fingerprint string with MD5 algorithm" do
+    config <<-CONFIG
+      filter {
+        fingerprint {
+          source => ["clientip"]
+          method => 'MD5'
+        }
+      }
+    CONFIG
+
+    sample("clientip" => "123.123.123.123") do
+      insist { subject.get("fingerprint") } == "ccdd8d3d940a01b2fb3258c059924c0d"
+    end
+  end
+
+  describe "fingerprint string with MD5 HMAC algorithm" do
     config <<-CONFIG
       filter {
         fingerprint {
@@ -200,7 +291,7 @@ describe LogStash::Filters::Fingerprint do
     end
   end
 
-  describe "fingerprint string with MD5 alogrithm and base64 encoding" do
+  describe "fingerprint string with MD5 HMAC algorithm and base64 encoding" do
     config <<-CONFIG
       filter {
         fingerprint {

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: logstash-filter-fingerprint
 version: !ruby/object:Gem::Version
-  version: 3.1.2
+  version: 3.2.0
 platform: ruby
 authors:
 - Elastic
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2017-11-07 00:00:00.000000000 Z
+date: 2018-06-20 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
@@ -78,7 +78,7 @@ files:
 - spec/filters/fingerprint_spec.rb
 homepage: http://www.elastic.co/guide/en/logstash/current/index.html
 licenses:
-- Apache License (2.0)
+- Apache-2.0
 metadata:
   logstash_plugin: 'true'
   logstash_group: filter
@@ -98,7 +98,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project:
-rubygems_version: 2.6.11
+rubygems_version: 2.6.13
 signing_key:
 specification_version: 4
 summary: Fingerprints fields by replacing values with a consistent hash