logstash-filter-cipher 2.0.2 → 2.0.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 3186e0367bb687d4d9fb244aec75e2961c16d675
-  data.tar.gz: c9b0629f9cd62859626220dd0120e13d1e40bd7b
+  metadata.gz: beeedcce9482610f01afe1d5a6abb3dadf1be9d8
+  data.tar.gz: e8ff3189814e38d63e9729893642978404386191
 SHA512:
-  metadata.gz: 40432e0006c8d3d5c7cc322175853fac6c086cc14f49c8a0b4854965ef587b56156aa4f7951e9de0fb0564ae8d21dea61028d72bdb61c9e6a1faa49e9240526d
-  data.tar.gz: d25448e42cc9d12d0d1cd6213ad36d96a2e9affc1e0367e3933aa31fd4f53a972bfa1181dd18b20e60a20ab1ec40c8633f8eba117a250b6123d7a515afdd3b80
+  metadata.gz: 90237b428e896c39f895e85b68d781225b1ff2d7edcbdb3103cbec327d2de918c8dd3843b799b98024bfc36762583f673bb3c4d637f4b42a787a565e44be6fa9
+  data.tar.gz: 7df726409a16267b2f71a9027b7a0a25fc4b962d7c8816534592f886f9d2842ccd51f39fd03fde880ff97f668c6cbb8b404c4fef27059b683a9ef71c7fe836cc

data/CHANGELOG.md

@@ -1,3 +1,6 @@
+## 2.0.3
+ - fixes base64 encoding issue, adds support for random IVs 
+
 ## 2.0.0
  - Plugins were updated to follow the new shutdown semantic, this mainly allows Logstash to instruct input plugins to terminate gracefully, 
    instead of using Thread.raise on the plugins' threads. Ref: https://github.com/elastic/logstash/pull/3895

data/README.md

@@ -1,5 +1,8 @@
 # Logstash Plugin
 
+[![Build
+Status](http://build-eu-00.elastic.co/view/LS%20Plugins/view/LS%20Filters/job/logstash-plugin-filter-cipher-unit/badge/icon)](http://build-eu-00.elastic.co/view/LS%20Plugins/view/LS%20Filters/job/logstash-plugin-filter-cipher-unit/)
+
 This is a plugin for [Logstash](https://github.com/elastic/logstash).
 
 It is fully free and fully open source. The license is Apache 2.0, meaning you are pretty much free to use it however you want in whatever way.

data/lib/logstash/filters/cipher.rb

@@ -1,6 +1,7 @@
 # encoding: utf-8
 require "logstash/filters/base"
 require "logstash/namespace"
+require "openssl"
 
 
 # This filter parses a source and apply a cipher or decipher before
@@ -31,23 +32,31 @@ class LogStash::Filters::Cipher < LogStash::Filters::Base
   config :base64, :validate => :boolean, :default => true
 
   # The key to use
+  #
+  # NOTE: If you encounter an error message at runtime containing the following:
+  #
+  # "java.security.InvalidKeyException: Illegal key size: possibly you need to install 
+  # Java Cryptography Extension (JCE) Unlimited Strength Jurisdiction Policy Files for your JRE"
+  #
+  # Please read the following: https://github.com/jruby/jruby/wiki/UnlimitedStrengthCrypto
+  #
   config :key, :validate => :string
 
   # The key size to pad
   #
-  # It depends of the cipher algorythm.I your key don't need
+  # It depends of the cipher algorithm. If your key doesn't need
   # padding, don't set this parameter
   #
-  # Example, for AES-256, we must have 32 char long key
+  # Example, for AES-128, we must have 16 char long key. AES-256 = 32 chars 
   # [source,ruby]
-  #     filter { cipher { key_size => 32 }
+  #     filter { cipher { key_size => 16 }
   #
-  config :key_size, :validate => :number, :default => 32
+  config :key_size, :validate => :number, :default => 16
 
   # The character used to pad the key
   config :key_pad, :default => "\0"
 
-  # The cipher algorythm
+  # The cipher algorithm
   #
   # A list of supported algorithms can be obtained by
   # [source,ruby]
@@ -59,12 +68,12 @@ class LogStash::Filters::Cipher < LogStash::Filters::Base
   # Valid values are encrypt or decrypt
   config :mode, :validate => :string, :required => true
 
-  # Cypher padding to use. Enables or disables padding. 
+  # Cipher padding to use. Enables or disables padding.
   #
-  # By default encryption operations are padded using standard block padding 
-  # and the padding is checked and removed when decrypting. If the pad 
-  # parameter is zero then no padding is performed, the total amount of data 
-  # encrypted or decrypted must then be a multiple of the block size or an 
+  # By default encryption operations are padded using standard block padding
+  # and the padding is checked and removed when decrypting. If the pad
+  # parameter is zero then no padding is performed, the total amount of data
+  # encrypted or decrypted must then be a multiple of the block size or an
   # error will occur.
   #
   # See EVP_CIPHER_CTX_set_padding for further information.
@@ -73,16 +82,57 @@ class LogStash::Filters::Cipher < LogStash::Filters::Base
   # If you want to change it, set this parameter. If you want to disable
   # it, Set this parameter to 0
   # [source,ruby]
-  #     filter { cipher { padding => 0 }}
+  #     filter { cipher { cipher_padding => 0 }}
   config :cipher_padding, :validate => :string
 
-  # The initialization vector to use
+  # The initialization vector to use (statically hard-coded). For
+  # a random IV see the iv_random_length property
+  #
+  # NOTE: If iv_random_length is set, it takes precedence over any value set for "iv"
   #
   # The cipher modes CBC, CFB, OFB and CTR all need an "initialization
   # vector", or short, IV. ECB mode is the only mode that does not require
   # an IV, but there is almost no legitimate use case for this mode
   # because of the fact that it does not sufficiently hide plaintext patterns.
-  config :iv, :validate => :string
+  #
+  # For AES algorithms set this to a 16 byte string.
+  # [source,ruby]
+  #     filter { cipher { iv => "1234567890123456" }}
+  #
+  # Deprecated: Please use `iv_random_length` instead
+  config :iv, :validate => :string, :deprecated => "Please use 'iv_random_length'"
+
+  # Force an random IV to be used per encryption invocation and specify
+  # the length of the random IV that will be generated via:
+  #
+  #       OpenSSL::Random.random_bytes(int_length)
+  #
+  # If iv_random_length is set, it takes precedence over any value set for "iv"
+  #
+  # Enabling this will force the plugin to generate a unique
+  # random IV for each encryption call. This random IV will be prepended to the
+  # encrypted result bytes and then base64 encoded. On decryption "iv_random_length" must
+  # also be set to utilize this feature. Random IV's are better than statically
+  # hardcoded IVs
+  #
+  # For AES algorithms you can set this to a 16
+  # [source,ruby]
+  #     filter { cipher { iv_random_length => 16 }}
+  config :iv_random_length, :validate => :number
+
+  # If this is set the internal Cipher instance will be
+  # re-used up to @max_cipher_reuse times before being
+  # reset() and re-created from scratch. This is an option
+  # for efficiency where lots of data is being encrypted
+  # and decrypted using this filter. This lets the filter
+  # avoid creating new Cipher instances over and over
+  # for each encrypt/decrypt operation.
+  #
+  # This is optional, the default is no re-use of the Cipher
+  # instance and max_cipher_reuse = 1 by default
+  # [source,ruby]
+  #     filter { cipher { max_cipher_reuse => 1000 }}
+  config :max_cipher_reuse, :validate => :number, :default => 1
 
   def register
     require 'base64' if @base64
@@ -96,30 +146,82 @@ class LogStash::Filters::Cipher < LogStash::Filters::Base
 
     #If decrypt or encrypt fails, we keep it it intact.
     begin
+
+      if (event[@source].nil? || event[@source].empty?)
+        @logger.debug("Event to filter, event 'source' field: " + @source + " was null(nil) or blank, doing nothing")
+        return
+      end
+
       #@logger.debug("Event to filter", :event => event)
       data = event[@source]
       if @mode == "decrypt"
-        data =  Base64.decode64(data) if @base64 == true
+        data =  Base64.strict_decode64(data) if @base64 == true
+
+        if !@iv_random_length.nil?
+          @random_iv = data.byteslice(0,@iv_random_length)
+          data = data.byteslice(@iv_random_length..data.length)
+        end
+
+      end
+
+      if !@iv_random_length.nil? and @mode == "encrypt"
+        @random_iv = OpenSSL::Random.random_bytes(@iv_random_length)
       end
+
+      # if iv_random_length is specified, generate a new one
+      # and force the cipher's IV = to the random value
+      if !@iv_random_length.nil?
+        @cipher.iv = @random_iv
+      end
+
       result = @cipher.update(data) + @cipher.final
+
       if @mode == "encrypt"
-        data =  Base64.encode64(data) if @base64 == true
+
+        # if we have a random_iv, prepend that to the crypted result
+        if !@random_iv.nil?
+          result = @random_iv + result
+        end
+
+        result =  Base64.strict_encode64(result).encode("utf-8") if @base64 == true
       end
+
     rescue => e
       @logger.warn("Exception catch on cipher filter", :event => event, :error => e)
+
+      # force a re-initialize on error to be safe
+      init_cipher
+
     else
+      @total_cipher_uses += 1
+
+      result = result.force_encoding("utf-8") if @mode == "decrypt"
+
       event[@target]= result
+
       #Is it necessary to add 'if !result.nil?' ? exception have been already catched.
       #In doubt, I keep it.
       filter_matched(event) if !result.nil?
-      #Too much bad result can be a problem, reinit cipher prevent this.
-      init_cipher
+
+      if !@max_cipher_reuse.nil? and @total_cipher_uses >= @max_cipher_reuse
+        @logger.debug("max_cipher_reuse["+@max_cipher_reuse.to_s+"] reached, total_cipher_uses = "+@total_cipher_uses.to_s)
+        init_cipher
+      end
+
     end
   end # def filter
 
   def init_cipher
 
+    if !@cipher.nil?
+      @cipher.reset
+      @cipher = nil
+    end
+
     @cipher = OpenSSL::Cipher.new(@algorithm)
+
+    @total_cipher_uses = 0
+
     if @mode == "encrypt"
       @cipher.encrypt
     elsif @mode == "decrypt"
@@ -136,11 +238,18 @@ class LogStash::Filters::Cipher < LogStash::Filters::Base
 
     @cipher.key = @key
 
-    @cipher.iv = @iv if @iv
+    if !@iv.nil? and !@iv.empty? and @iv_random_length.nil?
+      @cipher.iv = @iv if @iv
+
+    elsif !@iv_random_length.nil?
+      @logger.debug("iv_random_length is configured, ignoring any statically defined value for 'iv'", :iv_random_length => @iv_random_length)
+    else
+      raise "cipher plugin: either 'iv' or 'iv_random_length' must be configured, but not both; aborting"
+    end
 
     @cipher.padding = @cipher_padding if @cipher_padding
 
-    @logger.debug("Cipher initialisation done", :mode => @mode, :key => @key, :iv => @iv, :cipher_padding => @cipher_padding)
+    @logger.debug("Cipher initialisation done", :mode => @mode, :key => @key, :iv => @iv, :iv_random => @iv_random, :cipher_padding => @cipher_padding)
   end # def init_cipher
 
 

data/logstash-filter-cipher.gemspec

@@ -1,7 +1,7 @@
 Gem::Specification.new do |s|
 
   s.name            = 'logstash-filter-cipher'
-  s.version         = '2.0.2'
+  s.version         = '2.0.3'
   s.licenses        = ['Apache License (2.0)']
   s.summary         = "This filter parses a source and apply a cipher or decipher before storing it in the target"
   s.description     = "This gem is a logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/plugin install gemname. This gem is not a stand-alone program"

data/spec/filters/cipher_spec.rb

@@ -1,5 +1,215 @@
-require "logstash/devutils/rspec/spec_helper"
-require 'logstash/filters/cipher'
+    # encoding: utf-8
+
+    require "logstash/devutils/rspec/spec_helper"
+    require 'logstash/filters/cipher'
+
+    describe LogStash::Filters::Cipher do
+
+      let(:cleartext) do
+        'شسيبشن٤٤ت٥ت داھدساققبمر фывапролдзщшгнекутцйячсмить asdfghjklqpoiuztreyxcvbnm,.-öäü+ä123ß´yö.,;LÖÜ*O 來少精清皆人址法田手扌打表氵開日大木裝 1234567890#$%^&*()!@#;:\'?.>,<testAESEn+=_-~`}]'
+      end
+
+      let(:event) do
+        "{\"message\":\"#{cleartext}\"}"
+      end
+
+      let(:pipeline) { LogStash::Pipeline.new(config) }
+
+      let(:events) do
+        arr = event.is_a?(Array) ? event : [event]
+        arr.map do |evt|
+          LogStash::Event.new(evt.is_a?(String) ? LogStash::Json.load(evt) : evt)
+        end
+      end
+
+      let(:results) do
+        pipeline.instance_eval { @filters.each(&:register) }
+        results  = []
+        events.each do |evt|
+          # filter call the block on all filtered events, included new events added by the filter
+          pipeline.filter(evt) do |filtered_event|
+            results.push(filtered_event)
+          end
+        end
+        pipeline.flush_filters(:final => true) { |flushed_event| results << flushed_event }
+
+        results.select { |e| !e.cancelled? }
+      end
+
+      describe 'single event, encrypt/decrypt aes-128-cbc, 16b RANDOM IV, 16b key, b64 encode' do
+
+        let(:config) do
+          <<-CONFIG
+    filter {
+
+        cipher {
+            algorithm => "aes-128-cbc"
+            cipher_padding => 1
+            iv_random_length => 16
+            key => "1234567890123456"
+            key_size => 16
+            mode => "encrypt"
+            source => "message"
+            target => "message_crypted"
+            base64 => true
+            max_cipher_reuse => 1
+        }
+
+        cipher {
+            algorithm => "aes-128-cbc"
+            cipher_padding => 1
+            iv_random_length => 16
+            key => "1234567890123456"
+            key_size => 16
+            mode => "decrypt"
+            source => "message_crypted"
+            target => "message_decrypted"
+            base64 => true
+            max_cipher_reuse => 1
+        }
+    }
+    CONFIG
+        end
+
+        it 'validate initial cleartext message' do
+          result = results.first
+          expect(result["message"]).to eq(cleartext)
+        end
+
+        it 'validate decrypted message' do
+          result = results.first
+          expect(result["message_decrypted"]).to eq(result["message"])
+        end
+
+        it 'validate encrypted message is not equal to message' do
+          result = results.first
+          expect(result["message"]).not_to eq(result["message_crypted"])
+        end
+
+      end
+
+
+      describe 'single event, encrypt/decrypt aes-128-cbc, 16b STATIC IV, 16b key, b64 encode' do
+
+        let(:config) do
+          <<-CONFIG
+    filter {
+
+        cipher {
+            algorithm => "aes-128-cbc"
+            cipher_padding => 1
+            iv => "1234567890123456"
+            key => "1234567890123456"
+            key_size => 16
+            mode => "encrypt"
+            source => "message"
+            target => "message_crypted"
+            base64 => true
+            max_cipher_reuse => 1
+        }
+
+        cipher {
+            algorithm => "aes-128-cbc"
+            cipher_padding => 1
+            iv => "1234567890123456"
+            key => "1234567890123456"
+            key_size => 16
+            mode => "decrypt"
+            source => "message_crypted"
+            target => "message_decrypted"
+            base64 => true
+            max_cipher_reuse => 1
+        }
+    }
+    CONFIG
+        end
+
+        it 'validate initial cleartext message' do
+          result = results.first
+          expect(result["message"]).to eq(cleartext)
+        end
+
+        it 'validate decrypted message' do
+          result = results.first
+          expect(result["message_decrypted"]).to eq(result["message"])
+        end
+
+        it 'validate encrypted message is not equal to message' do
+          result = results.first
+          expect(result["message"]).not_to eq(result["message_crypted"])
+        end
+
+      end
+
+
+      describe '1000 events, 11 re-use, encrypt/decrypt aes-128-cbc, 16b RANDOM IV, 16b key, b64 encode' do
+
+        total_events = 1000
+
+        let(:event) do
+          events = []
+         (1..total_events).each do |i|
+            events.push("{\"message\":\"#{cleartext}\"}")
+          end
+          return events
+        end
+
+        let(:config) do
+          <<-CONFIG
+    filter {
+
+        cipher {
+            algorithm => "aes-128-cbc"
+            cipher_padding => 1
+            iv_random_length => 16
+            key => "1234567890123456"
+            key_size => 16
+            mode => "encrypt"
+            source => "message"
+            target => "message_crypted"
+            base64 => true
+            max_cipher_reuse => 11
+        }
+
+        cipher {
+            algorithm => "aes-128-cbc"
+            cipher_padding => 1
+            iv_random_length => 16
+            key => "1234567890123456"
+            key_size => 16
+            mode => "decrypt"
+            source => "message_crypted"
+            target => "message_decrypted"
+            base64 => true
+            max_cipher_reuse => 11
+        }
+    }
+    CONFIG
+        end
+
+        it 'validate total events' do
+          expect(results.length).to eq(total_events)
+        end
+
+        it 'validate initial cleartext message' do
+          results.each do |result|
+            expect(result["message"]).to eq(cleartext)
+          end
+        end
+
+        it 'validate decrypted message' do
+          results.each do |result|
+            expect(result["message_decrypted"]).to eq(result["message"])
+          end
+        end
+
+        it 'validate encrypted message is not equal to message' do
+          results.each do |result|
+            expect(result["message"]).not_to eq(result["message_crypted"])
+          end
+        end
+
+      end
+
+    end
 
-describe LogStash::Filters::Cipher do
-end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: logstash-filter-cipher
 version: !ruby/object:Gem::Version
-  version: 2.0.2
+  version: 2.0.3
 platform: ruby
 authors:
 - Elastic
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2015-10-14 00:00:00.000000000 Z
+date: 2016-01-27 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement