sensitive_data_filter 0.1.0 → 0.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 6e5a914234df291b0800d47279ad6dfe4b9405f7
-  data.tar.gz: 3450359e5f85f765783e4e142b6e2c4671a28765
+  metadata.gz: 70cedcd682fd58e3d8682c603bc8326c36e7b8ec
+  data.tar.gz: aee8dcf45f48b651e85507a6d0e0b02fe8fbb071
 SHA512:
-  metadata.gz: 8f155d08c7c6904f1bb5b9a683a3c528fd6e1b61377bf4edd96cc0b02b147b4342957e9f36ae95a0e489ff4358e05f5b142f72492a5c20a9172c8d12ee3d424c
-  data.tar.gz: 80262f7b96eb96c7564791459049c3e4d5a8a6508a85b1767004f6eaf0cb23e9c7709f7b52eb72d54cf4c331d5957951fb80fe106f4d2e672509afd7c5ffb697
+  metadata.gz: fc6618c4cdad98edb6899779d93b15a1cbfc5c8b650d2cce604f40b2c9178d24cf9000753ecf512ee82b99a17e580f8750cf3019ac6f0777532a417d32ffc8d9
+  data.tar.gz: 1f126f0eec67bc0c8bc74d7bb50e6121056a8113ce35454ba2fc7d1c36d6db74ede0736050515b96e53467d433844f6eef1a545554a01d58898fef0d117bf9eb

data/.rubocop.yml

@@ -16,6 +16,9 @@ Style/SignalException:
 Style/SafeNavigation:
   Enabled: false
 
+Style/RedundantFreeze:
+  Enabled: false
+
 Style/FileName:
   Exclude:
     - gemfiles/Gemfile.*.rb

data/CHANGELOG.md

@@ -3,6 +3,21 @@ All notable changes to this project will be documented in this file.
 This project adheres to [Semantic Versioning](http://semver.org/).
 This changelog adheres to [Keep a CHANGELOG](http://keepachangelog.com/).
 
+## [0.2.0] - 2016-12-13
+### Added
+- Occurrence exposes content type
+- Support for different content types
+- Native JSON parameter parsing
+- Allows defining parameter parsers
+- Scans and masks parameter keys
+- Adds credit card brand validation
+
+### Changed
+- Occurrence now exposes query and body params separately
+
+### Fixed
+- Skips scanning of file uploads
+
 ## [0.1.0] - 2016-12-09
 ### Added
 - Whitelisting of matches

data/README.md

@@ -44,24 +44,31 @@ SensitiveDataFilter.config do |config|
     # Report occurrence
   end
   config.whitelist pattern1, pattern2 # Allows specifying patterns to whitelist matches
+  config.register_parser('yaml', -> params { YAML.load params }, -> params { YAML.dump params })
 end
 ```
 
 An occurrence object has the following properties:
 
-* origin_ip:       the IP address that originated the request
-* request_method:  the HTTP method for the request (GET, POST, etc.)
-* url:             the URL of the request
-* original_params: the parameters sent with the request
-* filtered_params: the parameters sent with the request, with sensitive data filtered
-* session:         the session properties for the request
-* matches:         the matched sensitive data
-* matches_count:   the number of matches per data type, e.g. { 'CreditCard' => 1 }
+* origin_ip:             the IP address that originated the request
+* request_method:        the HTTP method for the request (GET, POST, etc.)
+* url:                   the URL of the request
+* content_type:          the Content-Type of the request
+* original_query_params: the query parameters sent with the request
+* original_body_params:  the body parameters sent with the request
+* filtered_query_params: the query parameters sent with the request, with sensitive data filtered
+* filtered_body_params:  the body parameters sent with the request, with sensitive data filtered
+* session:               the session properties for the request
+* matches:               the matched sensitive data
+* matches_count:         the number of matches per data type, e.g. { 'CreditCard' => 1 }
 
 It also exposes `to_h` and `to_s` methods for hash and string representation respectively.  
-Please note that these representations omit sensitive data, i.e. `original_params` and `matches` are not included.
+Please note that these representations omit sensitive data, 
+i.e. `original_query_params`, `original_body_params` and `matches` are not included.
 
-#### Important Note
+#### Important Notes
+
+Body parameters will not be parsed if a parser for the request's content type is not defined.
 
 You might want to filter sensitive parameters (e.g: passwords).
 In Rails you can do something like:
@@ -69,9 +76,29 @@ In Rails you can do something like:
 ```ruby
 filters = Rails.application.config.filter_parameters
 filter  = ActionDispatch::Http::ParameterFilter.new filters
-filter.filter @occurrence.filtered_params
+filtered_query_params = filter.filter @occurrence.filtered_query_params
+filtered_body_params = if @occurrence.filtered_body_params.is_a? Hash
+                         filter.filter @occurrence.filtered_body_params
+                       else
+                         @occurrence.filtered_body_params
+                       end
 ```
 
+#### Whitelisting
+
+A list of whitelisting patterns can be passed to `config.whitelist`. 
+Any sensitive data match which also matches any of these patterns will be ignored.
+
+#### Parameter Parsing
+
+Parsers for parameters encoded for a specific content type can be defined.
+The arguments for `config.register_parser` are:
+* a pattern to match the content type
+* a parser for the parameters
+* an unparser to convert parameters back to the encoded format
+ 
+The parser and unparser must be objects that respond to `call` and accept the parameters as an argument (e.g. procs or lambdas).
+
 ## Development
 
 After checking out the repo, run `bin/setup` to install dependencies. Then, run `rake spec` to run the tests. You can also run `bin/console` for an interactive prompt that will allow you to experiment.

data/gemfiles/Gemfile.ruby-2.1.rb

@@ -3,6 +3,8 @@ source 'https://rubygems.org'
 
 # ruby-2.1 compatible gems
 gem 'rack', '~> 1.4'
+gem 'activemodel', '>= 3', '< 5'
+gem 'activesupport', '>= 3', '< 5'
 
 # Specify your gem's dependencies in sensitive_data_filter.gemspec
 gemspec path: '../'

data/gemfiles/Gemfile.ruby-2.2.rb

@@ -3,6 +3,8 @@ source 'https://rubygems.org'
 
 # ruby-2.2 compatible gems
 gem 'rack', '~> 1.4'
+gem 'activemodel', '>= 3', '< 5'
+gem 'activesupport', '>= 3', '< 5'
 
 # Specify your gem's dependencies in sensitive_data_filter.gemspec
 gemspec path: '../'

data/lib/sensitive_data_filter/config.rb

@@ -44,5 +44,10 @@ module SensitiveDataFilter
     def whitelist_patterns
       @whitelist_patterns ||= []
     end
+
+    def register_parser(content_type, parser, unparser)
+      SensitiveDataFilter::Middleware::ParameterParser
+        .register_parser(content_type, parser, unparser)
+    end
   end
 end

data/lib/sensitive_data_filter/mask.rb

@@ -2,13 +2,17 @@
 module SensitiveDataFilter
   module Mask
     module_function def mask(value)
+      return mask_array(value) if value.is_a? Array
+      return mask_hash(value) if value.is_a? Hash
       SensitiveDataFilter.enabled_types.inject(value) { |acc, elem| elem.mask acc }
     end
 
+    module_function def mask_array(array)
+      array.map { |element| mask(element) }
+    end
+
     module_function def mask_hash(hash)
-      hash.map.with_object({}) { |(key, value), result|
-        result[key] = mask(value)
-      }
+      hash.map { |key, value| [mask(key), mask(value)] }.to_h
     end
   end
 end

data/lib/sensitive_data_filter/middleware.rb

@@ -4,8 +4,8 @@ module SensitiveDataFilter
   end
 end
 
+require 'sensitive_data_filter/middleware/parameter_parser'
 require 'sensitive_data_filter/middleware/env_parser'
-require 'sensitive_data_filter/middleware/parameter_scanner'
 require 'sensitive_data_filter/middleware/occurrence'
 require 'sensitive_data_filter/middleware/env_filter'
 require 'sensitive_data_filter/middleware/filter'

data/lib/sensitive_data_filter/middleware/env_filter.rb

@@ -9,8 +9,8 @@ module SensitiveDataFilter
       def initialize(env)
         @original_env_parser = EnvParser.new(env)
         @filtered_env_parser = @original_env_parser.copy
-        @scanner = ParameterScanner.new(@original_env_parser)
-        @filtered_env_parser.mask! if @scanner.sensitive_data?
+        @scan = build_scan
+        @filtered_env_parser.mask! if @scan.matches?
         @occurrence = build_occurrence
       end
 
@@ -25,8 +25,14 @@ module SensitiveDataFilter
       private
 
       def build_occurrence
-        return nil unless @scanner.sensitive_data?
-        Occurrence.new(@original_env_parser, @filtered_env_parser, @scanner.matches)
+        return nil unless @scan.matches?
+        Occurrence.new(@original_env_parser, @filtered_env_parser, @scan.matches)
+      end
+
+      def build_scan
+        SensitiveDataFilter::Scan.new(
+          [@original_env_parser.query_params, @original_env_parser.body_params]
+        )
       end
     end
   end

data/lib/sensitive_data_filter/middleware/env_parser.rb

@@ -1,7 +1,12 @@
 # frozen_string_literal: true
+require 'forwardable'
+
 module SensitiveDataFilter
   module Middleware
     class EnvParser
+      QUERY_STRING = 'QUERY_STRING'.freeze
+      RACK_INPUT   = 'rack.input'.freeze
+
       extend Forwardable
 
       attr_reader :env
@@ -9,6 +14,7 @@ module SensitiveDataFilter
       def initialize(env)
         @env = env
         @request = Rack::Request.new(@env)
+        @parameter_parser = ParameterParser.parser_for(@request.media_type)
       end
 
       def query_params
@@ -16,17 +22,18 @@ module SensitiveDataFilter
       end
 
       def body_params
+        return {} if file_upload?
         body = @request.body.read
         @request.body.rewind
-        Rack::Utils.parse_query(body)
+        @parameter_parser.parse(body)
       end
 
       def query_params=(new_params)
-        @env['QUERY_STRING'] = Rack::Utils.build_query(new_params)
+        @env[QUERY_STRING] = Rack::Utils.build_query(new_params)
       end
 
       def body_params=(new_params)
-        @env['rack.input'] = StringIO.new Rack::Utils.build_query(new_params)
+        @env[RACK_INPUT] = StringIO.new @parameter_parser.unparse(new_params)
       end
 
       def copy
@@ -34,11 +41,17 @@ module SensitiveDataFilter
       end
 
       def mask!
-        self.query_params = SensitiveDataFilter::Mask.mask_hash(query_params)
-        self.body_params  = SensitiveDataFilter::Mask.mask_hash(body_params)
+        self.query_params = SensitiveDataFilter::Mask.mask(query_params)
+        self.body_params  = SensitiveDataFilter::Mask.mask(body_params)
       end
 
-      def_delegators :@request, :ip, :request_method, :url, :params, :session
+      def_delegators :@request, :ip, :request_method, :url, :content_type, :session
+
+      private
+
+      def file_upload?
+        @request.media_type == 'multipart/form-data'
+      end
     end
   end
 end

data/lib/sensitive_data_filter/middleware/occurrence.rb

@@ -1,4 +1,5 @@
 # frozen_string_literal: true
+require 'forwardable'
 require 'facets/string/titlecase'
 
 module SensitiveDataFilter
@@ -18,15 +19,23 @@ module SensitiveDataFilter
         @original_env_parser.ip
       end
 
-      def original_params
-        @original_env_parser.params
+      def original_query_params
+        @original_env_parser.query_params
       end
 
-      def filtered_params
-        @filtered_env_parser.params
+      def original_body_params
+        @original_env_parser.body_params
       end
 
-      def_delegators :@original_env_parser, :request_method, :url, :session
+      def filtered_query_params
+        @filtered_env_parser.query_params
+      end
+
+      def filtered_body_params
+        @filtered_env_parser.body_params
+      end
+
+      def_delegators :@original_env_parser, :request_method, :url, :content_type, :session
 
       def matches_count
         @matches.map { |type, matches| [type, matches.count] }.to_h
@@ -34,12 +43,14 @@ module SensitiveDataFilter
 
       def to_h
         {
-          origin_ip:       origin_ip,
-          request_method:  request_method,
-          url:             url,
-          filtered_params: filtered_params,
-          session:         session,
-          matches_count:   matches_count
+          origin_ip:             origin_ip,
+          request_method:        request_method,
+          url:                   url,
+          content_type:          content_type,
+          filtered_query_params: filtered_query_params,
+          filtered_body_params:  filtered_body_params,
+          session:               session,
+          matches_count:         matches_count
         }
       end
 

data/lib/sensitive_data_filter/middleware/parameter_parser.rb

@@ -0,0 +1,48 @@
+# frozen_string_literal: true
+
+module SensitiveDataFilter
+  module Middleware
+    class ParameterParser
+      def self.register_parser(content_type, parse, unparse)
+        parsers.unshift new(content_type, parse, unparse)
+      end
+
+      def self.parsers
+        @parsers ||= DEFAULT_PARSERS.dup
+      end
+
+      def self.parser_for(content_type)
+        parsers.find { |parser| parser.can_parse? content_type } || NULL_PARSER
+      end
+
+      def initialize(content_type, parse, unparse)
+        @content_type = content_type
+        @parse        = parse
+        @unparse      = unparse
+      end
+
+      def can_parse?(content_type)
+        content_type.to_s.match @content_type
+      end
+
+      def parse(params)
+        @parse.call params
+      end
+
+      def unparse(params)
+        @unparse.call params
+      end
+
+      NULL_PARSER = new('', ->(params) { params }, ->(params) { params })
+
+      DEFAULT_PARSERS = [
+        new('urlencoded', # e.g.: 'application/x-www-form-urlencoded'
+            ->(params) { Rack::Utils.parse_query(params) },
+            ->(params) { Rack::Utils.build_query(params) }),
+        new('json', # e.g.: 'application/json'
+            ->(params) { JSON.parse(params) },
+            ->(params) { JSON.unparse(params) })
+      ].freeze
+    end
+  end
+end

data/lib/sensitive_data_filter/scan.rb

@@ -1,26 +1,39 @@
 # frozen_string_literal: true
 require 'facets/kernel/present'
+require 'facets/hash/collate'
 
 module SensitiveDataFilter
   class Scan
+    def self.scan(value)
+      return scan_array(value) if value.is_a? Array
+      return scan_hash(value) if value.is_a? Hash
+      SensitiveDataFilter.enabled_types.map.with_object({}) { |scanner, matches|
+        matches[scanner.name.split('::').last] = whitelist(scanner.scan(value))
+      }
+    end
+
+    def self.scan_array(array)
+      array.map { |element| scan(element) }.inject(:collate) || {}
+    end
+
+    def self.scan_hash(hash)
+      hash.map { |key, value| scan(key).collate(scan(value)) }.inject(:collate) || {}
+    end
+
+    def self.whitelist(matches)
+      matches.reject { |match| SensitiveDataFilter.whitelisted? match }
+    end
+
     def initialize(value)
       @value = value
     end
 
     def matches
-      @matches ||= SensitiveDataFilter.enabled_types.map.with_object({}) { |scanner, matches|
-        matches[scanner.name.split('::').last] = whitelist scanner.scan(@value)
-      }
+      @matches ||= self.class.scan(@value)
     end
 
     def matches?
       matches.values.any?(&:present?)
     end
-
-    private
-
-    def whitelist(matches)
-      matches.reject { |match| SensitiveDataFilter.whitelisted? match }
-    end
   end
 end

data/lib/sensitive_data_filter/types/credit_card.rb

@@ -1,4 +1,6 @@
 # frozen_string_literal: true
+require 'credit_card_validations'
+
 module SensitiveDataFilter
   module Types
     module CreditCard
@@ -21,7 +23,7 @@ module SensitiveDataFilter
       module_function def valid?(number)
         return false unless number.is_a? String
         return false unless number.match CARD
-        Luhn.new(number.gsub(SEPARATORS, '')).valid?
+        CreditCardValidations::Detector.new(number.gsub(SEPARATORS, '')).brand.present?
       end
 
       module_function def scan(value)
@@ -33,40 +35,6 @@ module SensitiveDataFilter
         return value unless value.is_a? String
         scan(value).inject(value) { |acc, elem| acc.gsub(elem, FILTERED) }
       end
-
-      # Adapted from https://github.com/rolfb/luhn-ruby/blob/master/lib/luhn.rb
-      class Luhn
-        def initialize(number)
-          @number = number
-        end
-
-        def valid?
-          numbers = split_digits(@number)
-          numbers.last == checksum(numbers[0..-2].join)
-        end
-
-        private
-
-        def checksum(number)
-          products = luhn_doubled(number)
-          sum      = products.inject(0) { |acc, elem| acc + sum_of(elem) }
-          checksum = 10 - (sum % 10)
-          checksum == 10 ? 0 : checksum
-        end
-
-        def luhn_doubled(number)
-          numbers = split_digits(number).reverse
-          numbers.map.with_index { |n, i| i.even? ? n * 2 : n * 1 }.reverse
-        end
-
-        def sum_of(number)
-          split_digits(number).inject(:+)
-        end
-
-        def split_digits(number)
-          number.to_s.split(//).map(&:to_i)
-        end
-      end
     end
   end
 end

data/lib/sensitive_data_filter/version.rb

@@ -1,4 +1,4 @@
 # frozen_string_literal: true
 module SensitiveDataFilter
-  VERSION = '0.1.0'
+  VERSION = '0.2.0'
 end

data/sensitive_data_filter.gemspec

@@ -25,6 +25,7 @@ Gem::Specification.new do |spec|
 
   spec.add_dependency 'rack', '>= 1.4'
   spec.add_dependency 'facets', '~> 3.1'
+  spec.add_dependency 'credit_card_validations', '~> 3.2'
 
   spec.add_development_dependency 'bundler', '~> 1.13'
   spec.add_development_dependency 'rake', '~> 10.0'

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: sensitive_data_filter
 version: !ruby/object:Gem::Version
-  version: 0.1.0
+  version: 0.2.0
 platform: ruby
 authors:
 - Alessandro Berardi
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2016-12-09 00:00:00.000000000 Z
+date: 2016-12-13 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rack
@@ -39,6 +39,20 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '3.1'
+- !ruby/object:Gem::Dependency
+  name: credit_card_validations
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.2'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.2'
 - !ruby/object:Gem::Dependency
   name: bundler
   requirement: !ruby/object:Gem::Requirement
@@ -182,7 +196,7 @@ files:
 - lib/sensitive_data_filter/middleware/env_parser.rb
 - lib/sensitive_data_filter/middleware/filter.rb
 - lib/sensitive_data_filter/middleware/occurrence.rb
-- lib/sensitive_data_filter/middleware/parameter_scanner.rb
+- lib/sensitive_data_filter/middleware/parameter_parser.rb
 - lib/sensitive_data_filter/scan.rb
 - lib/sensitive_data_filter/types.rb
 - lib/sensitive_data_filter/types/credit_card.rb

data/lib/sensitive_data_filter/middleware/parameter_scanner.rb

@@ -1,22 +0,0 @@
-# frozen_string_literal: true
-require 'facets/hash/collate'
-
-module SensitiveDataFilter
-  module Middleware
-    class ParameterScanner
-      def initialize(env_parser)
-        @env_parser = env_parser
-        @params     = @env_parser.query_params.values + @env_parser.body_params.values
-        @scans      = @params.map { |value| SensitiveDataFilter::Scan.new(value) }
-      end
-
-      def matches
-        @scans.map(&:matches).inject(:collate)
-      end
-
-      def sensitive_data?
-        @scans.any?(&:matches?)
-      end
-    end
-  end
-end