fluent-plugin-masking 1.0.2

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: 737c2cddcc0b796becb417b194c5ac9a8d25dc540125c81c8e9f1f4ae90c67fc
+  data.tar.gz: 15d0092825ca7b06b4d844f9d68042a56d081cf15f70327deddc06e63458c586
+SHA512:
+  metadata.gz: f956ccf1112547d5d1a4a9ef29368c6719cf7b67c5b57e1c2f95540067774e1d06a5fad3d55b5f2282d36ea68a14f1e098dcc7b704e4ba653d5d3a248e081d19
+  data.tar.gz: 71eb6075664087bb1d600b5fedf7be4d9fedea8c8e6f98516f1e8b19661198d7dc44de6aaf5e6e3956a24426b1cc827d909150109c393553d33910aad8508f47

data/Gemfile

@@ -0,0 +1,4 @@
+source 'https://rubygems.org'
+
+# Specify your gem's dependencies in fluent-plugin-masking.gemspec
+gemspec

data/Gemfile.lock

@@ -0,0 +1,55 @@
+PATH
+  remote: .
+  specs:
+    fluent-plugin-masking (1.0.0)
+      fluentd (>= 0.14.0, < 2)
+
+GEM
+  remote: https://rubygems.org/
+  specs:
+    cool.io (1.5.4)
+    dig_rb (1.0.1)
+    fluentd (1.6.3)
+      cool.io (>= 1.4.5, < 2.0.0)
+      dig_rb (~> 1.0.0)
+      http_parser.rb (>= 0.5.1, < 0.7.0)
+      msgpack (>= 0.7.0, < 2.0.0)
+      serverengine (>= 2.0.4, < 3.0.0)
+      sigdump (~> 0.2.2)
+      strptime (>= 0.2.2, < 1.0.0)
+      tzinfo (~> 1.0)
+      tzinfo-data (~> 1.0)
+      yajl-ruby (~> 1.0)
+    http_parser.rb (0.6.0)
+    msgpack (1.3.1)
+    power_assert (1.1.5)
+    rake (12.3.3)
+    rr (1.2.1)
+    serverengine (2.1.1)
+      sigdump (~> 0.2.2)
+    sigdump (0.2.4)
+    strptime (0.2.3)
+    test-unit (3.3.3)
+      power_assert
+    test-unit-rr (1.0.5)
+      rr (>= 1.1.1)
+      test-unit (>= 2.5.2)
+    thread_safe (0.3.6)
+    tzinfo (1.2.5)
+      thread_safe (~> 0.1)
+    tzinfo-data (1.2019.2)
+      tzinfo (>= 1.0.0)
+    yajl-ruby (1.4.1)
+
+PLATFORMS
+  ruby
+
+DEPENDENCIES
+  bundler
+  fluent-plugin-masking!
+  rake (~> 12.0)
+  test-unit (>= 3.1.0)
+  test-unit-rr
+
+BUNDLED WITH
+   2.0.2

data/Rakefile

@@ -0,0 +1,7 @@
+# in Rakefile
+require 'rake/testtask'
+  Rake::TestTask.new(:test) do |test|
+  test.libs << 'lib' << 'test'
+  test.pattern = 'test/**/test_*.rb'
+  test.verbose = true
+end

data/fluent-plugin-masking.gemspec

@@ -0,0 +1,25 @@
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+
+Gem::Specification.new do |spec|
+  spec.name          = "fluent-plugin-masking"
+  spec.version       = "1.0.2"
+
+  spec.authors       = ["Shai Moria", "Niv Lipetz"]
+  spec.email         = ["shai.moria@zooz.com", "niv.lipetz@zooz.com"]
+  spec.description   = "Fluentd Filter plugin to mask given fields in messages"
+  spec.summary       = "Fluentd Filter plugin to mask given fields in messages"
+  spec.homepage      = "https://github.com/zooz"
+
+  spec.files         = `git ls-files`.split($\)
+  spec.require_paths = ["lib"]
+  spec.license = "Apache-2.0"
+
+  spec.required_ruby_version = '>= 2.1'
+  
+  spec.add_runtime_dependency "fluentd", ">= 0.14.0", "< 2"
+  spec.add_development_dependency "bundler"
+  spec.add_development_dependency "rake", "~> 12.0"
+  spec.add_development_dependency "test-unit", ">= 3.1.0"
+  spec.add_development_dependency "test-unit-rr"
+end

data/lib/fluent/plugin/filter_masking.rb

@@ -0,0 +1,70 @@
+require 'fluent/filter'
+
+module Fluent
+  module Plugin
+    class MaskingFilter < Filter
+      Fluent::Plugin.register_filter("masking", self) # for "@type masking" in configuration
+
+      MASK_STRING = "*******"
+
+      def strToHash(str)
+        eval(str)
+      end
+
+      # returns the masked record
+      # error safe method - if any error occurs
+      # the original record is return
+      def maskRecord(record)
+        maskedRecord = record
+
+        begin
+          recordStr = record.to_s
+
+          @fields_to_mask.each do | fieldToMask |
+            recordStr = recordStr.gsub(/"#{fieldToMask}"=>.+?((?=,\s\")|(?=}{1}$))/m, "\"#{fieldToMask}\"=>\"#{MASK_STRING}\"")
+          end
+
+          maskedRecord = strToHash(recordStr)
+
+        rescue Exception => e
+          $log.error "Failed to mask record: #{e}"
+        end
+
+        maskedRecord
+      end
+
+      def initialize
+        super
+        @fields_to_mask = []
+      end
+
+      # this method only called ones (on startup time)
+      def configure(conf)
+        super
+        fieldsToMaskFilePath = conf['fieldsToMaskFilePath']
+
+        File.open(fieldsToMaskFilePath, "r") do |f|
+          f.each_line do |line|
+
+            value = line.to_s # make sure it's string
+            value = value.gsub(/\s+/, "") # remove spaces
+            value = value.gsub('\n', '') # remove line breakers
+
+            @fields_to_mask.push(value)
+          end
+        end
+
+        puts "black list fields:"
+        puts @fields_to_mask
+      end
+
+      def filter(tag, time, record)
+        # This method implements the filtering logic for individual filters
+        # It is internal to this class and called by filter_stream unless
+        # the user overrides filter_stream.
+        maskRecord(record)
+      end
+
+    end # end of MaskingFilter class definition
+  end
+end # end of 'module Fluent' definition

data/test/fields-to-mask

@@ -0,0 +1,3 @@
+email
+first_name
+last_name

data/test/test_filter_masking.rb

@@ -0,0 +1,90 @@
+# test/plugin/test_filter_your_own.rb
+
+require "test-unit"
+require "fluent/test"
+require "fluent/test/driver/filter"
+require "fluent/test/helpers"
+require "/Users/nivlipetz/Documents/sre/logs-infra/fluentd-docker/custom-plugins/filter_masking.rb"
+
+MASK_STRING = "*******"
+
+class YourOwnFilterTest < Test::Unit::TestCase
+  def setup
+    Fluent::Test.setup # this is required to setup router and others
+  end
+
+  # default configuration for tests
+  CONFIG = %[
+    fieldsToMaskFilePath test/fields-to-mask
+  ]
+
+  def create_driver(conf = CONFIG)
+    Fluent::Test::Driver::Filter.new(Fluent::Plugin::MaskingFilter).configure(conf)
+  end
+
+  def filter(config, messages)
+    d = create_driver(config)
+    d.run(default_tag: "input.access") do
+      messages.each do |message|
+        d.feed(message)
+      end
+    end
+    d.filtered_records
+  end
+
+  # sub_test_case 'configured with invalid configuration' do
+  #   test 'empty configuration' do
+  #     assert_raise(Fluent::ConfigError) do
+  #        create_driver('')
+  #     end
+  #   end
+
+  #   test 'param1 should reject too short string' do
+  #     conf = %[
+  #       param1 a
+  #     ]
+  #     assert_raise(Fluent::ConfigError) do
+  #        create_driver(conf)
+  #     end
+  #   end
+  #   # ...
+  # end
+
+  sub_test_case 'plugin will mask all fields that need masking' do
+    test 'mask first_name and last_name' do
+      conf = CONFIG
+      messages = [
+        { "first_name" => "mickey", "last_name" => "the-dog" }
+      ]
+      expected = [
+        { "first_name" => MASK_STRING, "last_name" => MASK_STRING }
+      ]
+      filtered_records = filter(conf, messages)
+      assert_equal(expected, filtered_records)
+    end
+
+    test 'mask only email' do
+      conf = CONFIG
+      messages = [
+        { "not_masked_field" => "mickey-the-dog", "email" => "mickey-the-dog@zooz.com" }
+      ]
+      expected = [
+        { "not_masked_field" => "mickey-the-dog", "email" => MASK_STRING }
+      ]
+      filtered_records = filter(conf, messages)
+      assert_equal(expected, filtered_records)
+    end
+
+    test 'mask nothing' do
+      conf = CONFIG
+      messages = [
+        { "not_masked_field_1" => "mickey-the-dog", "not_masked_field_2" => "nully_the_carpet" }
+      ]
+      expected = [
+        { "not_masked_field_1" => "mickey-the-dog", "not_masked_field_2" => "nully_the_carpet" }
+      ]
+      filtered_records = filter(conf, messages)
+      assert_equal(expected, filtered_records)
+    end
+  end
+end

metadata

@@ -0,0 +1,130 @@
+--- !ruby/object:Gem::Specification
+name: fluent-plugin-masking
+version: !ruby/object:Gem::Version
+  version: 1.0.2
+platform: ruby
+authors:
+- Shai Moria
+- Niv Lipetz
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2019-08-14 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: fluentd
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.14.0
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '2'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.14.0
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '2'
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '12.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '12.0'
+- !ruby/object:Gem::Dependency
+  name: test-unit
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 3.1.0
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 3.1.0
+- !ruby/object:Gem::Dependency
+  name: test-unit-rr
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+description: Fluentd Filter plugin to mask given fields in messages
+email:
+- shai.moria@zooz.com
+- niv.lipetz@zooz.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- Gemfile
+- Gemfile.lock
+- Rakefile
+- fluent-plugin-masking-1.0.0.gem
+- fluent-plugin-masking-1.0.1.gem
+- fluent-plugin-masking.gemspec
+- lib/fluent/plugin/filter_masking.rb
+- test/fields-to-mask
+- test/test_filter_masking.rb
+homepage: https://github.com/zooz
+licenses:
+- Apache-2.0
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '2.1'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubygems_version: 3.0.4
+signing_key: 
+specification_version: 4
+summary: Fluentd Filter plugin to mask given fields in messages
+test_files: []