fluent-plugin-string-scrub 0.0.3 → 1.0.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
-SHA1:
-  metadata.gz: cdab445162d666c325c977971529e5eb25a4ffc0
-  data.tar.gz: f7eeafe31bdaf17e838db80b51e232167391ce31
+SHA256:
+  metadata.gz: 3ec5362da8587dc2196f07f27308802fc95533ebea82e77fdca905744a907594
+  data.tar.gz: fad0bd42b3aeb35fcf2a980efd0de44db63d6b2886fad7569891e6d4c8773b85
 SHA512:
-  metadata.gz: f850bc87770dcec1b61e92392bf7f2ba075cbbea2782c0e8a1b9b8cb65b90c7977ac1868c1500a35efeba272045606b1e9fc7842990823c443f2d2821312ffb6
-  data.tar.gz: 3f8e0e2d4d51811b69cef252a55d18a635d151794716b04661d55537138b17e1b6ab6e523272e1103a0f63bfe7089492c1103159f845e1589a68a7ca6362584a
+  metadata.gz: 5d6cefbd65bfa5770618d9d0182aa8de32a2b066911b04b394d7b08f5553282426ffa4d77d04fa2db52c65ae43dcea3c6807698ca381d5e59b0b749d2ddcac7a
+  data.tar.gz: e3f0c80c0af477a33751e1375b576281ad051aef7f9a88cad893e3de9334e28b375cefe3e4b1a6cecfd963859ae1fc25b88708b8dcafcd1c7b65546c21fb96b2

data/.travis.yml

@@ -1,5 +1,18 @@
 language: ruby
+script: bundle exec rake
+sudo: false
 rvm:
-  - 1.9.3
-  - 2.0.0
-  - 2.1.0
+  - 2.1
+  - 2.2
+  - 2.3
+  - 2.4
+  - 2.5
+  - 2.6
+  - 2.7
+  - 3.0
+
+  #- ruby-head
+gemfile:
+  - Gemfile
+before_install:
+  - gem update bundler

data/Gemfile

@@ -1,4 +1,3 @@
 source 'https://rubygems.org'
 
-# Specify your gem's dependencies in fluent-plugin-string-scrub.gemspec
 gemspec

data/README.md

@@ -27,7 +27,7 @@ Or install it yourself as:
 
 ```
 <match **>
-  type string_scrub
+  @type string_scrub
   tag scrubbed.string
   replace_char ?
 </match>
@@ -37,23 +37,43 @@ Or install it yourself as:
 
 ```
 <source>
-  type forward
+  @type forward
 </source>
 
 <match raw.**>
-  type string_scrub
+  @type string_scrub
   remove_prefix raw
   add_prefix scrubbed
 </match>
 
 <match scrubbed.**>
-  type stdout
+  @type stdout
 </match>
 ```
 
+## Filter plugin
+
+Fluentd >= v0.12 can use filter plugin.
+
+```
+<source>
+  @type forward
+</source>
+
+<filter **>
+  @type string_scrub
+  replace_char ?
+</filter>
+
+<match **>
+  @type stdout
+</match>
+```
+
+
 ## Contributing
 
-1. Fork it ( https://github.com/[my-github-username]/fluent-plugin-string-scrub/fork )
+1. Fork it ( https://github.com/kataring/fluent-plugin-string-scrub/fork )
 2. Create your feature branch (`git checkout -b my-new-feature`)
 3. Commit your changes (`git commit -am 'Add some feature'`)
 4. Push to the branch (`git push origin my-new-feature`)

data/fluent-plugin-string-scrub.gemspec

@@ -3,7 +3,7 @@ lib = File.expand_path('../lib', __FILE__)
 
 Gem::Specification.new do |spec|
   spec.name          = "fluent-plugin-string-scrub"
-  spec.version       = "0.0.3"
+  spec.version       = "1.0.1"
   spec.authors       = ["Noriaki Katayama"]
   spec.email         = ["kataring@gmail.com"]
   spec.summary       = %q{Fluentd Output filter plugin.}
@@ -15,9 +15,9 @@ Gem::Specification.new do |spec|
   spec.executables   = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
   spec.test_files    = spec.files.grep(%r{^(test|spec|features)/})
   spec.require_paths = ["lib"]
+  spec.required_ruby_version = Gem::Requirement.new(">= 2.1".freeze)
 
   spec.add_development_dependency "rake"
   spec.add_development_dependency "test-unit"
-  spec.add_development_dependency "fluentd"
-  spec.add_runtime_dependency "string-scrub", "~> 0.0.5" if RUBY_VERSION.to_f < 2.1
+  spec.add_development_dependency "fluentd", [">= 0.14.0", "< 2"]
 end

data/lib/fluent/plugin/filter_string_scrub.rb

@@ -0,0 +1,63 @@
+require 'fluent/plugin/filter'
+
+class Fluent::Plugin::StringScrubFilter < Fluent::Plugin::Filter
+  Fluent::Plugin.register_filter('string_scrub', self)
+
+  config_param :replace_char, :string, :default => ''
+
+  def initialize
+    super
+  end
+
+  def configure(conf)
+    super
+
+    if @replace_char =~ /\\u\{*[A-F0-9]{4}\}*/
+      @replace_char = eval("\"#{@replace_char}\"")
+    end
+  end
+
+  def filter_stream(tag, es)
+    new_es = Fluent::MultiEventStream.new
+    es.each do |time,record|
+      begin
+        scrubbed = recv_record(record)
+        next if scrubbed.nil?
+        new_es.add(time, scrubbed)
+      rescue => e
+        router.emit_error_event(tag, time, record, e)
+      end
+    end
+
+    new_es
+  end
+
+  def recv_record(record)
+    scrubbed = {}
+    record.each do |k,v|
+      if v.instance_of? Hash
+        scrubbed[with_scrub(k)] = recv_record(v)
+      elsif v.instance_of? Integer
+        scrubbed[k] = v
+      else
+        scrubbed[with_scrub(k)] = with_scrub(v)
+      end
+    end
+    scrubbed
+  end
+
+  def with_scrub(string)
+    begin
+      string =~ //
+      return string
+    rescue ArgumentError => e
+      raise e unless e.message.index("invalid byte sequence in") == 0
+      if string.frozen?
+          string = string.dup.scrub!(@replace_char)
+      else
+          string.scrub!(@replace_char)
+      end
+      retry
+    end
+  end
+end

data/lib/fluent/plugin/out_string_scrub.rb

@@ -1,6 +1,10 @@
-class Fluent::StringScrubOutput < Fluent::Output
+require 'fluent/plugin/output'
+
+class Fluent::Plugin::StringScrubOutput < Fluent::Plugin::Output
   Fluent::Plugin.register_output('string_scrub', self)
 
+  helpers :event_emitter
+
   config_param :tag, :string, :default => nil
   config_param :remove_prefix, :string, :default => nil
   config_param :add_prefix, :string, :default => nil
@@ -8,31 +12,33 @@ class Fluent::StringScrubOutput < Fluent::Output
 
   def initialize
     super
-    require 'string/scrub' if RUBY_VERSION.to_f < 2.1
   end
 
   def configure(conf)
     super
 
-    if not @tag and not @remove_prefix and not @add_prefix
-      raise Fluent::ConfigError, "missing both of remove_prefix and add_prefix"
-    end
-    if @tag and (@remove_prefix or @add_prefix)
-      raise Fluent::ConfigError, "both of tag and remove_prefix/add_prefix must not be specified"
-    end
-    if @remove_prefix
-      @removed_prefix_string = @remove_prefix + '.'
-      @removed_length = @removed_prefix_string.length
-    end
-    if @add_prefix
-      @added_prefix_string = @add_prefix + '.'
+    if conf['@label'].nil?
+      if not @tag and not @remove_prefix and not @add_prefix
+         raise Fluent::ConfigError, "missing both of remove_prefix and add_prefix"
+      end
+      if @tag and (@remove_prefix or @add_prefix)
+          raise Fluent::ConfigError, "both of tag and remove_prefix/add_prefix must not be specified"
+      end
+      if @remove_prefix
+          @removed_prefix_string = @remove_prefix + '.'
+          @removed_length = @removed_prefix_string.length
+      end
+      if @add_prefix
+        @added_prefix_string = @add_prefix + '.'
+      end
     end
+
     if @replace_char =~ /\\u\{*[A-F0-9]{4}\}*/
       @replace_char = eval("\"#{@replace_char}\"")
     end
   end
 
-  def emit(tag, es, chain)
+  def process(tag, es)
     tag = if @tag
             @tag
           else
@@ -53,10 +59,8 @@ class Fluent::StringScrubOutput < Fluent::Output
     es.each do |time,record|
       scrubbed = recv_record(record)
       next if scrubbed.nil?
-      Fluent::Engine.emit(tag, time, scrubbed)
+      router.emit(tag, time, scrubbed)
     end
-
-    chain.next
   end
 
   def recv_record(record)
@@ -64,6 +68,8 @@ class Fluent::StringScrubOutput < Fluent::Output
     record.each do |k,v|
       if v.instance_of? Hash
         scrubbed[with_scrub(k)] = recv_record(v)
+      elsif v.instance_of? Integer
+        scrubbed[k] = v
       else
         scrubbed[with_scrub(k)] = with_scrub(v)
       end
@@ -77,7 +83,11 @@ class Fluent::StringScrubOutput < Fluent::Output
       return string
     rescue ArgumentError => e
       raise e unless e.message.index("invalid byte sequence in") == 0
-      string.scrub!(@replace_char)
+      if string.frozen?
+          string = string.dup.scrub!(@replace_char)
+      else
+          string.scrub!(@replace_char)
+      end
       retry
     end
   end

data/test/helper.rb

@@ -12,17 +12,11 @@ require 'test/unit'
 $LOAD_PATH.unshift(File.join(File.dirname(__FILE__), '..', 'lib'))
 $LOAD_PATH.unshift(File.dirname(__FILE__))
 require 'fluent/test'
-unless ENV.has_key?('VERBOSE')
-  nulllogger = Object.new
-  nulllogger.instance_eval {|obj|
-    def method_missing(method, *args)
-      # pass
-    end
-  }
-  $log = nulllogger
-end
-
+require 'fluent/version'
+require 'fluent/test/driver/output'
+require 'fluent/test/driver/filter'
 require 'fluent/plugin/out_string_scrub'
+require 'fluent/plugin/filter_string_scrub'
 
 class Test::Unit::TestCase
 end

data/test/plugin/test_filter_string-scrub.rb

@@ -0,0 +1,59 @@
+require 'helper'
+
+class StringScrubFilterTest < Test::Unit::TestCase
+  include Fluent
+
+  def setup
+    Fluent::Test.setup
+    @time = Fluent::Engine.now
+  end
+
+  CONFIG = %[
+    replace_char ?
+  ]
+
+  CONFIG_UNICODE_1 = %[
+    replace_char \uFFFD
+  ]
+
+  CONFIG_UNICODE_2 = %[
+    replace_char \u{FFFD}
+  ]
+
+  def create_driver(conf=CONFIG)
+    Fluent::Test::Driver::Filter.new(Fluent::Plugin::StringScrubFilter).configure(conf)
+  end
+
+  def filter(config, msgs)
+    d = create_driver(config)
+    d.run(default_tag: 'test.filter') {
+      msgs.each {|msg|
+        d.feed(@time, msg)
+      }
+    }
+    filtered = d.filtered
+    filtered.map {|m| m[1] }
+  end
+
+  def test_filter1
+    orig_message = 'testtesttest'
+    invalid_utf8 = "\xff".force_encoding('UTF-8')
+    msg = {"message" => orig_message + invalid_utf8}
+    filtered = filter(CONFIG, [msg])
+    assert_equal([{"message" => orig_message + '?'}], filtered)
+  end
+
+  def test_filter2_frozen_string
+    orig_message = 'testtesttest'
+    invalid_utf8 = "\xff".force_encoding('UTF-8')
+    msg = {"message" => (orig_message + invalid_utf8).freeze}
+    filtered = filter(CONFIG, [msg])
+    assert_equal([{"message" => orig_message + '?'}], filtered)
+
+  def test_filter_integer
+    orig_message = 123456789
+    msg = {"message" => orig_message}
+    filtered = filter(CONFIG, [msg])
+    assert_equal([{"message" => orig_message}], filtered)
+  end
+end

data/test/plugin/test_out_string-scrub.rb

@@ -28,8 +28,8 @@ class StringScrubOutputTest < Test::Unit::TestCase
     replace_char \u{FFFD}
   ]
 
-  def create_driver(conf=CONFIG,tag='test')
-    Fluent::Test::OutputTestDriver.new(Fluent::StringScrubOutput, tag).configure(conf)
+  def create_driver(conf = CONFIG)
+    Fluent::Test::Driver::Output.new(Fluent::Plugin::StringScrubOutput).configure(conf)
   end
 
   def test_configure
@@ -81,25 +81,25 @@ class StringScrubOutputTest < Test::Unit::TestCase
   def test_emit1_invalid_string
     orig_message = 'testtesttest'
     invalid_utf8 = "\xff".force_encoding('UTF-8')
-    d1 = create_driver(CONFIG, 'input.log')
-    d1.run do
-      d1.emit({'message' => orig_message + invalid_utf8})
+    d1 = create_driver(CONFIG)
+    d1.run(default_tag: 'input.log') do
+      d1.feed({'message' => orig_message + invalid_utf8})
     end
-    emits = d1.emits
+    emits = d1.events
     assert_equal 1, emits.length
-    
+
     e1 = emits[0]
     assert_equal "scrubbed.log", e1[0]
     assert_equal orig_message, e1[2]['message']
 
     invalid_ascii = "\xff".force_encoding('US-ASCII')
-    d2 = create_driver(CONFIG, 'input.log2')
-    d2.run do
-      d2.emit({'message' => orig_message + invalid_utf8})
+    d2 = create_driver(CONFIG)
+    d2.run(default_tag: 'input.log2') do
+      d2.feed({'message' => orig_message + invalid_utf8})
     end
-    emits = d2.emits
+    emits = d2.events
     assert_equal 1, emits.length
-    
+
     e2 = emits[0]
     assert_equal "scrubbed.log2", e2[0]
     assert_equal orig_message, e2[2]['message']
@@ -108,11 +108,11 @@ class StringScrubOutputTest < Test::Unit::TestCase
   def test_emit2_replace_char
     orig_message = 'testtesttest'
     invalid_utf8 = "\xff".force_encoding('UTF-8')
-    d1 = create_driver(CONFIG_REPLACE_CHAR, 'input.log')
-    d1.run do
-      d1.emit({'message' => orig_message + invalid_utf8})
+    d1 = create_driver(CONFIG_REPLACE_CHAR)
+    d1.run(default_tag: 'input.log') do
+      d1.feed({'message' => orig_message + invalid_utf8})
     end
-    emits = d1.emits
+    emits = d1.events
     assert_equal 1, emits.length
 
     e1 = emits[0]
@@ -120,11 +120,11 @@ class StringScrubOutputTest < Test::Unit::TestCase
     assert_equal orig_message + '?', e1[2]['message']
 
     invalid_ascii = "\xff".force_encoding('US-ASCII')
-    d2 = create_driver(CONFIG_REPLACE_CHAR, 'input.log2')
-    d2.run do
-        d2.emit({'message' => orig_message + invalid_utf8})
+    d2 = create_driver(CONFIG_REPLACE_CHAR)
+    d2.run(default_tag: 'input.log2') do
+        d2.feed({'message' => orig_message + invalid_utf8})
     end
-    emits = d2.emits
+    emits = d2.events
     assert_equal 1, emits.length
 
     e2 = emits[0]
@@ -135,11 +135,11 @@ class StringScrubOutputTest < Test::Unit::TestCase
   def test_emit3_struct_message
     orig_message = 'testtesttest'
     invalid_utf8 = "\xff".force_encoding('UTF-8')
-    d1 = create_driver(CONFIG_REPLACE_CHAR, 'input.log')
-    d1.run do
-      d1.emit({'message' => {'message_child' => orig_message + invalid_utf8}})
+    d1 = create_driver(CONFIG_REPLACE_CHAR)
+    d1.run(default_tag: 'input.log') do
+      d1.feed({'message' => {'message_child' => orig_message + invalid_utf8}})
     end
-    emits = d1.emits
+    emits = d1.events
     assert_equal 1, emits.length
 
     e1 = emits[0]
@@ -150,11 +150,11 @@ class StringScrubOutputTest < Test::Unit::TestCase
   def test_emit4_unicode1
     orig_message = 'testtesttest'
     invalid_utf8 = "\xff".force_encoding('UTF-8')
-    d1 = create_driver(CONFIG_UNICODE_1, 'input.log')
-    d1.run do
-      d1.emit({'message' => {'message_child' => orig_message + invalid_utf8}})
+    d1 = create_driver(CONFIG_UNICODE_1)
+    d1.run(default_tag: 'input.log') do
+      d1.feed({'message' => {'message_child' => orig_message + invalid_utf8}})
     end
-    emits = d1.emits
+    emits = d1.events
     assert_equal 1, emits.length
 
     e1 = emits[0]
@@ -163,15 +163,56 @@ class StringScrubOutputTest < Test::Unit::TestCase
 
     orig_message = 'testtesttest'
     invalid_utf8 = "\xff".force_encoding('UTF-8')
-    d1 = create_driver(CONFIG_UNICODE_2, 'input.log')
-    d1.run do
-      d1.emit({'message' => {'message_child' => orig_message + invalid_utf8}})
+    d1 = create_driver(CONFIG_UNICODE_2)
+    d1.run(default_tag: 'input.log') do
+      d1.feed({'message' => {'message_child' => orig_message + invalid_utf8}})
     end
-    emits = d1.emits
+    emits = d1.events
     assert_equal 1, emits.length
 
     e1 = emits[0]
     assert_equal "scrubbed.log", e1[0]
     assert_equal orig_message + "\uFFFD".force_encoding('UTF-8'), e1[2]['message']['message_child']
   end
+
+  def test_emit5_frozen_string
+    orig_message = 'testtesttest'
+    invalid_utf8 = "\xff".force_encoding('UTF-8')
+    d1 = create_driver(CONFIG)
+    d1.run(default_tag: 'input.log') do
+      d1.feed({'message' => (orig_message + invalid_utf8).freeze})
+    end
+    emits = d1.events
+    assert_equal 1, emits.length
+
+    e1 = emits[0]
+    assert_equal "scrubbed.log", e1[0]
+    assert_equal orig_message, e1[2]['message']
+
+    invalid_ascii = "\xff".force_encoding('US-ASCII')
+    d2 = create_driver(CONFIG)
+    d2.run(default_tag: 'input.log2') do
+      d2.feed({'message' => (orig_message + invalid_utf8).freeze})
+    end
+    emits = d2.events
+    assert_equal 1, emits.length
+
+    e2 = emits[0]
+    assert_equal "scrubbed.log2", e2[0]
+    assert_equal orig_message, e2[2]['message']
+  end
+
+  def test_emit6_integer
+    orig_message = 123456789
+    d1 = create_driver(CONFIG)
+    d1.run(default_tag: 'input.log') do
+      d1.feed({'message' => orig_message})
+    end
+    emits = d1.events
+    assert_equal 1, emits.length
+
+    e1 = emits[0]
+    assert_equal "scrubbed.log", e1[0]
+    assert_equal orig_message, e1[2]['message']
+  end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-string-scrub
 version: !ruby/object:Gem::Version
-  version: 0.0.3
+  version: 1.0.1
 platform: ruby
 authors:
 - Noriaki Katayama
-autorequire: 
+autorequire:
 bindir: bin
 cert_chain: []
-date: 2015-02-09 00:00:00.000000000 Z
+date: 2021-12-04 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rake
@@ -44,14 +44,20 @@ dependencies:
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: '0'
+        version: 0.14.0
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '2'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: '0'
+        version: 0.14.0
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '2'
 description: fluent plugin for string scrub.
 email:
 - kataring@gmail.com
@@ -66,14 +72,16 @@ files:
 - README.md
 - Rakefile
 - fluent-plugin-string-scrub.gemspec
+- lib/fluent/plugin/filter_string_scrub.rb
 - lib/fluent/plugin/out_string_scrub.rb
 - test/helper.rb
+- test/plugin/test_filter_string-scrub.rb
 - test/plugin/test_out_string-scrub.rb
 homepage: https://github.com/kataring/fluent-plugin-string-scrub
 licenses:
 - MIT
 metadata: {}
-post_install_message: 
+post_install_message:
 rdoc_options: []
 require_paths:
 - lib
@@ -81,18 +89,18 @@ required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
-      version: '0'
+      version: '2.1'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubyforge_project: 
-rubygems_version: 2.2.2
-signing_key: 
+rubygems_version: 3.2.22
+signing_key:
 specification_version: 4
 summary: Fluentd Output filter plugin.
 test_files:
 - test/helper.rb
+- test/plugin/test_filter_string-scrub.rb
 - test/plugin/test_out_string-scrub.rb