fluent-plugin-string-scrub 0.0.4 → 0.1.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 73d0da5e8e95386a46ed8d332e3ad698448b2c5f
-  data.tar.gz: 64bd7f44ed8164404d4793c6c947f8dff9cd86ab
+  metadata.gz: 22a7e6ce42d946054cdfb80294dbc339215242e5
+  data.tar.gz: 282dccc73abfd4f2c14209a3478f9d3bbe733ccd
 SHA512:
-  metadata.gz: c960490269222cdc16e9ce8d5c1addd1747436b73ed0cc9ef099c0d33989ec148cf9f133620b03f25cb9d070a996ec9dfff110b61f3e71891ae870be56d23aeb
-  data.tar.gz: 0b1443b6c15cd8ae04915f73230ca9287a2b0b033c266a446ad97eaa958ac488c07095d3336e8b43f6b99ce89e434ef85ab004e8fe908bf5f4d8ebf420560d0f
+  metadata.gz: 6f812ab0fbe3bc87f5e8a46e1a3d715a2da4950e0334631ca456606dbd77b8acd027517a334bea0afcb9e3a75180a5a746c4b0bd1dd82290620f62cb8fc93136
+  data.tar.gz: 8646f2982fe4ca357512f739f67d246c3a6808584798b6f64f16167316a8fab8e95725a9212109b5023fc2d9e0caeba6825def2d490e3e6f388c43f6db695147

data/.travis.yml

@@ -2,4 +2,7 @@ language: ruby
 rvm:
   - 1.9.3
   - 2.0.0
-  - 2.1.0
+  - 2.1.*
+gemfile:
+  - Gemfile
+  - Gemfile.fluentd.lt.0.12

data/Gemfile.fluentd.lt.0.12

@@ -0,0 +1,4 @@
+source "http://rubygems.org"
+
+gem 'fluentd', '~> 0.10.43'
+gemspec

data/README.md

@@ -51,6 +51,26 @@ Or install it yourself as:
 </match>
 ```
 
+## Filter plugin
+
+Fluentd >= v0.12 can use filter plugin.
+
+```
+<source>
+  type forward
+</source>
+
+<filter **>
+  type string_scrub
+  replace_char ?
+</filter>
+
+<match **>
+  type stdout
+</match>
+```
+
+
 ## Contributing
 
 1. Fork it ( https://github.com/[my-github-username]/fluent-plugin-string-scrub/fork )

data/fluent-plugin-string-scrub.gemspec

@@ -3,7 +3,7 @@ lib = File.expand_path('../lib', __FILE__)
 
 Gem::Specification.new do |spec|
   spec.name          = "fluent-plugin-string-scrub"
-  spec.version       = "0.0.4"
+  spec.version       = "0.1.0"
   spec.authors       = ["Noriaki Katayama"]
   spec.email         = ["kataring@gmail.com"]
   spec.summary       = %q{Fluentd Output filter plugin.}

data/lib/fluent/plugin/filter_string_scrub.rb

@@ -0,0 +1,60 @@
+class Fluent::StringScrubFilter < Fluent::Filter
+  Fluent::Plugin.register_filter('string_scrub', self)
+
+  config_param :replace_char, :string, :default => ''
+
+  def initialize
+    super
+    require 'string/scrub' if RUBY_VERSION.to_f < 2.1
+  end
+
+  def configure(conf)
+    super
+
+    if @replace_char =~ /\\u\{*[A-F0-9]{4}\}*/
+      @replace_char = eval("\"#{@replace_char}\"")
+    end
+  end
+
+  def filter_stream(tag, es)
+    new_es = Fluent::MultiEventStream.new
+    es.each do |time,record|
+      begin
+        scrubbed = recv_record(record)
+        next if scrubbed.nil?
+        new_es.add(time, record)
+      rescue => e
+        router.emit_error_event(tag, time, record, e)
+      end
+    end
+
+    new_es
+  end
+
+  def recv_record(record)
+    scrubbed = {}
+    record.each do |k,v|
+      if v.instance_of? Hash
+        scrubbed[with_scrub(k)] = recv_record(v)
+      else
+        scrubbed[with_scrub(k)] = with_scrub(v)
+      end
+    end
+    scrubbed
+  end
+
+  def with_scrub(string)
+    begin
+      string =~ //
+      return string
+    rescue ArgumentError => e
+      raise e unless e.message.index("invalid byte sequence in") == 0
+      if string.frozen?
+          string = string.dup.scrub!(@replace_char)
+      else
+          string.scrub!(@replace_char)
+      end
+      retry
+    end
+  end
+end if defined?(Fluent::Filter) # Support only >= v0.12

data/lib/fluent/plugin/out_string_scrub.rb

@@ -6,6 +6,11 @@ class Fluent::StringScrubOutput < Fluent::Output
   config_param :add_prefix, :string, :default => nil
   config_param :replace_char, :string, :default => ''
 
+  # Define `router` method of v0.12 to support v0.10 or earlier
+  unless method_defined?(:router)
+    define_method("router") { Fluent::Engine }
+  end
+
   def initialize
     super
     require 'string/scrub' if RUBY_VERSION.to_f < 2.1
@@ -14,19 +19,22 @@ class Fluent::StringScrubOutput < Fluent::Output
   def configure(conf)
     super
 
-    if not @tag and not @remove_prefix and not @add_prefix
-      raise Fluent::ConfigError, "missing both of remove_prefix and add_prefix"
-    end
-    if @tag and (@remove_prefix or @add_prefix)
-      raise Fluent::ConfigError, "both of tag and remove_prefix/add_prefix must not be specified"
-    end
-    if @remove_prefix
-      @removed_prefix_string = @remove_prefix + '.'
-      @removed_length = @removed_prefix_string.length
-    end
-    if @add_prefix
-      @added_prefix_string = @add_prefix + '.'
+    if conf['@label'].nil?
+      if not @tag and not @remove_prefix and not @add_prefix
+         raise Fluent::ConfigError, "missing both of remove_prefix and add_prefix"
+      end
+      if @tag and (@remove_prefix or @add_prefix)
+          raise Fluent::ConfigError, "both of tag and remove_prefix/add_prefix must not be specified"
+      end
+      if @remove_prefix
+          @removed_prefix_string = @remove_prefix + '.'
+          @removed_length = @removed_prefix_string.length
+      end
+      if @add_prefix
+        @added_prefix_string = @add_prefix + '.'
+      end
     end
+
     if @replace_char =~ /\\u\{*[A-F0-9]{4}\}*/
       @replace_char = eval("\"#{@replace_char}\"")
     end
@@ -53,7 +61,7 @@ class Fluent::StringScrubOutput < Fluent::Output
     es.each do |time,record|
       scrubbed = recv_record(record)
       next if scrubbed.nil?
-      Fluent::Engine.emit(tag, time, scrubbed)
+      router.emit(tag, time, scrubbed)
     end
 
     chain.next

data/test/helper.rb

@@ -23,6 +23,7 @@ unless ENV.has_key?('VERBOSE')
 end
 
 require 'fluent/plugin/out_string_scrub'
+require 'fluent/plugin/filter_string_scrub'
 
 class Test::Unit::TestCase
 end

data/test/plugin/test_filter_string-scrub.rb

@@ -0,0 +1,48 @@
+require 'helper'
+
+class StringScrubFilterTest < Test::Unit::TestCase
+  include Fluent
+
+  def setup
+    Fluent::Test.setup
+    @time = Fluent::Engine.now
+  end
+
+  CONFIG = %[
+    replace_char ?
+  ]
+
+  CONFIG_UNICODE_1 = %[
+    replace_char \uFFFD
+  ]
+
+  CONFIG_UNICODE_2 = %[
+    replace_char \u{FFFD}
+  ]
+
+  def create_driver(conf=CONFIG, tag='test.filter')
+    Fluent::Test::FilterTestDriver.new(Fluent::StringScrubFilter).configure(conf, tag)
+  end
+
+  def filter(config, msgs)
+    d = create_driver(config)
+    d.run {
+      msgs.each {|msg|
+        d.filter(msg, @time)
+      }
+    }
+    filtered = d.filtered_as_array
+    filtered.map {|m| m[2] }
+  end
+
+  def test_filter1
+    return unless defined? Fluent::Filter
+
+    orig_message = 'testtesttest'
+    invalid_utf8 = "\xff".force_encoding('UTF-8')
+    msg = {"message" => orig_message + invalid_utf8}
+    filtered = filter(CONFIG, [msg])
+    assert_equal([{"message" => orig_message + '?'}], filtered)
+  end
+
+end

data/test/plugin/test_out_string-scrub.rb

@@ -28,8 +28,8 @@ class StringScrubOutputTest < Test::Unit::TestCase
     replace_char \u{FFFD}
   ]
 
-  def create_driver(conf=CONFIG,tag='test')
-    Fluent::Test::OutputTestDriver.new(Fluent::StringScrubOutput, tag).configure(conf)
+  def create_driver(conf = CONFIG, tag = 'test', use_v1_config = true)
+    Fluent::Test::OutputTestDriver.new(Fluent::StringScrubOutput, tag).configure(conf, use_v1_config)
   end
 
   def test_configure
@@ -78,6 +78,26 @@ class StringScrubOutputTest < Test::Unit::TestCase
     }
   end
 
+  def test_emit_fluentd_0_12
+    if Fluent::VERSION >= "0.12"
+      in_tag = 'input.log'
+      orig_message = 'testtesttest'
+      Fluent::Engine.root_agent.add_label('@foo')
+      d = create_driver(%[
+                        @label @foo
+                        ], in_tag)
+      label = Fluent::Engine.root_agent.find_label('@foo')
+      assert_equal(label.event_router, d.instance.router)
+
+      d.run do
+        d.emit({'message' => orig_message})
+      end
+      emits = d.emits
+      tag, time, record = emits.first
+      assert_equal(in_tag, tag)
+    end
+  end
+
   def test_emit1_invalid_string
     orig_message = 'testtesttest'
     invalid_utf8 = "\xff".force_encoding('UTF-8')

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-string-scrub
 version: !ruby/object:Gem::Version
-  version: 0.0.4
+  version: 0.1.0
 platform: ruby
 authors:
 - Noriaki Katayama
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2015-03-24 00:00:00.000000000 Z
+date: 2015-05-22 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rake
@@ -62,12 +62,15 @@ files:
 - ".gitignore"
 - ".travis.yml"
 - Gemfile
+- Gemfile.fluentd.lt.0.12
 - LICENSE.txt
 - README.md
 - Rakefile
 - fluent-plugin-string-scrub.gemspec
+- lib/fluent/plugin/filter_string_scrub.rb
 - lib/fluent/plugin/out_string_scrub.rb
 - test/helper.rb
+- test/plugin/test_filter_string-scrub.rb
 - test/plugin/test_out_string-scrub.rb
 homepage: https://github.com/kataring/fluent-plugin-string-scrub
 licenses:
@@ -95,4 +98,5 @@ specification_version: 4
 summary: Fluentd Output filter plugin.
 test_files:
 - test/helper.rb
+- test/plugin/test_filter_string-scrub.rb
 - test/plugin/test_out_string-scrub.rb