fluent-plugin-grep 0.0.1 → 0.0.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 8bc4b1bc5d03ba51926b8d8dbc5039c0a45696c3
-  data.tar.gz: 4e6e77f9ef105730ee530720773ee98cc54f7e1c
+  metadata.gz: 01c08ca3372864d443be685dbe0ef91252644612
+  data.tar.gz: 6f7ec217ec20b890536a09de99a7fc28261c69e5
 SHA512:
-  metadata.gz: a9e7fdf029afe36c65abd1016c85ffbaa31701f0b1ba4e678c63e7e986fa79766b21c391c30120936d9f8120689e0ea81f52e749a459da11c48ab50402adb11e
-  data.tar.gz: 17984c87fc1b8ab48b4b7b334fa328c9ad6e702035188843b44d2bb2687ba3dd0765fdb28a9b5a437b150d3b911ca099e6d6fb19a51bcc495eb5dc08eadda449
+  metadata.gz: 7b9b72ba0eb1ae64d472f18f8c57afb53ec0f7383f20d5a2c68bd7face9eac46761376542d79b40206fe2acdfb36e786dd8008020153c7021a911c0300d1f24b
+  data.tar.gz: 444709c11deead3423c8b5601744ecf6b420f1f95277b4fb2ff10627875a5172b52023425d5da8fd11a7619139b0fb69961f0c7a344e2bf25dbfb3b0e79185f0

data/CHANGELOG.md

@@ -1,3 +1,9 @@
+## 0.0.2
+
+Features:
+
+- Add `replace_invalid_sequence` option
+
 ## 0.0.1
 
 First version

data/README.md

@@ -26,6 +26,12 @@ Then, output bocomes as belows:
     grep.syslog.host1: {"foo":"bar","message":"2013/01/13T07:02:13.232645 WARN POST /auth"}
     grep.syslog.host1: {"foo":"bar","message":"2013/01/13T07:02:43.632145 WARN POST /login"}
 
+## Parameters
+
+- replace_invalid_sequence
+
+    Replace invalid byte sequence in UTF-8 with '?' character if `true`
+
 ## ChangeLog
 
 See [CHANGELOG.md](CHANGELOG.md) for details.

data/fluent-plugin-grep.gemspec

@@ -3,7 +3,7 @@ $:.push File.expand_path("../lib", __FILE__)
 
 Gem::Specification.new do |s|
   s.name        = "fluent-plugin-grep"
-  s.version     = "0.0.1"
+  s.version     = "0.0.2"
   s.authors     = ["Naotoshi SEO"]
   s.email       = ["sonots@gmail.com"]
   s.homepage    = "https://github.com/sonots/fluent-plugin-grep"

data/lib/fluent/plugin/out_grep.rb

@@ -6,6 +6,7 @@ class Fluent::GrepOutput < Fluent::Output
   config_param :exclude, :string, :default => nil
   config_param :tag, :string, :default => nil
   config_param :add_tag_prefix, :string, :default => 'grep'
+  config_param :replace_invalid_sequence, :bool, :default => false
 
   def configure(conf)
     super
@@ -20,11 +21,35 @@ class Fluent::GrepOutput < Fluent::Output
 
     es.each do |time,record|
       value = record[@input_key]
-      next if @regexp and !@regexp.match(value)
-      next if @exclude and @exclude.match(value)
+      next unless match(value)
       Fluent::Engine.emit(emit_tag, time, record)
     end
 
     chain.next
   end
+
+  private
+
+  def match(string)
+    begin
+      return false if @regexp and !@regexp.match(string)
+      return false if @exclude and @exclude.match(string)
+    rescue ArgumentError => e
+      unless e.message.index("invalid byte sequence in") == 0
+        raise
+      end
+      string = replace_invalid_byte(string)
+      return false if @regexp and !@regexp.match(string)
+      return false if @exclude and @exclude.match(string)
+    end
+    return true
+  end
+
+  def replace_invalid_byte(string)
+    replace_options = { invalid: :replace, undef: :replace, replace: '?' }
+    original_encoding = string.encoding
+    temporal_encoding = (original_encoding == Encoding::UTF_8 ? Encoding::UTF_16BE : Encoding::UTF_8)
+    string.encode(temporal_encoding, original_encoding, replace_options).encode(original_encoding)
+  end
+
 end

data/spec/out_grep_spec.rb

@@ -114,6 +114,24 @@ describe Fluent::GrepOutput do
       end
       it { emit }
     end
+
+    context 'replace_invalid_sequence' do
+      let(:config) do
+        CONFIG + %[
+          regexp WARN
+          replace_invalid_sequence true
+        ]
+      end
+      let(:messages) do
+        [
+          "\xff".force_encoding('UTF-8'),
+        ]
+      end
+      before do
+        Fluent::Engine.stub(:now).and_return(time)
+      end
+      it { expect { emit }.not_to raise_error(ArgumentError) }
+    end
   end
 
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-grep
 version: !ruby/object:Gem::Version
-  version: 0.0.1
+  version: 0.0.2
 platform: ruby
 authors:
 - Naotoshi SEO