fluent-plugin-grepcounter 0.1.1 → 0.1.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 6e975b60c3a77833ed72a229d79c7c0cc64efa5a
-  data.tar.gz: 2aa97645c4645e08b2d0ce5cb0fc6576a688b53c
+  metadata.gz: 55fb88fa0e4693f083e614072f4e4b39a977e4dc
+  data.tar.gz: 6a34586f73d49333d97fddd3ce51c33c2033c40b
 SHA512:
-  metadata.gz: df73667b60738771cc4adb82d4e79dc48defd6f9b8db5056787b0e71721cb897065b69bf71a6bc3cd50414ece433d718aecbf8dde8de7685ea19cd150cfc6dfe
-  data.tar.gz: d70577b7bd642a1360b57b91a92f716cb4b470f3e26cd26fe68566ef1bf57536b827824cdb01a33ede8a0816862c86ef53d69de00a82c91606db61390ad7d1f0
+  metadata.gz: 01453ec71582eaed4d7a0d505dbcbfd93e3708debf9766ecb6c820679d5e85c4a1db0d8290981db628abc1fd6fc0d3ace701d097ee72ad57f722c178f1f1d65e
+  data.tar.gz: d3634e8a26b8dd3dbb04031248bc695c920d1987b7e2b62491cf36a5acd039d2ac683a19169025c7e795a360ca78d93890ffcccd15b9ccb7574cf7eda1ca15d7

data/CHANGELOG.md

@@ -1,3 +1,9 @@
+## 0.1.2 (2013/05/16)
+
+Features
+
+  - replace_invalid_sequence
+
 ## 0.1.1 (2013/05/05)
 
 Changes:

data/README.md

@@ -32,7 +32,7 @@ Then, output bocomes as belows (indented):
       "input_tag_last":"host1",
     }
 
-Another example of grepcounter configuration to use `output_delimiter`:
+Another example of grepcounter configuration to use `output_with_joined_delimiter`:
 
     <match syslog.**>
       type grepcounter
@@ -42,7 +42,7 @@ Another example of grepcounter configuration to use `output_delimiter`:
       exclude favicon.ico
       threshold 1
       add_tag_prefix warn.count
-      output_delimiter \n
+      output_with_joined_delimiter \n
     </source>
 
 Then, output bocomes as belows (indented). You can use the `message` field is joined with \n.

data/fluent-plugin-grepcounter.gemspec

@@ -3,7 +3,7 @@ $:.push File.expand_path("../lib", __FILE__)
 
 Gem::Specification.new do |s|
   s.name        = "fluent-plugin-grepcounter"
-  s.version     = "0.1.1"
+  s.version     = "0.1.2"
   s.authors     = ["Naotoshi SEO"]
   s.email       = ["sonots@gmail.com"]
   s.homepage    = "https://github.com/sonots/fluent-plugin-grepcounter"

data/lib/fluent/plugin/out_grepcounter.rb

@@ -11,6 +11,7 @@ class Fluent::GrepCounterOutput < Fluent::Output
   config_param :add_tag_prefix, :string, :default => 'count'
   config_param :output_with_joined_delimiter, :string, :default => nil
   config_param :aggregate, :string, :default => 'tag'
+  config_param :replace_invalid_sequence, :bool, :default => false
 
   attr_accessor :matches
   attr_accessor :last_checked
@@ -57,8 +58,7 @@ class Fluent::GrepCounterOutput < Fluent::Output
     # filter out and insert
     es.each do |time,record|
       value = record[@input_key]
-      next if @regexp and !@regexp.match(value)
-      next if @exclude and @exclude.match(value)
+      next unless match(value.to_s)
       matches << value
       count += 1
     end
@@ -71,6 +71,9 @@ class Fluent::GrepCounterOutput < Fluent::Output
     end
 
     chain.next
+  rescue => e
+    $log.warn e.message
+    $log.warn e.backtrace.join(', ')
   end
 
   # thread callback
@@ -79,10 +82,15 @@ class Fluent::GrepCounterOutput < Fluent::Output
     @last_checked = Fluent::Engine.now
     while true
       sleep 0.5
-      if Fluent::Engine.now - @last_checked >= @count_interval
-        now = Fluent::Engine.now
-        flush_emit(now - @last_checked)
-        @last_checked = now
+      begin
+        if Fluent::Engine.now - @last_checked >= @count_interval
+          now = Fluent::Engine.now
+          flush_emit(now - @last_checked)
+          @last_checked = now
+        end
+      rescue => e
+        $log.warn e.message
+        $log.warn e.backtrace.join(", ")
       end
     end
   end
@@ -112,6 +120,7 @@ class Fluent::GrepCounterOutput < Fluent::Output
   end
 
   def generate_output(count, matches, tag = nil)
+    return nil if count.nil?
     return nil if count < @threshold
     output = {}
     output['count'] = count
@@ -123,4 +132,25 @@ class Fluent::GrepCounterOutput < Fluent::Output
     output
   end
 
+  def match(string)
+    begin
+      return false if @regexp and !@regexp.match(string)
+      return false if @exclude and @exclude.match(string)
+    rescue ArgumentError => e
+      unless e.message.index("invalid byte sequence in") == 0
+        raise
+      end
+      string = replace_invalid_byte(string)
+      return false if @regexp and !@regexp.match(string)
+      return false if @exclude and @exclude.match(string)
+    end
+    return true
+  end
+
+  def replace_invalid_byte(string)
+    replace_options = { invalid: :replace, undef: :replace, replace: '?' }
+    original_encoding = string.encoding
+    temporal_encoding = (original_encoding == Encoding::UTF_8 ? Encoding::UTF_16BE : Encoding::UTF_8)
+    string.encode(temporal_encoding, original_encoding, replace_options).encode(original_encoding)
+  end
 end

data/spec/out_grepcounter_spec.rb

@@ -1,6 +1,13 @@
 # encoding: UTF-8
 require_relative 'spec_helper'
 
+class Fluent::Test::OutputTestDriver
+  def emit_with_tag(record, time=Time.now, tag = nil)
+    @tag = tag if tag
+    emit(record, time)
+  end
+end
+
 describe Fluent::GrepCounterOutput do
   before { Fluent::Test.setup }
   CONFIG = %[
@@ -202,6 +209,12 @@ describe Fluent::GrepCounterOutput do
     end
 
     context 'aggregate all' do
+      let(:emit) do
+        driver.run { messages.each {|message| driver.emit_with_tag({'message' => message}, time, 'foo.bar') } }
+        driver.run { messages.each {|message| driver.emit_with_tag({'message' => message}, time, 'foo.bar2') } }
+        driver.instance.flush_emit(0)
+      end
+
       let(:config) do
         CONFIG + %[
           regexp WARN
@@ -211,12 +224,30 @@ describe Fluent::GrepCounterOutput do
       end
       before do
         Fluent::Engine.stub(:now).and_return(time)
-        Fluent::Engine.should_receive(:emit).with("count", time, {"count"=>3, 
-          "message"=>["2013/01/13T07:02:13.232645 WARN POST /auth","2013/01/13T07:02:21.542145 WARN GET /favicon.ico","2013/01/13T07:02:43.632145 WARN POST /login"],
+        Fluent::Engine.should_receive(:emit).with("count", time, {"count"=>3*2,
+          "message"=>["2013/01/13T07:02:13.232645 WARN POST /auth","2013/01/13T07:02:21.542145 WARN GET /favicon.ico","2013/01/13T07:02:43.632145 WARN POST /login"]*2,
         })
       end
       it { emit }
     end
+
+    context 'replace_invalid_sequence' do
+      let(:config) do
+        CONFIG + %[
+          regexp WARN
+          replace_invalid_sequence true
+        ]
+      end
+      let(:messages) do
+        [
+          "\xff".force_encoding('UTF-8'),
+        ]
+      end
+      before do
+        Fluent::Engine.stub(:now).and_return(time)
+      end
+      it { expect { emit }.not_to raise_error(ArgumentError) }
+    end
   end
 end
 

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-grepcounter
 version: !ruby/object:Gem::Version
-  version: 0.1.1
+  version: 0.1.2
 platform: ruby
 authors:
 - Naotoshi SEO
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2013-05-05 00:00:00.000000000 Z
+date: 2013-05-16 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fluentd