fluentd 0.10.16 → 0.10.17

data/ChangeLog

@@ -1,4 +1,14 @@
 
+Release 0.10.17 - 2012/04/02
+
+* in_tail: don't ignore last line without LF [#44]
+* Added HandleTagNameMixin
+* Removed automake/conf style build support
+    Maintainig automake/conf style files are complex, but no user. We've decided
+    not to have those files anymore. Otherwise it's still used at rpm/deb packaging
+    scripts.
+
+
 Release 0.10.16 - 2012/03/26
 
 * Added in_gc plugin

data/VERSION

@@ -1 +1 @@
-0.10.16
+0.10.17

data/lib/fluent/mixin.rb

@@ -100,6 +100,32 @@ module RecordFilterMixin
   end
 end
 
+module HandleTagNameMixin
+  include RecordFilterMixin
+
+  attr_accessor :remove_tag_prefix, :remove_tag_suffix, :add_tag_prefix, :add_tag_suffix
+  def configure(conf)
+    super
+    if remove_tag_prefix = conf['remove_tag_prefix']
+      @remove_tag_prefix = Regexp.new('^' + Regexp.escape(remove_tag_prefix))
+    end
+
+    if remove_tag_suffix = conf['remove_tag_suffix']
+      @remove_tag_suffix = Regexp.new(Regexp.escape(remove_tag_suffix) + '$')
+    end
+
+    @add_tag_prefix = conf['add_tag_prefix']
+    @add_tag_suffix = conf['add_tag_suffix']
+  end
+
+  def filter_record(tag, time, record)
+    tag.sub!(@remove_tag_prefix, '') if @remove_tag_prefix
+    tag.sub!(@remove_tag_suffix, '') if @remove_tag_suffix
+    tag.insert(0, @add_tag_prefix) if @add_tag_prefix
+    tag << @add_tag_suffix if @add_tag_suffix
+    super(tag, time, record)
+  end
+end
 
 module SetTimeKeyMixin
   include RecordFilterMixin
@@ -152,7 +178,6 @@ module SetTimeKeyMixin
   end
 end
 
-
 module SetTagKeyMixin
   include RecordFilterMixin
 

data/lib/fluent/plugin/in_tail.rb

@@ -31,6 +31,8 @@ class TailInput < Input
   config_param :rotate_wait, :time, :default => 5
   config_param :pos_file, :string, :default => nil
 
+  attr_reader :paths
+
   def configure(conf)
     super
 
@@ -260,6 +262,7 @@ class TailInput < Input
         @pe = pe
         @receive_lines = receive_lines
         @buffer = ''
+        @iobuf = ''
       end
 
       attr_reader :io
@@ -267,18 +270,24 @@ class TailInput < Input
       def on_notify
         lines = []
 
-        while line = @io.gets
-          @buffer << line
-          @pos = @io.pos
-          unless @buffer[@buffer.length-1] == ?\n
+        while true
+          begin
+            if @buffer.empty?
+              @io.read_nonblock(2048, @buffer)
+            else
+              @buffer << @io.read_nonblock(2048, @iobuf)
+            end
+            while line = @buffer.slice!(/.*?\n/m)
+              lines << line
+            end
+          rescue EOFError
             break
           end
-          lines << line
         end
 
         unless lines.empty?
-          @pe.update_pos(@pos)
           @receive_lines.call(lines)
+          @pe.update_pos(@io.pos - @buffer.bytesize)
         end
       rescue
         $log.error $!.to_s

data/lib/fluent/version.rb

@@ -1,5 +1,5 @@
 module Fluent
 
-VERSION = '0.10.16'
+VERSION = '0.10.17'
 
 end

data/test/mixin.rb

@@ -155,4 +155,108 @@ module MixinTest
       d.run
     end
   end
+
+  class HandleTagMixinTest < Test::Unit::TestCase
+    include Utils
+
+    def test_add_tag_prefix
+      format_check({
+        'a' => 1
+      }, 'tag_prefix.test')
+
+      d = create_driver(Fluent::HandleTagNameMixin, %[
+        add_tag_prefix tag_prefix.
+        include_tag_key true
+      ])
+
+      d.emit({'a' => 1})
+      d.run
+    end
+
+    def test_add_tag_suffix
+      format_check({
+        'a' => 1
+      }, 'test.test_suffix')
+
+      d = create_driver(Fluent::HandleTagNameMixin, %[
+        add_tag_suffix .test_suffix
+        include_tag_key true
+      ])
+
+      d.emit({'a' => 1})
+      d.run
+    end
+
+    def test_remove_tag_prefix
+      format_check({
+        'a' => 1
+      }, 'st')
+
+      d = create_driver(Fluent::HandleTagNameMixin, %[
+        remove_tag_prefix te
+        include_tag_key true
+      ])
+
+      d.emit({'a' => 1})
+      d.run
+    end
+
+    def test_remove_tag_suffix
+      format_check({
+        'a' => 1
+      }, 'te')
+
+      d = create_driver(Fluent::HandleTagNameMixin, %[
+        remove_tag_suffix st
+        include_tag_key true
+      ])
+
+      d.emit({'a' => 1})
+      d.run
+    end
+
+    def test_mix_tag_handle
+      format_check({
+        'a' => 1
+      }, 'prefix.t')
+
+      d = create_driver(Fluent::HandleTagNameMixin, %[
+        remove_tag_prefix tes
+        add_tag_prefix prefix.
+      ])
+
+      d.emit({'a' => 1})
+      d.run
+    end
+
+    def test_with_set_tag_key_mixin
+      format_check({
+        'tag' => 'tag_prefix.test',
+        'a' => 1
+      }, 'tag_prefix.test')
+
+      d = create_driver([Fluent::SetTagKeyMixin, Fluent::HandleTagNameMixin], %[
+        add_tag_prefix tag_prefix.
+        include_tag_key true
+      ])
+
+      d.emit({'a' => 1})
+      d.run
+    end
+
+    def test_with_set_tag_key_mixin_include_order_reverse
+      format_check({
+        'tag' => 'tag_prefix.test',
+        'a' => 1
+      }, 'tag_prefix.test')
+
+      d = create_driver([Fluent::HandleTagNameMixin, Fluent::SetTagKeyMixin], %[
+        add_tag_prefix tag_prefix.
+        include_tag_key true
+      ])
+
+      d.emit({'a' => 1})
+      d.run
+    end
+  end
 end

data/test/plugin/in_tail.rb

@@ -0,0 +1,79 @@
+require 'fluent/test'
+require 'net/http'
+
+class TailInputTest < Test::Unit::TestCase
+  def setup
+    Fluent::Test.setup
+    FileUtils.rm_rf(TMP_DIR)
+    FileUtils.mkdir_p(TMP_DIR)
+  end
+
+  TMP_DIR = File.dirname(__FILE__) + "/../tmp"
+
+  CONFIG = %[
+    path #{TMP_DIR}/tail.txt
+    tag t1
+    rotate_wait 2s
+    pos_file #{TMP_DIR}/tail.pos
+    format /(?<message>.*)/
+  ]
+
+  def create_driver(conf=CONFIG)
+    Fluent::Test::InputTestDriver.new(Fluent::TailInput).configure(conf)
+  end
+
+  def test_configure
+    d = create_driver
+    assert_equal ["#{TMP_DIR}/tail.txt"], d.instance.paths
+    assert_equal "t1", d.instance.tag
+    assert_equal 2, d.instance.rotate_wait
+    assert_equal "#{TMP_DIR}/tail.pos", d.instance.pos_file
+  end
+
+  def test_emit
+    File.open("#{TMP_DIR}/tail.txt", "w") {|f|
+      f.puts "test1"
+      f.puts "test2"
+    }
+
+    d = create_driver
+
+    d.run do
+      sleep 1
+
+      File.open("#{TMP_DIR}/tail.txt", "a") {|f|
+        f.puts "test3"
+        f.puts "test4"
+      }
+      sleep 1
+    end
+
+    emits = d.emits
+    assert_equal(true, emits.length > 0)
+    assert_equal({"message"=>"test3"}, emits[0][2])
+    assert_equal({"message"=>"test4"}, emits[1][2])
+  end
+
+  def test_lf
+    File.open("#{TMP_DIR}/tail.txt", "w") {|f| }
+
+    d = create_driver
+
+    d.run do
+      File.open("#{TMP_DIR}/tail.txt", "a") {|f|
+        f.print "test3"
+      }
+      sleep 1
+
+      File.open("#{TMP_DIR}/tail.txt", "a") {|f|
+        f.puts "test4"
+      }
+      sleep 1
+    end
+
+    emits = d.emits
+    assert_equal(true, emits.length > 0)
+    assert_equal({"message"=>"test3test4"}, emits[0][2])
+  end
+end
+

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: fluentd
 version: !ruby/object:Gem::Version
-  version: 0.10.16
+  version: 0.10.17
   prerelease: 
 platform: ruby
 authors:
@@ -9,11 +9,11 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2012-03-27 00:00:00.000000000Z
+date: 2012-04-02 00:00:00.000000000Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: msgpack
-  requirement: &70356207478320 !ruby/object:Gem::Requirement
+  requirement: &70102411888840 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ~>
@@ -21,10 +21,10 @@ dependencies:
         version: 0.4.4
   type: :runtime
   prerelease: false
-  version_requirements: *70356207478320
+  version_requirements: *70102411888840
 - !ruby/object:Gem::Dependency
   name: json
-  requirement: &70356207476840 !ruby/object:Gem::Requirement
+  requirement: &70102411888180 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -32,10 +32,10 @@ dependencies:
         version: 1.4.3
   type: :runtime
   prerelease: false
-  version_requirements: *70356207476840
+  version_requirements: *70102411888180
 - !ruby/object:Gem::Dependency
   name: yajl-ruby
-  requirement: &70356207475080 !ruby/object:Gem::Requirement
+  requirement: &70102411887560 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ~>
@@ -43,10 +43,10 @@ dependencies:
         version: '1.0'
   type: :runtime
   prerelease: false
-  version_requirements: *70356207475080
+  version_requirements: *70102411887560
 - !ruby/object:Gem::Dependency
   name: cool.io
-  requirement: &70356207473980 !ruby/object:Gem::Requirement
+  requirement: &70102411886940 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ~>
@@ -54,10 +54,10 @@ dependencies:
         version: 1.1.0
   type: :runtime
   prerelease: false
-  version_requirements: *70356207473980
+  version_requirements: *70102411886940
 - !ruby/object:Gem::Dependency
   name: http_parser.rb
-  requirement: &70356207466200 !ruby/object:Gem::Requirement
+  requirement: &70102411886280 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ~>
@@ -65,10 +65,10 @@ dependencies:
         version: 0.5.1
   type: :runtime
   prerelease: false
-  version_requirements: *70356207466200
+  version_requirements: *70102411886280
 - !ruby/object:Gem::Dependency
   name: rake
-  requirement: &70356207981360 !ruby/object:Gem::Requirement
+  requirement: &70102411885660 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -76,10 +76,10 @@ dependencies:
         version: 0.9.2
   type: :development
   prerelease: false
-  version_requirements: *70356207981360
+  version_requirements: *70102411885660
 - !ruby/object:Gem::Dependency
   name: rr
-  requirement: &70356211841320 !ruby/object:Gem::Requirement
+  requirement: &70102411885040 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -87,10 +87,10 @@ dependencies:
         version: 1.0.0
   type: :development
   prerelease: false
-  version_requirements: *70356211841320
+  version_requirements: *70102411885040
 - !ruby/object:Gem::Dependency
   name: timecop
-  requirement: &70356212310240 !ruby/object:Gem::Requirement
+  requirement: &70102411884420 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -98,10 +98,10 @@ dependencies:
         version: 0.3.0
   type: :development
   prerelease: false
-  version_requirements: *70356212310240
+  version_requirements: *70102411884420
 - !ruby/object:Gem::Dependency
   name: jeweler
-  requirement: &70356224203900 !ruby/object:Gem::Requirement
+  requirement: &70102411883800 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -109,7 +109,7 @@ dependencies:
         version: 1.0.0
   type: :development
   prerelease: false
-  version_requirements: *70356224203900
+  version_requirements: *70102411883800
 description: 
 email: frsyuki@gmail.com
 executables:
@@ -185,6 +185,7 @@ files:
 - test/plugin/in_http.rb
 - test/plugin/in_object_space.rb
 - test/plugin/in_stream.rb
+- test/plugin/in_tail.rb
 - test/plugin/out_copy.rb
 - test/plugin/out_exec.rb
 - test/plugin/out_exec_filter.rb
@@ -229,6 +230,7 @@ test_files:
 - test/plugin/in_http.rb
 - test/plugin/in_object_space.rb
 - test/plugin/in_stream.rb
+- test/plugin/in_tail.rb
 - test/plugin/out_copy.rb
 - test/plugin/out_exec.rb
 - test/plugin/out_exec_filter.rb