fluent-plugin-scribe 0.10.5 → 0.10.6

data/ChangeLog

@@ -1,18 +1,26 @@
-Release 0.10.5 - 2011/11/16
+Release 0.10.6 - 2011/01/23
 
-* use DetachMultiProcessMixin and ObjectBufferedOutput, introduced from
-  fluentd v0.10.7
+* support add_prefix / remove_prefix option
+* update thrift to 0.8.0
+* support remove_newline option
 
 
-Release 0.10.4 - 2011/11/14
+Release 0.10.5 - 2011/11/16
+
+* fixed broken gemspec
 
 
-Release 0.10.3 - 2011/11/14
+Release 0.10.4 - 2011/11/14
 
 * fixed encoding problem (work-around for thrift-rb)
   - contributed by @tagomoris and @frsyuki
 
 
+Release 0.10.3 - 2011/10/16
+
+* modify summary of gemspec
+
+
 Release 0.10.2 - 2011/10/16
 
 * implement out_scribe plugin

data/README.rdoc

@@ -35,7 +35,11 @@ The category field is used as fluentd 'tag'.
 
 == How to use?
 
-Please add the following configurations to fluent.conf.
+fluent-plugin-scribe contains both input and output.
+
+=== Scribe Input
+
+Please add the following configurations to fluent.conf. This allows your Scribe client to import logs through port 1463.
 
   # Scribe input
   <source>
@@ -43,26 +47,33 @@ Please add the following configurations to fluent.conf.
     port 1463
   </source>
 
+These options are supported.
+
+* port: port number (default: 1463)
+* bind: bind address (default: 0.0.0.0)
+* server_type: server architecture either in 'simple', 'threaded', 'thread_pool', 'nonblocking' (default: nonblocking)
+* is_framed: use framed protocol or not (default: true)
+* add_prefix: prefix string, added to the tag (default: nil)
+
+=== Scribe Output
+
+Please add the following configurations to fluent.conf. This allows fluentd to output its logs into another Scribe server. Note that fluentd conveys semi-structured data while Scribe conveys unstructured data, thus, 'field_ref' parameter is prepared to specify which field will be transferred.
+
   # Scribe output
   <match *>
     type scribe
     host scribed-host.local
     port 1463
+    field_ref message
   </match>
 
 These options are supported.
 
-* (for input)
-* port: port number (default: 1463)
-* bind: bind address (default: 0.0.0.0)
-* server_type: server architecture either in 'simple', 'threaded', 'thread_pool', 'nonblocking' (default: nonblocking)
-* is_framed: use framed protocol or not (default: true)
-
-* (for output)
 * host: host name or address (default: localhost)
 * port: port number (default: 1463)
 * field_ref: field name which sent as scribe log message (default: message)
 * timeout: thrift protocol timeout (default: 30)
+* remove_prefix: prefix string, removed from the tag (default: nil)
 
 == For Developers
 

data/Rakefile

@@ -6,14 +6,14 @@ begin
   require 'jeweler'
   Jeweler::Tasks.new do |gemspec|
     gemspec.name = "fluent-plugin-scribe"
-    gemspec.summary = "Scribe Input/Output plugin for Fluent event collector"
+    gemspec.summary = "Scribe Input/Output plugin for Fluentd event collector"
     gemspec.author = "Kazuki Ohta"
     gemspec.email = "kazuki.ohta@gmail.com"
     gemspec.homepage = "https://github.com/fluent/fluent-plugin-scribe"
     gemspec.has_rdoc = false
     gemspec.require_paths = ["lib"]
-    gemspec.add_dependency "fluentd", "~> 0.10.7"
-    gemspec.add_dependency "thrift", "~> 0.7.0"
+    gemspec.add_dependency "fluentd", "~> 0.10.0"
+    gemspec.add_dependency "thrift", "~> 0.8.0"
     gemspec.test_files = Dir["test/**/*.rb"]
     gemspec.files = Dir["bin/**/*", "lib/**/*", "test/**/*.rb"] +
       %w[example.conf VERSION AUTHORS Rakefile fluent-plugin-scribe.gemspec]

data/VERSION

@@ -1 +1 @@
-0.10.5
+0.10.6

data/fluent-plugin-scribe.gemspec

@@ -5,14 +5,13 @@
 
 Gem::Specification.new do |s|
   s.name = %q{fluent-plugin-scribe}
-  s.version = "0.10.4"
+  s.version = "0.10.6"
 
   s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
-  s.authors = ["Kazuki Ohta"]
-  s.date = %q{2011-11-15}
-  s.default_executable = %q{fluent-scribe-remote}
+  s.authors = [%q{Kazuki Ohta}]
+  s.date = %q{2012-01-23}
   s.email = %q{kazuki.ohta@gmail.com}
-  s.executables = ["fluent-scribe-remote"]
+  s.executables = [%q{fluent-scribe-remote}]
   s.extra_rdoc_files = [
     "ChangeLog",
     "README.rdoc"
@@ -34,27 +33,28 @@ Gem::Specification.new do |s|
     "lib/fluent/plugin/thrift/scribe.thrift",
     "lib/fluent/plugin/thrift/scribe_constants.rb",
     "lib/fluent/plugin/thrift/scribe_types.rb",
-    "test/plugin/in_scribe.rb"
+    "test/plugin/in_scribe.rb",
+    "test/plugin/out_scribe.rb"
   ]
   s.homepage = %q{https://github.com/fluent/fluent-plugin-scribe}
-  s.require_paths = ["lib"]
-  s.rubygems_version = %q{1.6.2}
-  s.summary = %q{Scribe Input/Output plugin for Fluent event collector}
-  s.test_files = ["test/plugin/in_scribe.rb"]
+  s.require_paths = [%q{lib}]
+  s.rubygems_version = %q{1.8.6}
+  s.summary = %q{Scribe Input/Output plugin for Fluentd event collector}
+  s.test_files = [%q{test/plugin/in_scribe.rb}, %q{test/plugin/out_scribe.rb}]
 
   if s.respond_to? :specification_version then
     s.specification_version = 3
 
     if Gem::Version.new(Gem::VERSION) >= Gem::Version.new('1.2.0') then
       s.add_runtime_dependency(%q<fluentd>, ["~> 0.10.0"])
-      s.add_runtime_dependency(%q<thrift>, ["~> 0.7.0"])
+      s.add_runtime_dependency(%q<thrift>, ["~> 0.8.0"])
     else
       s.add_dependency(%q<fluentd>, ["~> 0.10.0"])
-      s.add_dependency(%q<thrift>, ["~> 0.7.0"])
+      s.add_dependency(%q<thrift>, ["~> 0.8.0"])
     end
   else
     s.add_dependency(%q<fluentd>, ["~> 0.10.0"])
-    s.add_dependency(%q<thrift>, ["~> 0.7.0"])
+    s.add_dependency(%q<thrift>, ["~> 0.8.0"])
   end
 end
 

data/lib/fluent/plugin/in_scribe.rb

@@ -20,13 +20,13 @@ module Fluent
 class ScribeInput < Input
   Plugin.register_input('scribe', self)
 
-  include DetachMultiProcessMixin
-
   config_param :port,            :integer, :default => 1463
   config_param :bind,            :string,  :default => '0.0.0.0'
   config_param :server_type,     :string,  :default => 'nonblocking'
   config_param :is_framed,       :bool,    :default => true
   config_param :body_size_limit, :size,    :default => 32*1024*1024  # TODO default
+  config_param :add_prefix,      :string,  :default => nil
+  config_param :remove_newline,  :bool,    :default => false
 
   def initialize
     require 'thrift'
@@ -48,6 +48,8 @@ class ScribeInput < Input
     $log.debug "listening scribe on #{@bind}:#{@port}"
 
     handler = FluentScribeHandler.new
+    handler.add_prefix = @add_prefix
+    handler.remove_newline = @remove_newline
     processor = Scribe::Processor.new handler
 
     @transport = Thrift::ServerSocket.new @bind, @port
@@ -82,16 +84,7 @@ class ScribeInput < Input
     else
       raise ConfigError, "in_scribe: unsupported server_type '#{@server_type}'"
     end
-
-    if @detach_process
-      @transport.listen
-      def @transport.listen
-      end
-    end
-
-    detach_multi_process do
-      @thread = Thread.new(&method(:run))
-    end
+    @thread = Thread.new(&method(:run))
   end
 
   def shutdown
@@ -107,21 +100,44 @@ class ScribeInput < Input
   end
 
   class FluentScribeHandler
+    attr_accessor :add_prefix
+    attr_accessor :remove_newline
+
     def Log(msgs)
       begin
-        tags = {}
-        now = Engine.now
-        msgs.each { |msg|
-          tag = msg.category
-          record = {
-            'message' => msg.message.force_encoding('UTF-8')
-          }
-          es = (tags[tag] ||= MultiEventStream.new)
-          es.add(now, record)
-        }
-        tags.each_pair { |tag, es|
-          Engine.emit_stream(tag, es)
-        }
+        if @add_prefix
+          if @remove_newline
+            msgs.each { |msg|
+              record = {
+                'message' => msg.message.force_encoding('UTF-8').chomp
+              }
+              Engine.emit(@add_prefix + '.' + msg.category, Engine.now, record)
+            }
+          else
+            msgs.each { |msg|
+              record = {
+                'message' => msg.message.force_encoding('UTF-8')
+              }
+              Engine.emit(@add_prefix + '.' + msg.category, Engine.now, record)
+            }
+          end
+        else
+          if @remove_newline
+            msgs.each { |msg|
+              record = {
+                'message' => msg.message.force_encoding('UTF-8').chomp
+              }
+              Engine.emit(msg.category, Engine.now, record)
+            }
+          else
+            msgs.each { |msg|
+              record = {
+                'message' => msg.message.force_encoding('UTF-8')
+              }
+              Engine.emit(msg.category, Engine.now, record)
+            }
+          end
+        end
         return ResultCode::OK
       rescue => e
         $log.error "unexpected error", :error=>$!.to_s

data/lib/fluent/plugin/out_scribe.rb

@@ -17,7 +17,7 @@
 #
 module Fluent
 
-class ScribeOutput < ObjectBufferedOutput
+class ScribeOutput < BufferedOutput
   Fluent::Plugin.register_output('scribe', self)
 
   config_param :host,      :string,  :default => 'localhost'
@@ -25,6 +25,9 @@ class ScribeOutput < ObjectBufferedOutput
   config_param :field_ref, :string,  :default => 'message'
   config_param :timeout,   :integer, :default => 30
 
+  config_param :remove_prefix,    :string, :default => nil
+  config_param :default_category, :string, :default => 'unknown'
+
   def initialize
     require 'thrift'
     $:.unshift File.join(File.dirname(__FILE__), 'thrift')
@@ -43,13 +46,33 @@ class ScribeOutput < ObjectBufferedOutput
 
   def start
     super
+
+    if @remove_prefix
+      @removed_prefix_string = @remove_prefix + '.'
+      @removed_length = @removed_prefix_string.length
+    end
   end
 
   def shutdown
     super
   end
 
-  def write_objects(tag, es)
+  def format(tag, time, record)
+    if @remove_prefix and
+        ( (tag[0, @removed_length] == @removed_prefix_string and tag.length > @removed_length) or
+          tag == @remove_prefix)
+      [(tag[@removed_length..-1] || @default_category), record].to_msgpack
+    else
+      [tag, record].to_msgpack
+    end
+  end
+
+  def write(chunk)
+    records = []
+    chunk.msgpack_each { |arr|
+      records << arr
+    }
+
     socket = Thrift::Socket.new @host, @port, @timeout
     transport = Thrift::FramedTransport.new socket
     protocol = Thrift::BinaryProtocol.new transport, false, false
@@ -58,7 +81,8 @@ class ScribeOutput < ObjectBufferedOutput
     transport.open
     begin
       entries = []
-      es.each { |time,record|
+      records.each { |r|
+        tag, record = r
         next unless record.has_key?(@field_ref)
         entry = LogEntry.new
         entry.category = tag

data/test/plugin/in_scribe.rb

@@ -20,6 +20,7 @@ class ScribeInputTest < Test::Unit::TestCase
     d = create_driver
     assert_equal 14630, d.instance.port
     assert_equal '127.0.0.1', d.instance.bind
+    assert_equal false, d.instance.remove_newline
   end
 
   def test_time
@@ -31,8 +32,82 @@ class ScribeInputTest < Test::Unit::TestCase
     d.expect_emit "tag1", time, {"message"=>'aiueo'}
     d.expect_emit "tag2", time, {"message"=>'aiueo'}
 
+    emits = [
+             ['tag1', time, {"message"=>'aiueo'}],
+             ['tag2', time, {"message"=>'aiueo'}],
+            ]
     d.run do
-      d.expected_emits.each { |tag, time, record|
+      emits.each { |tag, time, record|
+        res = send(tag, record['message'])
+        assert_equal ResultCode::OK, res
+      }
+    end
+  end
+
+  def test_add_prefix
+    d = create_driver(CONFIG + %[
+      add_prefix scribe
+    ])
+
+    time = Time.parse("2011-01-02 13:14:15 UTC").to_i
+    Fluent::Engine.now = time
+
+    d.expect_emit "scribe.tag1", time, {"message"=>'aiueo'}
+    d.expect_emit "scribe.tag2", time, {"message"=>'aiueo'}
+
+    emits = [
+             ['tag1', time, {"message"=>'aiueo'}],
+             ['tag2', time, {"message"=>'aiueo'}],
+            ]
+    d.run do
+      emits.each { |tag, time, record|
+        res = send(tag, record['message'])
+        assert_equal ResultCode::OK, res
+      }
+    end
+
+    d2 = create_driver(CONFIG + %[
+      add_prefix scribe.input
+    ])
+
+    time = Time.parse("2011-01-02 13:14:15 UTC").to_i
+    Fluent::Engine.now = time
+
+    d2.expect_emit "scribe.input.tag3", time, {"message"=>'aiueo'}
+    d2.expect_emit "scribe.input.tag4", time, {"message"=>'aiueo'}
+
+    emits = [
+             ['tag3', time, {"message"=>'aiueo'}],
+             ['tag4', time, {"message"=>'aiueo'}],
+            ]
+    d2.run do
+      emits.each { |tag, time, record|
+        res = send(tag, record['message'])
+        assert_equal ResultCode::OK, res
+      }
+    end
+  end
+
+  def test_remove_newline
+    d = create_driver(CONFIG + %[
+      remove_newline true
+    ])
+    assert_equal true, d.instance.remove_newline
+
+    time = Time.parse("2011-01-02 13:14:15 UTC").to_i
+    Fluent::Engine.now = time
+
+    d.expect_emit "tag1", time, {"message"=>'aiueo'}
+    d.expect_emit "tag2", time, {"message"=>'kakikukeko'}
+    d.expect_emit "tag3", time, {"message"=>'sasisuseso'}
+
+    emits = [
+             ['tag1', time, {"message"=>"aiueo\n"}],
+             ['tag2', time, {"message"=>"kakikukeko\n"}],
+             ['tag3', time, {"message"=>"sasisuseso"}],
+            ]
+    d.run do
+      emits.each { |tag, time, record|
         res = send(tag, record['message'])
         assert_equal ResultCode::OK, res
       }

data/test/plugin/out_scribe.rb

@@ -0,0 +1,155 @@
+require 'test/unit'
+require 'fluent/test'
+require 'lib/fluent/plugin/out_scribe'
+
+class ScribeOutputTest < Test::Unit::TestCase
+  CONFIG = %[
+    host 127.0.0.1
+    port 14630
+  ]
+
+  def create_driver(conf=CONFIG, tag='test')
+    Fluent::Test::BufferedOutputTestDriver.new(Fluent::ScribeOutput, tag).configure(conf)
+  end
+
+  def test_configure
+    d = create_driver('')
+
+    assert_equal 'localhost', d.instance.host
+    assert_equal 1463, d.instance.port
+    assert_equal 'message', d.instance.field_ref
+    assert_equal 30, d.instance.timeout
+    assert_equal 'unknown', d.instance.default_category
+    assert_nil d.instance.remove_prefix
+
+    d = create_driver
+
+    assert_equal '127.0.0.1', d.instance.host
+    assert_equal 14630, d.instance.port
+  end
+
+  def test_format
+    time = Time.parse("2011-12-21 13:14:15 UTC").to_i
+
+    d = create_driver
+    d.emit({"message" => "testing first", "message2" => "testing first another data"}, time)
+    d.emit({"message" => "testing second", "message2" => "testing second another data"}, time)
+    d.emit({"message" => "testing third", "message2" => "testing third another data"}, time)
+    d.expect_format [d.tag, {"message" => "testing first", "message2" => "testing first another data"}].to_msgpack
+    d.expect_format [d.tag, {"message" => "testing second", "message2" => "testing second another data"}].to_msgpack
+    d.expect_format [d.tag, {"message" => "testing third", "message2" => "testing third another data"}].to_msgpack
+    d.run
+
+    d = create_driver(CONFIG + %[
+field_ref message2
+remove_prefix test
+    ], 'test.scribeplugin')
+    assert_equal 'test.scribeplugin', d.tag
+
+    d.emit({"message" => "xxx testing first", "message2" => "xxx testing first another data"}, time)
+    d.emit({"message" => "xxx testing second", "message2" => "xxx testing second another data"}, time)
+    d.expect_format ['scribeplugin', {"message" => "xxx testing first", "message2" => "xxx testing first another data"}].to_msgpack
+    d.expect_format ['scribeplugin', {"message" => "xxx testing second", "message2" => "xxx testing second another data"}].to_msgpack
+    d.run
+
+    d = create_driver(CONFIG + %[
+field_ref message2
+remove_prefix test
+    ], 'xxx.test.scribeplugin')
+    assert_equal 'xxx.test.scribeplugin', d.tag
+    d.emit({"message" => "xxx testing first", "message2" => "xxx testing first another data"}, time)
+    d.expect_format ['xxx.test.scribeplugin', {"message" => "xxx testing first", "message2" => "xxx testing first another data"}].to_msgpack
+    d.run
+
+    d = create_driver(CONFIG + %[
+field_ref message2
+remove_prefix test
+    ], 'test')
+    assert_equal 'test', d.tag
+    d.emit({"message" => "xxx testing first", "message2" => "xxx testing first another data"}, time)
+    d.expect_format ['unknown', {"message" => "xxx testing first", "message2" => "xxx testing first another data"}].to_msgpack
+    d.run
+  end
+
+  def test_write
+    time = Time.parse("2011-12-21 13:14:15 UTC").to_i
+
+    d = create_driver
+    d.emit({"message" => "testing first", "message2" => "testing first another data"}, time)
+    d.emit({"message" => "testing second", "message2" => "testing second another data"}, time)
+    d.emit({"message" => "testing third", "message2" => "testing third another data"}, time)
+    result = d.run
+    assert_equal ResultCode::OK, result
+    assert_equal [[d.tag, 'testing first'], [d.tag, 'testing second'], [d.tag,'testing third']], $handler.last
+
+    d = create_driver(CONFIG + %[
+field_ref message2
+remove_prefix test
+    ], 'test.scribeplugin')
+    assert_equal 'test.scribeplugin', d.tag
+    d.emit({"message" => "xxx testing first", "message2" => "xxx testing first another data"}, time)
+    d.emit({"message" => "xxx testing second", "message2" => "xxx testing second another data"}, time)
+    result = d.run
+    assert_equal ResultCode::OK, result
+    assert_equal [['scribeplugin', 'xxx testing first another data'], ['scribeplugin', 'xxx testing second another data']], $handler.last
+
+    d = create_driver(CONFIG + %[
+field_ref message2
+remove_prefix test
+    ], 'xxx.test.scribeplugin')
+    assert_equal 'xxx.test.scribeplugin', d.tag
+    d.emit({"message" => "yyy testing first", "message2" => "yyy testing first another data"}, time)
+    result = d.run
+    assert_equal ResultCode::OK, result
+    assert_equal [['xxx.test.scribeplugin', 'yyy testing first another data']], $handler.last
+
+    d = create_driver(CONFIG + %[
+field_ref message2
+remove_prefix test
+    ], 'test')
+    assert_equal 'test', d.tag
+    d.emit({"message" => "zzz testing first", "message2" => "zzz testing first another data"}, time)
+    result = d.run
+    assert_equal ResultCode::OK, result
+    assert_equal [[d.instance.default_category, 'zzz testing first another data']], $handler.last
+  end
+
+  def setup
+    Fluent::Test.setup
+    $handler = TestScribeServerHandler.new
+    @dummy_server_thread = Thread.new do
+      begin
+        transport = Thrift::ServerSocket.new '127.0.0.1', 14630
+        processor = Scribe::Processor.new $handler
+        transport_factory = Thrift::FramedTransportFactory.new
+        protocol_factory = Thrift::BinaryProtocolFactory.new
+        protocol_factory.instance_eval {|obj|
+          def get_protocol(trans) # override
+            Thrift::BinaryProtocol.new(trans, strict_read=false, strict_write=false)
+          end
+        }
+        server = Thrift::SimpleServer.new processor, transport, transport_factory, protocol_factory
+        server.serve
+      ensure
+        transport.close unless transport.closed?
+      end
+    end
+    sleep 0.1 # boo...
+  end
+
+  def teardown
+    @dummy_server_thread.kill
+    @dummy_server_thread.join
+  end
+
+  class TestScribeServerHandler
+    attr :last
+    def initialize
+      @last = []
+    end
+    def Log(msgs)
+      @last = msgs.map{|msg| [msg.category, msg.message.force_encoding('UTF-8')]}
+      ResultCode::OK
+    end
+  end
+end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-scribe
 version: !ruby/object:Gem::Version
-  version: 0.10.5
+  version: 0.10.6
   prerelease: 
 platform: ruby
 authors:
@@ -9,30 +9,30 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2011-11-16 00:00:00.000000000Z
+date: 2012-01-23 00:00:00.000000000Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fluentd
-  requirement: &15568620 !ruby/object:Gem::Requirement
+  requirement: &2160486020 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
-        version: 0.10.7
+        version: 0.10.0
   type: :runtime
   prerelease: false
-  version_requirements: *15568620
+  version_requirements: *2160486020
 - !ruby/object:Gem::Dependency
   name: thrift
-  requirement: &15567980 !ruby/object:Gem::Requirement
+  requirement: &2160485540 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
-        version: 0.7.0
+        version: 0.8.0
   type: :runtime
   prerelease: false
-  version_requirements: *15567980
+  version_requirements: *2160485540
 description: 
 email: kazuki.ohta@gmail.com
 executables:
@@ -59,6 +59,7 @@ files:
 - lib/fluent/plugin/thrift/scribe_constants.rb
 - lib/fluent/plugin/thrift/scribe_types.rb
 - test/plugin/in_scribe.rb
+- test/plugin/out_scribe.rb
 - ChangeLog
 - README.rdoc
 homepage: https://github.com/fluent/fluent-plugin-scribe
@@ -81,9 +82,10 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 1.8.10
+rubygems_version: 1.8.6
 signing_key: 
 specification_version: 3
-summary: Scribe Input/Output plugin for Fluent event collector
+summary: Scribe Input/Output plugin for Fluentd event collector
 test_files:
 - test/plugin/in_scribe.rb
+- test/plugin/out_scribe.rb