fluent-plugin-scribe 0.10.7 → 0.10.8

data/ChangeLog

@@ -1,3 +1,8 @@
+Release 0.10.8 - 2011/04/27
+
+* support msg_format option at in_scribe
+
+
 Release 0.10.7 - 2011/02/04
 
 * support add_newline option at out_scribe

data/README.rdoc

@@ -54,6 +54,7 @@ These options are supported.
 * server_type: server architecture either in 'simple', 'threaded', 'thread_pool', 'nonblocking' (default: nonblocking)
 * is_framed: use framed protocol or not (default: true)
 * add_prefix: prefix string, added to the tag (default: nil)
+* msg_format: format of the messages either in 'text', 'json', 'url_param' (default: text)
 
 === Scribe Output
 
@@ -73,6 +74,7 @@ These options are supported.
 * port: port number (default: 1463)
 * field_ref: field name which sent as scribe log message (default: message)
 * timeout: thrift protocol timeout (default: 30)
+* body_format: parse a body message as a specified format (default: 'text', option: 'json')
 * remove_prefix: prefix string, removed from the tag (default: nil)
 
 == For Developers

data/VERSION

@@ -1 +1 @@
-0.10.7
+0.10.8

data/bin/fluent-scribe-remote

@@ -1,5 +1,6 @@
 #!/usr/bin/env ruby
 require 'thrift'
+require 'json'
 $:.unshift File.join(File.dirname(__FILE__), '../lib/fluent/plugin/thrift')
 require 'fb303_types'
 require 'fb303_constants'
@@ -24,7 +25,7 @@ raw_sock.setsockopt Socket::IPPROTO_TCP, Socket::TCP_NODELAY, 1
 
 entry = LogEntry.new
 entry.category = 'debug.hoge'
-entry.message = 'fuga'
+entry.message = {'a' => 'b'}.to_json
 p client.Log([entry])
 
 transport.close

data/fluent-plugin-scribe.gemspec

@@ -4,14 +4,14 @@
 # -*- encoding: utf-8 -*-
 
 Gem::Specification.new do |s|
-  s.name = %q{fluent-plugin-scribe}
-  s.version = "0.10.7"
+  s.name = "fluent-plugin-scribe"
+  s.version = "0.10.8"
 
   s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
-  s.authors = [%q{Kazuki Ohta}]
-  s.date = %q{2012-02-04}
-  s.email = %q{kazuki.ohta@gmail.com}
-  s.executables = [%q{fluent-scribe-remote}]
+  s.authors = ["Kazuki Ohta"]
+  s.date = "2012-04-27"
+  s.email = "kazuki.ohta@gmail.com"
+  s.executables = ["fluent-scribe-remote"]
   s.extra_rdoc_files = [
     "ChangeLog",
     "README.rdoc"
@@ -36,11 +36,11 @@ Gem::Specification.new do |s|
     "test/plugin/in_scribe.rb",
     "test/plugin/out_scribe.rb"
   ]
-  s.homepage = %q{https://github.com/fluent/fluent-plugin-scribe}
-  s.require_paths = [%q{lib}]
-  s.rubygems_version = %q{1.8.6}
-  s.summary = %q{Scribe Input/Output plugin for Fluentd event collector}
-  s.test_files = [%q{test/plugin/in_scribe.rb}, %q{test/plugin/out_scribe.rb}]
+  s.homepage = "https://github.com/fluent/fluent-plugin-scribe"
+  s.require_paths = ["lib"]
+  s.rubygems_version = "1.8.21"
+  s.summary = "Scribe Input/Output plugin for Fluentd event collector"
+  s.test_files = ["test/plugin/in_scribe.rb", "test/plugin/out_scribe.rb"]
 
   if s.respond_to? :specification_version then
     s.specification_version = 3

data/lib/fluent/plugin/in_scribe.rb

@@ -27,8 +27,10 @@ class ScribeInput < Input
   config_param :body_size_limit, :size,    :default => 32*1024*1024  # TODO default
   config_param :add_prefix,      :string,  :default => nil
   config_param :remove_newline,  :bool,    :default => false
+  config_param :msg_format,      :string,  :default => 'text'
 
   def initialize
+    require 'cgi'
     require 'thrift'
     $:.unshift File.join(File.dirname(__FILE__), 'thrift')
     require 'fb303_types'
@@ -50,6 +52,7 @@ class ScribeInput < Input
     handler = FluentScribeHandler.new
     handler.add_prefix = @add_prefix
     handler.remove_newline = @remove_newline
+    handler.msg_format = @msg_format
     processor = Scribe::Processor.new handler
 
     @transport = Thrift::ServerSocket.new @bind, @port
@@ -59,6 +62,10 @@ class ScribeInput < Input
       transport_factory = Thrift::BufferedTransportFactory.new
     end
 
+    unless ['text', 'json', 'url_param'].include? @msg_format
+      raise 'Unknown format: msg_format=#{@msg_format}'
+    end
+
     # 2011/09/29 Kazuki Ohta <kazuki.ohta@gmail.com>
     # This section is a workaround to set strict_read and strict_write option.
     # Ruby-Thrift 0.7 set them both 'true' in default, but Scribe protocol set
@@ -102,42 +109,18 @@ class ScribeInput < Input
   class FluentScribeHandler
     attr_accessor :add_prefix
     attr_accessor :remove_newline
+    attr_accessor :msg_format
 
     def Log(msgs)
       begin
-        if @add_prefix
-          if @remove_newline
-            msgs.each { |msg|
-              record = {
-                'message' => msg.message.force_encoding('UTF-8').chomp
-              }
-              Engine.emit(@add_prefix + '.' + msg.category, Engine.now, record)
-            }
-          else
-            msgs.each { |msg|
-              record = {
-                'message' => msg.message.force_encoding('UTF-8')
-              }
-              Engine.emit(@add_prefix + '.' + msg.category, Engine.now, record)
-            }
-          end
-        else
-          if @remove_newline
-            msgs.each { |msg|
-              record = {
-                'message' => msg.message.force_encoding('UTF-8').chomp
-              }
-              Engine.emit(msg.category, Engine.now, record)
-            }
+        msgs.each { |msg|
+          record = create_record(msg)
+          if @add_prefix
+            Engine.emit(@add_prefix + '.' + msg.category, Engine.now, record)
           else
-            msgs.each { |msg|
-              record = {
-                'message' => msg.message.force_encoding('UTF-8')
-              }
-              Engine.emit(msg.category, Engine.now, record)
-            }
+            Engine.emit(msg.category, Engine.now, record)
           end
-        end
+        }
         return ResultCode::OK
       rescue => e
         $log.error "unexpected error", :error=>$!.to_s
@@ -145,6 +128,31 @@ class ScribeInput < Input
         return ResultCode::TRY_LATER
       end
     end
+
+    private
+    def create_record(msg)
+      case @msg_format
+      when 'text'
+        if @remove_newline
+          return { 'message' => msg.message.force_encoding('UTF-8').chomp }
+        else
+          return { 'message' => msg.message.force_encoding('UTF-8') }
+        end
+      when 'json'
+        js = JSON.parse(msg.message.force_encoding('UTF-8'))
+        raise 'body must be a Hash, if json_body=true' unless js.is_a?(Hash)
+        return js
+      when 'url_param'
+        s = msg.message.force_encoding('UTF-8')
+        return Hash[ s.split('&').map { |kv|
+            k,v = kv.split('=', 2);
+            [CGI.unescape(k), CGI.unescape(v)]
+          }
+        ]
+      else
+        raise 'Invalid format: #{@msg_format}'
+      end
+    end
   end
 end
 

data/test/plugin/in_scribe.rb

@@ -114,6 +114,62 @@ class ScribeInputTest < Test::Unit::TestCase
     end
   end
 
+  def test_msg_format_json
+    d = create_driver(CONFIG + %[
+      msg_format json
+    ])
+    assert_equal 'json', d.instance.msg_format
+
+    time = Time.parse("2011-01-02 13:14:15 UTC").to_i
+    Fluent::Engine.now = time
+
+    d.expect_emit "tag1", time, {"a"=>1}
+    d.expect_emit "tag2", time, {"a"=>1, "b"=>2}
+    d.expect_emit "tag3", time, {"a"=>1, "b"=>2, "c"=>3}
+
+    emits = [
+             ['tag1', time, {"a"=>1}.to_json],
+             ['tag2', time, {"a"=>1, "b"=>2}.to_json],
+             ['tag3', time, {"a"=>1, "b"=>2, "c"=>3}.to_json],
+            ]
+    d.run do
+      emits.each { |tag, time, message|
+        res = send(tag, message)
+        assert_equal ResultCode::OK, res
+      }
+    end
+  end
+
+  def test_msg_format_url_param
+    d = create_driver(CONFIG + %[
+      msg_format url_param
+    ])
+    assert_equal 'url_param', d.instance.msg_format
+
+    time = Time.parse("2011-01-02 13:14:15 UTC").to_i
+    Fluent::Engine.now = time
+
+    d.expect_emit "tag0", time, {}
+    d.expect_emit "tag1", time, {"a"=>'1'}
+    d.expect_emit "tag2", time, {"a"=>'1', "b"=>'2'}
+    d.expect_emit "tag3", time, {"a"=>'1', "b"=>'2', "c"=>'3'}
+    d.expect_emit "tag4", time, {"a"=>'1', "b"=>'2', "c"=>'3=4'}
+
+    emits = [
+             ['tag0', time, ""],
+             ['tag1', time, "a=1"],
+             ['tag2', time, "a=1&b=2"],
+             ['tag3', time, "a=1&b=2&c=3"],
+             ['tag4', time, "a=1&b=2&c=3=4"],
+            ]
+    d.run do
+      emits.each { |tag, time, message|
+        res = send(tag, message)
+        assert_equal ResultCode::OK, res
+      }
+    end
+  end
+
   def send(tag, msg)
     socket = Thrift::Socket.new '127.0.0.1', 14630
     transport = Thrift::FramedTransport.new socket

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-scribe
 version: !ruby/object:Gem::Version
-  version: 0.10.7
+  version: 0.10.8
   prerelease: 
 platform: ruby
 authors:
@@ -9,11 +9,11 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2012-02-04 00:00:00.000000000Z
+date: 2012-04-27 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fluentd
-  requirement: &2156552400 !ruby/object:Gem::Requirement
+  requirement: !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ~>
@@ -21,10 +21,15 @@ dependencies:
         version: 0.10.0
   type: :runtime
   prerelease: false
-  version_requirements: *2156552400
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 0.10.0
 - !ruby/object:Gem::Dependency
   name: thrift
-  requirement: &2156551600 !ruby/object:Gem::Requirement
+  requirement: !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ~>
@@ -32,7 +37,12 @@ dependencies:
         version: 0.8.0
   type: :runtime
   prerelease: false
-  version_requirements: *2156551600
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 0.8.0
 description: 
 email: kazuki.ohta@gmail.com
 executables:
@@ -82,7 +92,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 1.8.6
+rubygems_version: 1.8.21
 signing_key: 
 specification_version: 3
 summary: Scribe Input/Output plugin for Fluentd event collector