fluent-plugin-scribe 0.10.12 → 0.10.13

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 4742edb4137955d08718e1cc275f2b163a0f7d98
-  data.tar.gz: 454ccd6f4ebb77898c159c25bb17e17c0f94526c
+  metadata.gz: e82d716df29ee314a3d51ee911e32f9b0de07a0f
+  data.tar.gz: 35be26bb3acbe72db1b38bad94b2801875d90e22
 SHA512:
-  metadata.gz: 5d607d8c5e8f579efe419e4575340caa1fca4b3103c48895e1208394eae5a30c609fe136875f8ad768b79484add797d12493e071747c6eb9e0d35f86598ac215
-  data.tar.gz: 5419ee2a3a620720172a909baa0087cf8e15da93ffd8a253a1aee2f891b0d40405272ec6e8e6600391877ac46eb7ca366fc3d670500eabf800b2bf13029f07ee
+  metadata.gz: f98b739988b9bdb0e69d34c56c9139e80cf7c9c1834c3d59f6ee22a449552230e471ee711fa3284487f6fb2735d296c6d42b182168653595526dc5a34fce1031
+  data.tar.gz: 9dc6b1104e4b3aace045c925961b3224e4cc215de001303191f3cd560a94828b7f6929bccc66a1168de4958383b99d140c6ec923c30c861280af3adae46bdedd

data/ChangeLog

@@ -1,3 +1,18 @@
+Release 0.10.13 - 2013/10/29
+* add validation for msg_format
+* use Yajl instead of JSON for parsing record
+* add ignore_invalid_record option to ignore invalid record and return OK to scribed
+
+Release 0.10.12 - 2013/09/04
+* support 'log_level' option
+* fix to do shutdown plugin instance not to duplicate listen
+
+Release 0.10.11 - 2013/09/04
+* add format_to_json bool option with default value false
+
+Release 0.10.10 - 2013/02/06
+* change 'written entries' log level from 'info' to 'debug'
+
 Release 0.10.9 - 2011/05/30
 * overwrite default value of buffer_chunk_limit to 1MB
 * improve memory usage efficiency

data/README.rdoc

@@ -1,8 +1,8 @@
-= Scribe input plugin for Fluent
+= Scribe input plugin for Fluentd[http://fluentd.org/]
 
 == Overview
 
-This is a plugin for fluentd[https://github.com/fluentd] event collector. This plugin adds the Scribe[https://github.com/facebook/scribe] compatible interface to fluentd.
+This is a plugin for fluentd[http://fluentd.org] data collector. This plugin adds the Scribe[https://github.com/facebook/scribe] compatible interface to fluentd.
 
 == What's Scribe?
 

data/fluent-plugin-scribe.gemspec

@@ -2,7 +2,7 @@
 
 Gem::Specification.new do |gem|
   gem.name          = "fluent-plugin-scribe"
-  gem.version       = "0.10.12"
+  gem.version       = "0.10.13"
 
   gem.authors       = ["Kazuki Ohta", "TAGOMORI Satoshi"]
   gem.email         = ["kazuki.ohta@gmail.com", "tagomoris@gmail.com"]
@@ -17,6 +17,7 @@ Gem::Specification.new do |gem|
   gem.require_paths = ["lib"]
 
   gem.add_development_dependency "rake"
+  gem.add_development_dependency "test-unit", '~> 3.0.2'
   gem.add_runtime_dependency "fluentd"
   gem.add_runtime_dependency "thrift", "~> 0.8.0"
 end

data/lib/fluent/plugin/in_scribe.rb

@@ -19,6 +19,12 @@ module Fluent
   class ScribeInput < Input
     Plugin.register_input('scribe', self)
 
+    SUPPORTED_FORMAT = {
+      'text' => :text,
+      'json' => :json,
+      'url_param' => :url_param,
+    }
+
     config_param :port,            :integer, :default => 1463
     config_param :bind,            :string,  :default => '0.0.0.0'
     config_param :server_type,     :string,  :default => 'nonblocking'
@@ -26,7 +32,12 @@ module Fluent
     config_param :body_size_limit, :size,    :default => 32*1024*1024  # TODO default
     config_param :add_prefix,      :string,  :default => nil
     config_param :remove_newline,  :bool,    :default => false
-    config_param :msg_format,      :string,  :default => 'text'
+    config_param :ignore_invalid_record, :bool, :default => false
+    config_param :msg_format, :default => :text do |val|
+      f = SUPPORTED_FORMAT[val]
+      raise ConfigError, "unsupported msg_format: #{val}" unless f
+      f
+    end
 
     unless method_defined?(:log)
       define_method(:log) { $log }
@@ -34,6 +45,7 @@ module Fluent
 
     def initialize
       require 'cgi'
+      require 'yajl'
       require 'thrift'
       $:.unshift File.join(File.dirname(__FILE__), 'thrift')
       require 'fb303_types'
@@ -56,6 +68,7 @@ module Fluent
       handler.add_prefix = @add_prefix
       handler.remove_newline = @remove_newline
       handler.msg_format = @msg_format
+      handler.ignore_invalid_record = @ignore_invalid_record
       handler.logger = log
       processor = Scribe::Processor.new handler
 
@@ -66,10 +79,6 @@ module Fluent
         transport_factory = Thrift::BufferedTransportFactory.new
       end
 
-      unless ['text', 'json', 'url_param'].include? @msg_format
-        raise 'Unknown format: msg_format=#{@msg_format}'
-      end
-
       # 2011/09/29 Kazuki Ohta <kazuki.ohta@gmail.com>
       # This section is a workaround to set strict_read and strict_write option.
       # Ruby-Thrift 0.7 set them both 'true' in default, but Scribe protocol set
@@ -105,8 +114,8 @@ module Fluent
 
     def run
       @server.serve
-    rescue
-      log.error "unexpected error", :error=>$!.to_s
+    rescue => e
+      log.error "unexpected error", :error => e.inspect
       log.error_backtrace
     end
 
@@ -114,12 +123,25 @@ module Fluent
       attr_accessor :add_prefix
       attr_accessor :remove_newline
       attr_accessor :msg_format
+      attr_accessor :ignore_invalid_record
       attr_accessor :logger # Use logger instead of log to avoid confusion with Log method
 
       def Log(msgs)
         begin
           msgs.each { |msg|
-            record = create_record(msg)
+            begin
+              record = create_record(msg)
+            rescue => e
+              if @ignore_invalid_record
+                # This warning can be disabled by 'log_level error'
+                logger.warn "got invalid record: #{msg}"
+                next
+              else
+                # Keep existence behaviour
+                raise e
+              end
+            end
+
             if @add_prefix
               Engine.emit(@add_prefix + '.' + msg.category, Engine.now, record)
             else
@@ -128,7 +150,7 @@ module Fluent
           }
           return ResultCode::OK
         rescue => e
-          logger.error "unexpected error", :error=>$!.to_s
+          logger.error "unexpected error", :error => e.inspect
           logger.error_backtrace
           return ResultCode::TRY_LATER
         end
@@ -137,17 +159,17 @@ module Fluent
       private
       def create_record(msg)
         case @msg_format
-        when 'text'
+        when :text
           if @remove_newline
             return { 'message' => msg.message.force_encoding('UTF-8').chomp }
           else
             return { 'message' => msg.message.force_encoding('UTF-8') }
           end
-        when 'json'
-          js = JSON.parse(msg.message.force_encoding('UTF-8'))
+        when :json
+          js = Yajl.load(msg.message.force_encoding('UTF-8'))
           raise 'body must be a Hash, if json_body=true' unless js.is_a?(Hash)
           return js
-        when 'url_param'
+        when :url_param
           s = msg.message.force_encoding('UTF-8')
           return Hash[ s.split('&').map { |kv|
               k,v = kv.split('=', 2);

data/test/plugin/test_in_scribe.rb

@@ -134,7 +134,7 @@ class ScribeInputTest < Test::Unit::TestCase
     d = create_driver(CONFIG + %[
       msg_format json
     ])
-    assert_equal 'json', d.instance.msg_format
+    assert_equal :json, d.instance.msg_format
 
     time = Time.parse("2011-01-02 13:14:15 UTC").to_i
     Fluent::Engine.now = time
@@ -158,11 +158,41 @@ class ScribeInputTest < Test::Unit::TestCase
     shutdown_driver(d)
   end
 
+  data do
+    Fluent::ScribeInput.new
+    {
+      'true'  => [ResultCode::OK, true],
+      'false' => [ResultCode::TRY_LATER, false]
+    }
+  end
+  def test_msg_format_json_with_ignore_invalid_record(data)
+    result, opt = data
+    d = create_driver(CONFIG + %[
+      msg_format json
+      ignore_invalid_record #{opt}
+    ])
+    assert_equal :json, d.instance.msg_format
+    assert_equal opt, d.instance.ignore_invalid_record
+
+    time = Time.parse("2011-01-02 13:14:15 UTC").to_i
+    Fluent::Engine.now = time
+
+    emits = [['tag1', time, '{"a":']]
+    d.run do
+      emits.each { |tag, time, message|
+        res = message_send(tag, message)
+        assert_equal result, res
+      }
+    end
+
+    shutdown_driver(d)
+  end
+
   def test_msg_format_url_param
     d = create_driver(CONFIG + %[
       msg_format url_param
     ])
-    assert_equal 'url_param', d.instance.msg_format
+    assert_equal :url_param, d.instance.msg_format
 
     time = Time.parse("2011-01-02 13:14:15 UTC").to_i
     Fluent::Engine.now = time

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-scribe
 version: !ruby/object:Gem::Version
-  version: 0.10.12
+  version: 0.10.13
 platform: ruby
 authors:
 - Kazuki Ohta
@@ -9,48 +9,62 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2014-02-05 00:00:00.000000000 Z
+date: 2014-10-28 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rake
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  name: test-unit
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 3.0.2
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 3.0.2
 - !ruby/object:Gem::Dependency
   name: fluentd
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
   name: thrift
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
         version: 0.8.0
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
         version: 0.8.0
 description: Fluentd input/output plugin to handle Facebook scribed thrift protocol
@@ -62,8 +76,8 @@ executables:
 extensions: []
 extra_rdoc_files: []
 files:
-- .gitignore
-- .travis.yml
+- ".gitignore"
+- ".travis.yml"
 - AUTHORS
 - ChangeLog
 - Gemfile
@@ -95,21 +109,20 @@ require_paths:
 - lib
 required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
-  - - '>='
+  - - ">="
     - !ruby/object:Gem::Version
       version: '0'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
-  - - '>='
+  - - ">="
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.0.3
+rubygems_version: 2.2.2
 signing_key: 
 specification_version: 4
 summary: Scribe Input/Output plugin for Fluentd event collector
 test_files:
 - test/plugin/test_in_scribe.rb
 - test/plugin/test_out_scribe.rb
-has_rdoc: