fluent-plugin-flume 0.1.3 → 0.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 4b78497ad732b1daed34f85264299c5f531b8862
-  data.tar.gz: 01d60de145ffe1f44c682aa74ca6448e596d8a9e
+  metadata.gz: d92aa659b736a1e4228653bcf7089cf0844ed412
+  data.tar.gz: 74f103fa74799fdb9ee89debad3a3ce803c360ae
 SHA512:
-  metadata.gz: 34fe016d453e36b285731ac4882ad4d2001702f56a65921017bef3c383bdf1894526520e7c869136d64d67ec9c43dcb5c0575e6630c8e216de3c754e23a441df
-  data.tar.gz: 779f483c0eeae828af4b693df7d54a525277961cfb6f421a27a92d673664ee0873cd382b18f0754d4733770c775edd766df435995a05e435cf1b6952dd8ac6a0
+  metadata.gz: d1ec6c1fdd6ab7cd36acc3bdef637f4186c4464adec65839abeea0116a1e464240dccf2f88c414127cf88ec6dbea340347eac0fc7f6fbd9dee87012820b50ef6
+  data.tar.gz: fe24084964f0b71d51073c807ede72419dfde1faca46f824b5dbbc7b73875e6c67fba1392905f55cf0254749ae8ae0d1a08d898ce7eb998fb0670f30fe9601b7

data/ChangeLog

@@ -1,3 +1,8 @@
+Release 0.2.0 - 2016/10/26
+
+	* Improve out_flume performance
+	* Deprecate in_flume for Flume-NG
+
 Release 0.1.3 - 2016/10/25
 
 	* Add format parameter for formatter plugins support

data/VERSION

@@ -1 +1 @@
-0.1.3
+0.2.0

data/lib/fluent/plugin/in_flume.rb

@@ -19,175 +19,171 @@
 require 'fluent/input'
 
 module Fluent
+  class FlumeInput < Input
+    Plugin.register_input('flume', self)
+
+    config_param :port,            :integer, :default => 56789
+    config_param :bind,            :string,  :default => '0.0.0.0'
+    config_param :server_type,     :string,  :default => 'simple'
+    config_param :is_framed,       :bool,    :default => false
+    config_param :body_size_limit, :size,    :default => 32 * 1024 * 1024
+    config_param :tag_field,	 :string,  :default => nil
+    config_param :default_tag,	 :string,  :default => 'category'
+    config_param :add_prefix,      :string,  :default => nil
+    config_param :msg_format,      :string,  :default => 'text'
+
+    unless method_defined?(:log)
+      define_method(:log) { $log }
+    end
 
-class FlumeInput < Input
-  Plugin.register_input('flume', self)
-
-  config_param :port,            :integer, :default => 56789
-  config_param :bind,            :string,  :default => '0.0.0.0'
-  config_param :server_type,     :string,  :default => 'simple'
-  config_param :is_framed,       :bool,    :default => false
-  config_param :body_size_limit, :size,    :default => 32 * 1024 * 1024
-  config_param :tag_field,	 :string,  :default => nil
-  config_param :default_tag,	 :string,  :default => 'category'
-  config_param :add_prefix,      :string,  :default => nil
-  config_param :msg_format,      :string,  :default => 'text'
-
-  unless method_defined?(:log)
-    define_method(:log) { $log }
-  end
-
-  def initialize
-    require 'thrift'
-    $:.unshift File.join(File.dirname(__FILE__), 'thrift')
-    require 'flume_types'
-    require 'flume_constants'
-    require 'thrift_flume_event_server'
+    def initialize
+      require 'thrift'
+      require 'fluent/plugin/old_thrift/flume_types'
+      $:.unshift File.join(File.dirname(__FILE__), 'old_thrift')
+      require 'flume_constants'
+      require 'thrift_flume_event_server'
+      super
+    end
 
-    super
-  end
+    def start
+      log.debug "listening flume on #{@bind}:#{@port}"
+      log.warn "[in_flume] This plugin is obsoleted for Flume NG. You should consider to use flume-ng-fluentd-sink."
+
+      handler = FluentFlumeHandler.new
+      handler.tag_field = @tag_field
+      handler.default_tag = @default_tag
+      handler.add_prefix = @add_prefix
+      handler.msg_format = @msg_format
+      handler.log = log
+      handler.router = router
+      processor = ThriftFlumeEventServer::Processor.new handler
+
+      @transport = Thrift::ServerSocket.new @bind, @port
+      unless @is_framed
+        transport_factory = Thrift::BufferedTransportFactory.new
+      else
+        raise ConfigError, "in_flume: unsupported is_framed '#{@is_framed}'"
+      end
 
-  def configure(conf)
-    super
-  end
+      unless ['text', 'json'].include? @msg_format
+        raise 'Unknown format: msg_format=#{@msg_format}'
+      end
 
-  def start
-    log.debug "listening flume on #{@bind}:#{@port}"
-
-    handler = FluentFlumeHandler.new
-    handler.tag_field = @tag_field
-    handler.default_tag = @default_tag
-    handler.add_prefix = @add_prefix
-    handler.msg_format = @msg_format
-    handler.log = log
-    processor = ThriftFlumeEventServer::Processor.new handler
-
-    @transport = Thrift::ServerSocket.new @bind, @port
-    unless @is_framed
-      transport_factory = Thrift::BufferedTransportFactory.new
-    else
-      raise ConfigError, "in_flume: unsupported is_framed '#{@is_framed}'"
+      # 2011/09/29 Kazuki Ohta <kazuki.ohta@gmail.com>
+      # This section is a workaround to set strict_read and strict_write option.
+      # Ruby-Thrift 0.7 set them both 'true' in default, but Flume protocol set
+      # them both 'false'.
+      protocol_factory = Thrift::BinaryProtocolFactory.new
+      protocol_factory.instance_eval {|obj|
+        def get_protocol(trans) # override
+          return Thrift::BinaryProtocol.new(trans,
+                                            strict_read=false,
+                                            strict_write=false)
+        end
+      }
+
+      case @server_type
+      when 'simple'
+        @server = Thrift::SimpleServer.new processor, @transport, transport_factory, protocol_factory
+      when 'threaded'
+        @server = Thrift::ThreadedServer.new processor, @transport, transport_factory, protocol_factory
+      when 'thread_pool'
+        @server = Thrift::ThreadPoolServer.new processor, @transport, transport_factory, protocol_factory
+      else
+        raise ConfigError, "in_flume: unsupported server_type '#{@server_type}'"
+      end
+      @thread = Thread.new(&method(:run))
     end
 
-    unless ['text', 'json'].include? @msg_format
-      raise 'Unknown format: msg_format=#{@msg_format}'
+    def shutdown
+      @transport.close unless @transport.closed?
+      #@thread.join
     end
 
-    # 2011/09/29 Kazuki Ohta <kazuki.ohta@gmail.com>
-    # This section is a workaround to set strict_read and strict_write option.
-    # Ruby-Thrift 0.7 set them both 'true' in default, but Flume protocol set
-    # them both 'false'.
-    protocol_factory = Thrift::BinaryProtocolFactory.new
-    protocol_factory.instance_eval {|obj|
-      def get_protocol(trans) # override
-        return Thrift::BinaryProtocol.new(trans,
-                                          strict_read=false,
-                                          strict_write=false)
-      end
-    }
-
-    case @server_type
-    when 'simple'
-      @server = Thrift::SimpleServer.new processor, @transport, transport_factory, protocol_factory
-    when 'threaded'
-      @server = Thrift::ThreadedServer.new processor, @transport, transport_factory, protocol_factory
-    when 'thread_pool'
-      @server = Thrift::ThreadPoolServer.new processor, @transport, transport_factory, protocol_factory
-    else
-      raise ConfigError, "in_flume: unsupported server_type '#{@server_type}'"
+    def run
+      log.debug "starting server: #{@server}"
+      @server.serve
+    rescue
+      log.error "unexpected error", :error=>$!.to_s
+      log.error_backtrace
     end
-    @thread = Thread.new(&method(:run))
-  end
 
-  def shutdown
-    @transport.close unless @transport.closed?
-    #@thread.join
-  end
-
-  def run
-    log.debug "starting server: #{@server}"
-    @server.serve
-  rescue
-    log.error "unexpected error", :error=>$!.to_s
-    log.error_backtrace
-  end
-
-  class FluentFlumeHandler
-    attr_accessor :tag_field
-    attr_accessor :default_tag
-    attr_accessor :add_prefix
-    attr_accessor :msg_format
-    attr_accessor :log
-
-    def append(evt)
-      begin
-        record = create_record(evt)
-        if @tag_field
-          tag = evt.fieldss[@tag_field] || @default_tag
-          unless tag
-            return # ignore
+    class FluentFlumeHandler
+      attr_accessor :tag_field
+      attr_accessor :default_tag
+      attr_accessor :add_prefix
+      attr_accessor :msg_format
+      attr_accessor :log
+      attr_accessor :router
+
+      def append(evt)
+        begin
+          record = create_record(evt)
+          if @tag_field
+            tag = evt.fieldss[@tag_field] || @default_tag
+            unless tag
+              return # ignore
+            end
+          else
+            tag = @default_tag
           end
-        else
-          tag = @default_tag
-        end
-        timestamp = evt.timestamp.to_i
-        if @add_prefix
-          Engine.emit(@add_prefix + '.' + tag, timestamp, record)
-        else
-          Engine.emit(tag, timestamp, record)
+          timestamp = evt.timestamp.to_i
+          if @add_prefix
+            router.emit(@add_prefix + '.' + tag, timestamp, record)
+          else
+            router.emit(tag, timestamp, record)
+          end
+        rescue => e
+          log.error "unexpected error", :error=>$!.to_s
+          log.error_backtrace
         end
-      rescue => e
-        log.error "unexpected error", :error=>$!.to_s
-        log.error_backtrace
       end
-    end
 
-    def rawAppend(evt)
-      log.error "rawAppend is not implemented yet: #{evt}"
-    end
+      def rawAppend(evt)
+        log.error "rawAppend is not implemented yet: #{evt}"
+      end
 
-    def ackedAppend(evt)
-      begin
-        record = create_record(evt)
-        if @tag_field
-          tag = evt.fieldss[@tag_field] || @default_tag
-          unless tag
-            return # ignore
+      def ackedAppend(evt)
+        begin
+          record = create_record(evt)
+          if @tag_field
+            tag = evt.fieldss[@tag_field] || @default_tag
+            unless tag
+              return # ignore
+            end
+          else
+            tag = @default_tag
           end
-        else
-          tag = @default_tag
-        end
-        timestamp = evt.timestamp.to_i
-        if @add_prefix
-          Engine.emit(@add_prefix + '.' + tag, timestamp, record)
-        else
-          Engine.emit(tag, timestamp, record)
+          timestamp = evt.timestamp.to_i
+          if @add_prefix
+            router.emit(@add_prefix + '.' + tag, timestamp, record)
+          else
+            router.emit(tag, timestamp, record)
+          end
+          return EventStatus::ACK
+        rescue => e
+          log.error "unexpected error", :error=>$!.to_s
+          log.error_backtrace
+          return EventStatus::ERR
         end
-        return EventStatus::ACK
-      rescue => e
-        log.error "unexpected error", :error=>$!.to_s
-        log.error_backtrace
-        return EventStatus::ERR
       end
-    end
 
-    def close()
-    end
+      def close()
+      end
 
-    private
-    def create_record(evt)
-      case @msg_format
-      when 'text'
-        return {'message'=>evt.body.force_encoding('UTF-8')}
-      when 'json'
-        js = JSON.parse(evt.body.force_encoding('UTF-8'))
-        raise 'body must be a Hash, if json_body=true' unless js.is_a?(Hash)
-        return js
-      else
-        raise 'Invalid format: #{@msg_format}'
+      private
+      def create_record(evt)
+        case @msg_format
+        when 'text'
+          return {'message'=>evt.body.force_encoding('UTF-8')}
+        when 'json'
+          js = JSON.parse(evt.body.force_encoding('UTF-8'))
+          raise 'body must be a Hash, if json_body=true' unless js.is_a?(Hash)
+          return js
+        else
+          raise 'Invalid format: #{@msg_format}'
+        end
       end
     end
   end
 end
-
-end

data/lib/fluent/plugin/old_thrift/flume.thrift

@@ -0,0 +1,72 @@
+/**
+ * Licensed to Cloudera, Inc. under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  Cloudera, Inc. licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+# flume.thrift
+#
+# This thrift interface and service defines a network transport mechanism to move events 
+# from one process/machine to another.  At the moment this mirrors the fields of an event 
+# as defined in c.c.f.core.Event.java's code . 
+#
+# This may change more fields are likely to be added, and the actual format is subject to change.
+
+# The server has two rpc methods 
+# -- append: which sends an event to the server,
+# -- close: shuts down this client's connection
+#
+# Currently append is oneway, requiring the thrift server to do flow control.  
+
+namespace java com.cloudera.flume.handlers.thrift
+
+typedef i64 Timestamp
+
+enum Priority { 
+  FATAL = 0,
+  ERROR = 1,
+  WARN = 2,
+  INFO = 3,
+  DEBUG = 4,
+  TRACE = 5
+}
+
+enum EventStatus {
+  ACK = 0,
+  COMMITED = 1,
+  ERR = 2
+}
+
+struct ThriftFlumeEvent {
+  1: Timestamp timestamp,
+  2: Priority priority,
+  3: binary body,
+  4: i64 nanos,
+  5: string host,
+  6: map<string,binary> fieldss
+}
+
+# Instead of using thrift's serialization, we just assume the contents are serialized already.
+struct RawEvent {
+  1: binary raw
+}
+
+service ThriftFlumeEventServer {
+  oneway void append( 1:ThriftFlumeEvent evt ),
+  oneway void rawAppend( 1:RawEvent evt),
+  EventStatus ackedAppend( 1: ThriftFlumeEvent evt ), 
+    
+  void close(), 
+}
+

data/lib/fluent/plugin/old_thrift/flume_constants.rb

@@ -0,0 +1,8 @@
+#
+# Autogenerated by Thrift Compiler (0.9.3)
+#
+# DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING
+#
+
+require 'thrift'
+require 'flume_types'

data/lib/fluent/plugin/old_thrift/flume_types.rb

@@ -0,0 +1,72 @@
+#
+# Autogenerated by Thrift Compiler (0.9.3)
+#
+# DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING
+#
+
+require 'thrift'
+
+module Priority
+  FATAL = 0
+  ERROR = 1
+  WARN = 2
+  INFO = 3
+  DEBUG = 4
+  TRACE = 5
+  VALUE_MAP = {0 => "FATAL", 1 => "ERROR", 2 => "WARN", 3 => "INFO", 4 => "DEBUG", 5 => "TRACE"}
+  VALID_VALUES = Set.new([FATAL, ERROR, WARN, INFO, DEBUG, TRACE]).freeze
+end
+
+module EventStatus
+  ACK = 0
+  COMMITED = 1
+  ERR = 2
+  VALUE_MAP = {0 => "ACK", 1 => "COMMITED", 2 => "ERR"}
+  VALID_VALUES = Set.new([ACK, COMMITED, ERR]).freeze
+end
+
+class ThriftFlumeEvent
+  include ::Thrift::Struct, ::Thrift::Struct_Union
+  TIMESTAMP = 1
+  PRIORITY = 2
+  BODY = 3
+  NANOS = 4
+  HOST = 5
+  FIELDSS = 6
+
+  FIELDS = {
+    TIMESTAMP => {:type => ::Thrift::Types::I64, :name => 'timestamp'},
+    PRIORITY => {:type => ::Thrift::Types::I32, :name => 'priority', :enum_class => ::Priority},
+    BODY => {:type => ::Thrift::Types::STRING, :name => 'body', :binary => true},
+    NANOS => {:type => ::Thrift::Types::I64, :name => 'nanos'},
+    HOST => {:type => ::Thrift::Types::STRING, :name => 'host'},
+    FIELDSS => {:type => ::Thrift::Types::MAP, :name => 'fieldss', :key => {:type => ::Thrift::Types::STRING}, :value => {:type => ::Thrift::Types::STRING, :binary => true}}
+  }
+
+  def struct_fields; FIELDS; end
+
+  def validate
+    unless @priority.nil? || ::Priority::VALID_VALUES.include?(@priority)
+      raise ::Thrift::ProtocolException.new(::Thrift::ProtocolException::UNKNOWN, 'Invalid value of field priority!')
+    end
+  end
+
+  ::Thrift::Struct.generate_accessors self
+end
+
+class RawEvent
+  include ::Thrift::Struct, ::Thrift::Struct_Union
+  RAW = 1
+
+  FIELDS = {
+    RAW => {:type => ::Thrift::Types::STRING, :name => 'raw', :binary => true}
+  }
+
+  def struct_fields; FIELDS; end
+
+  def validate
+  end
+
+  ::Thrift::Struct.generate_accessors self
+end
+

data/lib/fluent/plugin/old_thrift/thrift_flume_event_server.rb

@@ -0,0 +1,219 @@
+#
+# Autogenerated by Thrift Compiler (0.9.3)
+#
+# DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING
+#
+
+require 'thrift'
+require 'flume_types'
+
+module ThriftFlumeEventServer
+  class Client
+    include ::Thrift::Client
+
+    def append(evt)
+      send_append(evt)
+    end
+
+    def send_append(evt)
+      send_oneway_message('append', Append_args, :evt => evt)
+    end
+    def rawAppend(evt)
+      send_rawAppend(evt)
+    end
+
+    def send_rawAppend(evt)
+      send_oneway_message('rawAppend', RawAppend_args, :evt => evt)
+    end
+    def ackedAppend(evt)
+      send_ackedAppend(evt)
+      return recv_ackedAppend()
+    end
+
+    def send_ackedAppend(evt)
+      send_message('ackedAppend', AckedAppend_args, :evt => evt)
+    end
+
+    def recv_ackedAppend()
+      result = receive_message(AckedAppend_result)
+      return result.success unless result.success.nil?
+      raise ::Thrift::ApplicationException.new(::Thrift::ApplicationException::MISSING_RESULT, 'ackedAppend failed: unknown result')
+    end
+
+    def close()
+      send_close()
+      recv_close()
+    end
+
+    def send_close()
+      send_message('close', Close_args)
+    end
+
+    def recv_close()
+      result = receive_message(Close_result)
+      return
+    end
+
+  end
+
+  class Processor
+    include ::Thrift::Processor
+
+    def process_append(seqid, iprot, oprot)
+      args = read_args(iprot, Append_args)
+      @handler.append(args.evt)
+      return
+    end
+
+    def process_rawAppend(seqid, iprot, oprot)
+      args = read_args(iprot, RawAppend_args)
+      @handler.rawAppend(args.evt)
+      return
+    end
+
+    def process_ackedAppend(seqid, iprot, oprot)
+      args = read_args(iprot, AckedAppend_args)
+      result = AckedAppend_result.new()
+      result.success = @handler.ackedAppend(args.evt)
+      write_result(result, oprot, 'ackedAppend', seqid)
+    end
+
+    def process_close(seqid, iprot, oprot)
+      args = read_args(iprot, Close_args)
+      result = Close_result.new()
+      @handler.close()
+      write_result(result, oprot, 'close', seqid)
+    end
+
+  end
+
+  # HELPER FUNCTIONS AND STRUCTURES
+
+  class Append_args
+    include ::Thrift::Struct, ::Thrift::Struct_Union
+    EVT = 1
+
+    FIELDS = {
+      EVT => {:type => ::Thrift::Types::STRUCT, :name => 'evt', :class => ::ThriftFlumeEvent}
+    }
+
+    def struct_fields; FIELDS; end
+
+    def validate
+    end
+
+    ::Thrift::Struct.generate_accessors self
+  end
+
+  class Append_result
+    include ::Thrift::Struct, ::Thrift::Struct_Union
+
+    FIELDS = {
+
+    }
+
+    def struct_fields; FIELDS; end
+
+    def validate
+    end
+
+    ::Thrift::Struct.generate_accessors self
+  end
+
+  class RawAppend_args
+    include ::Thrift::Struct, ::Thrift::Struct_Union
+    EVT = 1
+
+    FIELDS = {
+      EVT => {:type => ::Thrift::Types::STRUCT, :name => 'evt', :class => ::RawEvent}
+    }
+
+    def struct_fields; FIELDS; end
+
+    def validate
+    end
+
+    ::Thrift::Struct.generate_accessors self
+  end
+
+  class RawAppend_result
+    include ::Thrift::Struct, ::Thrift::Struct_Union
+
+    FIELDS = {
+
+    }
+
+    def struct_fields; FIELDS; end
+
+    def validate
+    end
+
+    ::Thrift::Struct.generate_accessors self
+  end
+
+  class AckedAppend_args
+    include ::Thrift::Struct, ::Thrift::Struct_Union
+    EVT = 1
+
+    FIELDS = {
+      EVT => {:type => ::Thrift::Types::STRUCT, :name => 'evt', :class => ::ThriftFlumeEvent}
+    }
+
+    def struct_fields; FIELDS; end
+
+    def validate
+    end
+
+    ::Thrift::Struct.generate_accessors self
+  end
+
+  class AckedAppend_result
+    include ::Thrift::Struct, ::Thrift::Struct_Union
+    SUCCESS = 0
+
+    FIELDS = {
+      SUCCESS => {:type => ::Thrift::Types::I32, :name => 'success', :enum_class => ::EventStatus}
+    }
+
+    def struct_fields; FIELDS; end
+
+    def validate
+      unless @success.nil? || ::EventStatus::VALID_VALUES.include?(@success)
+        raise ::Thrift::ProtocolException.new(::Thrift::ProtocolException::UNKNOWN, 'Invalid value of field success!')
+      end
+    end
+
+    ::Thrift::Struct.generate_accessors self
+  end
+
+  class Close_args
+    include ::Thrift::Struct, ::Thrift::Struct_Union
+
+    FIELDS = {
+
+    }
+
+    def struct_fields; FIELDS; end
+
+    def validate
+    end
+
+    ::Thrift::Struct.generate_accessors self
+  end
+
+  class Close_result
+    include ::Thrift::Struct, ::Thrift::Struct_Union
+
+    FIELDS = {
+
+    }
+
+    def struct_fields; FIELDS; end
+
+    def validate
+    end
+
+    ::Thrift::Struct.generate_accessors self
+  end
+
+end

data/lib/fluent/plugin/out_flume.rb

@@ -19,91 +19,90 @@
 require 'fluent/output'
 
 module Fluent
+  class FlumeOutput < BufferedOutput
+    Fluent::Plugin.register_output('flume', self)
+
+    config_param :host,      :string,  :default => 'localhost'
+    config_param :port,      :integer, :default => 35863
+    config_param :timeout,   :integer, :default => 30
+    config_param :remove_prefix,    :string, :default => nil
+    config_param :default_category, :string, :default => 'unknown'
+    desc "The format of the thrift body. (default: json)"
+    config_param :format, :string, default: 'json'
+    config_param :trim_nl, :bool, default: true
+
+    unless method_defined?(:log)
+      define_method(:log) { $log }
+    end
 
-class FlumeOutput < BufferedOutput
-  Fluent::Plugin.register_output('flume', self)
-
-  config_param :host,      :string,  :default => 'localhost'
-  config_param :port,      :integer, :default => 35863
-  config_param :timeout,   :integer, :default => 30
-  config_param :remove_prefix,    :string, :default => nil
-  config_param :default_category, :string, :default => 'unknown'
-  desc "The format of the thrift body. (default: json)"
-  config_param :format, :string, default: 'json'
-  config_param :trim_nl, :bool, default: true
-
-  unless method_defined?(:log)
-    define_method(:log) { $log }
-  end
-
-  def initialize
-    require 'thrift'
-    $:.unshift File.join(File.dirname(__FILE__), 'thrift')
-    require 'flume_types'
-    require 'flume_constants'
-    require 'thrift_source_protocol'
-    super
-  end
+    def initialize
+      require 'thrift'
+      require 'fluent/plugin/thrift/flume_types'
+      $:.unshift File.join(File.dirname(__FILE__), 'thrift')
+      require 'flume_constants'
+      require 'thrift_source_protocol'
+      super
+    end
 
-  def configure(conf)
-    # override default buffer_chunk_limit
-    conf['buffer_chunk_limit'] ||= '1m'
+    def configure(conf)
+      # override default buffer_chunk_limit
+      conf['buffer_chunk_limit'] ||= '1m'
 
-    super
+      super
 
-    @formatter = Plugin.new_formatter(@format)
-    @formatter.configure(conf)
-  end
+      @formatter = Plugin.new_formatter(@format)
+      @formatter.configure(conf)
+    end
 
-  def start
-    super
+    def start
+      super
 
-    if @remove_prefix
-      @removed_prefix_string = @remove_prefix + '.'
-      @removed_length = @removed_prefix_string.length
+      if @remove_prefix
+        @removed_prefix_string = @remove_prefix + '.'
+        @removed_length = @removed_prefix_string.length
+      end
     end
-  end
-
-  def shutdown
-    super
-  end
 
-  def format(tag, time, record)
-    if @remove_prefix and
-        ((tag[0, @removed_length] == @removed_prefix_string and tag.length > @removed_length) or tag == @remove_prefix)
-      tag = (tag[@removed_length..-1] || @default_category)
+    def emit(tag, es, chain)
+      if @remove_prefix and
+         ((tag[0, @removed_length] == @removed_prefix_string and tag.length > @removed_length) or tag == @remove_prefix)
+        tag = (tag[@removed_length..-1] || @default_category)
+      end
+      super(tag, es, chain, tag)
     end
-    fr = @formatter.format(tag, time, record)
-    fr.chomp! if @trim_nl
-    [tag, time, fr].to_msgpack
-  end
 
-  def write(chunk)
-    socket = Thrift::Socket.new @host, @port, @timeout
-    transport = Thrift::FramedTransport.new socket
-    #protocol = Thrift::BinaryProtocol.new transport, false, false
-    protocol = Thrift::CompactProtocol.new transport
-    client = ThriftSourceProtocol::Client.new protocol
+    def format(tag, time, record)
+      fr = @formatter.format(tag, time, record)
+      fr.chomp! if @trim_nl
+      [time, fr].to_msgpack
+    end
 
-    count = 0
-    header = {}
-    transport.open
-    log.debug "thrift client opened: #{client}"
-    begin
-      chunk.msgpack_each { |tag, time, record|
-        header['timestamp'.freeze] = time.to_s
-        header['tag'.freeze] = tag
-        entry = ThriftFlumeEvent.new(:body    => record.force_encoding('UTF-8'),
-                                     :headers => header)
-        client.append entry
-        count += 1
-      }
-      log.debug "Writing #{count} entries to flume"
-    ensure
-      log.debug "thrift client closing: #{client}"
-      transport.close
+    def write(chunk)
+      socket = Thrift::Socket.new @host, @port, @timeout
+      transport = Thrift::FramedTransport.new socket
+      #protocol = Thrift::BinaryProtocol.new transport, false, false
+      protocol = Thrift::CompactProtocol.new transport
+      client = ThriftSourceProtocol::Client.new protocol
+
+      tag = chunk.key
+      count = 0
+      header = {}
+      transport.open
+      log.debug "thrift client opened: #{client}"
+      begin
+        chunk.msgpack_each { |time, record|
+          header['timestamp'.freeze] = time.to_s
+          header['tag'.freeze] = tag
+          entry = ThriftFlumeEvent.new(:body    => record.force_encoding('UTF-8'),
+                                       :headers => header)
+          client.append entry
+          count += 1
+        }
+        log.debug "Writing #{count} entries to flume"
+      ensure
+        log.debug "thrift client closing: #{client}"
+        transport.close
+      end
     end
   end
 end
-
-end

data/test/plugin/test_in_flume.rb

@@ -160,9 +160,9 @@ class FlumeInputTest < Test::Unit::TestCase
   def test_message_format_json
     d = create_driver(CONFIG + %[
       tag_field category
-      message_format json
+      msg_format json
     ])
-    assert_equal 'json', d.instance.message_format
+    assert_equal 'json', d.instance.msg_format
 
     time = Time.parse("2011-01-02 13:14:15 UTC").to_i
     Fluent::Engine.now = time
@@ -185,8 +185,6 @@ class FlumeInputTest < Test::Unit::TestCase
   end
 
   def setup
-    omit("Latest flume thrift protocol does not have event server.")
-
     Fluent::FlumeInput.new
     Fluent::Test.setup
   end

data/test/plugin/test_out_flume.rb

@@ -41,9 +41,8 @@ class FlumeOutputTest < Test::Unit::TestCase
     d = create_driver
     d.emit({"k11"=>"v11", "k12"=>"v12"}, time)
     d.emit({"k21"=>"v21", "k22"=>"v22"}, time)
-    puts ({"k11" => "v11", "k12" => "v12"}.to_json)
-    d.expect_format [d.tag, time, {"k11"=>"v11", "k12"=>"v12"}.to_json].to_msgpack
-    d.expect_format [d.tag, time, {"k21"=>"v21", "k22"=>"v22"}.to_json].to_msgpack
+    d.expect_format [time, {"k11"=>"v11", "k12"=>"v12"}.to_json].to_msgpack
+    d.expect_format [time, {"k21"=>"v21", "k22"=>"v22"}.to_json].to_msgpack
     d.run
 
     d = create_driver(CONFIG + %[
@@ -53,8 +52,8 @@ class FlumeOutputTest < Test::Unit::TestCase
 
     d.emit({"k11"=>"v11", "k12"=>"v12"}, time)
     d.emit({"k21"=>"v21", "k22"=>"v22"}, time)
-    d.expect_format ['flumeplugin', time, {"k11"=>"v11", "k12"=>"v12"}.to_json].to_msgpack
-    d.expect_format ['flumeplugin', time, {"k21"=>"v21", "k22"=>"v22"}.to_json].to_msgpack
+    d.expect_format [time, {"k11"=>"v11", "k12"=>"v12"}.to_json].to_msgpack
+    d.expect_format [time, {"k21"=>"v21", "k22"=>"v22"}.to_json].to_msgpack
     d.run
 
     d = create_driver(CONFIG + %[
@@ -62,7 +61,7 @@ class FlumeOutputTest < Test::Unit::TestCase
     ], 'xxx.test.flumeplugin')
     assert_equal 'xxx.test.flumeplugin', d.tag
     d.emit({"k11"=>"v11", "k12"=>"v12"}, time)
-    d.expect_format ['xxx.test.flumeplugin', time, {"k11"=>"v11", "k12"=>"v12"}.to_json].to_msgpack
+    d.expect_format [time, {"k11"=>"v11", "k12"=>"v12"}.to_json].to_msgpack
     d.run
 
     d = create_driver(CONFIG + %[
@@ -70,7 +69,7 @@ class FlumeOutputTest < Test::Unit::TestCase
     ], 'test')
     assert_equal 'test', d.tag
     d.emit({"k11"=>"v11", "k12"=>"v12"}, time)
-    d.expect_format ['unknown', time, {"k11"=>"v11", "k12"=>"v12"}.to_json].to_msgpack
+    d.expect_format [time, {"k11"=>"v11", "k12"=>"v12"}.to_json].to_msgpack
     d.run
 
     d = create_driver(CONFIG + %[
@@ -79,8 +78,8 @@ class FlumeOutputTest < Test::Unit::TestCase
     assert_equal 'test.flumeplugin', d.tag
     d.emit({"k11"=>"v11", "k12"=>"v12"}, time)
     d.emit({"k21"=>"v21", "k22"=>"v22"}, time)
-    d.expect_format ['flumeplugin', time, {"k11"=>"v11", "k12"=>"v12"}.to_json].to_msgpack
-    d.expect_format ['flumeplugin', time, {"k21"=>"v21", "k22"=>"v22"}.to_json].to_msgpack
+    d.expect_format [time, {"k11"=>"v11", "k12"=>"v12"}.to_json].to_msgpack
+    d.expect_format [time, {"k21"=>"v21", "k22"=>"v22"}.to_json].to_msgpack
     d.run
   end
 

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-flume
 version: !ruby/object:Gem::Version
-  version: 0.1.3
+  version: 0.2.0
 platform: ruby
 authors:
 - Muga Nishizawa
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2016-10-24 00:00:00.000000000 Z
+date: 2016-10-25 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fluentd
@@ -86,6 +86,10 @@ files:
 - example-out-flume.conf
 - fluent-plugin-flume.gemspec
 - lib/fluent/plugin/in_flume.rb
+- lib/fluent/plugin/old_thrift/flume.thrift
+- lib/fluent/plugin/old_thrift/flume_constants.rb
+- lib/fluent/plugin/old_thrift/flume_types.rb
+- lib/fluent/plugin/old_thrift/thrift_flume_event_server.rb
 - lib/fluent/plugin/out_flume.rb
 - lib/fluent/plugin/thrift/flume.thrift
 - lib/fluent/plugin/thrift/flume.thrift.orig