fluent-plugin-flume 0.1.0

data/lib/fluent/plugin/thrift/flume.thrift

@@ -0,0 +1,72 @@
+/**
+ * Licensed to Cloudera, Inc. under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  Cloudera, Inc. licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+# flume.thrift
+#
+# This thrift interface and service defines a network transport mechanism to move events 
+# from one process/machine to another.  At the moment this mirrors the fields of an event 
+# as defined in c.c.f.core.Event.java's code . 
+#
+# This may change more fields are likely to be added, and the actual format is subject to change.
+
+# The server has two rpc methods 
+# -- append: which sends an event to the server,
+# -- close: shuts down this client's connection
+#
+# Currently append is oneway, requiring the thrift server to do flow control.  
+
+namespace java com.cloudera.flume.handlers.thrift
+
+typedef i64 Timestamp
+
+enum Priority { 
+  FATAL = 0,
+  ERROR = 1,
+  WARN = 2,
+  INFO = 3,
+  DEBUG = 4,
+  TRACE = 5
+}
+
+enum EventStatus {
+  ACK = 0,
+  COMMITED = 1,
+  ERR = 2
+}
+
+struct ThriftFlumeEvent {
+  1: Timestamp timestamp,
+  2: Priority priority,
+  3: binary body,
+  4: i64 nanos,
+  5: string host,
+  6: map<string,binary> fieldss
+}
+
+# Instead of using thrift's serialization, we just assume the contents are serialized already.
+struct RawEvent {
+  1: binary raw
+}
+
+service ThriftFlumeEventServer {
+  oneway void append( 1:ThriftFlumeEvent evt ),
+  oneway void rawAppend( 1:RawEvent evt),
+  EventStatus ackedAppend( 1: ThriftFlumeEvent evt ), 
+    
+  void close(), 
+}
+

data/lib/fluent/plugin/thrift/flume.thrift.orig

@@ -0,0 +1,72 @@
+/**
+ * Licensed to Cloudera, Inc. under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  Cloudera, Inc. licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+# flume.thrift
+#
+# This thrift interface and service defines a network transport mechanism to move events 
+# from one process/machine to another.  At the moment this mirrors the fields of an event 
+# as defined in c.c.f.core.Event.java's code . 
+#
+# This may change more fields are likely to be added, and the actual format is subject to change.
+
+# The server has two rpc methods 
+# -- append: which sends an event to the server,
+# -- close: shuts down this client's connection
+#
+# Currently append is oneway, requiring the thrift server to do flow control.  
+
+namespace java com.cloudera.flume.handlers.thrift
+
+typedef i64 Timestamp
+
+enum Priority { 
+  FATAL = 0,
+  ERROR = 1,
+  WARN = 2,
+  INFO = 3,
+  DEBUG = 4,
+  TRACE = 5
+}
+
+enum EventStatus {
+  ACK = 0,
+  COMMITED = 1,
+  ERR = 2
+}
+
+struct ThriftFlumeEvent {
+  1: Timestamp timestamp,
+  2: Priority priority,
+  3: binary body,
+  4: i64 nanos,
+  5: string host,
+  6: map<string,binary> fields
+}
+
+# Instead of using thrift's serialization, we just assume the contents are serialized already.
+struct RawEvent {
+  1: binary raw
+}
+
+service ThriftFlumeEventServer {
+  oneway void append( 1:ThriftFlumeEvent evt ),
+  oneway void rawAppend( 1:RawEvent evt),
+  EventStatus ackedAppend( 1: ThriftFlumeEvent evt ), 
+    
+  void close(), 
+}
+

data/lib/fluent/plugin/thrift/flume_constants.rb

@@ -0,0 +1,8 @@
+#
+# Autogenerated by Thrift
+#
+# DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING
+#
+
+require 'flume_types'
+

data/lib/fluent/plugin/thrift/flume_types.rb

@@ -0,0 +1,71 @@
+#
+# Autogenerated by Thrift
+#
+# DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING
+#
+
+
+module Priority
+  FATAL = 0
+  ERROR = 1
+  WARN = 2
+  INFO = 3
+  DEBUG = 4
+  TRACE = 5
+  VALUE_MAP = {0 => "FATAL", 1 => "ERROR", 2 => "WARN", 3 => "INFO", 4 => "DEBUG", 5 => "TRACE"}
+  VALID_VALUES = Set.new([FATAL, ERROR, WARN, INFO, DEBUG, TRACE]).freeze
+end
+
+module EventStatus
+  ACK = 0
+  COMMITED = 1
+  ERR = 2
+  VALUE_MAP = {0 => "ACK", 1 => "COMMITED", 2 => "ERR"}
+  VALID_VALUES = Set.new([ACK, COMMITED, ERR]).freeze
+end
+
+class ThriftFlumeEvent
+  include ::Thrift::Struct, ::Thrift::Struct_Union
+  TIMESTAMP = 1
+  PRIORITY = 2
+  BODY = 3
+  NANOS = 4
+  HOST = 5
+  FIELDSS = 6
+
+  FIELDS = {
+    TIMESTAMP => {:type => ::Thrift::Types::I64, :name => 'timestamp'},
+    PRIORITY => {:type => ::Thrift::Types::I32, :name => 'priority', :enum_class => Priority},
+    BODY => {:type => ::Thrift::Types::STRING, :name => 'body', :binary => true},
+    NANOS => {:type => ::Thrift::Types::I64, :name => 'nanos'},
+    HOST => {:type => ::Thrift::Types::STRING, :name => 'host'},
+    FIELDSS => {:type => ::Thrift::Types::MAP, :name => 'fieldss', :key => {:type => ::Thrift::Types::STRING}, :value => {:type => ::Thrift::Types::STRING, :binary => true}}
+  }
+
+  def struct_fields; FIELDS; end
+
+  def validate
+    unless @priority.nil? || Priority::VALID_VALUES.include?(@priority)
+      raise ::Thrift::ProtocolException.new(::Thrift::ProtocolException::UNKNOWN, 'Invalid value of field priority!')
+    end
+  end
+
+  ::Thrift::Struct.generate_accessors self
+end
+
+class RawEvent
+  include ::Thrift::Struct, ::Thrift::Struct_Union
+  RAW = 1
+
+  FIELDS = {
+    RAW => {:type => ::Thrift::Types::STRING, :name => 'raw', :binary => true}
+  }
+
+  def struct_fields; FIELDS; end
+
+  def validate
+  end
+
+  ::Thrift::Struct.generate_accessors self
+end
+

data/lib/fluent/plugin/thrift/thrift_flume_event_server.rb

@@ -0,0 +1,220 @@
+#
+# Autogenerated by Thrift
+#
+# DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING
+#
+
+require 'thrift'
+require 'flume_types'
+
+module ThriftFlumeEventServer
+  class Client
+    include ::Thrift::Client
+
+    def append(evt)
+      send_append(evt)
+    end
+
+    def send_append(evt)
+      send_message('append', Append_args, :evt => evt)
+    end
+    def rawAppend(evt)
+      send_rawAppend(evt)
+    end
+
+    def send_rawAppend(evt)
+      send_message('rawAppend', RawAppend_args, :evt => evt)
+    end
+    def ackedAppend(evt)
+      send_ackedAppend(evt)
+      return recv_ackedAppend()
+    end
+
+    def send_ackedAppend(evt)
+      send_message('ackedAppend', AckedAppend_args, :evt => evt)
+    end
+
+    def recv_ackedAppend()
+      result = receive_message(AckedAppend_result)
+      return result.success unless result.success.nil?
+      raise ::Thrift::ApplicationException.new(::Thrift::ApplicationException::MISSING_RESULT, 'ackedAppend failed: unknown result')
+    end
+
+    def close()
+      send_close()
+      recv_close()
+    end
+
+    def send_close()
+      send_message('close', Close_args)
+    end
+
+    def recv_close()
+      result = receive_message(Close_result)
+      return
+    end
+
+  end
+
+  class Processor
+    include ::Thrift::Processor
+
+    def process_append(seqid, iprot, oprot)
+      args = read_args(iprot, Append_args)
+      @handler.append(args.evt)
+      return
+    end
+
+    def process_rawAppend(seqid, iprot, oprot)
+      args = read_args(iprot, RawAppend_args)
+      @handler.rawAppend(args.evt)
+      return
+    end
+
+    def process_ackedAppend(seqid, iprot, oprot)
+      args = read_args(iprot, AckedAppend_args)
+      result = AckedAppend_result.new()
+      result.success = @handler.ackedAppend(args.evt)
+      write_result(result, oprot, 'ackedAppend', seqid)
+    end
+
+    def process_close(seqid, iprot, oprot)
+      args = read_args(iprot, Close_args)
+      result = Close_result.new()
+      @handler.close()
+      write_result(result, oprot, 'close', seqid)
+    end
+
+  end
+
+  # HELPER FUNCTIONS AND STRUCTURES
+
+  class Append_args
+    include ::Thrift::Struct, ::Thrift::Struct_Union
+    EVT = 1
+
+    FIELDS = {
+      EVT => {:type => ::Thrift::Types::STRUCT, :name => 'evt', :class => ThriftFlumeEvent}
+    }
+
+    def struct_fields; FIELDS; end
+
+    def validate
+    end
+
+    ::Thrift::Struct.generate_accessors self
+  end
+
+  class Append_result
+    include ::Thrift::Struct, ::Thrift::Struct_Union
+
+    FIELDS = {
+
+    }
+
+    def struct_fields; FIELDS; end
+
+    def validate
+    end
+
+    ::Thrift::Struct.generate_accessors self
+  end
+
+  class RawAppend_args
+    include ::Thrift::Struct, ::Thrift::Struct_Union
+    EVT = 1
+
+    FIELDS = {
+      EVT => {:type => ::Thrift::Types::STRUCT, :name => 'evt', :class => RawEvent}
+    }
+
+    def struct_fields; FIELDS; end
+
+    def validate
+    end
+
+    ::Thrift::Struct.generate_accessors self
+  end
+
+  class RawAppend_result
+    include ::Thrift::Struct, ::Thrift::Struct_Union
+
+    FIELDS = {
+
+    }
+
+    def struct_fields; FIELDS; end
+
+    def validate
+    end
+
+    ::Thrift::Struct.generate_accessors self
+  end
+
+  class AckedAppend_args
+    include ::Thrift::Struct, ::Thrift::Struct_Union
+    EVT = 1
+
+    FIELDS = {
+      EVT => {:type => ::Thrift::Types::STRUCT, :name => 'evt', :class => ThriftFlumeEvent}
+    }
+
+    def struct_fields; FIELDS; end
+
+    def validate
+    end
+
+    ::Thrift::Struct.generate_accessors self
+  end
+
+  class AckedAppend_result
+    include ::Thrift::Struct, ::Thrift::Struct_Union
+    SUCCESS = 0
+
+    FIELDS = {
+      SUCCESS => {:type => ::Thrift::Types::I32, :name => 'success', :enum_class => EventStatus}
+    }
+
+    def struct_fields; FIELDS; end
+
+    def validate
+      unless @success.nil? || EventStatus::VALID_VALUES.include?(@success)
+        raise ::Thrift::ProtocolException.new(::Thrift::ProtocolException::UNKNOWN, 'Invalid value of field success!')
+      end
+    end
+
+    ::Thrift::Struct.generate_accessors self
+  end
+
+  class Close_args
+    include ::Thrift::Struct, ::Thrift::Struct_Union
+
+    FIELDS = {
+
+    }
+
+    def struct_fields; FIELDS; end
+
+    def validate
+    end
+
+    ::Thrift::Struct.generate_accessors self
+  end
+
+  class Close_result
+    include ::Thrift::Struct, ::Thrift::Struct_Union
+
+    FIELDS = {
+
+    }
+
+    def struct_fields; FIELDS; end
+
+    def validate
+    end
+
+    ::Thrift::Struct.generate_accessors self
+  end
+
+end
+

data/test/plugin/test_in_flume.rb

@@ -0,0 +1,209 @@
+require 'test/unit'
+require 'fluent/test'
+require 'fluent/plugin/in_flume'
+
+class FlumeInputTest < Test::Unit::TestCase
+  CONFIG = %[
+    port 56790
+    bind 127.0.0.1
+  ]
+
+  def create_driver(conf=CONFIG)
+    Fluent::Test::InputTestDriver.new(Fluent::FlumeInput).configure(conf)
+  end
+
+  def test_configure
+    d = create_driver ''
+    assert_equal 56789, d.instance.port
+    assert_equal '0.0.0.0', d.instance.bind
+
+    d = create_driver
+    assert_equal 56790, d.instance.port
+    assert_equal '127.0.0.1', d.instance.bind
+  end
+
+  def test_time
+    d = create_driver
+
+    time = Time.parse("2011-01-02 13:14:15 UTC").to_i
+    Fluent::Engine.now = time
+
+    d.expect_emit "category", time, {"message"=>'aiueo'}
+    d.expect_emit "category", time, {"message"=>'aiueo'}
+
+    emits = [
+      ['tag1', time, {"message"=>'aiueo'}],
+      ['tag2', time, {"message"=>'aiueo'}]
+    ]
+
+    d.run do
+      emits.each { |tag, time, record|
+        send('tag', tag, time, record['message'])
+        sleep 0.01
+      }
+    end
+
+    d2 = create_driver(CONFIG + %[
+      default_tag flume
+    ])
+
+    time = Time.parse("2011-01-02 13:14:15 UTC").to_i
+    Fluent::Engine.now = time
+
+    d2.expect_emit "flume", time, {"message"=>'aiueo'}
+    d2.expect_emit "flume", time, {"message"=>'aiueo'}
+
+    emits = [
+      ['tag1', time, {"message"=>'aiueo'}],
+      ['tag2', time, {"message"=>'aiueo'}]
+    ]
+
+    d2.run do
+      emits.each { |tag, time, record|
+        send('tag', tag, time, record['message'])
+        sleep 0.01
+      }
+    end
+
+    d3 = create_driver(CONFIG + %[
+      tag_field flume
+    ])
+
+    time = Time.parse("2011-01-02 13:14:15 UTC").to_i
+    Fluent::Engine.now = time
+
+    d3.expect_emit "tag1", time, {"message"=>'aiueo'}
+    d3.expect_emit "tag2", time, {"message"=>'aiueo'}
+
+    emits = [
+      ['tag1', time, {"message"=>'aiueo'}],
+      ['tag2', time, {"message"=>'aiueo'}]
+    ]
+
+    d3.run do
+      emits.each { |tag, time, record|
+        send('flume', tag, time, record['message'])
+        sleep 0.01
+      }
+    end
+
+    d4 = create_driver(CONFIG + %[
+      default_tag fluent
+      tag_field flume
+    ])
+
+    time = Time.parse("2011-01-02 13:14:15 UTC").to_i
+    Fluent::Engine.now = time
+
+    d4.expect_emit "tag1", time, {"message"=>'aiueo'}
+    d4.expect_emit "tag2", time, {"message"=>'aiueo'}
+
+    emits = [
+      ['tag1', time, {"message"=>'aiueo'}],
+      ['tag2', time, {"message"=>'aiueo'}]
+    ]
+
+    d4.run do
+      emits.each { |tag, time, record|
+        send('flume', tag, time, record['message'])
+        sleep 0.01
+      }
+    end
+  end
+
+  def test_add_prefix
+    d = create_driver(CONFIG + %[
+      add_prefix flume
+      tag_field category
+    ])
+
+    time = Time.parse("2011-01-02 13:14:15 UTC").to_i
+    Fluent::Engine.now = time
+
+    d.expect_emit "flume.tag1", time, {"message"=>'aiueo'}
+    d.expect_emit "flume.tag2", time, {"message"=>'aiueo'}
+
+    emits = [
+      ['tag1', time, {"message"=>'aiueo'}],
+      ['tag2', time, {"message"=>'aiueo'}],
+    ]
+    d.run do
+      emits.each { |tag, time, record|
+        send('category', tag, time, record['message'])
+        sleep 0.01
+      }
+    end
+
+    d2 = create_driver(CONFIG + %[
+      add_prefix flume.input
+      tag_field category
+    ])
+
+    time = Time.parse("2011-01-02 13:14:15 UTC").to_i
+    Fluent::Engine.now = time
+
+    d2.expect_emit "flume.input.tag3", time, {"message"=>'aiueo'}
+    d2.expect_emit "flume.input.tag4", time, {"message"=>'aiueo'}
+
+    emits = [
+      ['tag3', time, {"message"=>'aiueo'}],
+      ['tag4', time, {"message"=>'aiueo'}],
+    ]
+    d2.run do
+      emits.each { |tag, time, record|
+        send('category', tag, time, record['message'])
+        sleep 0.01
+      }
+    end
+  end
+
+  def test_message_format_json
+    d = create_driver(CONFIG + %[
+      tag_field category
+      message_format json
+    ])
+    assert_equal 'json', d.instance.message_format
+
+    time = Time.parse("2011-01-02 13:14:15 UTC").to_i
+    Fluent::Engine.now = time
+
+    d.expect_emit "tag1", time, {"a"=>1}
+    d.expect_emit "tag2", time, {"a"=>1, "b"=>2}
+    d.expect_emit "tag3", time, {"a"=>1, "b"=>2, "c"=>3}
+
+    emits = [
+      ['tag1', time, {"a"=>1}.to_json],
+      ['tag2', time, {"a"=>1, "b"=>2}.to_json],
+      ['tag3', time, {"a"=>1, "b"=>2, "c"=>3}.to_json],
+    ]
+    d.run do
+      emits.each { |tag, time, message|
+        send('category', tag, time, message)
+        sleep 0.01
+      }
+    end
+  end
+
+  def setup
+    Fluent::FlumeInput.new
+    Fluent::Test.setup
+  end
+
+  def send(tag_field, tag, time, msg)
+    socket = Thrift::Socket.new '127.0.0.1', 56790
+    #transport = Thrift::Socket.new '127.0.0.1', 56790
+    transport = Thrift::BufferedTransport.new socket
+    protocol = Thrift::BinaryProtocol.new transport
+    client = ThriftFlumeEventServer::Client.new protocol
+    transport.open
+    raw_sock = socket.to_io
+    raw_sock.setsockopt Socket::IPPROTO_TCP, Socket::TCP_NODELAY, 1
+
+    entry = ThriftFlumeEvent.new(:body=>msg.to_s,
+                                 :priority=>Priority::INFO,
+                                 :timestamp=>time,
+                                 :fieldss=>{tag_field=>tag})
+    client.append entry
+    transport.close
+  end
+end