fluent-plugin-flume 0.1.0

data/AUTHORS

@@ -0,0 +1 @@
+Muga Nishizawa <muga.nishizawa _at_ gmail.com>

data/ChangeLog

@@ -0,0 +1 @@
+Release 0.1.0 - 2012/05/02

data/README.rdoc

@@ -0,0 +1,105 @@
+= Flume input/output plugin for Fluent
+
+== Overview
+
+This is a plugin for fluentd[https://github.com/fluentd] event collector.  This plugin adds the Flume[https://github.com/apache/flume] compatible interface to fluentd.
+
+== What's Flume?
+
+Flume[https://github.com/apache/flume] is a distributed, reliable, and available service for efficiently collecting, aggregating, and moving large amounts of log data.
+
+It uses Thrift[http://thrift.apache.org/], a cross-language RPC framework, to communicate between clients and servers.
+
+== What's Flume plugin for fluent?
+
+The Flume plugin for fluentd, which enables fluentd to talk the Flume protocol.  Flume protocol is defined as follows, in Thrift-IDL format:
+
+  typedef i64 Timestamp
+  
+  enum Priority {
+    FATAL = 0,
+    ERROR = 1,
+    WARN = 2,
+    INFO = 3,
+    DEBUG = 4,
+    TRACE = 5
+  }
+  
+  enum EventStatus {
+    ACK = 0,
+    COMMITED = 1,
+    ERR = 2
+  }
+  
+  struct ThriftFlumeEvent {
+    1: Timestamp timestamp,
+    2: Priority priority,
+    3: binary body,
+    4: i64 nanos,
+    5: string host,
+    6: map<string,binary> fields
+  }
+  
+  # Instead of using thrift's serialization, we just assume the contents are serialized already.
+  struct RawEvent {
+    1: binary raw
+  }
+  
+  service ThriftFlumeEventServer {
+    oneway void append( 1:ThriftFlumeEvent evt ),
+    oneway void rawAppend( 1:RawEvent evt),
+    EventStatus ackedAppend( 1: ThriftFlumeEvent evt ),
+    
+    void close(),
+  }
+
+A value that is stored in the ThriftFlumeEvent.fields map is used as fluentd 'tag'.  A key of the value enables be specified by users as configuration parameter.
+
+== How to use?
+
+fluent-plugin-flume contains both input and output.
+
+== Flume Input
+
+Please add the following configurations to fluent.conf.
+
+  # Flume input
+  <source>
+    type flume
+    port 56789
+  </source>
+
+These options are supported.
+
+* port: port number (default: 56789)
+* bind: bind address (default: 0.0.0.0)
+* server_type: server architecture either in 'simple', 'threaded', 'thread_pool', (default: simple)
+* is_framed: use framed protocol or not (default: false)
+* tag_field: key name of fluentd 'tag' that is stored in ThriftFlumeEvent.fields (default: nil)
+* default_tag: default fluentd 'tag' (default: 'category')
+* add_prefix: prefix string, added to the tag (default: nil)
+
+== Flume Output
+
+Please add the following configurations to fluent.conf.  This allows fluentd to output its logs into another Flume server.  Note that fluentd conveys semi-structured data while Flume conveys unstructured data.  Thus the plugin translates semi-structured data into JSON data and conveys it to Flume.
+
+  # Flume output
+  <match *>
+    type flume
+    host flume-host.local
+    port 56789
+  </match>
+
+These options are supported.
+
+* host: host name or address (default: localhost)
+* port: port number (default: 35863)
+* timeout: thrift protocol timeout (default: 30)
+* remove_prefix: prefix string, removed from the tag (default: nil)
+
+== Contributors
+
+== Copyright
+
+Copyright:: Copyright (c) 2012 Treasure Data, Inc.
+License::   Apache License, Version 2.0

data/Rakefile

@@ -0,0 +1,57 @@
+require 'rake'
+require 'rake/testtask'
+require 'rake/clean'
+
+begin
+  require 'jeweler'
+  Jeweler::Tasks.new do |gemspec|
+    gemspec.name = "fluent-plugin-flume"
+    gemspec.summary = "Flume Input/Output plugin for Fluentd event collector"
+    gemspec.author = "Muga Nishizawa"
+    gemspec.email = "muga.nishizawa@gmail.com"
+    gemspec.homepage = "https://github.com/muga/fluent-plugin-flume"
+    gemspec.has_rdoc = false
+    gemspec.require_paths = ["lib"]
+    gemspec.add_dependency "fluentd", "~> 0.10.16"
+    gemspec.add_dependency "thrift", "~> 0.6.0"
+    gemspec.test_files = Dir["test/**/*.rb"]
+    gemspec.files = Dir["bin/**/*", "lib/**/*", "test/**/*.rb"] +
+      %w[example.conf VERSION AUTHORS Rakefile fluent-plugin-flume.gemspec]
+    gemspec.executables = ['fluent-flume-remote']
+  end
+  Jeweler::GemcutterTasks.new
+rescue LoadError
+  puts "Jeweler not available. Install it with: gem install jeweler"
+end
+
+task "thrift_gen" do
+  system "mkdir -p tmp"
+  system "thrift --gen rb -o tmp lib/fluent/plugin/thrift/flume.thrift"
+  system "mv tmp/gen-rb/* lib/fluent/plugin/thrift/"
+  system "rm -fR tmp"
+end
+
+Rake::TestTask.new(:test) do |t|
+  t.test_files = Dir['test/plugin/*.rb']
+  t.ruby_opts = ['-rubygems'] if defined? Gem
+  t.ruby_opts << '-I.'
+end
+
+#VERSION_FILE = "lib/fluent/version.rb"
+#
+#file VERSION_FILE => ["VERSION"] do |t|
+#  version = File.read("VERSION").strip
+#  File.open(VERSION_FILE, "w") {|f|
+#    f.write <<EOF
+#module Fluent
+#
+#VERSION = '#{version}'
+#
+#end
+#EOF
+#  }
+#end
+#
+#task :default => [VERSION_FILE, :build]
+
+task :default => [:build]

data/VERSION

@@ -0,0 +1 @@
+0.1.0

data/bin/fluent-flume-remote

@@ -0,0 +1,28 @@
+#!/usr/bin/env ruby
+require 'thrift'
+require 'json'
+$:.unshift File.join(File.dirname(__FILE__), '../lib/fluent/plugin/thrift')
+require 'flume_types'
+require 'flume_constants'
+require 'thrift_flume_event_server'
+
+host = 'localhost'
+port = 3586
+
+socket = Thrift::Socket.new host, port.to_i
+transport = Thrift::BufferedTransport.new socket
+protocol = Thrift::BinaryProtocol.new transport
+client = ThriftFlumeEventServer::Client.new protocol
+transport.open
+
+# 2011/09/02 Kazuki Ohta <kazuki.ohta@gmail.com>
+# explicitly specify TCP_NODELAY for low-latency communication.
+raw_sock = socket.to_io
+raw_sock.setsockopt Socket::IPPROTO_TCP, Socket::TCP_NODELAY, 1
+
+entry = ThriftFlumeEvent.new(:body=>{'a'=>'b'}.to_json,
+                             :priority=>Priority::INFO,
+                             :timestamp=>(Time.now.to_i * 1000))
+client.append entry
+
+transport.close

data/fluent-plugin-flume.gemspec

@@ -0,0 +1,56 @@
+# Generated by jeweler
+# DO NOT EDIT THIS FILE DIRECTLY
+# Instead, edit Jeweler::Tasks in Rakefile, and run 'rake gemspec'
+# -*- encoding: utf-8 -*-
+
+Gem::Specification.new do |s|
+  s.name = "fluent-plugin-flume"
+  s.version = "0.0.0"
+
+  s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
+  s.authors = ["Muga Nishizawa"]
+  s.date = "2012-04-07"
+  s.email = "muga.nishizawa@gmail.com"
+  s.executables = ["fluent-flume-remote"]
+  s.extra_rdoc_files = [
+    "ChangeLog",
+    "README.rdoc"
+  ]
+  s.files = [
+    "AUTHORS",
+    "Rakefile",
+    "VERSION",
+    "bin/fluent-flume-remote",
+    "example.conf",
+    "fluent-plugin-flume.gemspec",
+    "lib/fluent/plugin/in_flume.rb",
+    "lib/fluent/plugin/out_flume.rb",
+    "lib/fluent/plugin/thrift/flume.thrift",
+    "lib/fluent/plugin/thrift/flume.thrift.orig",
+    "lib/fluent/plugin/thrift/flume_constants.rb",
+    "lib/fluent/plugin/thrift/flume_types.rb",
+    "lib/fluent/plugin/thrift/thrift_flume_event_server.rb",
+    "test/plugin/out_flume.rb"
+  ]
+  s.homepage = "https://github.com/muga/fluent-plugin-flume"
+  s.require_paths = ["lib"]
+  s.rubygems_version = "1.8.15"
+  s.summary = "Flume Input/Output plugin for Fluentd event collector"
+  s.test_files = ["test/plugin/out_flume.rb"]
+
+  if s.respond_to? :specification_version then
+    s.specification_version = 3
+
+    if Gem::Version.new(Gem::VERSION) >= Gem::Version.new('1.2.0') then
+      s.add_runtime_dependency(%q<fluentd>, ["~> 0.10.16"])
+      s.add_runtime_dependency(%q<thrift>, ["~> 0.6.0"])
+    else
+      s.add_dependency(%q<fluentd>, ["~> 0.10.16"])
+      s.add_dependency(%q<thrift>, ["~> 0.6.0"])
+    end
+  else
+    s.add_dependency(%q<fluentd>, ["~> 0.10.16"])
+    s.add_dependency(%q<thrift>, ["~> 0.6.0"])
+  end
+end
+

data/lib/fluent/plugin/in_flume.rb

@@ -0,0 +1,164 @@
+#
+# Fluent
+#
+# Copyright (C) 2012 Muga Nishizawa
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+module Fluent
+
+class FlumeInput < Input
+  Plugin.register_input('flume', self)
+
+  config_param :port,            :integer, :default => 56789
+  config_param :bind,            :string,  :default => '0.0.0.0'
+  config_param :server_type,     :string,  :default => 'simple'
+  config_param :is_framed,       :bool,    :default => false
+  config_param :body_size_limit, :size,    :default => 32*1024*1024  # TODO default
+  config_param :tag_field,	 :string,  :default => nil
+  config_param :default_tag,	 :string,  :default => 'category'
+  config_param :add_prefix,      :string,  :default => nil
+  config_param :message_format,  :string,  :default => 'text'
+
+  def initialize
+    require 'thrift'
+    $:.unshift File.join(File.dirname(__FILE__), 'thrift')
+    require 'flume_types'
+    require 'flume_constants'
+    require 'thrift_flume_event_server'
+
+    super
+  end
+
+  def configure(conf)
+    super
+  end
+
+  def start
+    $log.debug "listening flume on #{@bind}:#{@port}"
+
+    handler = FluentFlumeHandler.new
+    handler.tag_field = @tag_field
+    handler.default_tag = @default_tag
+    handler.add_prefix = @add_prefix
+    handler.message_format = @message_format
+    processor = ThriftFlumeEventServer::Processor.new handler
+
+    @transport = Thrift::ServerSocket.new @bind, @port
+    unless @is_framed
+      transport_factory = Thrift::BufferedTransportFactory.new
+    else
+      raise ConfigError, "in_flume: unsupported is_framed '#{@is_framed}'"
+    end
+
+    unless ['text', 'json'].include? @message_format
+      raise 'Unknown format: message_format=#{@message_format}'
+    end
+
+    # 2011/09/29 Kazuki Ohta <kazuki.ohta@gmail.com>
+    # This section is a workaround to set strict_read and strict_write option.
+    # Ruby-Thrift 0.7 set them both 'true' in default, but Flume protocol set
+    # them both 'false'.
+    protocol_factory = Thrift::BinaryProtocolFactory.new
+    protocol_factory.instance_eval {|obj|
+      def get_protocol(trans) # override
+        return Thrift::BinaryProtocol.new(trans,
+                                          strict_read=false,
+                                          strict_write=false)
+      end
+    }
+
+    case @server_type
+    when 'simple'
+      @server = Thrift::SimpleServer.new processor, @transport, transport_factory, protocol_factory
+    when 'threaded'
+      @server = Thrift::ThreadedServer.new processor, @transport, transport_factory, protocol_factory
+    when 'thread_pool'
+      @server = Thrift::ThreadPoolServer.new processor, @transport, transport_factory, protocol_factory
+    else
+      raise ConfigError, "in_flume: unsupported server_type '#{@server_type}'"
+    end
+    @thread = Thread.new(&method(:run))
+  end
+
+  def shutdown
+    @transport.close unless @transport.closed? # TODO
+    #@thread.join # TODO
+  end
+
+  def run
+    $log.debug "starting server: #{@server}"
+    @server.serve
+  rescue
+    $log.error "unexpected error", :error=>$!.to_s
+    $log.error_backtrace
+  end
+
+  class FluentFlumeHandler
+    attr_accessor :tag_field
+    attr_accessor :default_tag
+    attr_accessor :add_prefix
+    attr_accessor :message_format
+
+    def append(evt)
+      begin
+        record = create_record(evt)
+        if @tag_field
+          tag = evt.fieldss[@tag_field] || @default_tag
+          unless tag
+            return # ignore
+          end
+        else
+          tag = @default_tag
+        end
+        timestamp = evt.timestamp.to_i
+        if @add_prefix
+          Engine.emit(@add_prefix + '.' + tag, timestamp, record)
+        else
+          Engine.emit(tag, timestamp, record)
+        end
+      rescue => e
+        $log.error "unexpected error", :error=>$!.to_s
+        $log.error_backtrace
+      end
+    end
+
+    def rawAppend(evt)
+      $log.error "rawAppend is not implemented yet: #{evt}"
+    end
+
+    def ackedAppend(evt)
+      $log.error "ackedAppend is not implemented yet: #{evt}"
+      EventStatus::OK
+    end
+
+    def close()
+    end
+
+    private
+    def create_record(evt)
+      case @message_format
+      when 'text'
+        return {'message'=>evt.body.force_encoding('UTF-8')}
+      when 'json'
+        js = JSON.parse(evt.body.force_encoding('UTF-8'))
+        raise 'body must be a Hash, if json_body=true' unless js.is_a?(Hash)
+        return js
+      else
+        raise 'Invalid format: #{@message_format}'
+      end
+    end
+  end
+end
+
+end

data/lib/fluent/plugin/out_flume.rb

@@ -0,0 +1,96 @@
+#
+# Fluent
+#
+# Copyright (C) 2012 Muga Nishizawa
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+module Fluent
+
+class FlumeOutput < BufferedOutput
+  Fluent::Plugin.register_output('flume', self)
+
+  config_param :host,      :string,  :default => 'localhost'
+  config_param :port,      :integer, :default => 35863
+  config_param :timeout,   :integer, :default => 30
+  config_param :remove_prefix,    :string, :default => nil
+  config_param :default_category, :string, :default => 'unknown'
+
+  def initialize
+    require 'thrift'
+    $:.unshift File.join(File.dirname(__FILE__), 'thrift')
+    require 'flume_types'
+    require 'flume_constants'
+    require 'thrift_flume_event_server'
+
+    super
+  end
+
+  def configure(conf)
+    super
+  end
+
+  def start
+    super
+
+    if @remove_prefix
+      @removed_prefix_string = @remove_prefix + '.'
+      @removed_length = @removed_prefix_string.length
+    end
+  end
+
+  def shutdown
+    super
+  end
+
+  def format(tag, time, record)
+    if @remove_prefix and
+        ( (tag[0, @removed_length] == @removed_prefix_string and tag.length > @removed_length) or
+          tag == @remove_prefix)
+      [(tag[@removed_length..-1] || @default_category), time, record].to_msgpack
+    else
+      [tag, time, record].to_msgpack
+    end
+  end
+
+  def write(chunk)
+    records = []
+    chunk.msgpack_each { |arr|
+      records << arr
+    }
+
+    transport = Thrift::Socket.new @host, @port, @timeout
+    #transport = Thrift::FramedTransport.new socket
+    #protocol = Thrift::BinaryProtocol.new transport, false, false
+    protocol = Thrift::BinaryProtocol.new transport
+    client = ThriftFlumeEventServer::Client.new protocol
+
+    transport.open
+    $log.debug "thrift client opend: #{client}"
+    begin
+      records.each { |r|
+        tag, time, record = r
+        entry = ThriftFlumeEvent.new(:body=>record.to_json.to_s.force_encoding('UTF-8'),
+                                     :priority=>Priority::INFO,
+                                     :timestamp=>time,
+                                     :fieldss=>{'category'=>tag})
+        client.append entry
+      }
+    ensure
+      $log.debug "thrift client closing: #{client}"
+      transport.close
+    end
+  end
+end
+
+end