fluent-plugin-flume 0.1.1 → 0.1.2

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 7572fd137da7ba6c6668e09fdb489ca58f417781
+  data.tar.gz: ec492fdb5fadfaddf89a21ea4b78f76c126bcd05
+SHA512:
+  metadata.gz: b53cfe600d060d4537c2cac0b96b5cc4b020e9ad05b81b56e2bede3923836fb52612d7022cd6f90c64c211665c3499a07b1db8b97131188d19ef84b1c0fab1be
+  data.tar.gz: 7287e3c0988a562da41eac18cbbd535265d738af90b61015c639601b43c8bddcee5c9d2d476f7c15239242431f8b604869d208bdb3b7b136683189cd2940bf2e

data/.gitignore

@@ -0,0 +1,5 @@
+*~
+*.gem
+.bundle
+Gemfile.lock
+pkg/*

data/ChangeLog

@@ -1,4 +1,11 @@
+Release 0.1.2 - 2015/11/30
+
+	* Updated thrift to 0.9.0
+	* Use latest flume thrift protocol in flume output
+
 Release 0.1.1 - 20XX/XX/XX
-* updated thrift to 0.8.0
+
+	* updated thrift to 0.8.0
+	* overrode buffer_chunk_limit for more reliability (ISSUE-1)
 
 Release 0.1.0 - 2012/05/02

data/Gemfile

@@ -0,0 +1,3 @@
+source "http://rubygems.org"
+
+gemspec

data/README.rdoc

@@ -1,8 +1,8 @@
-= Flume input/output plugin for Fluent
+= Flume input/output plugin for Fluentd[http://fluentd.org/]
 
 == Overview
 
-This is a plugin for fluentd[https://github.com/fluentd] event collector.  This plugin adds the Flume[https://github.com/apache/flume] compatible interface to fluentd.
+This is a plugin for fluentd[http://fluentd.org/] data collector.  This plugin adds the Flume[https://github.com/apache/flume] compatible interface to fluentd.
 
 == What's Flume?
 
@@ -79,6 +79,8 @@ These options are supported.
 * default_tag: default fluentd 'tag' (default: 'category')
 * add_prefix: prefix string, added to the tag (default: nil)
 
+Use flume-ng-fluentd-sink[https://github.com/cosmo0920/flume-ng-fluentd-sink] to send events from flume-ng to fluentd.
+
 == Flume Output
 
 Please add the following configurations to fluent.conf.  This allows fluentd to output its logs into another Flume server.  Note that fluentd conveys semi-structured data while Flume conveys unstructured data.  Thus the plugin translates semi-structured data into JSON data and conveys it to Flume.
@@ -101,5 +103,5 @@ These options are supported.
 
 == Copyright
 
-Copyright:: Copyright (c) 2012 Treasure Data, Inc.
+Copyright:: Copyright (c) 2012 - 2013 Treasure Data, Inc.
 License::   Apache License, Version 2.0

data/Rakefile

@@ -1,29 +1,10 @@
+require 'bundler'
+Bundler::GemHelper.install_tasks
+
 require 'rake'
 require 'rake/testtask'
 require 'rake/clean'
 
-begin
-  require 'jeweler'
-  Jeweler::Tasks.new do |gemspec|
-    gemspec.name = "fluent-plugin-flume"
-    gemspec.summary = "Flume Input/Output plugin for Fluentd event collector"
-    gemspec.author = "Muga Nishizawa"
-    gemspec.email = "muga.nishizawa@gmail.com"
-    gemspec.homepage = "https://github.com/muga/fluent-plugin-flume"
-    gemspec.has_rdoc = false
-    gemspec.require_paths = ["lib"]
-    gemspec.add_dependency "fluentd", "~> 0.10.16"
-    gemspec.add_dependency "thrift", "~> 0.8.0"
-    gemspec.test_files = Dir["test/**/*.rb"]
-    gemspec.files = Dir["bin/**/*", "lib/**/*", "test/**/*.rb"] +
-      %w[example.conf VERSION AUTHORS Rakefile fluent-plugin-flume.gemspec]
-    gemspec.executables = ['fluent-flume-remote']
-  end
-  Jeweler::GemcutterTasks.new
-rescue LoadError
-  puts "Jeweler not available. Install it with: gem install jeweler"
-end
-
 task "thrift_gen" do
   system "mkdir -p tmp"
   system "thrift --gen rb -o tmp lib/fluent/plugin/thrift/flume.thrift"
@@ -32,6 +13,7 @@ task "thrift_gen" do
 end
 
 Rake::TestTask.new(:test) do |t|
+  t.libs << 'lib' << 'test'
   t.test_files = Dir['test/plugin/*.rb']
   t.ruby_opts = ['-rubygems'] if defined? Gem
   t.ruby_opts << '-I.'

data/VERSION

@@ -1 +1 @@
-0.1.1
+0.1.2

data/bin/fluent-flume-remote

@@ -4,15 +4,15 @@ require 'json'
 $:.unshift File.join(File.dirname(__FILE__), '../lib/fluent/plugin/thrift')
 require 'flume_types'
 require 'flume_constants'
-require 'thrift_flume_event_server'
+require 'thrift_source_protocol'
 
 host = 'localhost'
 port = 3586
 
 socket = Thrift::Socket.new host, port.to_i
-transport = Thrift::BufferedTransport.new socket
-protocol = Thrift::BinaryProtocol.new transport
-client = ThriftFlumeEventServer::Client.new protocol
+transport = Thrift::FramedTransport.new socket
+protocol = Thrift::CompactProtocol.new transport
+client = ThriftSourceProtocol::Client.new protocol
 transport.open
 
 # 2011/09/02 Kazuki Ohta <kazuki.ohta@gmail.com>
@@ -21,8 +21,10 @@ raw_sock = socket.to_io
 raw_sock.setsockopt Socket::IPPROTO_TCP, Socket::TCP_NODELAY, 1
 
 entry = ThriftFlumeEvent.new(:body=>{'a'=>'b'}.to_json,
-                             :priority=>Priority::INFO,
-                             :timestamp=>(Time.now.to_i * 1000))
+                             :headers => {
+                               'timestamp' => (Time.now.to_i * 1000).to_s,
+                               'tag'       => tag,
+                             })
 client.append entry
 
 transport.close

data/example-in-flume.conf

@@ -0,0 +1,10 @@
+# Flume input
+<source>
+  type flume
+  port 56789
+</source>
+
+# match tag=debug.** and dump to console
+<match **>
+  type stdout
+</match>

data/example-out-flume.conf

@@ -0,0 +1,11 @@
+# built-in TCP input
+# $ echo <json> | fluent-cat <tag>
+<source>
+  type tcp
+</source>
+
+<match **>
+  type flume
+  host localhost
+  port 56789
+</match>

data/fluent-plugin-flume.gemspec

@@ -1,56 +1,24 @@
-# Generated by jeweler
-# DO NOT EDIT THIS FILE DIRECTLY
-# Instead, edit Jeweler::Tasks in Rakefile, and run 'rake gemspec'
-# -*- encoding: utf-8 -*-
+$:.push File.expand_path('../lib', __FILE__)
 
 Gem::Specification.new do |s|
-  s.name = "fluent-plugin-flume"
-  s.version = "0.1.1"
+  s.name          = 'fluent-plugin-flume'
+  s.version       = File.read("VERSION").strip
+  s.authors       = ["Muga Nishizawa"]
+  s.email         = ["muga.nishizawa@gmail.com"]
+  s.description   = "Flume Input/Output plugin for Fluentd event collector"
+  s.summary       = s.description
+  s.homepage      = 'https://github.com/fluent/fluent-plugin-flume'
+  s.license       = 'MIT'
+  s.has_rdoc    = false
 
-  s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
-  s.authors = ["Muga Nishizawa"]
-  s.date = "2012-05-20"
-  s.email = "muga.nishizawa@gmail.com"
-  s.executables = ["fluent-flume-remote"]
-  s.extra_rdoc_files = [
-    "ChangeLog",
-    "README.rdoc"
-  ]
-  s.files = [
-    "AUTHORS",
-    "Rakefile",
-    "VERSION",
-    "bin/fluent-flume-remote",
-    "fluent-plugin-flume.gemspec",
-    "lib/fluent/plugin/in_flume.rb",
-    "lib/fluent/plugin/out_flume.rb",
-    "lib/fluent/plugin/thrift/flume.thrift",
-    "lib/fluent/plugin/thrift/flume.thrift.orig",
-    "lib/fluent/plugin/thrift/flume_constants.rb",
-    "lib/fluent/plugin/thrift/flume_types.rb",
-    "lib/fluent/plugin/thrift/thrift_flume_event_server.rb",
-    "test/plugin/test_in_flume.rb",
-    "test/plugin/test_out_flume.rb"
-  ]
-  s.homepage = "https://github.com/muga/fluent-plugin-flume"
-  s.require_paths = ["lib"]
-  s.rubygems_version = "1.8.15"
-  s.summary = "Flume Input/Output plugin for Fluentd event collector"
-  s.test_files = ["test/plugin/test_in_flume.rb", "test/plugin/test_out_flume.rb"]
+  s.files         = `git ls-files`.split($/)
+  s.executables   = s.files.grep(%r{^bin/}).map{ |f| File.basename(f) }
+  s.test_files    = s.files.grep(%r{^(test|spec|features)/})
+  s.require_paths = ['lib']
 
-  if s.respond_to? :specification_version then
-    s.specification_version = 3
+  s.add_runtime_dependency 'fluentd', '>= 0.10.43'
+  s.add_runtime_dependency 'thrift', '~> 0.9.0'
 
-    if Gem::Version.new(Gem::VERSION) >= Gem::Version.new('1.2.0') then
-      s.add_runtime_dependency(%q<fluentd>, ["~> 0.10.16"])
-      s.add_runtime_dependency(%q<thrift>, ["~> 0.8.0"])
-    else
-      s.add_dependency(%q<fluentd>, ["~> 0.10.16"])
-      s.add_dependency(%q<thrift>, ["~> 0.8.0"])
-    end
-  else
-    s.add_dependency(%q<fluentd>, ["~> 0.10.16"])
-    s.add_dependency(%q<thrift>, ["~> 0.8.0"])
-  end
+  s.add_development_dependency 'rake', '>= 0.9.2'
+  s.add_development_dependency 'test-unit', '~> 3.1.0'
 end
-

data/lib/fluent/plugin/in_flume.rb

@@ -1,7 +1,7 @@
 #
 # Fluent
 #
-# Copyright (C) 2012 Muga Nishizawa
+# Copyright (C) 2012 - 2013 Muga Nishizawa
 #
 #    Licensed under the Apache License, Version 2.0 (the "License");
 #    you may not use this file except in compliance with the License.
@@ -24,11 +24,15 @@ class FlumeInput < Input
   config_param :bind,            :string,  :default => '0.0.0.0'
   config_param :server_type,     :string,  :default => 'simple'
   config_param :is_framed,       :bool,    :default => false
-  config_param :body_size_limit, :size,    :default => 32*1024*1024  # TODO default
+  config_param :body_size_limit, :size,    :default => 32 * 1024 * 1024
   config_param :tag_field,	 :string,  :default => nil
   config_param :default_tag,	 :string,  :default => 'category'
   config_param :add_prefix,      :string,  :default => nil
-  config_param :message_format,  :string,  :default => 'text'
+  config_param :msg_format,      :string,  :default => 'text'
+
+  unless method_defined?(:log)
+    define_method(:log) { $log }
+  end
 
   def initialize
     require 'thrift'
@@ -45,13 +49,14 @@ class FlumeInput < Input
   end
 
   def start
-    $log.debug "listening flume on #{@bind}:#{@port}"
+    log.debug "listening flume on #{@bind}:#{@port}"
 
     handler = FluentFlumeHandler.new
     handler.tag_field = @tag_field
     handler.default_tag = @default_tag
     handler.add_prefix = @add_prefix
-    handler.message_format = @message_format
+    handler.msg_format = @msg_format
+    handler.log = log
     processor = ThriftFlumeEventServer::Processor.new handler
 
     @transport = Thrift::ServerSocket.new @bind, @port
@@ -61,8 +66,8 @@ class FlumeInput < Input
       raise ConfigError, "in_flume: unsupported is_framed '#{@is_framed}'"
     end
 
-    unless ['text', 'json'].include? @message_format
-      raise 'Unknown format: message_format=#{@message_format}'
+    unless ['text', 'json'].include? @msg_format
+      raise 'Unknown format: msg_format=#{@msg_format}'
     end
 
     # 2011/09/29 Kazuki Ohta <kazuki.ohta@gmail.com>
@@ -92,23 +97,24 @@ class FlumeInput < Input
   end
 
   def shutdown
-    @transport.close unless @transport.closed? # TODO
-    #@thread.join # TODO
+    @transport.close unless @transport.closed?
+    #@thread.join
   end
 
   def run
-    $log.debug "starting server: #{@server}"
+    log.debug "starting server: #{@server}"
     @server.serve
   rescue
-    $log.error "unexpected error", :error=>$!.to_s
-    $log.error_backtrace
+    log.error "unexpected error", :error=>$!.to_s
+    log.error_backtrace
   end
 
   class FluentFlumeHandler
     attr_accessor :tag_field
     attr_accessor :default_tag
     attr_accessor :add_prefix
-    attr_accessor :message_format
+    attr_accessor :msg_format
+    attr_accessor :log
 
     def append(evt)
       begin
@@ -128,18 +134,38 @@ class FlumeInput < Input
           Engine.emit(tag, timestamp, record)
         end
       rescue => e
-        $log.error "unexpected error", :error=>$!.to_s
-        $log.error_backtrace
+        log.error "unexpected error", :error=>$!.to_s
+        log.error_backtrace
       end
     end
 
     def rawAppend(evt)
-      $log.error "rawAppend is not implemented yet: #{evt}"
+      log.error "rawAppend is not implemented yet: #{evt}"
     end
 
     def ackedAppend(evt)
-      $log.error "ackedAppend is not implemented yet: #{evt}"
-      EventStatus::OK
+      begin
+        record = create_record(evt)
+        if @tag_field
+          tag = evt.fieldss[@tag_field] || @default_tag
+          unless tag
+            return # ignore
+          end
+        else
+          tag = @default_tag
+        end
+        timestamp = evt.timestamp.to_i
+        if @add_prefix
+          Engine.emit(@add_prefix + '.' + tag, timestamp, record)
+        else
+          Engine.emit(tag, timestamp, record)
+        end
+        return EventStatus::ACK
+      rescue => e
+        log.error "unexpected error", :error=>$!.to_s
+        log.error_backtrace
+        return EventStatus::ERR
+      end
     end
 
     def close()
@@ -147,7 +173,7 @@ class FlumeInput < Input
 
     private
     def create_record(evt)
-      case @message_format
+      case @msg_format
       when 'text'
         return {'message'=>evt.body.force_encoding('UTF-8')}
       when 'json'
@@ -155,7 +181,7 @@ class FlumeInput < Input
         raise 'body must be a Hash, if json_body=true' unless js.is_a?(Hash)
         return js
       else
-        raise 'Invalid format: #{@message_format}'
+        raise 'Invalid format: #{@msg_format}'
       end
     end
   end

data/lib/fluent/plugin/out_flume.rb

@@ -1,7 +1,7 @@
 #
 # Fluent
 #
-# Copyright (C) 2012 Muga Nishizawa
+# Copyright (C) 2012 - 2013 Muga Nishizawa
 #
 #    Licensed under the Apache License, Version 2.0 (the "License");
 #    you may not use this file except in compliance with the License.
@@ -26,17 +26,22 @@ class FlumeOutput < BufferedOutput
   config_param :remove_prefix,    :string, :default => nil
   config_param :default_category, :string, :default => 'unknown'
 
+  unless method_defined?(:log)
+    define_method(:log) { $log }
+  end
+
   def initialize
     require 'thrift'
     $:.unshift File.join(File.dirname(__FILE__), 'thrift')
     require 'flume_types'
     require 'flume_constants'
-    require 'thrift_flume_event_server'
-
+    require 'thrift_source_protocol'
     super
   end
 
   def configure(conf)
+    # override default buffer_chunk_limit
+    conf['buffer_chunk_limit'] ||= '1m'
     super
   end
 
@@ -64,30 +69,28 @@ class FlumeOutput < BufferedOutput
   end
 
   def write(chunk)
-    records = []
-    chunk.msgpack_each { |arr|
-      records << arr
-    }
-
-    transport = Thrift::Socket.new @host, @port, @timeout
-    #transport = Thrift::FramedTransport.new socket
+    socket = Thrift::Socket.new @host, @port, @timeout
+    transport = Thrift::FramedTransport.new socket
     #protocol = Thrift::BinaryProtocol.new transport, false, false
-    protocol = Thrift::BinaryProtocol.new transport
-    client = ThriftFlumeEventServer::Client.new protocol
+    protocol = Thrift::CompactProtocol.new transport
+    client = ThriftSourceProtocol::Client.new protocol
 
+    count = 0
     transport.open
-    $log.debug "thrift client opend: #{client}"
+    log.debug "thrift client opened: #{client}"
     begin
-      records.each { |r|
-        tag, time, record = r
-        entry = ThriftFlumeEvent.new(:body=>record.to_json.to_s.force_encoding('UTF-8'),
-                                     :priority=>Priority::INFO,
-                                     :timestamp=>time,
-                                     :fieldss=>{'category'=>tag})
+      chunk.msgpack_each { |tag, time, record|
+        entry = ThriftFlumeEvent.new(:body      => record.to_json.to_s.force_encoding('UTF-8'),
+                                     :headers   => {
+                                       'timestamp' => time.to_s,
+                                       'tag'       => tag,
+                                     })
         client.append entry
+        count += 1
       }
+      log.debug "Writing #{count} entries to flume"
     ensure
-      $log.debug "thrift client closing: #{client}"
+      log.debug "thrift client closing: #{client}"
       transport.close
     end
   end