fluentd 0.10.43 → 0.10.44

data/.travis.yml

@@ -3,15 +3,20 @@ language: ruby
 rvm:
   - 1.9.3
   - 2.0.0
-  - rbx-19mode
+  - 2.1.0
+  - rbx
 
 branches:
   only:
     - master
     - v11
 
+gemfile:
+  - Gemfile
+  - Gemfile.cool.io.1.1.1
+
 script: bundle exec rake
 
 matrix:
   allow_failures:
-    - rvm: rbx-19mode
+    - rvm: rbx

data/ChangeLog

@@ -1,4 +1,13 @@
-Release 0.10.43 - 2013/02/03
+Release 0.10.44 - 2014/02/26
+
+* in_forward: Print remote address and port in trace log
+* Remove several methods from fluentd-debug to call remote object method directly
+* Add --gemfile option to manage Fluentd and plugins using Gemfile and Bundler
+* Add <include path /> directive to include the configuration file under the elements
+* Add 'ruby' keyword to call Kernel method in configuration DSL
+* Relax Cool.io gem version to allow 1.2.x
+
+Release 0.10.43 - 2014/02/03
 
 * Input/Output: Add 'log_level' option to plugins
 * parser/in_tail: Fix RegexpParser forget to call configure method

data/Gemfile.cool.io.1.1.1

@@ -0,0 +1,4 @@
+source 'https://rubygems.org/'
+
+gem 'cool.io', '= 1.1.1'
+gemspec

data/README.md

@@ -0,0 +1,39 @@
+Fluentd: Open-Source Data Collector
+===================================
+
+[<img src="https://travis-ci.org/fluent/fluentd.png" />](https://travis-ci.org/fluent/fluentd) [<img src="https://codeclimate.com/github/fluent/fluentd.png " />](https://codeclimate.com/github/fluent/fluentd)
+
+
+[Fluentd](http://fluentd.org/) collects events from various data sources and writes them to files, database or other types of storages. You can simplify your data stream, and have robust data collection mechanism instantly:
+
+<p align="center">
+<img src="http://docs.fluentd.org/images/fluentd-architecture.png" width="500px"/>
+</p>
+
+An event consists of *tag*, *time* and *record*. Tag is a string separated with '.' (e.g. myapp.access). It is used to categorize events. Time is a UNIX time recorded at occurrence of an event. Record is a JSON object.
+
+
+## Quick Start
+
+    $ gem install fluentd
+    $ fluentd -s conf
+    $ fluentd -c conf/fluent.conf &
+    $ echo '{"json":"message"}' | fluent-cat debug.test
+
+## More Information
+
+- Web site:  http://fluentd.org/
+- Documents: http://docs.fluentd.org/
+- Source repository: http://github.com/fluent
+- Discussion: http://groups.google.com/group/fluentd
+- News Letters: http://go.treasuredata.com/Fluentd_education
+- Author: Sadayuki Furuhashi
+- Copyright: (c) 2011 FURUHASHI Sadayuki
+- License: Apache License, Version 2.0
+
+## Contributors:
+
+Patches contributed by [great developers](https://github.com/fluent/fluentd/contributors).
+
+[<img src="https://ga-beacon.appspot.com/UA-24890265-6/fluent/fluentd" />](https://github.com/fluent/fluentd)
+

data/fluentd.gemspec

@@ -18,15 +18,15 @@ Gem::Specification.new do |gem|
 
   gem.required_ruby_version = '>= 1.9.2'
 
-  gem.add_runtime_dependency(%q<msgpack>, [">= 0.4.4", "!= 0.5.0", "!= 0.5.1", "!= 0.5.2", "!= 0.5.3", "< 0.6.0"])
-  gem.add_runtime_dependency(%q<json>, [">= 1.4.3"])
-  gem.add_runtime_dependency(%q<yajl-ruby>, ["~> 1.0"])
-  gem.add_runtime_dependency(%q<cool.io>, ["~> 1.1.1"])
-  gem.add_runtime_dependency(%q<http_parser.rb>, ["~> 0.5.1"])
-  gem.add_runtime_dependency(%q<sigdump>, ["~> 0.2.2"])
+  gem.add_runtime_dependency("msgpack", [">= 0.4.4", "!= 0.5.0", "!= 0.5.1", "!= 0.5.2", "!= 0.5.3", "< 0.6.0"])
+  gem.add_runtime_dependency("json", [">= 1.4.3"])
+  gem.add_runtime_dependency("yajl-ruby", ["~> 1.0"])
+  gem.add_runtime_dependency("cool.io", [">= 1.1.1", "!= 1.2.0", "< 2.0.0"])
+  gem.add_runtime_dependency("http_parser.rb", ["~> 0.5.1"])
+  gem.add_runtime_dependency("sigdump", ["~> 0.2.2"])
 
-  gem.add_development_dependency(%q<rake>, [">= 0.9.2"])
-  gem.add_development_dependency(%q<parallel_tests>, [">= 0.15.3"])
-  gem.add_development_dependency(%q<rr>, [">= 1.0.0"])
-  gem.add_development_dependency(%q<timecop>, [">= 0.3.0"])
+  gem.add_development_dependency("rake", [">= 0.9.2"])
+  gem.add_development_dependency("parallel_tests", [">= 0.15.3"])
+  gem.add_development_dependency("rr", [">= 1.0.0"])
+  gem.add_development_dependency("timecop", [">= 0.3.0"])
 end

data/lib/fluent/command/bundler_injection.rb

@@ -0,0 +1,31 @@
+#
+# Fluentd
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+
+system("bundle install")
+unless $?.success?
+  exit $?.exitstatus
+end
+
+cmdline = [
+  'bundle',
+  'exec',
+  RbConfig.ruby,
+  File.expand_path(File.join(File.dirname(__FILE__), 'fluentd.rb')),
+] + ARGV
+
+exec *cmdline
+exit! 127
+

data/lib/fluent/command/debug.rb

@@ -59,6 +59,13 @@ require 'fluent/load'
 $log = Fluent::Log.new(STDERR, Fluent::Log::LEVEL_TRACE)
 Fluent::Engine.init
 
+DRb::DRbObject.class_eval do
+  undef_method :methods
+  undef_method :instance_eval
+  undef_method :instance_variables
+  undef_method :instance_variable_get
+end
+
 remote_engine = DRb::DRbObject.new_with_uri(uri)
 
 Fluent.module_eval do

data/lib/fluent/command/fluentd.rb

@@ -107,6 +107,14 @@ op.on('--suppress-config-dump', "suppress config dumping when fluentd starts", T
   opts[:suppress_config_dump] = b
 }
 
+op.on('-g', '--gemfile GEMFILE', "Gemfile path") {|s|
+  opts[:gemfile] = s
+}
+
+op.on('-G', '--gem-path GEM_INSTALL_PATH', "Gemfile install path (default: $(dirname $gemfile)/vendor/bundle)") {|s|
+  opts[:gem_install_path] = s
+}
+
 (class<<self;self;end).module_eval do
   define_method(:usage) do |msg|
     puts op.to_s
@@ -116,9 +124,9 @@ op.on('--suppress-config-dump', "suppress config dumping when fluentd starts", T
 end
 
 begin
-  op.parse!(ARGV)
+  rest = op.parse(ARGV)
 
-  if ARGV.length != 0
+  if rest.length != 0
     usage nil
   end
 rescue
@@ -126,6 +134,20 @@ rescue
 end
 
 
+##
+## Bundler injection
+#
+if ENV['FLUENTD_DISABLE_BUNDLER_INJECTION'] != '1' && gemfile = opts[:gemfile]
+  ENV['BUNDLE_GEMFILE'] = gemfile
+  if path = opts[:gem_install_path]
+    ENV['BUNDLE_PATH'] = path
+  else
+    ENV['BUNDLE_PATH'] = File.expand_path(File.join(File.dirname(gemfile), 'vendor/bundle'))
+  end
+  ENV['FLUENTD_DISABLE_BUNDLER_INJECTION'] = '1'
+  load File.expand_path(File.join(File.dirname(__FILE__), 'bundler_injection.rb'))
+end
+
 if setup_path = opts[:setup_path]
   require 'fileutils'
   FileUtils.mkdir_p File.join(setup_path, "plugin")

data/lib/fluent/config.rb

@@ -171,6 +171,9 @@ module Fluent
           line.gsub!(/\s*(?:\#.*)?$/,'')
           if line.empty?
             next
+          elsif m = /^\<include\s*(.*)\s*\/\>$/.match(line)
+            value = m[1].strip
+            process_include(attrs, elems, value, allow_include)
           elsif m = /^\<([a-zA-Z0-9_]+)\s*(.+?)?\>$/.match(line)
             e_name = m[1]
             e_arg = m[2] || ""
@@ -197,7 +200,7 @@ module Fluent
         return attrs, elems
       end
 
-      def process_include(attrs, elems, uri)
+      def process_include(attrs, elems, uri, allow_include = true)
         u = URI.parse(uri)
         if u.scheme == 'file' || u.path == uri  # file path
           path = u.path
@@ -207,11 +210,11 @@ module Fluent
             pattern = path
           end
 
-          Dir.glob(pattern).each {|path|
+          Dir.glob(pattern).sort.each {|path|
             basepath = File.dirname(path)
             fname = File.basename(path)
             File.open(path) {|f|
-              Parser.new(basepath, f.each_line, fname).parse!(true, nil, attrs, elems)
+              Parser.new(basepath, f.each_line, fname).parse!(allow_include, nil, attrs, elems)
             }
           }
 
@@ -220,7 +223,7 @@ module Fluent
           fname = path
           require 'open-uri'
           open(uri) {|f|
-            Parser.new(basepath, f.each_line, fname).parse!(true, nil, attrs, elems)
+            Parser.new(basepath, f.each_line, fname).parse!(allow_include, nil, attrs, elems)
           }
         end
 

data/lib/fluent/config_dsl.rb

@@ -81,6 +81,24 @@ module Fluent
           ::Kernel.raise ::ArgumentError, "#{name} block requires arguments for match pattern" if args.nil? || args.size != 1
           @proxy.add_element('match', args.first, block)
         end
+
+        def self.const_missing(name)
+          return ::Kernel.const_get(name) if ::Kernel.const_defined?(name)
+
+          if name.to_s =~ /^Fluent::Config::DSL::Element::(.*)$/
+            name = "#{$1}".to_sym
+            return ::Kernel.const_get(name) if ::Kernel.const_defined?(name)
+          end
+          ::Kernel.eval("#{name}")
+        end
+
+        def ruby(&block)
+          if block
+            @proxy.instance_exec(&block)
+          else
+            ::Kernel
+          end
+        end
       end
     end
   end

data/lib/fluent/plugin/in_forward.rb

@@ -55,7 +55,9 @@ module Fluent
       @loop.stop
       @usock.close
       listen_address = (@bind == '0.0.0.0' ? '127.0.0.1' : @bind)
-      TCPSocket.open(listen_address, @port) {|sock| }  # FIXME @thread.join blocks without this line
+      # This line is for connecting listen socket to stop the event loop.
+      # We should use more better approach, e.g. using pipe, fixing cool.io with timeout, etc.
+      TCPSocket.open(listen_address, @port) {|sock| } # FIXME @thread.join blocks without this line
       @thread.join
       @lsock.close
     end
@@ -151,7 +153,10 @@ module Fluent
         end
         @on_message = on_message
         @log = log
-        @log.trace { "accepted fluent socket object_id=#{self.object_id}" }
+        @log.trace {
+          remote_port, remote_addr = *Socket.unpack_sockaddr_in(@_io.getpeername) rescue nil
+          "accepted fluent socket from '#{remote_addr}:#{remote_port}': object_id=#{self.object_id}"
+        }
       end
 
       def on_connect

data/lib/fluent/plugin/in_stream.rb

@@ -112,7 +112,10 @@ module Fluent
         end
         @on_message = on_message
         @log = log
-        @log.trace { "accepted fluent socket object_id=#{self.object_id}" }
+        @log.trace {
+          remote_port, remote_addr = *Socket.unpack_sockaddr_in(@_io.getpeername) rescue nil
+          "accepted fluent socket from '#{remote_addr}:#{remote_port}': object_id=#{self.object_id}"
+        }
       end
 
       def on_connect

data/lib/fluent/plugin/out_stream.rb

@@ -89,7 +89,7 @@ module Fluent
     def initialize
       super
       $log.warn "'tcp' output is obsoleted and will be removed. Use 'forward' instead."
-      $log.warn "see 'forward' section in http://fluentd.org/doc/plugin.html for the high-availability configuration."
+      $log.warn "see 'forward' section in http://docs.fluentd.org/ for the high-availability configuration."
     end
 
     config_param :port, :integer, :default => DEFAULT_LISTEN_PORT

data/lib/fluent/version.rb

@@ -1,5 +1,5 @@
 module Fluent
 
-  VERSION = '0.10.43'
+  VERSION = '0.10.44'
 
 end

data/test/config.rb

@@ -15,7 +15,7 @@ class ConfigTest < Test::Unit::TestCase
       include dir/config_test_2.conf  #
       include #{TMP_DIR}/config_test_4.conf
       include file://#{TMP_DIR}/config_test_5.conf
-      include config.d/*.conf
+      <include config.d/*.conf />
     ]
     write_config "#{TMP_DIR}/dir/config_test_2.conf", %[
       k2 relative_path_include
@@ -32,13 +32,31 @@ class ConfigTest < Test::Unit::TestCase
     ]
     write_config "#{TMP_DIR}/config.d/config_test_6.conf", %[
       k6 wildcard_include_1
-      <elem name>
+      <elem1 name>
         include normal_parameter
-      </elem>
+      </elem1>
     ]
     write_config "#{TMP_DIR}/config.d/config_test_7.conf", %[
       k7 wildcard_include_2
     ]
+    write_config "#{TMP_DIR}/config.d/config_test_8.conf", %[
+      <elem2 name>
+        <include ../dir/config_test_9.conf />
+      </elem2>
+    ]
+    write_config "#{TMP_DIR}/dir/config_test_9.conf", %[
+      k9 embeded
+      <elem3 name>
+        nested nested_value
+        include hoge
+      </elem3>
+    ]
+    write_config "#{TMP_DIR}/config.d/00_config_test_8.conf", %[
+      k8 wildcard_include_3
+      <elem4 name>
+        include normal_parameter
+      </elem4>
+    ]
 
   end
 
@@ -52,10 +70,33 @@ class ConfigTest < Test::Unit::TestCase
     assert_equal 'uri_include', c['k5']
     assert_equal 'wildcard_include_1', c['k6']
     assert_equal 'wildcard_include_2', c['k7']
-    assert_equal 'elem', c.elements.first.name
-    assert_equal 'name', c.elements.first.arg
-    assert_equal 'normal_parameter', c.elements.first['include']
-    
+    assert_equal 'wildcard_include_3', c['k8']
+    assert_equal [
+      'k1',
+      'k2',
+      'k3',
+      'k4',
+      'k5',
+      'k8', # Because of the file name this comes first.
+      'k6',
+      'k7',
+    ], c.keys
+
+    elem1 = c.elements.find { |e| e.name == 'elem1' }
+    assert_not_nil elem1
+    assert_equal 'name', elem1.arg
+    assert_equal 'normal_parameter', elem1['include']
+
+    elem2 = c.elements.find { |e| e.name == 'elem2' }
+    assert_not_nil elem2
+    assert_equal 'name', elem2.arg
+    assert_equal 'embeded', elem2['k9']
+    assert !elem2.has_key?('include')
+
+    elem3 = elem2.elements.find { |e| e.name == 'elem3' }
+    assert_not_nil elem3
+    assert_equal 'nested_value', elem3['nested']
+    assert_equal 'hoge', elem3['include']
   end
 
   def test_check_not_fetchd

data/test/configdsl.rb

@@ -104,4 +104,45 @@ match('aa')
       Fluent::Config::DSL::Parser.parse(TEST_DSL_CONFIG5)
     }
   end
+
+  def test_with_ruby_keyword
+    uname_string = `uname -a`
+    root1 = Fluent::Config::DSL::Parser.parse(<<DSL)
+uname_str = ruby.open('|uname -a'){|out| out.read}
+source {
+  uname uname_str
+}
+DSL
+    source1 = root1.elements.first
+    assert_equal 'source', source1.name
+    assert_equal 1, source1.keys.size
+    assert_equal uname_string, source1['uname']
+
+    root2 = Fluent::Config::DSL::Parser.parse(<<DSL)
+ruby_version = ruby {
+  require 'erb'
+  ERB.new('<%= RUBY_VERSION %> from erb').result
+}
+source {
+  version ruby_version
+}
+DSL
+    source2 = root2.elements.first
+    assert_equal 'source', source2.name
+    assert_equal 1, source2.keys.size
+    assert_equal "#{RUBY_VERSION} from erb", source2['version']
+
+    # Parser#parse raises NoMethodError when configuration dsl elements are written in ruby block
+    conf3 = <<DSL
+ruby {
+  source {
+    type "tail"
+  }
+}
+source {
+  uname uname_str
+}
+DSL
+    assert_raise (NoMethodError) { Fluent::Config::DSL::Parser.parse(conf3) }
+  end
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: fluentd
 version: !ruby/object:Gem::Version
-  version: 0.10.43
+  version: 0.10.44
   prerelease: 
 platform: ruby
 authors:
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2014-02-03 00:00:00.000000000 Z
+date: 2014-02-26 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: msgpack
@@ -94,17 +94,29 @@ dependencies:
   requirement: !ruby/object:Gem::Requirement
     none: false
     requirements:
-    - - ~>
+    - - ! '>='
       - !ruby/object:Gem::Version
         version: 1.1.1
+    - - ! '!='
+      - !ruby/object:Gem::Version
+        version: 1.2.0
+    - - <
+      - !ruby/object:Gem::Version
+        version: 2.0.0
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     none: false
     requirements:
-    - - ~>
+    - - ! '>='
       - !ruby/object:Gem::Version
         version: 1.1.1
+    - - ! '!='
+      - !ruby/object:Gem::Version
+        version: 1.2.0
+    - - <
+      - !ruby/object:Gem::Version
+        version: 2.0.0
 - !ruby/object:Gem::Dependency
   name: http_parser.rb
   requirement: !ruby/object:Gem::Requirement
@@ -220,7 +232,8 @@ files:
 - COPYING
 - ChangeLog
 - Gemfile
-- README.rdoc
+- Gemfile.cool.io.1.1.1
+- README.md
 - Rakefile
 - bin/fluent-cat
 - bin/fluent-debug
@@ -229,6 +242,7 @@ files:
 - fluent.conf
 - fluentd.gemspec
 - lib/fluent/buffer.rb
+- lib/fluent/command/bundler_injection.rb
 - lib/fluent/command/cat.rb
 - lib/fluent/command/debug.rb
 - lib/fluent/command/fluentd.rb
@@ -324,7 +338,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
       segments:
       - 0
-      hash: -936965180993264578
+      hash: -3906480416745103078
 requirements: []
 rubyforge_project: 
 rubygems_version: 1.8.23

data/README.rdoc

@@ -1,69 +0,0 @@
-= Fluentd
-
-{<img src="https://travis-ci.org/fluent/fluentd.png" />}[https://travis-ci.org/fluent/fluentd] {<img src="https://codeclimate.com/github/fluent/fluentd.png " />}[https://codeclimate.com/github/fluent/fluentd]
-
-Fluentd is an event collector system. It is a generalized version of syslogd, which handles JSON objects for its log messages.
-
-== Architecture
-
-Fluentd collects events from various data sources and writes them to files, database or other types of storages:
-
-    
-    Web apps  ---+                 +--> file
-                 |                 |
-                 +-->           ---+
-    /var/log  ------>  Fluentd  ------> mail
-                 +-->           ---+
-                 |                 |
-    Apache    ----                 +--> Fluentd
-    
-
-Fluent also supports log transfer:
-
-    
-    Web server
-    +---------+
-    | Fluentd -------
-    +---------+|     |
-     +---------+     |
-                     |
-    Proxy server     |    Log server, Amazon S3, HDFS, ...
-    +---------+      +--> +---------+
-    | Fluentd ----------> | Fluentd ||
-    +---------+|     +--> +---------+|
-     +---------+     |     +---------+
-                     |
-    Database server  |
-    +---------+      |
-    | Fluentd ---------> mail
-    +---------+|
-     +---------+
-    
-
-An event consists of *tag*, *time* and *record*. Tag is a string separated with '.' (e.g. myapp.access). It is used to categorize events. Time is a UNIX time recorded at occurrence of an event. Record is a JSON object.
-
-
-== Quick Start
-
-  $ gem install fluentd
-  $ # install sample configuration file to the directory
-  $ fluentd -s conf
-  $ fluentd -c conf/fluent.conf &
-  $ echo '{"json":"message"}' | fluent-cat debug.test
-
-== Meta
-
-Web site::  http://fluentd.org/
-Documents:: http://docs.fluentd.org/
-Source repository:: http://github.com/fluent
-Discussion:: http://groups.google.com/group/fluentd
-Author:: Sadayuki Furuhashi
-Copyright:: (c) 2011 FURUHASHI Sadayuki
-License:: Apache License, Version 2.0
-
-== Contributors:
-
-Patches contributed by {those people}[https://github.com/fluent/fluentd/contributors].
-
-{<img src="https://ga-beacon.appspot.com/UA-24890265-6/fluent/fluentd" />}[https://github.com/fluent/fluentd]
-