fluentd 0.10.2 → 0.10.3

data/ChangeLog

@@ -1,11 +1,17 @@
 
+Release 0.10.3 - 2011/10/21
+
+  * Supports 'include' in config file
+    * Supports "http://" schema
+	* Supports wildcards
+
+
 Release 0.10.2 - 2011/10/18
 
   * Fixed EventStream#to_msgpack_stream (thanks CkReal)
   * Added gemspec.required_ruby_version = '~> 1.9.2' (thanks sakuro)
 
 
-
 Release 0.10.1 - 2011/10/16
 
   * SetTimeKeyMixin accepts include_time_key parameter

data/VERSION

@@ -1 +1 @@
-0.10.2
+0.10.3

data/fluentd.gemspec

@@ -5,11 +5,11 @@
 
 Gem::Specification.new do |s|
   s.name = %q{fluentd}
-  s.version = "0.10.2"
+  s.version = "0.10.3"
 
   s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
   s.authors = ["Sadayuki Furuhashi"]
-  s.date = %q{2011-10-18}
+  s.date = %q{2011-10-21}
   s.email = %q{frsyuki@gmail.com}
   s.executables = ["fluentd", "fluent-cat", "fluent-gem"]
   s.extra_rdoc_files = [
@@ -60,6 +60,7 @@ Gem::Specification.new do |s|
      "lib/fluent/test/input_test.rb",
      "lib/fluent/test/output_test.rb",
      "lib/fluent/version.rb",
+     "test/config.rb",
      "test/helper.rb",
      "test/match.rb",
      "test/plugin/in_http.rb",
@@ -76,7 +77,8 @@ Gem::Specification.new do |s|
   s.rubygems_version = %q{1.3.7}
   s.summary = %q{Fluent event collector}
   s.test_files = [
-    "test/helper.rb",
+    "test/config.rb",
+     "test/helper.rb",
      "test/match.rb",
      "test/plugin/in_http.rb",
      "test/plugin/in_stream.rb",

data/lib/fluent/config.rb

@@ -92,13 +92,11 @@ module Config
   end
 
   def self.read(path)
-    parse(File.read(path), File.basename(path))
+    Parser.read(path)
   end
 
-  def self.parse(str, fname)
-    lines = str.split("\n")
-    i, attrs, elems = parse_element('end', lines, 0, fname)
-    Element.new('ROOT', '', attrs, elems)
+  def self.parse(str, fname, basepath=Dir.pwd)
+    Parser.parse(str, fname, basepath)
   end
 
   def self.new(name='')
@@ -147,33 +145,89 @@ module Config
   end
 
   private
-  def self.parse_element(name, lines, i, fname)
-    attrs = {}
-    elems = []
-    while i < lines.length
-      line = lines[i]
-      line.lstrip!
-      line.gsub!(/\s*(?:\#.*)?$/,'')
+  class Parser
+    def self.read(path)
+      path = File.expand_path(path)
+      File.open(path) {|io|
+        parse(io, File.basename(path), File.dirname(path))
+      }
+    end
+
+    def self.parse(io, fname, basepath=Dir.pwd)
+      attrs, elems = Parser.new(basepath, io.each_line, fname).parse!(true)
+      Element.new('ROOT', '', attrs, elems)
+    end
+
+    def initialize(basepath, iterator, fname, i=0)
+      @basepath = basepath
+      @iterator = iterator
+      @i = i
+      @fname = fname
+    end
+
+    def parse!(allow_include, elem_name=nil, attrs={}, elems=[])
+      while line = @iterator.next
+        @i += 1
+        line.lstrip!
+        line.gsub!(/\s*(?:\#.*)?$/,'')
         if line.empty?
-          i += 1
           next
         elsif m = /^\<([a-zA-Z0-9_]+)\s*(.+?)?\>$/.match(line)
           e_name = m[1]
           e_arg = m[2] || ""
-          i, e_attrs, e_elems = parse_element(e_name, lines, i+1, fname)
+          e_attrs, e_elems = parse!(false, e_name)
           elems << Element.new(e_name, e_arg, e_attrs, e_elems)
-        elsif line == "</#{name}>"
-          i += 1
+        elsif line == "</#{elem_name}>"
           break
-        elsif m = /^([a-zA-Z0-9_]+)\s*(.+)?$/.match(line)
-          attrs[m[1]] = m[2] || ""
-          i += 1
+        elsif m = /^([a-zA-Z0-9_]+)\s*(.*)$/.match(line)
+          key = m[1]
+          value = m[2]
+          if allow_include && key == 'include'
+            process_include(attrs, elems, value)
+          else
+            attrs[key] = value
+          end
           next
         else
-          raise ConfigParseError, "parse error at #{fname}:#{i}"
+          raise ConfigParseError, "parse error at #{@fname} line #{@i}"
+        end
+      end
+
+      return attrs, elems
+    rescue StopIteration
+      return attrs, elems
+    end
+
+    def process_include(attrs, elems, uri)
+      u = URI.parse(uri)
+      if u.scheme == 'file' || u.path == uri  # file path
+        path = u.path
+        if path[0] != ?/
+          pattern = File.expand_path("#{@basepath}/#{path}")
+        else
+          pattern = path
         end
+
+        Dir.glob(pattern).each {|path|
+          basepath = File.dirname(path)
+          fname = File.basename(path)
+          File.open(path) {|f|
+            Parser.new(basepath, f.each_line, fname).parse!(true, nil, attrs, elems)
+          }
+        }
+
+      else
+        basepath = '/'
+        fname = path
+        require 'open-uri'
+        open(uri) {|f|
+          Parser.new(basepath, f.each_line, fname).parse!(true, nil, attrs, elems)
+        }
+      end
+
+    rescue SystemCallError
+      raise ConfigParseError, "include error at #{@fname} line #{@i}: #{$!.to_s}"
     end
-    return i, attrs, elems
   end
 end
 

data/lib/fluent/load.rb

@@ -5,6 +5,7 @@ require 'monitor'
 require 'stringio'
 require 'fileutils'
 require 'json'
+require 'uri'
 require 'msgpack'
 require 'cool.io'
 require 'cool.io/eventmachine'

data/lib/fluent/version.rb

@@ -1,5 +1,5 @@
 module Fluent
 
-VERSION = '0.10.2'
+VERSION = '0.10.3'
 
 end

data/test/config.rb

@@ -0,0 +1,60 @@
+require File.dirname(__FILE__) + '/helper'
+
+require 'fluent/config'
+require 'fileutils'
+
+class ConfigTest < Test::Unit::TestCase
+  include Fluent
+
+  TMP_DIR = File.dirname(__FILE__) + "/tmp"
+
+  def test_include
+    write_config "#{TMP_DIR}/config_test_1.conf", %[
+      k1 root_config
+      include dir/config_test_2.conf  #
+      include #{TMP_DIR}/config_test_4.conf
+      include file://#{TMP_DIR}/config_test_5.conf
+      include config.d/*.conf
+    ]
+    write_config "#{TMP_DIR}/dir/config_test_2.conf", %[
+      k2 relative_path_include
+      include ../config_test_3.conf
+    ]
+    write_config "#{TMP_DIR}/config_test_3.conf", %[
+      k3 relative_include_in_included_file
+    ]
+    write_config "#{TMP_DIR}/config_test_4.conf", %[
+      k4 absolute_path_include
+    ]
+    write_config "#{TMP_DIR}/config_test_5.conf", %[
+      k5 uri_include
+    ]
+    write_config "#{TMP_DIR}/config.d/config_test_6.conf", %[
+      k6 wildcard_include_1
+      <elem name>
+        include normal_parameter
+      </elem>
+    ]
+    write_config "#{TMP_DIR}/config.d/config_test_7.conf", %[
+      k7 wildcard_include_2
+    ]
+
+    c = Config.read("#{TMP_DIR}/config_test_1.conf")
+    assert_equal 'root_config', c['k1']
+    assert_equal 'relative_path_include', c['k2']
+    assert_equal 'relative_include_in_included_file', c['k3']
+    assert_equal 'absolute_path_include', c['k4']
+    assert_equal 'uri_include', c['k5']
+    assert_equal 'wildcard_include_1', c['k6']
+    assert_equal 'wildcard_include_2', c['k7']
+    assert_equal 'elem', c.elements.first.name
+    assert_equal 'name', c.elements.first.arg
+    assert_equal 'normal_parameter', c.elements.first['include']
+  end
+
+  def write_config(path, data)
+    FileUtils.mkdir_p(File.dirname(path))
+    File.open(path, "w") {|f| f.write data }
+  end
+end
+

metadata

@@ -1,13 +1,13 @@
 --- !ruby/object:Gem::Specification 
 name: fluentd
 version: !ruby/object:Gem::Version 
-  hash: 51
+  hash: 49
   prerelease: false
   segments: 
   - 0
   - 10
-  - 2
-  version: 0.10.2
+  - 3
+  version: 0.10.3
 platform: ruby
 authors: 
 - Sadayuki Furuhashi
@@ -15,7 +15,7 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2011-10-18 00:00:00 +09:00
+date: 2011-10-21 00:00:00 +09:00
 default_executable: 
 dependencies: 
 - !ruby/object:Gem::Dependency 
@@ -169,6 +169,7 @@ files:
 - lib/fluent/test/input_test.rb
 - lib/fluent/test/output_test.rb
 - lib/fluent/version.rb
+- test/config.rb
 - test/helper.rb
 - test/match.rb
 - test/plugin/in_http.rb
@@ -216,6 +217,7 @@ signing_key:
 specification_version: 3
 summary: Fluent event collector
 test_files: 
+- test/config.rb
 - test/helper.rb
 - test/match.rb
 - test/plugin/in_http.rb