fluentd 0.10.62 → 0.12.0.pre.1

data/lib/fluent/buffer.rb

@@ -1,7 +1,5 @@
 #
-# Fluent
-#
-# Copyright (C) 2011 FURUHASHI Sadayuki
+# Fluentd
 #
 #    Licensed under the Apache License, Version 2.0 (the "License");
 #    you may not use this file except in compliance with the License.
@@ -223,15 +221,11 @@ module Fluent
 
     def total_queued_chunk_size
       total = 0
-      synchronize {
-        @map.each_value {|c|
-          total += c.size
-        }
-        @queue.synchronize {
-          @queue.each {|c|
-            total += c.size
-          }
-        }
+      @map.each_value {|c|
+        total += c.size
+      }
+      @queue.each {|c|
+        total += c.size
       }
       total
     end

data/lib/fluent/command/cat.rb

@@ -1,7 +1,5 @@
 #
-# Fluent cat
-#
-# Copyright (C) 2011 FURUHASHI Sadayuki
+# Fluentd
 #
 #    Licensed under the Apache License, Version 2.0 (the "License");
 #    you may not use this file except in compliance with the License.

data/lib/fluent/command/debug.rb

@@ -1,7 +1,5 @@
 #
-# Fluent
-#
-# Copyright (C) 2011 FURUHASHI Sadayuki
+# Fluentd
 #
 #    Licensed under the Apache License, Version 2.0 (the "License");
 #    you may not use this file except in compliance with the License.

data/lib/fluent/command/fluentd.rb

@@ -1,8 +1,6 @@
 #
 # Fluentd
 #
-# Copyright (C) 2011 FURUHASHI Sadayuki
-#
 #    Licensed under the Apache License, Version 2.0 (the "License");
 #    you may not use this file except in compliance with the License.
 #    You may obtain a copy of the License at
@@ -52,10 +50,6 @@ op.on('-d', '--daemon PIDFILE', "daemonize fluent process") {|s|
   opts[:daemonize] = s
 }
 
-op.on('--no-supervisor', "run without fluent supervisor") {
-  opts[:supervise] = false
-}
-
 op.on('--user USER', "change user") {|s|
   opts[:chuser] = s
 }
@@ -88,10 +82,6 @@ op.on('--use-v1-config', "Use v1 configuration format", TrueClass) {|b|
   opts[:use_v1_config] = b
 }
 
-op.on('--use-v0-config', "Use v0 configuration format (default)", TrueClass) {|b|
-  opts[:use_v1_config] = !b
-}
-
 op.on('-v', '--verbose', "increase verbose level (-v: debug, -vv: trace)", TrueClass) {|b|
   if b
     opts[:log_level] = [opts[:log_level] - 1, Fluent::Log::LEVEL_TRACE].max

data/lib/fluent/config.rb

@@ -1,8 +1,6 @@
 #
 # Fluent
 #
-# Copyright (C) 2011 FURUHASHI Sadayuki
-#
 #    Licensed under the Apache License, Version 2.0 (the "License");
 #    you may not use this file except in compliance with the License.
 #    You may obtain a copy of the License at
@@ -15,6 +13,7 @@
 #    See the License for the specific language governing permissions and
 #    limitations under the License.
 #
+
 module Fluent
   require 'fluent/config/error'
   require 'fluent/config/element'
@@ -44,10 +43,17 @@ module Fluent
 
   module PluginId
     def configure(conf)
-      @id = conf['@id'] || conf['id']
+      @id = conf['id']
       super
     end
 
+    def require_id
+      unless @id
+        raise ConfigError, "'id' parameter is required"
+      end
+      @id
+    end
+
     def plugin_id
       @id ? @id : "object:#{object_id.to_s(16)}"
     end

data/lib/fluent/config/basic_parser.rb

@@ -13,6 +13,7 @@
 #    See the License for the specific language governing permissions and
 #    limitations under the License.
 #
+
 module Fluent
   module Config
 
@@ -26,8 +27,6 @@ module Fluent
 
       LINE_END = /(?:[ \t]*(?:\#.*)?(?:\z|[\r\n]))+/
       SPACING = /(?:[ \t\r\n]|\z|\#.*?(?:\z|[\r\n]))+/
-      ZERO_OR_MORE_SPACING = /(?:[ \t\r\n]|\z|\#.*?(?:\z|[\r\n]))*/
-      SPACING_WITHOUT_COMMENT = /(?:[ \t\r\n]|\z)+/
 
       module ClassMethods
         def symbol(string)
@@ -79,10 +78,6 @@ module Fluent
         skip(SPACING)
       end
 
-      def spacing_without_comment
-        skip(SPACING_WITHOUT_COMMENT)
-      end
-
       def parse_error!(message)
         raise ConfigParseError, "#{message} at #{error_sample}"
       end

data/lib/fluent/config/configure_proxy.rb

@@ -1,7 +1,23 @@
+#
+# Fluentd
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+
 module Fluent
   module Config
     class ConfigureProxy
-      attr_accessor :name, :final, :param_name, :required, :multi, :alias, :argument, :params, :defaults, :sections
+      attr_accessor :name, :param_name, :required, :multi, :alias, :argument, :params, :defaults, :sections
       # config_param :desc, :string, :default => '....'
       # config_set_default :buffer_type, :memory
       #
@@ -22,7 +38,6 @@ module Fluent
 
       def initialize(name, opts = {})
         @name = name.to_sym
-        @final = opts.fetch(:final, false)
 
         @param_name = (opts[:param_name] || @name).to_sym
         @required = opts[:required]
@@ -43,13 +58,7 @@ module Fluent
         @multi.nil? ? true : @multi
       end
 
-      def final?
-        @final
-      end
-
       def merge(other) # self is base class, other is subclass
-        return merge_for_finalized(other) if self.final?
-
         options = {
           param_name: other.param_name,
           required: (other.required.nil? ? self.required : other.required),
@@ -60,52 +69,7 @@ module Fluent
         merged.argument = other.argument || self.argument
         merged.params = self.params.merge(other.params)
         merged.defaults = self.defaults.merge(other.defaults)
-        merged.sections = {}
-        (self.sections.keys + other.sections.keys).uniq.each do |section_key|
-          self_section = self.sections[section_key]
-          other_section = other.sections[section_key]
-          merged_section = if self_section && other_section
-                             self_section.merge(other_section)
-                           elsif self_section || other_section
-                             self_section || other_section
-                           else
-                             raise "BUG: both of self and other section are nil"
-                           end
-          merged.sections[section_key] = merged_section
-        end
-
-        merged
-      end
-
-      def merge_for_finalized(other)
-        # list what subclass can do for finalized section
-        #  * overwrite param_name to escape duplicated name of instance variable
-        #  * append params/defaults/sections which are missing in superclass
-
-        options = {
-          param_name: other.param_name,
-          required: (self.required.nil? ? other.required : self.required),
-          multi: (self.multi.nil? ? other.multi : self.multi),
-          final: true,
-        }
-        merged = self.class.new(other.name, options)
-
-        merged.argument = self.argument || other.argument
-        merged.params = other.params.merge(self.params)
-        merged.defaults = other.defaults.merge(self.defaults)
-        merged.sections = {}
-        (self.sections.keys + other.sections.keys).uniq.each do |section_key|
-          self_section = self.sections[section_key]
-          other_section = other.sections[section_key]
-          merged_section = if self_section && other_section
-                             other_section.merge(self_section)
-                           elsif self_section || other_section
-                             self_section || other_section
-                           else
-                             raise "BUG: both of self and other section are nil"
-                           end
-          merged.sections[section_key] = merged_section
-        end
+        merged.sections = self.sections.merge(other.sections)
 
         merged
       end
@@ -120,13 +84,13 @@ module Fluent
           elsif a.is_a?(Hash)
             opts.merge!(a)
           else
-            raise ArgumentError, "#{self.name}: wrong number of arguments (#{1 + args.length} for #{block ? 2 : 3})"
+            raise ArgumentError, "wrong number of arguments (#{1 + args.length} for #{block ? 2 : 3})"
           end
         }
 
         type = opts[:type]
         if block && type
-          raise ArgumentError, "#{self.name}: both of block and type cannot be specified"
+          raise ArgumentError, "both of block and type cannot be specified"
         end
 
         begin
@@ -134,7 +98,7 @@ module Fluent
           block ||= Configurable.lookup_type(type)
         rescue ConfigError
           # override error message
-          raise ArgumentError, "#{self.name}: unknown config_argument type `#{type}'"
+          raise ArgumentError, "unknown config_argument type `#{type}'"
         end
 
         if opts.has_key?(:default)
@@ -146,7 +110,7 @@ module Fluent
 
       def config_argument(name, *args, &block)
         if @argument
-          raise ArgumentError, "#{self.name}: config_argument called twice"
+          raise ArgumentError, "config_argument called twice"
         end
         name, block, opts = parameter_configuration(name, *args, &block)
 
@@ -166,7 +130,7 @@ module Fluent
         name = name.to_sym
 
         if @defaults.has_key?(name)
-          raise ArgumentError, "#{self.name}: default value specified twice for #{name}"
+          raise ArgumentError, "default value specified twice for #{name}"
         end
 
         @defaults[name] = defval
@@ -175,13 +139,13 @@ module Fluent
 
       def config_section(name, *args, &block)
         unless block_given?
-          raise ArgumentError, "#{self.name}: config_section requires block parameter"
+          raise ArgumentError, "config_section requires block parameter"
         end
         name = name.to_sym
 
         opts = {}
         unless args.empty? || args.size == 1 && args.first.is_a?(Hash)
-          raise ArgumentError, "#{self.name}: unknown config_section arguments: #{args.inspect}"
+          raise ArgumentError, "unknown config_section arguments: #{args.inspect}"
         end
 
         sub_proxy = ConfigureProxy.new(name, *args)

data/lib/fluent/config/dsl.rb

@@ -1,3 +1,19 @@
+#
+# Fluentd
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+
 require 'fluent/config'
 
 module Fluent

data/lib/fluent/config/element.rb

@@ -1,3 +1,19 @@
+#
+# Fluentd
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+
 module Fluent
   require 'fluent/config/error'
 
@@ -29,7 +45,6 @@ module Fluent
         "name:#{@name}, arg:#{@arg}, " + attrs + ", " + @elements.inspect
       end
 
-      # This method assumes _o_ is an Element object. Should return false for nil or other object
       def ==(o)
         self.name == o.name && self.arg == o.arg &&
           self.keys.size == o.keys.size &&
@@ -87,7 +102,11 @@ module Fluent
           out << "#{indent}<#{@name} #{@arg}>\n"
         end
         each_pair { |k, v|
-          out << "#{nindent}#{k} #{v}\n"
+          if @v1_config
+            out << "#{nindent}#{k} #{Element.unescape_parameter(v)}\n"
+          else
+            out << "#{nindent}#{k} #{v}\n"
+          end
         }
         @elements.each { |e|
           out << e.to_s(nest + 1)

data/lib/fluent/config/error.rb

@@ -1,3 +1,19 @@
+#
+# Fluentd
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+
 module Fluent
   class ConfigError < StandardError
   end

data/lib/fluent/config/literal_parser.rb

@@ -13,6 +13,7 @@
 #    See the License for the specific language governing permissions and
 #    limitations under the License.
 #
+
 module Fluent
   module Config
 
@@ -50,7 +51,7 @@ module Fluent
       end
 
       def parse_literal(string_boundary_charset = LINE_END)
-        spacing_without_comment
+        spacing
 
         value = if skip(/\[/)
                   scan_json(true)
@@ -64,15 +65,13 @@ module Fluent
 
       def scan_string(string_boundary_charset = LINE_END)
         if skip(/\"/)
-          return scan_double_quoted_string
-        elsif skip(/\'/)
-          return scan_single_quoted_string
+          return scan_quoted_string
         else
           return scan_nonquoted_string(string_boundary_charset)
         end
       end
 
-      def scan_double_quoted_string
+      def scan_quoted_string
         string = []
         while true
           if skip(/\"/)
@@ -85,24 +84,7 @@ module Fluent
           elsif s = scan(/./)
             string << s
           else
-            parse_error! "unexpected end of file in a double quoted string"
-          end
-        end
-      end
-
-      def scan_single_quoted_string
-        string = []
-        while true
-          if skip(/\'/)
-            return string.join
-          elsif s = scan(/\\'/)
-            string << "'"
-          elsif s = scan(/\\\\/)
-            string << "\\"
-          elsif s = scan(/./)
-            string << s
-          else
-            parse_error! "unexpected end of file in a signle quoted string"
+            parse_error! "unexpected end of file in a quoted string"
           end
         end
       end
@@ -112,8 +94,8 @@ module Fluent
 
         string = []
         while true
-          if s = scan(/\#/)
-            string << '#'
+          if s = scan(/\\./)
+            string << eval_escape_char(s[1,1])
           elsif s = scan(charset)
             string << s
           else
@@ -146,7 +128,7 @@ module Fluent
 
         @ss.pos += code.length
 
-        '"#{' + code + '}"'
+        code
       end
 
       def eval_embedded_code(code)
@@ -232,7 +214,7 @@ module Fluent
         end
 
         unless result
-          parse_error! "got incomplete JSON #{is_array ? 'array' : 'hash'} configuration"
+          parse_error! "got incomplete #{is_array ? 'array' : 'hash'} configuration"
         end
 
         JSON.dump(result)