fluentd 0.14.17-x64-mingw32 → 1.3.1-x64-mingw32

data/lib/fluent/config/configure_proxy.rb

@@ -370,6 +370,12 @@ module Fluent
 
       def dump_config_definition
         dumped_config = {}
+        if @argument
+          argument_name, _block, options = @argument
+          options[:required] = !@defaults.key?(argument_name)
+          options[:argument] = true
+          dumped_config[argument_name] = options
+        end
         @params.each do |name, config|
           dumped_config[name] = config[1]
           dumped_config[name][:required] = !@defaults.key?(name)
@@ -378,7 +384,7 @@ module Fluent
         end
         # Overwrite by config_set_default
         @defaults.each do |name, value|
-          if @params.key?(name)
+          if @params.key?(name) || (@argument && @argument.first == name)
             dumped_config[name][:default] = value
           else
             dumped_config[name] = { default: value }

data/lib/fluent/config/dsl.rb

@@ -22,6 +22,8 @@ require 'fluent/config/element'
 module Fluent
   module Config
     module DSL
+      RESERVED_PARAMETERS = [:type, :id, :log_level, :label] # Need '@' prefix for reserved parameters
+
       module Parser
         def self.read(path)
           path = File.expand_path(path)
@@ -95,11 +97,12 @@ module Fluent
             proxy.element.instance_exec(&block)
             @elements.push(proxy.to_config_element)
           else
-            @attrs[name.to_s] = if value.is_a?(Array) || value.is_a?(Hash)
-                                  JSON.dump(value)
-                                else
-                                  value.to_s
-                                end
+            param_name = RESERVED_PARAMETERS.include?(name) ? "@#{name}" : name.to_s
+            @attrs[param_name] = if value.is_a?(Array) || value.is_a?(Hash)
+                                   JSON.dump(value)
+                                 else
+                                   value.to_s
+                                 end
           end
 
           self

data/lib/fluent/config/element.rb

@@ -76,6 +76,11 @@ module Fluent
         "name:#{@name}, arg:#{@arg}, " + attrs + ", " + @elements.inspect
       end
 
+      # Used by PP and Pry
+      def pretty_print(q)
+        q.text(inspect)
+      end
+
       # This method assumes _o_ is an Element object. Should return false for nil or other object
       def ==(o)
         self.name == o.name && self.arg == o.arg &&

data/lib/fluent/config/literal_parser.rb

@@ -18,6 +18,7 @@ require 'stringio'
 
 require 'json'
 require 'yajl'
+require 'socket'
 require 'irb/ruby-lex'  # RubyLex
 
 require 'fluent/config/basic_parser'
@@ -162,6 +163,11 @@ module Fluent
         if @eval_context.nil?
           parse_error! "embedded code is not allowed in this file"
         end
+        # Add hostname and worker_id to code for preventing unused warnings
+        code = <<EOM + code
+hostname = Socket.gethostname
+worker_id = ENV['SERVERENGINE_WORKER_ID'] || ''
+EOM
         @eval_context.instance_eval(code)
       end
 
@@ -192,7 +198,7 @@ module Fluent
 
       def scan_json(is_array)
         result = nil
-        # Yajl does not raise ParseError for imcomplete json string, like '[1', '{"h"', '{"h":' or '{"h1":1'
+        # Yajl does not raise ParseError for incomplete json string, like '[1', '{"h"', '{"h":' or '{"h1":1'
         # This is the reason to use JSON module.
 
         buffer = (is_array ? "[" : "{")

data/lib/fluent/config/types.rb

@@ -60,11 +60,35 @@ module Fluent
       when ''
         true
       else
-        nil
+        # Current parser passes comment without actual values, e.g. "param #foo".
+        # parser should pass empty string in this case but changing behaviour may break existing environment so keep parser behaviour. Just ignore comment value in boolean handling for now.
+        if str.respond_to?('start_with?') && str.start_with?('#')
+          true
+        else
+          nil
+        end
+      end
+    end
+
+    def self.regexp_value(str)
+      return nil unless str
+      return Regexp.compile(str) unless str.start_with?("/")
+      right_slash_position = str.rindex("/")
+      if right_slash_position < str.size - 3
+        raise Fluent::ConfigError, "invalid regexp: missing right slash: #{str}"
       end
+      options = str[(right_slash_position + 1)..-1]
+      option = 0
+      option |= Regexp::IGNORECASE if options.include?("i")
+      option |= Regexp::MULTILINE if options.include?("m")
+      Regexp.compile(str[1...right_slash_position], option)
     end
 
-    STRING_TYPE = Proc.new { |val, opts| val.to_s.force_encoding(Encoding::UTF_8) }
+    STRING_TYPE = Proc.new { |val, opts|
+      v = val.to_s
+      v = v.frozen? ? v.dup : v # config_param can't assume incoming string is mutable
+      v.force_encoding(Encoding::UTF_8)
+    }
     ENUM_TYPE = Proc.new { |val, opts|
       s = val.to_sym
       list = opts[:list]
@@ -79,6 +103,7 @@ module Fluent
     SIZE_TYPE = Proc.new { |val, opts| Config.size_value(val) }
     BOOL_TYPE = Proc.new { |val, opts| Config.bool_value(val) }
     TIME_TYPE = Proc.new { |val, opts| Config.time_value(val) }
+    REGEXP_TYPE = Proc.new { |val, opts| Config.regexp_value(val) }
 
     REFORMAT_VALUE = ->(type, value) {
       if value.nil?
@@ -91,6 +116,7 @@ module Fluent
         when :size then Config.size_value(value)
         when :bool then Config.bool_value(value)
         when :time then Config.time_value(value)
+        when :regexp then Config.regexp_value(value)
         else
           raise "unknown type in REFORMAT: #{type}"
         end

data/lib/fluent/config/v1_parser.rb

@@ -174,9 +174,8 @@ module Fluent
           ss = StringScanner.new(data)
           V1Parser.new(ss, basepath, fname, @eval_context).parse_element(true, nil, attrs, elems)
         end
-
       rescue SystemCallError => e
-        cpe = ConfigParseError.new("include error #{uri}")
+        cpe = ConfigParseError.new("include error #{uri} - #{e}")
         cpe.set_backtrace(e.backtrace)
         raise cpe
       end

data/lib/fluent/configurable.rb

@@ -128,6 +128,7 @@ module Fluent
       time: Config::TIME_TYPE,
       hash: Config::HASH_TYPE,
       array: Config::ARRAY_TYPE,
+      regexp: Config::REGEXP_TYPE,
     }.each do |name, type|
       register_type(name, type)
     end

data/lib/fluent/counter.rb

@@ -0,0 +1,23 @@
+#
+# Fluentd
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+
+require 'fluent/counter/client'
+require 'fluent/counter/server'
+
+module Fluent
+  module Counter
+  end
+end

data/lib/fluent/counter/base_socket.rb

@@ -0,0 +1,46 @@
+#
+# Fluentd
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+
+require 'cool.io'
+require 'fluent/msgpack_factory'
+
+module Fluent
+  module Counter
+    class BaseSocket < Coolio::TCPSocket
+      include Fluent::MessagePackFactory::Mixin
+
+      def packed_write(data)
+        write pack(data)
+      end
+
+      def on_read(data)
+        msgpack_unpacker.feed_each(data) do |d|
+          on_message d
+        end
+      end
+
+      def on_message(data)
+        raise NotImplementedError
+      end
+
+      private
+
+      def pack(data)
+        msgpack_packer.pack(data)
+      end
+    end
+  end
+end

data/lib/fluent/counter/client.rb

@@ -0,0 +1,297 @@
+#
+# Fluentd
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+
+require 'cool.io'
+require 'fluent/counter/base_socket'
+require 'fluent/counter/error'
+require 'timeout'
+
+module Fluent
+  module Counter
+    class Client
+      DEFAULT_PORT = 24321
+      DEFAULT_ADDR = '127.0.0.1'
+      DEFAULT_TIMEOUT = 5
+      ID_LIMIT_COUNT = 1 << 31
+
+      def initialize(loop = nil, opt = {})
+        @loop = loop || Coolio::Loop.new
+        @port = opt[:port] || DEFAULT_PORT
+        @host = opt[:host] || DEFAULT_ADDR
+        @log = opt[:log] || $log
+        @timeout = opt[:timeout] || DEFAULT_TIMEOUT
+        @conn = Connection.connect(@host, @port, method(:on_message))
+        @responses = {}
+        @id = 0
+        @id_mutex = Mutex.new
+        @loop_mutex = Mutex.new
+      end
+
+      def start
+        @loop.attach(@conn)
+        @log.debug("starting counter client: #{@host}:#{@port}")
+        self
+      rescue => e
+        if @log
+          @log.error e
+        else
+          STDERR.puts e
+        end
+      end
+
+      def stop
+        @conn.close
+        @log.debug("calling stop in counter client: #{@host}:#{@port}")
+      end
+
+      def establish(scope)
+        scope = Timeout.timeout(@timeout) {
+          response = send_request('establish', nil, [scope])
+          Fluent::Counter.raise_error(response.errors.first) if response.errors?
+          data = response.data
+          data.first
+        }
+        @scope = scope
+      rescue Timeout::Error
+        raise "Can't establish the connection to counter server due to timeout"
+      end
+
+      # === Example
+      # `init` receives various arguments.
+      #
+      # 1. init(name: 'name')
+      # 2. init({ name: 'name',reset_interval: 20 }, options: {})
+      # 3. init([{ name: 'name1',reset_interval: 20 }, { name: 'name2',reset_interval: 20 }])
+      # 4. init([{ name: 'name1',reset_interval: 20 }, { name: 'name2',reset_interval: 20 }], options: {})
+      # 5. init([{ name: 'name1',reset_interval: 20 }, { name: 'name2',reset_interval: 20 }]) { |res| ... }
+      def init(params, options: {})
+        exist_scope!
+        params = [params] unless params.is_a?(Array)
+        res = send_request('init', @scope, params, options)
+
+        # if `async` is false or missing, block at this method and return a Future::Result object.
+        if block_given?
+          Thread.start do
+            yield res.get
+          end
+        else
+          res
+        end
+      end
+
+      def delete(*params, options: {})
+        exist_scope!
+        res = send_request('delete', @scope, params, options)
+
+        if block_given?
+          Thread.start do
+            yield res.get
+          end
+        else
+          res
+        end
+      end
+
+      # === Example
+      # `inc` receives various arguments.
+      #
+      # 1. inc(name: 'name')
+      # 2. inc({ name: 'name',value: 20 }, options: {})
+      # 3. inc([{ name: 'name1',value: 20 }, { name: 'name2',value: 20 }])
+      # 4. inc([{ name: 'name1',value: 20 }, { name: 'name2',value: 20 }], options: {})
+      def inc(params, options: {})
+        exist_scope!
+        params = [params] unless params.is_a?(Array)
+        res = send_request('inc', @scope, params, options)
+
+        if block_given?
+          Thread.start do
+            yield res.get
+          end
+        else
+          res
+        end
+      end
+
+      def get(*params, options: {})
+        exist_scope!
+        res = send_request('get', @scope, params, options)
+
+        if block_given?
+          Thread.start do
+            yield res.get
+          end
+        else
+          res
+        end
+      end
+
+      def reset(*params, options: {})
+        exist_scope!
+        res = send_request('reset', @scope, params, options)
+
+        if block_given?
+          Thread.start do
+            yield res.get
+          end
+        else
+          res
+        end
+      end
+
+      private
+
+      def exist_scope!
+        raise 'Call `establish` method to get a `scope` before calling this method' unless @scope
+      end
+
+      def on_message(data)
+        if response = @responses.delete(data['id'])
+          response.set(data)
+        else
+          @log.warn("Receiving missing id data: #{data}")
+        end
+      end
+
+      def send_request(method, scope, params, opt = {})
+        id = generate_id
+        res = Future.new(@loop, @loop_mutex)
+        @responses[id] = res # set a response value to this future object at `on_message`
+        request = build_request(method, id, scope, params, opt)
+        @log.debug(request)
+        @conn.send_data request
+        res
+      end
+
+      def build_request(method, id, scope = nil, params = nil, options = nil)
+        r = { id: id, method: method }
+        r[:scope] = scope if scope
+        r[:params] = params if params
+        r[:options] = options if options
+        r
+      end
+
+      def generate_id
+        id = 0
+        @id_mutex.synchronize do
+          id = @id
+          @id += 1
+          @id = 0 if ID_LIMIT_COUNT < @id
+        end
+        id
+      end
+    end
+
+    class Connection < Fluent::Counter::BaseSocket
+      def initialize(io, on_message)
+        super(io)
+        @connection = false
+        @buffer = ''
+        @on_message = on_message
+      end
+
+      def send_data(data)
+        if @connection
+          packed_write data
+        else
+          @buffer += pack(data)
+        end
+      end
+
+      def on_connect
+        @connection = true
+        write @buffer
+        @buffer = ''
+      end
+
+      def on_close
+        @connection = false
+      end
+
+      def on_message(data)
+        @on_message.call(data)
+      end
+    end
+
+    class Future
+      class Result
+        attr_reader :data, :errors
+
+        def initialize(result)
+          @errors = result['errors']
+          @data = result['data']
+        end
+
+        def success?
+          @errors.nil? || @errors.empty?
+        end
+
+        def error?
+          !success?
+        end
+      end
+
+      def initialize(loop, mutex)
+        @set = false
+        @result = nil
+        @mutex = mutex
+        @loop = loop
+      end
+
+      def set(v)
+        @result = Result.new(v)
+        @set = true
+      end
+
+      def errors
+        get.errors
+      end
+
+      def errors?
+        es = errors
+        es && !es.empty?
+      end
+
+      def data
+        get.data
+      end
+
+      def get
+        # Block until `set` method is called and @result is set
+        join if @result.nil?
+        @result
+      end
+
+      def wait
+        res = get
+        if res.error?
+          Fluent::Counter.raise_error(res.errors.first)
+        end
+        res
+      end
+
+      private
+
+      def join
+        until @set
+          @mutex.synchronize do
+            @loop.run_once(0.0001) # retun a lock as soon as possible
+          end
+        end
+      end
+    end
+  end
+end