fluentd 1.14.4-x64-mingw-ucrt

data/lib/fluent/config.rb

@@ -0,0 +1,76 @@
+#
+# Fluent
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+
+require 'fluent/config/error'
+require 'fluent/config/element'
+require 'fluent/configurable'
+
+module Fluent
+  module Config
+    # @param config_path [String] config file path
+    # @param encoding [String] encoding of config file
+    # @param additional_config [String] config which is added to last of config body
+    # @param use_v1_config [Bool] config is formatted with v1 or not
+    # @return [Fluent::Config]
+    def self.build(config_path:, encoding: 'utf-8', additional_config: nil, use_v1_config: true)
+      config_fname = File.basename(config_path)
+      config_basedir = File.dirname(config_path)
+      config_data = File.open(config_path, "r:#{encoding}:utf-8") do |f|
+        s = f.read
+        if additional_config
+          c = additional_config.gsub("\\n", "\n")
+          s += "\n#{c}"
+        end
+        s
+      end
+      Fluent::Config.parse(config_data, config_fname, config_basedir, use_v1_config)
+    end
+
+    def self.parse(str, fname, basepath = Dir.pwd, v1_config = nil, syntax: :v1)
+      parser = if fname =~ /\.rb$/ || syntax == :ruby
+                 :ruby
+               elsif v1_config.nil?
+                 case syntax
+                 when :v1 then :v1
+                 when :v0 then :v0
+                 else
+                   raise ArgumentError, "Unknown Fluentd configuration syntax: '#{syntax}'"
+                 end
+               elsif v1_config then :v1
+               else :v0
+               end
+      case parser
+      when :v1
+        require 'fluent/config/v1_parser'
+        V1Parser.parse(str, fname, basepath, Kernel.binding)
+      when :v0
+        # TODO: show deprecated message in v1
+        require 'fluent/config/parser'
+        Parser.parse(str, fname, basepath)
+      when :ruby
+        require 'fluent/config/dsl'
+        $log.warn("Ruby DSL configuration format is deprecated. Please use original configuration format. https://docs.fluentd.org/configuration/config-file") if $log
+        Config::DSL::Parser.parse(str, File.join(basepath, fname))
+      else
+        raise "[BUG] unknown configuration parser specification:'#{parser}'"
+      end
+    end
+
+    def self.new(name = '')
+      Element.new(name, '', {}, [])
+    end
+  end
+end

data/lib/fluent/configurable.rb

@@ -0,0 +1,201 @@
+#
+# Fluentd
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+
+require 'fluent/config/configure_proxy'
+require 'fluent/config/section'
+require 'fluent/config/error'
+require 'fluent/registry'
+require 'fluent/plugin'
+require 'fluent/config/types'
+
+module Fluent
+  module Configurable
+    def self.included(mod)
+      mod.extend(ClassMethods)
+    end
+
+    def initialize
+      super
+      # to simulate implicit 'attr_accessor' by config_param / config_section and its value by config_set_default
+      proxy = self.class.merged_configure_proxy
+      proxy.params.keys.each do |name|
+        next if name.to_s.start_with?('@')
+        if proxy.defaults.has_key?(name)
+          instance_variable_set("@#{name}".to_sym, proxy.defaults[name])
+        end
+      end
+      proxy.sections.keys.each do |name|
+        next if name.to_s.start_with?('@')
+        subproxy = proxy.sections[name]
+        if subproxy.multi?
+          instance_variable_set("@#{subproxy.variable_name}".to_sym, [])
+        else
+          instance_variable_set("@#{subproxy.variable_name}".to_sym, nil)
+        end
+      end
+    end
+
+    def configure_proxy_generate
+      proxy = self.class.merged_configure_proxy
+
+      if self.respond_to?(:owner) && self.owner
+        owner_proxy = owner.class.merged_configure_proxy
+        if proxy.configured_in_section
+          owner_proxy = owner_proxy.sections[proxy.configured_in_section]
+        end
+        proxy.overwrite_defaults(owner_proxy) if owner_proxy
+      end
+
+      proxy
+    end
+
+    def configured_section_create(name, conf = nil)
+      conf ||= Fluent::Config::Element.new(name.to_s, '', {}, [])
+      root_proxy = configure_proxy_generate
+      proxy = if name.nil? # root
+                root_proxy
+              else
+                root_proxy.sections[name]
+              end
+      # take care to raise Fluent::ConfigError if conf mismatched to proxy
+      Fluent::Config::SectionGenerator.generate(proxy, conf, nil, nil)
+    end
+
+    def configure(conf, strict_config_value=false)
+      @config = conf
+
+      logger = if self.respond_to?(:log)
+                 self.log
+               elsif self.respond_to?(:owner) && self.owner.respond_to?(:log)
+                 self.owner.log
+               elsif defined?($log)
+                 $log
+               else
+                 nil
+               end
+      proxy = configure_proxy_generate
+      conf.corresponding_proxies << proxy
+
+      # In the nested section, can't get plugin class through proxies so get plugin class here
+      plugin_class = Fluent::Plugin.lookup_type_from_class(proxy.name.to_s)
+      root = Fluent::Config::SectionGenerator.generate(proxy, conf, logger, plugin_class, [], strict_config_value)
+      @config_root_section = root
+
+      root.instance_eval{ @params.keys }.each do |param_name|
+        next if param_name.to_s.start_with?('@')
+        varname = "@#{param_name}".to_sym
+        instance_variable_set(varname, root[param_name])
+      end
+
+      self
+    end
+
+    def config
+      @masked_config ||= @config.to_masked_element
+    end
+
+    CONFIG_TYPE_REGISTRY = Registry.new(:config_type, 'fluent/plugin/type_')
+
+    def self.register_type(type, callable = nil, &block)
+      callable ||= block
+      CONFIG_TYPE_REGISTRY.register(type, callable)
+    end
+
+    def self.lookup_type(type)
+      CONFIG_TYPE_REGISTRY.lookup(type)
+    end
+
+    {
+      string: Config::STRING_TYPE,
+      enum: Config::ENUM_TYPE,
+      integer: Config::INTEGER_TYPE,
+      float: Config::FLOAT_TYPE,
+      size: Config::SIZE_TYPE,
+      bool: Config::BOOL_TYPE,
+      time: Config::TIME_TYPE,
+      hash: Config::HASH_TYPE,
+      array: Config::ARRAY_TYPE,
+      regexp: Config::REGEXP_TYPE,
+    }.each do |name, type|
+      register_type(name, type)
+    end
+
+    module ClassMethods
+      def configure_proxy_map
+        map = {}
+        self.define_singleton_method(:configure_proxy_map){ map }
+        map
+      end
+
+      def configure_proxy(mod_name)
+        map = configure_proxy_map
+        unless map[mod_name]
+          type_lookup = ->(type) { Fluent::Configurable.lookup_type(type) }
+          proxy = Fluent::Config::ConfigureProxy.new(mod_name, root: true, required: true, multi: false, type_lookup: type_lookup)
+          map[mod_name] = proxy
+        end
+        map[mod_name]
+      end
+
+      def configured_in(section_name)
+        configure_proxy(self.name).configured_in(section_name)
+      end
+
+      def config_param(name, type = nil, **kwargs, &block)
+        configure_proxy(self.name).config_param(name, type, **kwargs, &block)
+        # reserved names '@foo' are invalid as attr_accessor name
+        attr_accessor(name) unless kwargs[:skip_accessor] || Fluent::Config::Element::RESERVED_PARAMETERS.include?(name.to_s)
+      end
+
+      def config_set_default(name, defval)
+        configure_proxy(self.name).config_set_default(name, defval)
+      end
+
+      def config_set_desc(name, desc)
+        configure_proxy(self.name).config_set_desc(name, desc)
+      end
+
+      def config_section(name, **kwargs, &block)
+        section_already_exists = !!merged_configure_proxy.sections[name]
+        configure_proxy(self.name).config_section(name, **kwargs, &block)
+        variable_name = configure_proxy(self.name).sections[name].variable_name
+        if !section_already_exists && !self.respond_to?(variable_name)
+          attr_accessor variable_name
+        end
+      end
+
+      def desc(description)
+        configure_proxy(self.name).desc(description)
+      end
+
+      def merged_configure_proxy
+        configurables = ancestors.reverse.select{ |a| a.respond_to?(:configure_proxy) }
+
+        # 'a.object_id.to_s' is to support anonymous class
+        #   which created in tests to overwrite original behavior temporally
+        #
+        # p Module.new.name   #=> nil
+        # p Class.new.name    #=> nil
+        # p AnyGreatClass.dup.name #=> nil
+        configurables.map{ |a| a.configure_proxy(a.name || a.object_id.to_s) }.reduce(:merge)
+      end
+
+      def dump_config_definition
+        configure_proxy_map[self.to_s].dump_config_definition
+      end
+    end
+  end
+end

data/lib/fluent/counter/base_socket.rb

@@ -0,0 +1,44 @@
+#
+# Fluentd
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+
+require 'cool.io'
+require 'fluent/msgpack_factory'
+
+module Fluent
+  module Counter
+    class BaseSocket < Coolio::TCPSocket
+      def packed_write(data)
+        write pack(data)
+      end
+
+      def on_read(data)
+        Fluent::MessagePackFactory.msgpack_unpacker.feed_each(data) do |d|
+          on_message d
+        end
+      end
+
+      def on_message(data)
+        raise NotImplementedError
+      end
+
+      private
+
+      def pack(data)
+        Fluent::MessagePackFactory.msgpack_packer.pack(data)
+      end
+    end
+  end
+end

data/lib/fluent/counter/client.rb

@@ -0,0 +1,297 @@
+#
+# Fluentd
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+
+require 'cool.io'
+require 'fluent/counter/base_socket'
+require 'fluent/counter/error'
+require 'timeout'
+
+module Fluent
+  module Counter
+    class Client
+      DEFAULT_PORT = 24321
+      DEFAULT_ADDR = '127.0.0.1'
+      DEFAULT_TIMEOUT = 5
+      ID_LIMIT_COUNT = 1 << 31
+
+      def initialize(loop = nil, opt = {})
+        @loop = loop || Coolio::Loop.new
+        @port = opt[:port] || DEFAULT_PORT
+        @host = opt[:host] || DEFAULT_ADDR
+        @log = opt[:log] || $log
+        @timeout = opt[:timeout] || DEFAULT_TIMEOUT
+        @conn = Connection.connect(@host, @port, method(:on_message))
+        @responses = {}
+        @id = 0
+        @id_mutex = Mutex.new
+        @loop_mutex = Mutex.new
+      end
+
+      def start
+        @loop.attach(@conn)
+        @log.debug("starting counter client: #{@host}:#{@port}")
+        self
+      rescue => e
+        if @log
+          @log.error e
+        else
+          STDERR.puts e
+        end
+      end
+
+      def stop
+        @conn.close
+        @log.debug("calling stop in counter client: #{@host}:#{@port}")
+      end
+
+      def establish(scope)
+        scope = Timeout.timeout(@timeout) {
+          response = send_request('establish', nil, [scope])
+          Fluent::Counter.raise_error(response.errors.first) if response.errors?
+          data = response.data
+          data.first
+        }
+        @scope = scope
+      rescue Timeout::Error
+        raise "Can't establish the connection to counter server due to timeout"
+      end
+
+      # === Example
+      # `init` receives various arguments.
+      #
+      # 1. init(name: 'name')
+      # 2. init({ name: 'name',reset_interval: 20 }, options: {})
+      # 3. init([{ name: 'name1',reset_interval: 20 }, { name: 'name2',reset_interval: 20 }])
+      # 4. init([{ name: 'name1',reset_interval: 20 }, { name: 'name2',reset_interval: 20 }], options: {})
+      # 5. init([{ name: 'name1',reset_interval: 20 }, { name: 'name2',reset_interval: 20 }]) { |res| ... }
+      def init(params, options: {})
+        exist_scope!
+        params = [params] unless params.is_a?(Array)
+        res = send_request('init', @scope, params, options)
+
+        # if `async` is false or missing, block at this method and return a Future::Result object.
+        if block_given?
+          Thread.start do
+            yield res.get
+          end
+        else
+          res
+        end
+      end
+
+      def delete(*params, options: {})
+        exist_scope!
+        res = send_request('delete', @scope, params, options)
+
+        if block_given?
+          Thread.start do
+            yield res.get
+          end
+        else
+          res
+        end
+      end
+
+      # === Example
+      # `inc` receives various arguments.
+      #
+      # 1. inc(name: 'name')
+      # 2. inc({ name: 'name',value: 20 }, options: {})
+      # 3. inc([{ name: 'name1',value: 20 }, { name: 'name2',value: 20 }])
+      # 4. inc([{ name: 'name1',value: 20 }, { name: 'name2',value: 20 }], options: {})
+      def inc(params, options: {})
+        exist_scope!
+        params = [params] unless params.is_a?(Array)
+        res = send_request('inc', @scope, params, options)
+
+        if block_given?
+          Thread.start do
+            yield res.get
+          end
+        else
+          res
+        end
+      end
+
+      def get(*params, options: {})
+        exist_scope!
+        res = send_request('get', @scope, params, options)
+
+        if block_given?
+          Thread.start do
+            yield res.get
+          end
+        else
+          res
+        end
+      end
+
+      def reset(*params, options: {})
+        exist_scope!
+        res = send_request('reset', @scope, params, options)
+
+        if block_given?
+          Thread.start do
+            yield res.get
+          end
+        else
+          res
+        end
+      end
+
+      private
+
+      def exist_scope!
+        raise 'Call `establish` method to get a `scope` before calling this method' unless @scope
+      end
+
+      def on_message(data)
+        if response = @responses.delete(data['id'])
+          response.set(data)
+        else
+          @log.warn("Receiving missing id data: #{data}")
+        end
+      end
+
+      def send_request(method, scope, params, opt = {})
+        id = generate_id
+        res = Future.new(@loop, @loop_mutex)
+        @responses[id] = res # set a response value to this future object at `on_message`
+        request = build_request(method, id, scope, params, opt)
+        @log.debug(request)
+        @conn.send_data request
+        res
+      end
+
+      def build_request(method, id, scope = nil, params = nil, options = nil)
+        r = { id: id, method: method }
+        r[:scope] = scope if scope
+        r[:params] = params if params
+        r[:options] = options if options
+        r
+      end
+
+      def generate_id
+        id = 0
+        @id_mutex.synchronize do
+          id = @id
+          @id += 1
+          @id = 0 if ID_LIMIT_COUNT < @id
+        end
+        id
+      end
+    end
+
+    class Connection < Fluent::Counter::BaseSocket
+      def initialize(io, on_message)
+        super(io)
+        @connection = false
+        @buffer = ''
+        @on_message = on_message
+      end
+
+      def send_data(data)
+        if @connection
+          packed_write data
+        else
+          @buffer += pack(data)
+        end
+      end
+
+      def on_connect
+        @connection = true
+        write @buffer
+        @buffer = ''
+      end
+
+      def on_close
+        @connection = false
+      end
+
+      def on_message(data)
+        @on_message.call(data)
+      end
+    end
+
+    class Future
+      class Result
+        attr_reader :data, :errors
+
+        def initialize(result)
+          @errors = result['errors']
+          @data = result['data']
+        end
+
+        def success?
+          @errors.nil? || @errors.empty?
+        end
+
+        def error?
+          !success?
+        end
+      end
+
+      def initialize(loop, mutex)
+        @set = false
+        @result = nil
+        @mutex = mutex
+        @loop = loop
+      end
+
+      def set(v)
+        @result = Result.new(v)
+        @set = true
+      end
+
+      def errors
+        get.errors
+      end
+
+      def errors?
+        es = errors
+        es && !es.empty?
+      end
+
+      def data
+        get.data
+      end
+
+      def get
+        # Block until `set` method is called and @result is set
+        join if @result.nil?
+        @result
+      end
+
+      def wait
+        res = get
+        if res.error?
+          Fluent::Counter.raise_error(res.errors.first)
+        end
+        res
+      end
+
+      private
+
+      def join
+        until @set
+          @mutex.synchronize do
+            @loop.run_once(0.0001) # return a lock as soon as possible
+          end
+        end
+      end
+    end
+  end
+end

data/lib/fluent/counter/error.rb

@@ -0,0 +1,86 @@
+#
+# Fluentd
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+
+module Fluent
+  module Counter
+    class BaseError < StandardError
+      def to_hash
+        { 'code' => code, 'message' => message }
+      end
+
+      def code
+        raise NotImplementedError
+      end
+    end
+
+    class InvalidParams < BaseError
+      def code
+        'invalid_params'
+      end
+    end
+
+    class UnknownKey < BaseError
+      def code
+        'unknown_key'
+      end
+    end
+
+    class ParseError < BaseError
+      def code
+        'parse_error'
+      end
+    end
+
+    class InvalidRequest < BaseError
+      def code
+        'invalid_request'
+      end
+    end
+
+    class MethodNotFound < BaseError
+      def code
+        'method_not_found'
+      end
+    end
+
+    class InternalServerError < BaseError
+      def code
+        'internal_server_error'
+      end
+    end
+
+    def raise_error(response)
+      msg = response['message']
+      case response['code']
+      when 'invalid_params'
+        raise InvalidParams.new(msg)
+      when 'unknown_key'
+        raise UnknownKey.new(msg)
+      when 'parse_error'
+        raise ParseError.new(msg)
+      when 'invalid_request'
+        raise InvalidRequest.new(msg)
+      when 'method_not_found'
+        raise MethodNotFound.new(msg)
+      when 'internal_server_error'
+        raise InternalServerError.new(msg)
+      else
+        raise "Unknown code: #{response['code']}"
+      end
+    end
+    module_function :raise_error
+  end
+end