fluentd-hubspot 0.14.14.1

data/lib/fluent/config/v1_parser.rb

@@ -0,0 +1,190 @@
+#
+# Fluentd
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+
+require 'strscan'
+require 'uri'
+
+require 'fluent/config/error'
+require 'fluent/config/basic_parser'
+require 'fluent/config/literal_parser'
+require 'fluent/config/element'
+
+module Fluent
+  module Config
+    class V1Parser < LiteralParser
+      ELEMENT_NAME = /[a-zA-Z0-9_]+/
+
+      def self.parse(data, fname, basepath = Dir.pwd, eval_context = nil)
+        ss = StringScanner.new(data)
+        ps = V1Parser.new(ss, basepath, fname, eval_context)
+        ps.parse!
+      end
+
+      def initialize(strscan, include_basepath, fname, eval_context)
+        super(strscan, eval_context)
+        @include_basepath = include_basepath
+        @fname = fname
+      end
+
+      def parse!
+        attrs, elems = parse_element(true, nil)
+        root = Element.new('ROOT', '', attrs, elems)
+        root.v1_config = true
+
+        spacing
+        unless eof?
+          parse_error! "expected EOF"
+        end
+
+        return root
+      end
+
+      ELEM_SYMBOLS = ['match', 'source', 'filter', 'system']
+      RESERVED_PARAMS = %W(@type @id @label @log_level)
+
+      def parse_element(root_element, elem_name, attrs = {}, elems = [])
+        while true
+          spacing
+          if eof?
+            if root_element
+              break
+            end
+            parse_error! "expected end tag '</#{elem_name}>' but got end of file"
+          end
+
+          if skip(/\<\//)
+            e_name = scan(ELEMENT_NAME)
+            spacing
+            unless skip(/\>/)
+              parse_error! "expected character in tag name"
+            end
+            unless line_end
+              parse_error! "expected end of line after end tag"
+            end
+            if e_name != elem_name
+              parse_error! "unmatched end tag"
+            end
+            break
+
+          elsif skip(/\</)
+            e_name = scan(ELEMENT_NAME)
+            spacing
+            e_arg = scan_nonquoted_string(/(?:#{ZERO_OR_MORE_SPACING}\>)/)
+            spacing
+            unless skip(/\>/)
+              parse_error! "expected '>'"
+            end
+            unless line_end
+              parse_error! "expected end of line after tag"
+            end
+            e_arg ||= ''
+            # call parse_element recursively
+            e_attrs, e_elems = parse_element(false, e_name)
+            new_e = Element.new(e_name, e_arg, e_attrs, e_elems)
+            new_e.v1_config = true
+            elems << new_e
+
+          elsif root_element && skip(/(\@include|include)#{SPACING}/)
+            if !prev_match.start_with?('@')
+              $log.warn "'include' is deprecated. Use '@include' instead"
+            end
+            parse_include(attrs, elems)
+
+          else
+            k = scan_string(SPACING)
+            spacing_without_comment
+            if prev_match.include?("\n") || eof? # support 'tag_mapped' like "without value" configuration
+              attrs[k] = ""
+            else
+              if k == '@include'
+                parse_include(attrs, elems)
+              elsif RESERVED_PARAMS.include?(k)
+                v = parse_literal
+                unless line_end
+                  parse_error! "expected end of line"
+                end
+                attrs[k] = v
+              else
+                if k.start_with?('@')
+                  if root_element || ELEM_SYMBOLS.include?(elem_name)
+                    parse_error! "'@' is the system reserved prefix. Don't use '@' prefix parameter in the configuration: #{k}"
+                  else
+                    # TODO: This is for backward compatibility. It will throw an error in the future.
+                    $log.warn "'@' is the system reserved prefix. It works in the nested configuration for now but it will be rejected: #{k}"
+                  end
+                end
+
+                v = parse_literal
+                unless line_end
+                  parse_error! "expected end of line"
+                end
+                attrs[k] = v
+              end
+            end
+          end
+        end
+
+        return attrs, elems
+      end
+
+      def parse_include(attrs, elems)
+        uri = scan_string(LINE_END)
+        eval_include(attrs, elems, uri)
+        line_end
+      end
+
+      def eval_include(attrs, elems, uri)
+        u = URI.parse(uri)
+        if u.scheme == 'file' || (!u.scheme.nil? && u.scheme.length == 1) || u.path == uri # file path
+          # When the Windows absolute path then u.scheme.length == 1
+          # e.g. C:
+          path = u.path
+          if path[0] != ?/
+            pattern = File.expand_path("#{@include_basepath}/#{path}")
+          else
+            pattern = path
+          end
+          Dir.glob(pattern).sort.each { |entry|
+            basepath = File.dirname(entry)
+            fname = File.basename(entry)
+            data = File.read(entry)
+            data.force_encoding('UTF-8')
+            ss = StringScanner.new(data)
+            V1Parser.new(ss, basepath, fname, @eval_context).parse_element(true, nil, attrs, elems)
+          }
+        else
+          require 'open-uri'
+          basepath = '/'
+          fname = path
+          data = open(uri) { |f| f.read }
+          data.force_encoding('UTF-8')
+          ss = StringScanner.new(data)
+          V1Parser.new(ss, basepath, fname, @eval_context).parse_element(true, nil, attrs, elems)
+        end
+
+      rescue SystemCallError => e
+        cpe = ConfigParseError.new("include error #{uri}")
+        cpe.set_backtrace(e.backtrace)
+        raise cpe
+      end
+
+      # override
+      def error_sample
+        "#{@fname} #{super}"
+      end
+    end
+  end
+end

data/lib/fluent/configurable.rb

@@ -0,0 +1,200 @@
+#
+# Fluentd
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+
+require 'fluent/config/configure_proxy'
+require 'fluent/config/section'
+require 'fluent/config/error'
+require 'fluent/registry'
+require 'fluent/plugin'
+require 'fluent/config/types'
+
+module Fluent
+  module Configurable
+    def self.included(mod)
+      mod.extend(ClassMethods)
+    end
+
+    def initialize
+      super
+      # to simulate implicit 'attr_accessor' by config_param / config_section and its value by config_set_default
+      proxy = self.class.merged_configure_proxy
+      proxy.params.keys.each do |name|
+        next if name.to_s.start_with?('@')
+        if proxy.defaults.has_key?(name)
+          instance_variable_set("@#{name}".to_sym, proxy.defaults[name])
+        end
+      end
+      proxy.sections.keys.each do |name|
+        next if name.to_s.start_with?('@')
+        subproxy = proxy.sections[name]
+        if subproxy.multi?
+          instance_variable_set("@#{subproxy.variable_name}".to_sym, [])
+        else
+          instance_variable_set("@#{subproxy.variable_name}".to_sym, nil)
+        end
+      end
+    end
+
+    def configure_proxy_generate
+      proxy = self.class.merged_configure_proxy
+
+      if self.respond_to?(:owner) && self.owner
+        owner_proxy = owner.class.merged_configure_proxy
+        if proxy.configured_in_section
+          owner_proxy = owner_proxy.sections[proxy.configured_in_section]
+        end
+        proxy.overwrite_defaults(owner_proxy) if owner_proxy
+      end
+
+      proxy
+    end
+
+    def configured_section_create(name, conf = nil)
+      conf ||= Fluent::Config::Element.new(name.to_s, '', {}, [])
+      root_proxy = configure_proxy_generate
+      proxy = if name.nil? # root
+                root_proxy
+              else
+                root_proxy.sections[name]
+              end
+      # take care to raise Fluent::ConfigError if conf mismatched to proxy
+      Fluent::Config::SectionGenerator.generate(proxy, conf, nil, nil)
+    end
+
+    def configure(conf)
+      @config = conf
+
+      logger = if self.respond_to?(:log)
+                 self.log
+               elsif self.respond_to?(:owner) && self.owner.respond_to?(:log)
+                 self.owner.log
+               elsif defined?($log)
+                 $log
+               else
+                 nil
+               end
+      proxy = configure_proxy_generate
+      conf.corresponding_proxies << proxy
+
+      # In the nested section, can't get plugin class through proxies so get plugin class here
+      plugin_class = Fluent::Plugin.lookup_type_from_class(proxy.name.to_s)
+      root = Fluent::Config::SectionGenerator.generate(proxy, conf, logger, plugin_class)
+      @config_root_section = root
+
+      root.instance_eval{ @params.keys }.each do |param_name|
+        next if param_name.to_s.start_with?('@')
+        varname = "@#{param_name}".to_sym
+        instance_variable_set(varname, root[param_name])
+      end
+
+      self
+    end
+
+    def config
+      @masked_config ||= @config.to_masked_element
+    end
+
+    CONFIG_TYPE_REGISTRY = Registry.new(:config_type, 'fluent/plugin/type_')
+
+    def self.register_type(type, callable = nil, &block)
+      callable ||= block
+      CONFIG_TYPE_REGISTRY.register(type, callable)
+    end
+
+    def self.lookup_type(type)
+      CONFIG_TYPE_REGISTRY.lookup(type)
+    end
+
+    {
+      string: Config::STRING_TYPE,
+      enum: Config::ENUM_TYPE,
+      integer: Config::INTEGER_TYPE,
+      float: Config::FLOAT_TYPE,
+      size: Config::SIZE_TYPE,
+      bool: Config::BOOL_TYPE,
+      time: Config::TIME_TYPE,
+      hash: Config::HASH_TYPE,
+      array: Config::ARRAY_TYPE,
+    }.each do |name, type|
+      register_type(name, type)
+    end
+
+    module ClassMethods
+      def configure_proxy_map
+        map = {}
+        self.define_singleton_method(:configure_proxy_map){ map }
+        map
+      end
+
+      def configure_proxy(mod_name)
+        map = configure_proxy_map
+        unless map[mod_name]
+          type_lookup = ->(type) { Fluent::Configurable.lookup_type(type) }
+          proxy = Fluent::Config::ConfigureProxy.new(mod_name, root: true, required: true, multi: false, type_lookup: type_lookup)
+          map[mod_name] = proxy
+        end
+        map[mod_name]
+      end
+
+      def configured_in(section_name)
+        configure_proxy(self.name).configured_in(section_name)
+      end
+
+      def config_param(name, type = nil, **kwargs, &block)
+        configure_proxy(self.name).config_param(name, type, **kwargs, &block)
+        # reserved names '@foo' are invalid as attr_accessor name
+        attr_accessor(name) unless kwargs[:skip_accessor] || Fluent::Config::Element::RESERVED_PARAMETERS.include?(name.to_s)
+      end
+
+      def config_set_default(name, defval)
+        configure_proxy(self.name).config_set_default(name, defval)
+      end
+
+      def config_set_desc(name, desc)
+        configure_proxy(self.name).config_set_desc(name, desc)
+      end
+
+      def config_section(name, **kwargs, &block)
+        section_already_exists = !!merged_configure_proxy.sections[name]
+        configure_proxy(self.name).config_section(name, **kwargs, &block)
+        variable_name = configure_proxy(self.name).sections[name].variable_name
+        if !section_already_exists && !self.respond_to?(variable_name)
+          attr_accessor variable_name
+        end
+      end
+
+      def desc(description)
+        configure_proxy(self.name).desc(description)
+      end
+
+      def merged_configure_proxy
+        configurables = ancestors.reverse.select{ |a| a.respond_to?(:configure_proxy) }
+
+        # 'a.object_id.to_s' is to support anonymous class
+        #   which created in tests to overwrite original behavior temporally
+        #
+        # p Module.new.name   #=> nil
+        # p Class.new.name    #=> nil
+        # p AnyGreatClass.dup.name #=> nil
+        configurables.map{ |a| a.configure_proxy(a.name || a.object_id.to_s) }.reduce(:merge)
+      end
+
+      def dump_config_definition
+        configure_proxy_map[self.to_s].dump_config_definition
+      end
+    end
+  end
+end

data/lib/fluent/daemon.rb

@@ -0,0 +1,15 @@
+#!/usr/bin/env ruby
+# -*- coding: utf-8 -*-
+
+here = File.dirname(__FILE__)
+$LOAD_PATH << File.expand_path(File.join(here, '..'))
+
+require 'serverengine'
+require 'fluent/supervisor'
+
+server_module = Fluent.const_get(ARGV[0])
+worker_module = Fluent.const_get(ARGV[1])
+# it doesn't call ARGV in block because when reloading config, params will be initialized and then it can't use previous config.
+config_path = ARGV[2]
+params = JSON.parse(ARGV[3])
+ServerEngine::Daemon.run_server(server_module, worker_module) { Fluent::Supervisor.load_config(config_path, params) }

data/lib/fluent/engine.rb

@@ -0,0 +1,266 @@
+#
+# Fluentd
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+
+require 'fluent/config'
+require 'fluent/event'
+require 'fluent/event_router'
+require 'fluent/msgpack_factory'
+require 'fluent/root_agent'
+require 'fluent/time'
+require 'fluent/system_config'
+require 'fluent/plugin'
+
+module Fluent
+  class EngineClass
+    include Fluent::MessagePackFactory::Mixin
+
+    def initialize
+      @root_agent = nil
+      @default_loop = nil
+      @engine_stopped = false
+
+      @log_event_router = nil
+      @log_emit_thread = nil
+      @log_event_loop_stop = false
+      @log_event_loop_graceful_stop = false
+      @log_event_queue = []
+      @log_event_verbose = false
+
+      @suppress_config_dump = false
+
+      @system_config = SystemConfig.new
+    end
+
+    MAINLOOP_SLEEP_INTERVAL = 0.3
+
+    MATCH_CACHE_SIZE = 1024
+    LOG_EMIT_INTERVAL = 0.1
+
+    attr_reader :root_agent
+    attr_reader :matches, :sources
+    attr_reader :system_config
+
+    def init(system_config)
+      @system_config = system_config
+
+      suppress_interval(system_config.emit_error_log_interval) unless system_config.emit_error_log_interval.nil?
+      @suppress_config_dump = system_config.suppress_config_dump unless system_config.suppress_config_dump.nil?
+      @without_source = system_config.without_source unless system_config.without_source.nil?
+
+      @log_event_verbose = system_config.log_event_verbose unless system_config.log_event_verbose.nil?
+
+      @root_agent = RootAgent.new(log: log, system_config: @system_config)
+
+      MessagePackFactory.init
+
+      self
+    end
+
+    def log
+      $log
+    end
+
+    def suppress_interval(interval_time)
+      @suppress_emit_error_log_interval = interval_time
+      @next_emit_error_log_time = Time.now.to_i
+    end
+
+    def parse_config(io, fname, basepath = Dir.pwd, v1_config = false)
+      if fname =~ /\.rb$/
+        require 'fluent/config/dsl'
+        Config::DSL::Parser.parse(io, File.join(basepath, fname))
+      else
+        Config.parse(io, fname, basepath, v1_config)
+      end
+    end
+
+    def run_configure(conf)
+      configure(conf)
+      conf.check_not_fetched { |key, e|
+        parent_name, plugin_name = e.unused_in
+        if parent_name
+          message = if plugin_name
+                      "section <#{e.name}> is not used in <#{parent_name}> of #{plugin_name} plugin"
+                    else
+                      "section <#{e.name}> is not used in <#{parent_name}>"
+                    end
+          $log.warn :worker0, message
+          next
+        end
+        unless e.name == 'system'
+          unless @without_source && e.name == 'source'
+            $log.warn :worker0, "parameter '#{key}' in #{e.to_s.strip} is not used."
+          end
+        end
+      }
+    end
+
+    def configure(conf)
+      # plugins / configuration dumps
+      Gem::Specification.find_all.select{|x| x.name =~ /^fluent(d|-(plugin|mixin)-.*)$/}.each do |spec|
+        $log.info :worker0, "gem '#{spec.name}' version '#{spec.version}'"
+      end
+
+      @root_agent.configure(conf)
+
+      begin
+        log_event_agent = @root_agent.find_label(Fluent::Log::LOG_EVENT_LABEL)
+        log_event_router = log_event_agent.event_router
+
+        # suppress mismatched tags only for <label @FLUENT_LOG> label.
+        # it's not suppressed in default event router for non-log-event events
+        log_event_router.suppress_missing_match!
+
+        @log_event_router = log_event_router
+
+        unmatched_tags = Fluent::Log.event_tags.select{|t| !@log_event_router.match?(t) }
+        unless unmatched_tags.empty?
+          $log.warn :worker0, "match for some tags of log events are not defined (to be ignored)", tags: unmatched_tags
+        end
+      rescue ArgumentError # ArgumentError "#{label_name} label not found"
+        # use default event router if <label @FLUENT_LOG> is missing in configuration
+        log_event_router = @root_agent.event_router
+
+        if Fluent::Log.event_tags.any?{|t| log_event_router.match?(t) }
+          @log_event_router = log_event_router
+
+          unmatched_tags = Fluent::Log.event_tags.select{|t| !@log_event_router.match?(t) }
+          unless unmatched_tags.empty?
+            $log.warn :worker0, "match for some tags of log events are not defined (to be ignored)", tags: unmatched_tags
+          end
+        end
+      end
+
+      $log.enable_event(true) if @log_event_router
+
+      unless @suppress_config_dump
+        $log.info :worker0, "using configuration file: #{conf.to_s.rstrip}"
+      end
+    end
+
+    def add_plugin_dir(dir)
+      Plugin.add_plugin_dir(dir)
+    end
+
+    def emit(tag, time, record)
+      raise "BUG: use router.emit instead of Engine.emit"
+    end
+
+    def emit_array(tag, array)
+      raise "BUG: use router.emit_array instead of Engine.emit_array"
+    end
+
+    def emit_stream(tag, es)
+      raise "BUG: use router.emit_stream instead of Engine.emit_stream"
+    end
+
+    def flush!
+      @root_agent.flush!
+    end
+
+    def now
+      # TODO thread update
+      Fluent::EventTime.now
+    end
+
+    def log_event_loop
+      $log.disable_events(Thread.current)
+
+      while sleep(LOG_EMIT_INTERVAL)
+        break if @log_event_loop_stop
+        break if @log_event_loop_graceful_stop && @log_event_queue.empty?
+        next if @log_event_queue.empty?
+
+        # NOTE: thead-safe of slice! depends on GVL
+        events = @log_event_queue.slice!(0..-1)
+        next if events.empty?
+
+        events.each {|tag,time,record|
+          begin
+            @log_event_router.emit(tag, time, record)
+          rescue => e
+            # This $log.error doesn't emit log events, because of `$log.disable_events(Thread.current)` above
+            $log.error "failed to emit fluentd's log event", tag: tag, event: record, error: e
+          end
+        }
+      end
+    end
+
+    def run
+      # if ENV doesn't have SERVERENGINE_WORKER_ID, it is a worker under --no-supervisor or in tests
+      # so it's (almost) a single worker, worker_id=0
+      worker_id = (ENV['SERVERENGINE_WORKER_ID'] || 0).to_i
+
+      begin
+        $log.info "starting fluentd worker", pid: Process.pid, ppid: Process.ppid, worker: worker_id
+        start
+
+        if @log_event_router
+          $log.enable_event(true)
+          @log_emit_thread = Thread.new(&method(:log_event_loop))
+          @log_emit_thread.abort_on_exception = true
+        end
+
+        $log.info "fluentd worker is now running", worker: worker_id
+        sleep MAINLOOP_SLEEP_INTERVAL until @engine_stopped
+        $log.info "fluentd worker is now stopping", worker: worker_id
+
+      rescue Exception => e
+        $log.error "unexpected error", error: e
+        $log.error_backtrace
+        raise
+      end
+
+      unless @log_event_verbose
+        $log.enable_event(false)
+        if @log_emit_thread
+          # to make sure to emit all log events into router, before shutting down
+          @log_event_loop_graceful_stop = true
+          @log_emit_thread.join
+          @log_emit_thread = nil
+        end
+      end
+      $log.info "shutting down fluentd worker", worker: worker_id
+      shutdown
+      if @log_emit_thread
+        @log_event_loop_stop = true
+        @log_emit_thread.join
+      end
+    end
+
+    def stop
+      @engine_stopped = true
+      nil
+    end
+
+    def push_log_event(tag, time, record)
+      return if @log_emit_thread.nil?
+      @log_event_queue.push([tag, time, record])
+    end
+
+    private
+
+    def start
+      @root_agent.start
+    end
+
+    def shutdown
+      @root_agent.shutdown
+    end
+  end
+
+  Engine = EngineClass.new
+end