fluentd 1.20.0-aarch64-mingw-ucrt

data/lib/fluent/rpc.rb

@@ -0,0 +1,95 @@
+#
+# Fluentd
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+
+require 'webrick'
+
+module Fluent
+  module RPC
+    class Server
+      def initialize(endpoint, log)
+        m = endpoint.match(/^\[?(?<host>[0-9a-zA-Z:\-\.]+)\]?:(?<port>[0-9]+)$/)
+        raise Fluent::ConfigError, "Invalid rpc_endpoint: #{endpoint}" unless m
+        @bind = m[:host]
+        @port = m[:port]
+        @log = log
+
+        @server = WEBrick::HTTPServer.new(
+          BindAddress: @bind,
+          Port: @port,
+          Logger: WEBrick::Log.new(STDERR, WEBrick::Log::FATAL),
+          AccessLog: [],
+        )
+      end
+
+      def mount(path, servlet, *args)
+        @server.mount(path, servlet, *args)
+        @log.debug "register #{path} RPC servlet"
+      end
+
+      def mount_proc(path, &block)
+        @server.mount_proc(path) { |req, res|
+          begin
+            code, header, response = block.call(req, res)
+          rescue => e
+            @log.warn "failed to handle RPC request", path: path, error: e.to_s
+            @log.warn_backtrace e.backtrace
+
+            code = 500
+            body = {
+              'message '=> 'Internal Server Error',
+              'error' => "#{e}",
+              'backtrace'=> e.backtrace,
+            }
+          end
+
+          code = 200 if code.nil?
+          header = {'Content-Type' => 'application/json'} if header.nil?
+          body = if response.nil?
+                   '{"ok":true}'
+                 else
+                   response.body['ok'] = code == 200
+                   response.body.to_json
+                 end
+
+          res.status = code
+          header.each_pair { |k, v|
+            res[k] = v
+          }
+          res.body = body
+        }
+        @log.debug "register #{path} RPC handler"
+      end
+
+      def start
+        @log.debug "listening RPC http server on http://#{@bind}:#{@port}/"
+        @thread = Thread.new {
+          @server.start
+        }
+      end
+
+      def shutdown
+        if @server
+          @server.shutdown
+          @server = nil
+        end
+        if @thread
+          @thread.join
+          @thread = nil
+        end
+      end
+    end
+  end
+end

data/lib/fluent/source_only_buffer_agent.rb

@@ -0,0 +1,102 @@
+#
+# Fluentd
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+
+require 'fluent/agent'
+require 'fluent/system_config'
+
+module Fluent
+  class SourceOnlyBufferAgent < Agent
+    # Use INSTANCE_ID to use the same base dir as the other workers.
+    # This will make recovery easier.
+    BUFFER_DIR_NAME = Fluent::INSTANCE_ID
+
+    def initialize(log:, system_config:)
+      super(log: log)
+
+      @default_buffer_path = File.join(system_config.root_dir || DEFAULT_BACKUP_DIR, 'source-only-buffer', BUFFER_DIR_NAME)
+      @optional_buffer_config = system_config.source_only_buffer.to_h.transform_keys(&:to_s)
+      @base_buffer_dir = nil
+      @actual_buffer_dir = nil
+    end
+
+    def configure(flush: false)
+      buffer_config = @optional_buffer_config.compact
+      buffer_config['flush_at_shutdown'] = flush ? 'true' : 'false'
+      buffer_config['flush_thread_count'] = 0 unless flush
+      buffer_config['path'] ||= @default_buffer_path
+
+      super(
+        Config::Element.new('SOURCE_ONLY_BUFFER', '', {}, [
+          Config::Element.new('match', '**', {'@type' => 'buffer', '@label' => '@ROOT'}, [
+            Config::Element.new('buffer', '', buffer_config, [])
+          ])
+        ])
+      )
+
+      @base_buffer_dir = buffer_config['path']
+      # It can be "#{@base_buffer_dir}/worker#{fluentd_worker_id}/" when using multiple workers
+      @actual_buffer_dir = File.dirname(outputs[0].buffer.path)
+
+      unless flush
+        log.info "with-source-only: the emitted data will be stored in the buffer files under" +
+                 " #{@base_buffer_dir}. You can send SIGWINCH to the supervisor process to cancel" +
+                 " with-source-only mode and process data."
+      end
+    end
+
+    def cleanup
+      unless (Dir.empty?(@actual_buffer_dir) rescue true)
+        log.warn "some buffer files remain in #{@base_buffer_dir}." +
+                 " Please consider recovering or saving the buffer files in the directory." +
+                 " To recover them, you can set the buffer path manually to system config and" +
+                 " retry, i.e., restart Fluentd with with-source-only mode and send SIGWINCH again." +
+                 " Config Example:\n#{config_example_to_recover(@base_buffer_dir)}"
+        return
+      end
+
+      begin
+        FileUtils.remove_dir(@base_buffer_dir)
+      rescue Errno::ENOENT
+        # This worker doesn't need to do anything. Another worker may remove the dir first.
+      rescue => e
+        log.warn "failed to remove the buffer directory: #{@base_buffer_dir}", error: e
+      end
+    end
+
+    def emit_error_event(tag, time, record, error)
+      error_info = {error: error, location: (error.backtrace ? error.backtrace.first : nil), tag: tag, time: time, record: record}
+      log.warn "SourceOnlyBufferAgent: dump an error event:", error_info
+    end
+
+    def handle_emits_error(tag, es, error)
+      error_info = {error: error, location: (error.backtrace ? error.backtrace.first : nil), tag: tag}
+      log.warn "SourceOnlyBufferAgent: emit transaction failed:", error_info
+      log.warn_backtrace
+    end
+
+    private
+
+    def config_example_to_recover(path)
+      <<~EOC
+        <system>
+          <source_only_buffer>
+            path #{path}
+          </source_only_buffer>
+        </system>
+      EOC
+    end
+  end
+end

data/lib/fluent/static_config_analysis.rb

@@ -0,0 +1,195 @@
+#
+# Fluentd
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+
+require 'fluent/config'
+require 'fluent/plugin'
+
+module Fluent
+  # Static Analysis means analysing all plugins and Fluent::Element without invoking Plugin#configure
+  class StaticConfigAnalysis
+    module Elem
+      Input = Struct.new(:plugin, :config)
+      Output = Struct.new(:plugin, :config)
+      Filter = Struct.new(:plugin, :config)
+      Label = Struct.new(:name, :config, :nodes)
+      Worker = Struct.new(:ids, :config, :nodes)
+    end
+
+    Result = Struct.new(:tree, :outputs, :inputs, :filters, :labels) do
+      def all_plugins
+        (outputs + inputs + filters).map(&:plugin)
+      end
+    end
+
+    # @param workers [Integer] Number of workers
+    # @return [Fluent::StaticConfigAnalysis::Result]
+    def self.call(conf, workers: 1)
+      new(workers).call(conf)
+    end
+
+    def initialize(workers)
+      @workers = workers
+
+      reset
+    end
+
+    def call(config)
+      reset
+
+      tree = [
+        static_worker_analyse(config),
+        static_label_analyse(config),
+        static_filter_and_output_analyse(config),
+        static_input_analyse(config),
+      ].flatten
+
+      Result.new(tree, @outputs, @inputs, @filters, @labels.values)
+    end
+
+    private
+
+    def reset
+      @outputs = []
+      @inputs = []
+      @filters = []
+      @labels = {}
+    end
+
+    def static_worker_analyse(conf)
+      available_worker_ids = [*0...@workers]
+
+      ret = []
+      supported_directives = %w[source match filter label]
+      conf.elements(name: 'worker').each do |config|
+        ids = parse_worker_id(config)
+        ids.each do |id|
+          if available_worker_ids.include?(id)
+            available_worker_ids.delete(id)
+          else
+            raise Fluent::ConfigError, "specified worker_id<#{id}> collisions is detected on <worker> directive. Available worker id(s): #{available_worker_ids}"
+          end
+        end
+
+        config.elements.each do |elem|
+          unless supported_directives.include?(elem.name)
+            raise Fluent::ConfigError, "<worker> section cannot have <#{elem.name}> directive"
+          end
+        end
+
+        nodes = [
+          static_label_analyse(config),
+          static_filter_and_output_analyse(config),
+          static_input_analyse(config),
+        ].flatten
+        ret << Elem::Worker.new(ids, config, nodes)
+      end
+
+      ret
+    end
+
+    def parse_worker_id(conf)
+      worker_id_str = conf.arg
+
+      if worker_id_str.empty?
+        raise Fluent::ConfigError, 'Missing worker id on <worker> directive'
+      end
+
+      l, r =
+         begin
+           worker_id_str.split('-', 2).map { |v| Integer(v) }
+         rescue TypeError, ArgumentError
+           raise Fluent::ConfigError, "worker id should be integer: #{worker_id_str}"
+         end
+
+      if l < 0 || l >= @workers
+        raise Fluent::ConfigError, "worker id #{l} specified by <worker> directive is not allowed. Available worker id is between 0 and #{@workers-1}"
+      end
+
+      # e.g. specified one worker id like `<worker 0>`
+      if r.nil?
+        return [l]
+      end
+
+      if r < 0 || r >= @workers
+        raise Fluent::ConfigError, "worker id #{r} specified by <worker> directive is not allowed. Available worker id is between 0 and #{@workers-1}"
+      end
+
+      if l > r
+        raise Fluent::ConfigError, "greater first_worker_id<#{l}> than last_worker_id<#{r}> specified by <worker> directive is not allowed. Available multi worker assign syntax is <smaller_worker_id>-<greater_worker_id>"
+      end
+
+      [l, r]
+    end
+
+    def static_label_analyse(conf)
+      ret = []
+      conf.elements(name: 'label').each do |e|
+        name = e.arg
+        if name.empty?
+          raise ConfigError, 'Missing symbol argument on <label> directive'
+        end
+
+        if @labels[name]
+          raise ConfigError, "Section <label #{name}> appears twice"
+        end
+
+        l = Elem::Label.new(name, e, static_filter_and_output_analyse(e))
+        ret << l
+        @labels[name] = l
+      end
+
+      ret
+    end
+
+    def static_filter_and_output_analyse(conf)
+      ret = []
+      conf.elements('filter', 'match').each do |e|
+        type = e['@type']
+        if type.nil? || type.empty?
+          raise Fluent::ConfigError, "Missing '@type' parameter on <#{e.name}> directive"
+        end
+
+        if e.name == 'filter'
+          f = Elem::Filter.new(Fluent::Plugin.new_filter(type), e)
+          ret << f
+          @filters << f
+        else
+          o = Elem::Output.new(Fluent::Plugin.new_output(type), e)
+          ret << o
+          @outputs << o
+        end
+      end
+
+      ret
+    end
+
+    def static_input_analyse(conf)
+      ret = []
+      conf.elements(name: 'source').each do |e|
+        type = e['@type']
+        if type.nil? || type.empty?
+          raise Fluent::ConfigError, "Missing '@type' parameter on <#{e.name}> directive"
+        end
+
+        i = Elem::Input.new(Fluent::Plugin.new_input(type), e)
+        @inputs << i
+        ret << i
+      end
+
+      ret
+    end
+  end
+end