fluentd222 1.16.2-x86_64-linux

data/lib/fluent/plugin/in_tcp.rb

@@ -0,0 +1,226 @@
+#
+# Fluentd
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+
+require 'fluent/plugin/input'
+
+module Fluent::Plugin
+  class TcpInput < Input
+    Fluent::Plugin.register_input('tcp', self)
+
+    helpers :server, :parser, :extract, :compat_parameters
+
+    desc 'Tag of output events.'
+    config_param :tag, :string
+    desc 'The port to listen to.'
+    config_param :port, :integer, default: 5170
+    desc 'The bind address to listen to.'
+    config_param :bind, :string, default: '0.0.0.0'
+
+    desc "The field name of the client's hostname."
+    config_param :source_host_key, :string, default: nil, deprecated: "use source_hostname_key instead."
+    desc "The field name of the client's hostname."
+    config_param :source_hostname_key, :string, default: nil
+    desc "The field name of the client's address."
+    config_param :source_address_key, :string, default: nil
+
+    # Setting default to nil for backward compatibility
+    desc "The max bytes of message."
+    config_param :message_length_limit, :size, default: nil
+
+    config_param :blocking_timeout, :time, default: 0.5
+
+    desc 'The payload is read up to this character.'
+    config_param :delimiter, :string, default: "\n" # syslog family add "\n" to each message and this seems only way to split messages in tcp stream
+    desc 'Check the remote connection is still available by sending a keepalive packet if this value is true.'
+    config_param :send_keepalive_packet, :bool, default: false
+
+    # in_forward like host/network restriction
+    config_section :security, required: false, multi: false do
+      config_section :client, param_name: :clients, required: true, multi: true do
+        desc 'The IP address or host name of the client'
+        config_param :host, :string, default: nil
+        desc 'Network address specification'
+        config_param :network, :string, default: nil
+      end
+    end
+
+    def configure(conf)
+      compat_parameters_convert(conf, :parser)
+      parser_config = conf.elements('parse').first
+      unless parser_config
+        raise Fluent::ConfigError, "<parse> section is required."
+      end
+      super
+      @_event_loop_blocking_timeout = @blocking_timeout
+      @source_hostname_key ||= @source_host_key if @source_host_key
+
+      @nodes = nil
+      if @security
+        @nodes = []
+        @security.clients.each do |client|
+          if client.host && client.network
+            raise Fluent::ConfigError, "both of 'host' and 'network' are specified for client"
+          end
+          if !client.host && !client.network
+            raise Fluent::ConfigError, "Either of 'host' and 'network' must be specified for client"
+          end
+          source = nil
+          if client.host
+            begin
+              source = IPSocket.getaddress(client.host)
+            rescue SocketError
+              raise Fluent::ConfigError, "host '#{client.host}' cannot be resolved"
+            end
+          end
+          source_addr = begin
+                          IPAddr.new(source || client.network)
+                        rescue ArgumentError
+                          raise Fluent::ConfigError, "network '#{client.network}' address format is invalid"
+                        end
+          @nodes.push(source_addr)
+        end
+      end
+
+      @parser = parser_create(conf: parser_config)
+    end
+
+    def multi_workers_ready?
+      true
+    end
+
+    def start
+      super
+
+      log.info "listening tcp socket", bind: @bind, port: @port
+      del_size = @delimiter.length
+      discard_till_next_delimiter = false
+      if @_extract_enabled && @_extract_tag_key
+        server_create(:in_tcp_server_single_emit, @port, bind: @bind, resolve_name: !!@source_hostname_key, send_keepalive_packet: @send_keepalive_packet) do |data, conn|
+          unless check_client(conn)
+            conn.close
+            next
+          end
+
+          conn.buffer << data
+          buf = conn.buffer
+          pos = 0
+          while i = buf.index(@delimiter, pos)
+            msg = buf[pos...i]
+            pos = i + del_size
+
+            if discard_till_next_delimiter
+              discard_till_next_delimiter = false
+              next
+            end
+
+            if !@message_length_limit.nil? && @message_length_limit < msg.bytesize
+              log.info "The received data is larger than 'message_length_limit', dropped:", limit: @message_length_limit, size: msg.bytesize, head: msg[...32]
+              next
+            end
+
+            @parser.parse(msg) do |time, record|
+              unless time && record
+                log.warn "pattern not matched", message: msg
+                next
+              end
+
+              tag = extract_tag_from_record(record)
+              tag ||= @tag
+              time ||= extract_time_from_record(record) || Fluent::EventTime.now
+              record[@source_address_key] = conn.remote_addr if @source_address_key
+              record[@source_hostname_key] = conn.remote_host if @source_hostname_key
+              router.emit(tag, time, record)
+            end
+          end
+          buf.slice!(0, pos) if pos > 0
+          # If the buffer size exceeds the limit here, it means that the next message will definitely exceed the limit.
+          # So we should clear the buffer here. Otherwise, it will keep storing useless data until the next delimiter comes.
+          if !@message_length_limit.nil? && @message_length_limit < buf.bytesize
+            log.info "The buffer size exceeds 'message_length_limit', cleared:", limit: @message_length_limit, size: buf.bytesize, head: buf[...32]
+            buf.clear
+            # We should discard the subsequent data until the next delimiter comes.
+            discard_till_next_delimiter = true
+            next
+          end
+        end
+      else
+        server_create(:in_tcp_server_batch_emit, @port, bind: @bind, resolve_name: !!@source_hostname_key, send_keepalive_packet: @send_keepalive_packet) do |data, conn|
+          unless check_client(conn)
+            conn.close
+            next
+          end
+
+          conn.buffer << data
+          buf = conn.buffer
+          pos = 0
+          es = Fluent::MultiEventStream.new
+          while i = buf.index(@delimiter, pos)
+            msg = buf[pos...i]
+            pos = i + del_size
+
+            if discard_till_next_delimiter
+              discard_till_next_delimiter = false
+              next
+            end
+
+            if !@message_length_limit.nil? && @message_length_limit < msg.bytesize
+              log.info "The received data is larger than 'message_length_limit', dropped:", limit: @message_length_limit, size: msg.bytesize, head: msg[...32]
+              next
+            end
+
+            @parser.parse(msg) do |time, record|
+              unless time && record
+                log.warn "pattern not matched", message: msg
+                next
+              end
+
+              time ||= extract_time_from_record(record) || Fluent::EventTime.now
+              record[@source_address_key] = conn.remote_addr if @source_address_key
+              record[@source_hostname_key] = conn.remote_host if @source_hostname_key
+              es.add(time, record)
+            end
+          end
+          router.emit_stream(@tag, es)
+          buf.slice!(0, pos) if pos > 0
+          # If the buffer size exceeds the limit here, it means that the next message will definitely exceed the limit.
+          # So we should clear the buffer here. Otherwise, it will keep storing useless data until the next delimiter comes.
+          if !@message_length_limit.nil? && @message_length_limit < buf.bytesize
+            log.info "The buffer size exceeds 'message_length_limit', cleared:", limit: @message_length_limit, size: buf.bytesize, head: buf[...32]
+            buf.clear
+            # We should discard the subsequent data until the next delimiter comes.
+            discard_till_next_delimiter = true
+            next
+          end
+        end
+      end
+    end
+
+    private
+
+    def check_client(conn)
+      if @nodes
+        remote_addr = conn.remote_addr
+        node = @nodes.find { |n| n.include?(remote_addr) rescue false }
+        unless node
+          log.warn "anonymous client '#{remote_addr}' denied"
+          return false
+        end
+      end
+
+      true
+    end
+  end
+end

data/lib/fluent/plugin/in_udp.rb

@@ -0,0 +1,92 @@
+#
+# Fluentd
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+
+require 'fluent/plugin/input'
+
+module Fluent::Plugin
+  class UdpInput < Input
+    Fluent::Plugin.register_input('udp', self)
+
+    helpers :server, :parser, :extract, :compat_parameters
+
+    desc 'Tag of output events.'
+    config_param :tag, :string
+    desc 'The port to listen to.'
+    config_param :port, :integer, default: 5160
+    desc 'The bind address to listen to.'
+    config_param :bind, :string, default: '0.0.0.0'
+
+    desc "The field name of the client's hostname."
+    config_param :source_host_key, :string, default: nil, deprecated: "use source_hostname_key instead."
+    desc "The field name of the client's hostname."
+    config_param :source_hostname_key, :string, default: nil
+    desc "The field name of the client's address."
+    config_param :source_address_key, :string, default: nil
+
+    desc "Deprecated parameter. Use message_length_limit instead"
+    config_param :body_size_limit, :size, default: nil, deprecated: "use message_length_limit instead."
+    desc "The max bytes of message"
+    config_param :message_length_limit, :size, default: 4096
+    desc "Remove newline from the end of incoming payload"
+    config_param :remove_newline, :bool, default: true
+    desc "The max size of socket receive buffer. SO_RCVBUF"
+    config_param :receive_buffer_size, :size, default: nil
+
+    config_param :blocking_timeout, :time, default: 0.5
+
+    def configure(conf)
+      compat_parameters_convert(conf, :parser)
+      parser_config = conf.elements('parse').first
+      unless parser_config
+        raise Fluent::ConfigError, "<parse> section is required."
+      end
+      super
+      @_event_loop_blocking_timeout = @blocking_timeout
+      @source_hostname_key ||= @source_host_key if @source_host_key
+      @message_length_limit = @body_size_limit if @body_size_limit
+
+      @parser = parser_create(conf: parser_config)
+    end
+
+    def multi_workers_ready?
+      true
+    end
+
+    def start
+      super
+
+      log.info "listening udp socket", bind: @bind, port: @port
+      server_create(:in_udp_server, @port, proto: :udp, bind: @bind, resolve_name: !!@source_hostname_key, max_bytes: @message_length_limit, receive_buffer_size: @receive_buffer_size) do |data, sock|
+        data.chomp! if @remove_newline
+        begin
+          @parser.parse(data) do |time, record|
+            unless time && record
+              log.warn "pattern not matched", data: data
+              next
+            end
+
+            tag = extract_tag_from_record(record)
+            tag ||= @tag
+            time ||= extract_time_from_record(record) || Fluent::EventTime.now
+            record[@source_address_key] = sock.remote_addr if @source_address_key
+            record[@source_hostname_key] = sock.remote_host if @source_hostname_key
+            router.emit(tag, time, record)
+          end
+        end
+      end
+    end
+  end
+end

data/lib/fluent/plugin/in_unix.rb

@@ -0,0 +1,195 @@
+#
+# Fluentd
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+
+require 'fluent/env'
+require 'fluent/plugin/input'
+require 'fluent/msgpack_factory'
+
+require 'cool.io'
+require 'yajl'
+require 'fileutils'
+require 'socket'
+
+module Fluent::Plugin
+  # TODO: This plugin will be 3rd party plugin
+  class UnixInput < Input
+    Fluent::Plugin.register_input('unix', self)
+
+    helpers :event_loop
+
+    def initialize
+      super
+
+      @lsock = nil
+    end
+
+    desc 'The path to your Unix Domain Socket.'
+    config_param :path, :string, default: Fluent::DEFAULT_SOCKET_PATH
+    desc 'The backlog of Unix Domain Socket.'
+    config_param :backlog, :integer, default: nil
+    desc "New tag instead of incoming tag"
+    config_param :tag, :string, default: nil
+
+    def configure(conf)
+      super
+    end
+
+    def start
+      super
+
+      @lsock = listen
+      event_loop_attach(@lsock)
+    end
+
+    def shutdown
+      if @lsock
+        event_loop_detach(@lsock)
+        @lsock.close
+      end
+
+      super
+    end
+
+    def listen
+      if File.exist?(@path)
+        log.warn "Found existing '#{@path}'. Remove this file for in_unix plugin"
+        File.unlink(@path)
+      end
+      FileUtils.mkdir_p(File.dirname(@path))
+
+      log.info "listening fluent socket on #{@path}"
+      s = Coolio::UNIXServer.new(@path, Handler, log, method(:on_message))
+      s.listen(@backlog) unless @backlog.nil?
+      s
+    end
+
+    # message Entry {
+    #   1: long time
+    #   2: object record
+    # }
+    #
+    # message Forward {
+    #   1: string tag
+    #   2: list<Entry> entries
+    # }
+    #
+    # message PackedForward {
+    #   1: string tag
+    #   2: raw entries  # msgpack stream of Entry
+    # }
+    #
+    # message Message {
+    #   1: string tag
+    #   2: long? time
+    #   3: object record
+    # }
+    def on_message(msg)
+      unless msg.is_a?(Array)
+        log.warn "incoming data is broken:", msg: msg
+        return
+      end
+
+      tag = @tag || (msg[0].to_s)
+      entries = msg[1]
+
+      case entries
+      when String
+        # PackedForward
+        es = Fluent::MessagePackEventStream.new(entries)
+        router.emit_stream(tag, es)
+
+      when Array
+        # Forward
+        es = Fluent::MultiEventStream.new
+        entries.each {|e|
+          record = e[1]
+          next if record.nil?
+          time = convert_time(e[0])
+          es.add(time, record)
+        }
+        router.emit_stream(tag, es)
+
+      else
+        # Message
+        record = msg[2]
+        return if record.nil?
+
+        time = convert_time(msg[1])
+        router.emit(tag, time, record)
+      end
+    end
+
+    def convert_time(time)
+      case time
+      when nil, 0
+        Fluent::EventTime.now
+      when Fluent::EventTime
+        time
+      else
+        Fluent::EventTime.from_time(Time.at(time))
+      end
+    end
+
+    class Handler < Coolio::Socket
+      def initialize(io, log, on_message)
+        super(io)
+
+        @on_message = on_message
+        @log = log
+      end
+
+      def on_connect
+      end
+
+      def on_read(data)
+        first = data[0]
+        if first == '{'.freeze || first == '['.freeze
+          m = method(:on_read_json)
+          @parser = Yajl::Parser.new
+          @parser.on_parse_complete = @on_message
+        else
+          m = method(:on_read_msgpack)
+          @parser = Fluent::MessagePackFactory.msgpack_unpacker
+        end
+
+        singleton_class.module_eval do
+          define_method(:on_read, m)
+        end
+        m.call(data)
+      end
+
+      def on_read_json(data)
+        @parser << data
+      rescue => e
+        @log.error "unexpected error in json payload", error: e.to_s
+        @log.error_backtrace
+        close
+      end
+
+      def on_read_msgpack(data)
+        @parser.feed_each(data, &@on_message)
+      rescue => e
+        @log.error "unexpected error in msgpack payload", error: e.to_s
+        @log.error_backtrace
+        close
+      end
+
+      def on_close
+        @log.trace { "closed fluent socket object_id=#{self.object_id}" }
+      end
+    end
+  end
+end

data/lib/fluent/plugin/input.rb

@@ -0,0 +1,75 @@
+#
+# Fluentd
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+
+require 'fluent/plugin/base'
+
+require 'fluent/log'
+require 'fluent/plugin_id'
+require 'fluent/plugin_helper'
+
+module Fluent
+  module Plugin
+    class Input < Base
+      include PluginId
+      include PluginLoggerMixin
+      include PluginHelper::Mixin
+
+      helpers_internal :event_emitter, :metrics
+
+      def initialize
+        super
+        @emit_records_metrics = nil
+        @emit_size_metrics = nil
+        @counter_mutex = Mutex.new
+        @enable_size_metrics = false
+      end
+
+      def emit_records
+        @emit_records_metrics.get
+      end
+
+      def emit_size
+        @emit_size_metrics.get
+      end
+
+      def configure(conf)
+        super
+
+        @emit_records_metrics = metrics_create(namespace: "fluentd", subsystem: "input", name: "emit_records", help_text: "Number of count emit records")
+        @emit_size_metrics = metrics_create(namespace: "fluentd", subsystem: "input", name: "emit_size", help_text: "Total size of emit events")
+        @enable_size_metrics = !!system_config.enable_size_metrics
+      end
+
+      def statistics
+        stats = {
+          'emit_records' => @emit_records_metrics.get,
+          'emit_size' => @emit_size_metrics.get,
+        }
+
+        { 'input' => stats }
+      end
+
+      def metric_callback(es)
+        @emit_records_metrics.add(es.size)
+        @emit_size_metrics.add(es.to_msgpack_stream.bytesize) if @enable_size_metrics
+      end
+
+      def multi_workers_ready?
+        false
+      end
+    end
+  end
+end

data/lib/fluent/plugin/metrics.rb

@@ -0,0 +1,119 @@
+#
+# Fluentd
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+
+require 'socket'
+
+require 'fluent/plugin/base'
+
+require 'fluent/log'
+require 'fluent/unique_id'
+require 'fluent/plugin_id'
+
+module Fluent
+  module Plugin
+    class Metrics < Base
+      include PluginId
+      include PluginLoggerMixin
+      include UniqueId::Mixin
+
+      DEFAULT_TYPE = 'local'
+
+      configured_in :metrics
+
+      config_param :default_labels, :hash, default: {agent: "Fluentd", hostname: "#{Socket.gethostname}"}
+      config_param :labels, :hash, default: {}
+
+      attr_reader :use_gauge_metric
+      attr_reader :has_methods_for_gauge, :has_methods_for_counter
+
+      def initialize
+        super
+
+        @has_methods_for_counter = false
+        @has_methods_for_gauge = false
+        @use_gauge_metric = false
+      end
+
+      def configure(conf)
+        super
+
+        if use_gauge_metric
+          @has_methods_for_gauge = has_methods_for_gauge?
+        else
+          @has_methods_for_counter = has_methods_for_counter?
+        end
+      end
+
+      # Some metrics should be counted by gauge.
+      # ref: https://prometheus.io/docs/concepts/metric_types/#gauge
+      def use_gauge_metric=(use_gauge_metric=false)
+        @use_gauge_metric = use_gauge_metric
+      end
+
+      def create(namespace:, subsystem:,name:,help_text:,labels: {})
+        # This API is for cmetrics type.
+      end
+
+      def get
+        raise NotImplementedError, "Implement this method in child class"
+      end
+
+      def inc
+        raise NotImplementedError, "Implement this method in child class"
+      end
+
+      def dec
+        raise NotImplementedError, "Implement this method in child class"
+      end
+
+      def add(value)
+        raise NotImplementedError, "Implement this method in child class"
+      end
+
+      def sub(value)
+        raise NotImplementedError, "Implement this method in child class"
+      end
+
+      def set(value)
+        raise NotImplementedError, "Implement this method in child class"
+      end
+
+      private
+
+      def has_methods_for_counter?
+        implemented_methods = self.class.instance_methods(false)
+
+        if [:get, :inc, :add].all? {|e| implemented_methods.include?(e)} &&
+           [:set].all?{|e| self.class.method_defined?(e)}
+          true
+        else
+          raise "BUG: metrics plugin on counter mode MUST implement `get`, `inc`, `add` methods. And aliased `set` methods should be aliased from another method"
+        end
+      end
+
+      def has_methods_for_gauge?
+        implemented_methods = self.class.instance_methods(false)
+
+        if [:get, :inc, :add].all? {|e| implemented_methods.include?(e)} &&
+           [:set, :dec, :sub].all?{|e| self.class.method_defined?(e)}
+          true
+        else
+          raise "BUG: metrics plugin on gauge mode MUST implement `get`, `inc`, and `add` methods. And `dec`, `sub`, and `set` methods should be aliased from other methods"
+        end
+      end
+    end
+  end
+end