fluentd-hubspot 0.14.14.1

data/lib/fluent/plugin/in_tcp.rb

@@ -0,0 +1,85 @@
+#
+# Fluentd
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+
+require 'fluent/plugin/input'
+
+module Fluent::Plugin
+  class TcpInput < Input
+    Fluent::Plugin.register_input('tcp', self)
+
+    helpers :server, :parser, :extract, :compat_parameters
+
+    desc 'Tag of output events.'
+    config_param :tag, :string
+    desc 'The port to listen to.'
+    config_param :port, :integer, default: 5170
+    desc 'The bind address to listen to.'
+    config_param :bind, :string, default: '0.0.0.0'
+
+    desc "The field name of the client's hostname."
+    config_param :source_host_key, :string, default: nil, deprecated: "use source_hostname_key instead."
+    desc "The field name of the client's hostname."
+    config_param :source_hostname_key, :string, default: nil
+
+    config_param :blocking_timeout, :time, default: 0.5
+
+    desc 'The payload is read up to this character.'
+    config_param :delimiter, :string, default: "\n" # syslog family add "\n" to each message and this seems only way to split messages in tcp stream
+
+    def configure(conf)
+      compat_parameters_convert(conf, :parser)
+      super
+      @_event_loop_blocking_timeout = @blocking_timeout
+      @source_hostname_key ||= @source_host_key if @source_host_key
+
+      @parser = parser_create
+    end
+
+    def multi_workers_ready?
+      true
+    end
+
+    def start
+      super
+
+      @buffer = ''
+      server_create(:in_tcp_server, @port, bind: @bind) do |data, conn|
+        @buffer << data
+        begin
+          pos = 0
+          while i = @buffer.index(@delimiter, pos)
+            msg = @buffer[pos...i]
+            pos = i + @delimiter.length
+
+            @parser.parse(msg) do |time, record|
+              unless time && record
+                log.warn "pattern not match", message: msg
+                next
+              end
+
+              tag = extract_tag_from_record(record)
+              tag ||= @tag
+              time ||= extract_time_from_record(record) || Fluent::EventTime.now
+              record[@source_hostname_key] = conn.remote_host if @source_hostname_key
+              router.emit(tag, time, record)
+            end
+          end
+          @buffer.slice!(0, pos) if pos > 0
+        end
+      end
+    end
+  end
+end

data/lib/fluent/plugin/in_udp.rb

@@ -0,0 +1,81 @@
+#
+# Fluentd
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+
+require 'fluent/plugin/input'
+
+module Fluent::Plugin
+  class UdpInput < Input
+    Fluent::Plugin.register_input('udp', self)
+
+    helpers :server, :parser, :extract, :compat_parameters
+
+    desc 'Tag of output events.'
+    config_param :tag, :string
+    desc 'The port to listen to.'
+    config_param :port, :integer, default: 5160
+    desc 'The bind address to listen to.'
+    config_param :bind, :string, default: '0.0.0.0'
+
+    desc "The field name of the client's hostname."
+    config_param :source_host_key, :string, default: nil, deprecated: "use source_hostname_key instead."
+    desc "The field name of the client's hostname."
+    config_param :source_hostname_key, :string, default: nil
+
+    desc "Deprecated parameter. Use message_length_limit instead"
+    config_param :body_size_limit, :size, default: nil, deprecated: "use message_length_limit instead."
+    desc "The max bytes of message"
+    config_param :message_length_limit, :size, default: 4096
+
+    config_param :blocking_timeout, :time, default: 0.5
+
+    def configure(conf)
+      compat_parameters_convert(conf, :parser)
+      super
+      @_event_loop_blocking_timeout = @blocking_timeout
+      @source_hostname_key ||= @source_host_key if @source_host_key
+      @message_length_limit = @body_size_limit if @body_size_limit
+
+      @parser = parser_create
+    end
+
+    def multi_workers_ready?
+      true
+    end
+
+    def start
+      super
+
+      log.info "listening udp socket", bind: @bind, port: @port
+      server_create(:in_udp_server, @port, proto: :udp, bind: @bind, max_bytes: @message_length_limit) do |data, sock|
+        data.chomp!
+        begin
+          @parser.parse(data) do |time, record|
+            unless time && record
+              log.warn "pattern not match", data: data
+              next
+            end
+
+            tag = extract_tag_from_record(record)
+            tag ||= @tag
+            time ||= extract_time_from_record(record) || Fluent::EventTime.now
+            record[@source_hostname_key] = sock.remote_host if @source_hostname_key
+            router.emit(tag, time, record)
+          end
+        end
+      end
+    end
+  end
+end

data/lib/fluent/plugin/in_unix.rb

@@ -0,0 +1,201 @@
+#
+# Fluentd
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+
+require 'fileutils'
+require 'socket'
+
+require 'cool.io'
+require 'yajl'
+
+require 'fluent/input'
+require 'fluent/event'
+
+module Fluent
+  # obsolete
+  class StreamInput < Input
+    config_param :blocking_timeout, :time, default: 0.5
+
+    def initialize
+      require 'socket'
+      require 'yajl'
+      super
+    end
+
+    def start
+      super
+
+      @loop = Coolio::Loop.new
+      @lsock = listen
+      @loop.attach(@lsock)
+      @thread = Thread.new(&method(:run))
+    end
+
+    def shutdown
+      @loop.watchers.each {|w| w.detach }
+      @loop.stop
+      @lsock.close
+      @thread.join
+
+      super
+    end
+
+    #def listen
+    #end
+
+    def run
+      @loop.run(@blocking_timeout)
+    rescue
+      log.error "unexpected error", error: $!.to_s
+      log.error_backtrace
+    end
+
+    private
+
+    # message Entry {
+    #   1: long time
+    #   2: object record
+    # }
+    #
+    # message Forward {
+    #   1: string tag
+    #   2: list<Entry> entries
+    # }
+    #
+    # message PackedForward {
+    #   1: string tag
+    #   2: raw entries  # msgpack stream of Entry
+    # }
+    #
+    # message Message {
+    #   1: string tag
+    #   2: long? time
+    #   3: object record
+    # }
+    def on_message(msg)
+      # TODO format error
+      tag = msg[0].to_s
+      entries = msg[1]
+
+      if entries.class == String
+        # PackedForward
+        es = MessagePackEventStream.new(entries)
+        router.emit_stream(tag, es)
+
+      elsif entries.class == Array
+        # Forward
+        es = MultiEventStream.new
+        entries.each {|e|
+          record = e[1]
+          next if record.nil?
+          time = e[0]
+          time = (now ||= Engine.now) if time.to_i == 0
+          es.add(time, record)
+        }
+        router.emit_stream(tag, es)
+
+      else
+        # Message
+        record = msg[2]
+        return if record.nil?
+
+        time = msg[1]
+        time = Engine.now if time.to_i == 0
+        router.emit(tag, time, record)
+      end
+    end
+
+    class Handler < Coolio::Socket
+      def initialize(io, log, on_message)
+        super(io)
+        if io.is_a?(TCPSocket)
+          opt = [1, @timeout.to_i].pack('I!I!')  # { int l_onoff; int l_linger; }
+          io.setsockopt(Socket::SOL_SOCKET, Socket::SO_LINGER, opt)
+        end
+        @on_message = on_message
+        @log = log
+        @log.trace {
+          remote_port, remote_addr = *Socket.unpack_sockaddr_in(@_io.getpeername) rescue nil
+          "accepted fluent socket from '#{remote_addr}:#{remote_port}': object_id=#{self.object_id}"
+        }
+      end
+
+      def on_connect
+      end
+
+      def on_read(data)
+        first = data[0]
+        if first == '{' || first == '['
+          m = method(:on_read_json)
+          @y = Yajl::Parser.new
+          @y.on_parse_complete = @on_message
+        else
+          m = method(:on_read_msgpack)
+          @u = Fluent::Engine.msgpack_factory.unpacker
+        end
+
+        (class << self; self; end).module_eval do
+          define_method(:on_read, m)
+        end
+        m.call(data)
+      end
+
+      def on_read_json(data)
+        @y << data
+      rescue
+        @log.error "unexpected error", error: $!.to_s
+        @log.error_backtrace
+        close
+      end
+
+      def on_read_msgpack(data)
+        @u.feed_each(data, &@on_message)
+      rescue
+        @log.error "unexpected error", error: $!.to_s
+        @log.error_backtrace
+        close
+      end
+
+      def on_close
+        @log.trace { "closed fluent socket object_id=#{self.object_id}" }
+      end
+    end
+  end
+
+  class UnixInput < StreamInput
+    Plugin.register_input('unix', self)
+
+    desc 'The path to your Unix Domain Socket.'
+    config_param :path, :string, default: DEFAULT_SOCKET_PATH
+    desc 'The backlog of Unix Domain Socket.'
+    config_param :backlog, :integer, default: nil
+
+    def configure(conf)
+      super
+      #log.warn "'unix' input is obsoleted and will be removed. Use 'forward' instead."
+    end
+
+    def listen
+      if File.exist?(@path)
+        File.unlink(@path)
+      end
+      FileUtils.mkdir_p File.dirname(@path)
+      log.info "listening fluent socket on #{@path}"
+      s = Coolio::UNIXServer.new(@path, Handler, log, method(:on_message))
+      s.listen(@backlog) unless @backlog.nil?
+      s
+    end
+  end
+end

data/lib/fluent/plugin/input.rb

@@ -0,0 +1,37 @@
+#
+# Fluentd
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+
+require 'fluent/plugin/base'
+
+require 'fluent/log'
+require 'fluent/plugin_id'
+require 'fluent/plugin_helper'
+
+module Fluent
+  module Plugin
+    class Input < Base
+      include PluginId
+      include PluginLoggerMixin
+      include PluginHelper::Mixin
+
+      helpers_internal :event_emitter
+
+      def multi_workers_ready?
+        false
+      end
+    end
+  end
+end

data/lib/fluent/plugin/multi_output.rb

@@ -0,0 +1,157 @@
+#
+# Fluentd
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+
+require 'fluent/plugin/base'
+require 'fluent/log'
+require 'fluent/plugin_id'
+require 'fluent/plugin_helper'
+
+module Fluent
+  module Plugin
+    class MultiOutput < Base
+      include PluginId
+      include PluginLoggerMixin
+      include PluginHelper::Mixin # for event_emitter
+
+      helpers :event_emitter # to get router from agent, which will be supplied to child plugins
+
+      config_section :store, param_name: :stores, multi: true, required: true do
+        config_param :@type, :string, default: nil
+      end
+
+      attr_reader :outputs, :outputs_statically_created
+
+      def process(tag, es)
+        raise NotImplementedError, "BUG: output plugins MUST implement this method"
+      end
+
+      def initialize
+        super
+        @outputs = []
+        @outputs_statically_created = false
+
+        @counters_monitor = Monitor.new
+        # TODO: well organized counters
+        @num_errors = 0
+        @emit_count = 0
+        @emit_records = 0
+        # @write_count = 0
+        # @rollback_count = 0
+      end
+
+      def multi_output?
+        true
+      end
+
+      def configure(conf)
+        super
+
+        @stores.each do |store|
+          store_conf = store.corresponding_config_element
+          type = store_conf['@type']
+          unless type
+            raise Fluent::ConfigError, "Missing '@type' parameter in <store> section"
+          end
+
+          log.debug "adding store", type: type
+
+          output = Fluent::Plugin.new_output(type)
+          output.context_router = self.context_router
+          output.configure(store_conf)
+          @outputs << output
+        end
+      end
+
+      def static_outputs
+        @outputs_statically_created = true
+        @outputs
+      end
+
+      # Child plugin's lifecycles are controlled by agent automatically.
+      # It calls `outputs` to traverse plugins, and invoke start/stop/*shutdown/close/terminate on these directly.
+      # * `start` of this plugin will be called after child plugins
+      # * `stop`, `*shutdown`, `close` and `terminate` of this plugin will be called before child plugins
+
+      # But when MultiOutput plugins are created dynamically (by forest plugin or others), agent cannot find
+      # sub-plugins. So child plugins' lifecycles MUST be controlled by MultiOutput plugin itself.
+      # TODO: this hack will be removed at v2.
+      def call_lifecycle_method(method_name, checker_name)
+        return if @outputs_statically_created
+        @outputs.each do |o|
+          begin
+            log.debug "calling #{method_name} on output plugin dynamically created", type: Fluent::Plugin.lookup_type_from_class(o.class), plugin_id: o.plugin_id
+            o.send(method_name) unless o.send(checker_name)
+          rescue Exception => e
+            log.warn "unexpected error while calling #{method_name} on output plugin dynamically created", plugin: o.class, plugin_id: o.plugin_id, error: e
+            log.warn_backtrace
+          end
+        end
+      end
+
+      def start
+        super
+        call_lifecycle_method(:start, :started?)
+      end
+
+      def after_start
+        super
+        call_lifecycle_method(:after_start, :after_started?)
+      end
+
+      def stop
+        super
+        call_lifecycle_method(:stop, :stopped?)
+      end
+
+      def before_shutdown
+        super
+        call_lifecycle_method(:before_shutdown, :before_shutdown?)
+      end
+
+      def shutdown
+        super
+        call_lifecycle_method(:shutdown, :shutdown?)
+      end
+
+      def after_shutdown
+        super
+        call_lifecycle_method(:after_shutdown, :after_shutdown?)
+      end
+
+      def close
+        super
+        call_lifecycle_method(:close, :closed?)
+      end
+
+      def terminate
+        super
+        call_lifecycle_method(:terminate, :terminated?)
+      end
+
+      def emit_sync(tag, es)
+        @counters_monitor.synchronize{ @emit_count += 1 }
+        begin
+          process(tag, es)
+          @counters_monitor.synchronize{ @emit_records += es.size }
+        rescue
+          @counters_monitor.synchronize{ @num_errors += 1 }
+          raise
+        end
+      end
+      alias :emit_events :emit_sync
+    end
+  end
+end