fluentd 0.14.4-x86-mingw32

data/lib/fluent/plugin/in_tcp.rb

@@ -0,0 +1,41 @@
+#
+# Fluentd
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+
+require 'cool.io'
+
+require 'fluent/plugin/socket_util'
+
+module Fluent
+  class TcpInput < SocketUtil::BaseInput
+    Plugin.register_input('tcp', self)
+
+    config_set_default :port, 5170
+    desc 'The payload is read up to this character.'
+    config_param :delimiter, :string, default: "\n" # syslog family add "\n" to each message and this seems only way to split messages in tcp stream
+
+    def listen(callback)
+      log.info "listening tcp socket on #{@bind}:#{@port}"
+
+      socket_manager_path = ENV['SERVERENGINE_SOCKETMANAGER_PATH']
+      if Fluent.windows?
+        socket_manager_path = socket_manager_path.to_i
+      end
+      client = ServerEngine::SocketManager::Client.new(socket_manager_path)
+      lsock = client.listen_tcp(@bind, @port)
+      Coolio::TCPServer.new(lsock, nil, SocketUtil::TcpHandler, log, @delimiter, callback)
+    end
+  end
+end

data/lib/fluent/plugin/in_udp.rb

@@ -0,0 +1,37 @@
+#
+# Fluentd
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+
+require 'fluent/plugin/socket_util'
+
+module Fluent
+  class UdpInput < SocketUtil::BaseInput
+    Plugin.register_input('udp', self)
+
+    config_set_default :port, 5160
+    config_param :body_size_limit, :size, default: 4096
+
+    def listen(callback)
+      log.info "listening udp socket on #{@bind}:#{@port}"
+      socket_manager_path = ENV['SERVERENGINE_SOCKETMANAGER_PATH']
+      if Fluent.windows?
+        socket_manager_path = socket_manager_path.to_i
+      end
+      client = ServerEngine::SocketManager::Client.new(socket_manager_path)
+      @usock = client.listen_udp(@bind, @port)
+      SocketUtil::UdpHandler.new(@usock, log, @body_size_limit, callback)
+    end
+  end
+end

data/lib/fluent/plugin/in_unix.rb

@@ -0,0 +1,201 @@
+#
+# Fluentd
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+
+require 'fileutils'
+require 'socket'
+
+require 'cool.io'
+require 'yajl'
+
+require 'fluent/input'
+require 'fluent/event'
+
+module Fluent
+  # obsolete
+  class StreamInput < Input
+    config_param :blocking_timeout, :time, default: 0.5
+
+    def initialize
+      require 'socket'
+      require 'yajl'
+      super
+    end
+
+    def start
+      super
+
+      @loop = Coolio::Loop.new
+      @lsock = listen
+      @loop.attach(@lsock)
+      @thread = Thread.new(&method(:run))
+    end
+
+    def shutdown
+      @loop.watchers.each {|w| w.detach }
+      @loop.stop
+      @lsock.close
+      @thread.join
+
+      super
+    end
+
+    #def listen
+    #end
+
+    def run
+      @loop.run(@blocking_timeout)
+    rescue
+      log.error "unexpected error", error: $!.to_s
+      log.error_backtrace
+    end
+
+    private
+
+    # message Entry {
+    #   1: long time
+    #   2: object record
+    # }
+    #
+    # message Forward {
+    #   1: string tag
+    #   2: list<Entry> entries
+    # }
+    #
+    # message PackedForward {
+    #   1: string tag
+    #   2: raw entries  # msgpack stream of Entry
+    # }
+    #
+    # message Message {
+    #   1: string tag
+    #   2: long? time
+    #   3: object record
+    # }
+    def on_message(msg)
+      # TODO format error
+      tag = msg[0].to_s
+      entries = msg[1]
+
+      if entries.class == String
+        # PackedForward
+        es = MessagePackEventStream.new(entries)
+        router.emit_stream(tag, es)
+
+      elsif entries.class == Array
+        # Forward
+        es = MultiEventStream.new
+        entries.each {|e|
+          record = e[1]
+          next if record.nil?
+          time = e[0]
+          time = (now ||= Engine.now) if time.to_i == 0
+          es.add(time, record)
+        }
+        router.emit_stream(tag, es)
+
+      else
+        # Message
+        record = msg[2]
+        return if record.nil?
+
+        time = msg[1]
+        time = Engine.now if time.to_i == 0
+        router.emit(tag, time, record)
+      end
+    end
+
+    class Handler < Coolio::Socket
+      def initialize(io, log, on_message)
+        super(io)
+        if io.is_a?(TCPSocket)
+          opt = [1, @timeout.to_i].pack('I!I!')  # { int l_onoff; int l_linger; }
+          io.setsockopt(Socket::SOL_SOCKET, Socket::SO_LINGER, opt)
+        end
+        @on_message = on_message
+        @log = log
+        @log.trace {
+          remote_port, remote_addr = *Socket.unpack_sockaddr_in(@_io.getpeername) rescue nil
+          "accepted fluent socket from '#{remote_addr}:#{remote_port}': object_id=#{self.object_id}"
+        }
+      end
+
+      def on_connect
+      end
+
+      def on_read(data)
+        first = data[0]
+        if first == '{' || first == '['
+          m = method(:on_read_json)
+          @y = Yajl::Parser.new
+          @y.on_parse_complete = @on_message
+        else
+          m = method(:on_read_msgpack)
+          @u = Fluent::Engine.msgpack_factory.unpacker
+        end
+
+        (class << self; self; end).module_eval do
+          define_method(:on_read, m)
+        end
+        m.call(data)
+      end
+
+      def on_read_json(data)
+        @y << data
+      rescue
+        @log.error "unexpected error", error: $!.to_s
+        @log.error_backtrace
+        close
+      end
+
+      def on_read_msgpack(data)
+        @u.feed_each(data, &@on_message)
+      rescue
+        @log.error "unexpected error", error: $!.to_s
+        @log.error_backtrace
+        close
+      end
+
+      def on_close
+        @log.trace { "closed fluent socket object_id=#{self.object_id}" }
+      end
+    end
+  end
+
+  class UnixInput < StreamInput
+    Plugin.register_input('unix', self)
+
+    desc 'The path to your Unix Domain Socket.'
+    config_param :path, :string, default: DEFAULT_SOCKET_PATH
+    desc 'The backlog of Unix Domain Socket.'
+    config_param :backlog, :integer, default: nil
+
+    def configure(conf)
+      super
+      #log.warn "'unix' input is obsoleted and will be removed. Use 'forward' instead."
+    end
+
+    def listen
+      if File.exist?(@path)
+        File.unlink(@path)
+      end
+      FileUtils.mkdir_p File.dirname(@path)
+      log.info "listening fluent socket on #{@path}"
+      s = Coolio::UNIXServer.new(@path, Handler, log, method(:on_message))
+      s.listen(@backlog) unless @backlog.nil?
+      s
+    end
+  end
+end

data/lib/fluent/plugin/input.rb

@@ -0,0 +1,33 @@
+#
+# Fluentd
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+
+require 'fluent/plugin/base'
+
+require 'fluent/log'
+require 'fluent/plugin_id'
+require 'fluent/plugin_helper'
+
+module Fluent
+  module Plugin
+    class Input < Base
+      include PluginId
+      include PluginLoggerMixin
+      include PluginHelper::Mixin
+
+      helpers :event_emitter
+    end
+  end
+end

data/lib/fluent/plugin/multi_output.rb

@@ -0,0 +1,95 @@
+#
+# Fluentd
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+
+require 'fluent/plugin/base'
+require 'fluent/log'
+require 'fluent/plugin_id'
+require 'fluent/plugin_helper'
+
+module Fluent
+  module Plugin
+    class MultiOutput < Base
+      include PluginId
+      include PluginLoggerMixin
+      include PluginHelper::Mixin # for event_emitter
+
+      helpers :event_emitter # to get router from agent, which will be supplied to child plugins
+
+      config_section :store, param_name: :stores, multi: true, required: true do
+        config_param :@type, :string, default: nil
+      end
+
+      attr_reader :outputs
+
+      def process(tag, es)
+        raise NotImplementedError, "BUG: output plugins MUST implement this method"
+      end
+
+      def initialize
+        super
+        @outputs = []
+
+        @compat = false
+
+        @counters_monitor = Monitor.new
+        # TODO: well organized counters
+        @num_errors = 0
+        @emit_count = 0
+        @emit_records = 0
+        # @write_count = 0
+        # @rollback_count = 0
+      end
+
+      def configure(conf)
+        super
+
+        @stores.each do |store|
+          store_conf = store.corresponding_config_element
+          type = store_conf['@type']
+          unless type
+            raise Fluent::ConfigError, "Missing '@type' parameter in <store> section"
+          end
+
+          log.debug "adding store", type: type
+
+          output = Fluent::Plugin.new_output(type)
+          if output.has_router?
+            output.router = router
+          end
+          output.configure(store_conf)
+          @outputs << output
+        end
+      end
+
+      # Child plugin's lifecycles are controlled by agent automatically.
+      # It calls `outputs` to traverse plugins, and invoke start/stop/*shutdown/close/terminate on these directly.
+      # * `start` of this plugin will be called after child plugins
+      # * `stop`, `*shutdown`, `close` and `terminate` of this plugin will be called before child plugins
+
+      def emit_sync(tag, es)
+        @counters_monitor.synchronize{ @emit_count += 1 }
+        begin
+          process(tag, es)
+          @counters_monitor.synchronize{ @emit_records += es.size }
+        rescue
+          @counters_monitor.synchronize{ @num_errors += 1 }
+          raise
+        end
+      end
+      alias :emit_events :emit_sync
+    end
+  end
+end

data/lib/fluent/plugin/out_buffered_null.rb

@@ -0,0 +1,59 @@
+#
+# Fluentd
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+
+require 'fluent/plugin/output'
+
+module Fluent::Plugin
+  class BufferedNullOutput < Output
+    # This plugin is for tests of buffer plugins
+
+    Fluent::Plugin.register_output('buffered_null', self)
+
+    config_section :buffer do
+      config_set_default :chunk_keys, ['tag']
+      config_set_default :flush_at_shutdown, true
+      config_set_default :chunk_limit_size, 10 * 1024
+    end
+
+    attr_accessor :feed_proc, :delayed
+
+    def initialize
+      super
+      @delayed = false
+      @feed_proc = nil
+    end
+
+    def prefer_delayed_commit
+      @delayed
+    end
+
+    def write(chunk)
+      if @feed_proc
+        @feed_proc.call(chunk)
+      else
+        # ignore chunk.read
+      end
+    end
+
+    def try_write(chunk)
+      if @feed_proc
+        @feed_proc.call(chunk)
+      else
+        # ignore chunk.read
+      end
+    end
+  end
+end