fluentd-hubspot 0.14.14.1

data/lib/fluent/command/bundler_injection.rb

@@ -0,0 +1,45 @@
+#
+# Fluentd
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+
+require 'rbconfig'
+
+if ENV['BUNDLE_BIN_PATH']
+  puts 'error: You seem to use `bundle exec` already.'
+  exit 1
+else
+  begin
+    bundle_bin = Gem::Specification.find_by_name('bundler').bin_file('bundle')
+  rescue Gem::LoadError => e
+    puts "error: #{e}"
+    exit 1
+  end
+  ruby_bin = RbConfig.ruby
+  system("#{ruby_bin} #{bundle_bin} install")
+  unless $?.success?
+    exit $?.exitstatus
+  end
+
+  cmdline = [
+    ruby_bin,
+    bundle_bin,
+    'exec',
+    ruby_bin,
+    File.expand_path(File.join(File.dirname(__FILE__), 'fluentd.rb')),
+  ] + ARGV
+
+  exec *cmdline
+  exit! 127
+end

data/lib/fluent/command/cat.rb

@@ -0,0 +1,330 @@
+#
+# Fluentd
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+
+require 'optparse'
+require 'fluent/env'
+require 'fluent/time'
+require 'fluent/msgpack_factory'
+
+op = OptionParser.new
+
+op.banner += " <tag>"
+
+port = 24224
+host = '127.0.0.1'
+unix = false
+socket_path = Fluent::DEFAULT_SOCKET_PATH
+
+config_path = Fluent::DEFAULT_CONFIG_PATH
+format = 'json'
+message_key = 'message'
+time_as_integer = false
+
+op.on('-p', '--port PORT', "fluent tcp port (default: #{port})", Integer) {|i|
+  port = i
+}
+
+op.on('-h', '--host HOST', "fluent host (default: #{host})") {|s|
+  host = s
+}
+
+op.on('-u', '--unix', "use unix socket instead of tcp", TrueClass) {|b|
+  unix = b
+}
+
+op.on('-s', '--socket PATH', "unix socket path (default: #{socket_path})") {|s|
+  socket_path = s
+}
+
+op.on('-f', '--format FORMAT', "input format (default: #{format})") {|s|
+  format = s
+}
+
+op.on('--json', "same as: -f json", TrueClass) {|b|
+  format = 'json'
+}
+
+op.on('--msgpack', "same as: -f msgpack", TrueClass) {|b|
+  format = 'msgpack'
+}
+
+op.on('--none', "same as: -f none", TrueClass) {|b|
+  format = 'none'
+}
+
+op.on('--message-key KEY', "key field for none format (default: #{message_key})") {|s|
+  message_key = s
+}
+
+op.on('--time-as-integer', "Send time as integer for v0.12 or earlier", TrueClass) { |b|
+  time_as_integer = true
+}
+
+(class << self; self; end).module_eval do
+  define_method(:usage) do |msg|
+    puts op.to_s
+    puts "error: #{msg}" if msg
+    exit 1
+  end
+end
+
+begin
+  op.parse!(ARGV)
+
+  if ARGV.length != 1
+    usage nil
+  end
+
+  tag = ARGV.shift
+
+rescue
+  usage $!.to_s
+end
+
+
+require 'thread'
+require 'monitor'
+require 'socket'
+require 'yajl'
+require 'msgpack'
+
+
+class Writer
+  include MonitorMixin
+
+  class TimerThread
+    def initialize(writer)
+      @writer = writer
+    end
+
+    def start
+      @finish = false
+      @thread = Thread.new(&method(:run))
+    end
+
+    def shutdown
+      @finish = true
+      @thread.join
+    end
+
+    def run
+      until @finish
+        sleep 1
+        @writer.on_timer
+      end
+    end
+  end
+
+  def initialize(tag, connector, time_as_integer: false)
+    @tag = tag
+    @connector = connector
+    @socket = false
+
+    @socket_time = Time.now.to_i
+    @socket_ttl = 10  # TODO
+    @error_history = []
+
+    @pending = []
+    @pending_limit = 1024  # TODO
+    @retry_wait = 1
+    @retry_limit = 5  # TODO
+    @time_as_integer = time_as_integer
+
+    super()
+  end
+
+  def write(record)
+    if record.class != Hash
+      raise ArgumentError, "Input must be a map (got #{record.class})"
+    end
+
+    time = Fluent::EventTime.now
+    time = time.to_i if @time_as_integer
+    entry = [time, record]
+    synchronize {
+      unless write_impl([entry])
+        # write failed
+        @pending.push(entry)
+
+        while @pending.size > @pending_limit
+          # exceeds pending limit; trash oldest record
+          time, record = @pending.shift
+          abort_message(time, record)
+        end
+      end
+    }
+  end
+
+  def on_timer
+    now = Time.now.to_i
+
+    synchronize {
+      unless @pending.empty?
+        # flush pending records
+        if write_impl(@pending)
+          # write succeeded
+          @pending.clear
+        end
+      end
+
+      if @socket && @socket_time + @socket_ttl < now
+        # socket is not used @socket_ttl seconds
+        close
+      end
+    }
+  end
+
+  def close
+    @socket.close
+    @socket = nil
+  end
+
+  def start
+    @timer = TimerThread.new(self)
+    @timer.start
+    self
+  end
+
+  def shutdown
+    @timer.shutdown
+  end
+
+  private
+  def write_impl(array)
+    socket = get_socket
+    unless socket
+      return false
+    end
+
+    begin
+      packer = Fluent::MessagePackFactory.packer
+      socket.write packer.pack([@tag, array])
+      socket.flush
+    rescue
+      $stderr.puts "write failed: #{$!}"
+      close
+      return false
+    end
+
+    return true
+  end
+
+  def get_socket
+    unless @socket
+      unless try_connect
+        return nil
+      end
+    end
+
+    @socket_time = Time.now.to_i
+    return @socket
+  end
+
+  def try_connect
+    now = Time.now.to_i
+
+    unless @error_history.empty?
+      # wait before re-connecting
+      wait = @retry_wait * (2 ** (@error_history.size-1))
+      if now <= @socket_time + wait
+        return false
+      end
+    end
+
+    begin
+      @socket = @connector.call
+      @error_history.clear
+      return true
+
+    rescue
+      $stderr.puts "connect failed: #{$!}"
+      @error_history << $!
+      @socket_time = now
+
+      if @retry_limit < @error_history.size
+        # abort all pending records
+        @pending.each {|(time, record)|
+          abort_message(time, record)
+        }
+        @pending.clear
+        @error_history.clear
+      end
+
+      return false
+    end
+  end
+
+  def abort_message(time, record)
+    $stdout.puts "!#{time}:#{Yajl.dump(record)}"
+  end
+end
+
+
+if unix
+  connector = Proc.new {
+    UNIXSocket.open(socket_path)
+  }
+else
+  connector = Proc.new {
+    TCPSocket.new(host, port)
+  }
+end
+
+w = Writer.new(tag, connector, time_as_integer: time_as_integer)
+w.start
+
+case format
+when 'json'
+  begin
+    while line = $stdin.gets
+      record = Yajl.load(line)
+      w.write(record)
+    end
+  rescue
+    $stderr.puts $!
+    exit 1
+  end
+
+when 'msgpack'
+  require 'fluent/engine'
+
+  begin
+    u = Fluent::Engine.msgpack_factory.unpacker($stdin)
+    u.each {|record|
+      w.write(record)
+    }
+  rescue EOFError
+  rescue
+    $stderr.puts $!
+    exit 1
+  end
+
+when 'none'
+  begin
+    while line = $stdin.gets
+      record = { message_key => line.chomp }
+      w.write(record)
+    end
+  rescue
+    $stderr.puts $!
+    exit 1
+  end
+
+else
+  $stderr.puts "Unknown format '#{format}'"
+  exit 1
+end
+

data/lib/fluent/command/debug.rb

@@ -0,0 +1,102 @@
+#
+# Fluentd
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+
+require 'optparse'
+
+op = OptionParser.new
+
+host = '127.0.0.1'
+port = 24230
+unix = nil
+
+op.on('-h', '--host HOST', "fluent host (default: #{host})") {|s|
+  host = s
+}
+
+op.on('-p', '--port PORT', "debug_agent tcp port (default: #{port})", Integer) {|i|
+  port = i
+}
+
+op.on('-u', '--unix PATH', "use unix socket instead of tcp") {|b|
+  unix = b
+}
+
+(class << self; self; end).module_eval do
+  define_method(:usage) do |msg|
+    puts op.to_s
+    puts "error: #{msg}" if msg
+    exit 1
+  end
+end
+
+begin
+  op.parse!(ARGV)
+
+  if ARGV.length != 0
+    usage nil
+  end
+rescue
+  usage $!.to_s
+end
+
+require 'drb/drb'
+
+if unix
+  uri = "drbunix:#{unix}"
+else
+  uri = "druby://#{host}:#{port}"
+end
+
+require 'fluent/log'
+require 'fluent/engine'
+require 'fluent/system_config'
+require 'serverengine'
+
+include Fluent::SystemConfig::Mixin
+
+dl_opts = {}
+dl_opts[:log_level] = ServerEngine::DaemonLogger::TRACE
+logger = ServerEngine::DaemonLogger.new(STDERR, dl_opts)
+$log = Fluent::Log.new(logger)
+Fluent::Engine.init(system_config)
+
+DRb::DRbObject.class_eval do
+  undef_method :methods
+  undef_method :instance_eval
+  undef_method :instance_variables
+  undef_method :instance_variable_get
+end
+
+remote_engine = DRb::DRbObject.new_with_uri(uri)
+
+Fluent.module_eval do
+  remove_const(:Engine)
+  const_set(:Engine, remote_engine)
+end
+
+include Fluent
+
+puts "Connected to #{uri}."
+puts "Usage:"
+puts "    Fluent::Engine.match('some.tag').output  : get an output plugin instance"
+puts "    Fluent::Engine.sources[i]                : get input plugin instances"
+puts "    Fluent::Plugin.load_plugin(type,name)    : load plugin class (use this if you get DRb::DRbUnknown)"
+puts ""
+
+Encoding.default_internal = nil if Encoding.respond_to?(:default_internal)
+
+require 'irb'
+IRB.start