logstash-core 1.5.0.beta2-java

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 1c512265bcb6873ce0def467329ba6b4a058aea5
+  data.tar.gz: a376414ac39751f9c4e7b67fbb109410fe4d04b8
+SHA512:
+  metadata.gz: 5520cb37d6482b810ab0501e370bb08e859365d7b5e4217683fc065ecde3c859dde0f3da0c8447ff5b28ab19f9264db9cc4c8e5e5523a6a53bb8c57f3054608a
+  data.tar.gz: b722a0fe0d550a82adafc116bd579489f7871c633d22dce7d9724ca6617c107e09b138a92632babf75eebb704a9f2fb34022e05ed544d0b0381cfc05b4befd64

data/lib/logstash-event.rb

@@ -0,0 +1,2 @@
+# encoding: utf-8
+require "logstash/event"

data/lib/logstash.rb

@@ -0,0 +1,4 @@
+# encoding: utf-8
+require "logstash/agent"
+require "logstash/event"
+require "logstash/namespace"

data/lib/logstash/JRUBY-PR1448.rb

@@ -0,0 +1,32 @@
+# This patch fixes a problem that exists in JRuby prior to 1.7.11 where the
+# ruby binary path used by rubygems is malformed on Windows, causing
+# dependencies to not install cleanly when using `.\bin\logstash.bat deps`.
+# This monkeypatch can probably be removed once it's unlikely that people
+# are still using JRuby older than 1.7.11.
+  class << Gem
+    def ruby
+      ruby_path = original_ruby
+      ruby_path = "java -jar #{jar_path(ruby_path)}" if jarred_path?(ruby_path)
+      ruby_path
+    end
+
+    def jarred_path?(p)
+      p =~ /^file:/
+    end
+
+    # A jar path looks like this on non-Windows platforms:
+    #   file:/path/to/file.jar!/path/within/jar/to/file.txt
+    # and like this on Windows:
+    #   file:/C:/path/to/file.jar!/path/within/jar/to/file.txt
+    #
+    # This method returns:
+    #   /path/to/file.jar
+    # or
+    #   C:/path/to/file.jar
+    # as appropriate.
+    def jar_path(p)
+      path = p.sub(/^file:/, "").sub(/!.*/, "")
+      path = path.sub(/^\//, "") if win_platform? && path =~ /^\/[A-Za-z]:/
+      path
+    end
+  end

data/lib/logstash/agent.rb

@@ -0,0 +1,355 @@
+# encoding: utf-8
+require "clamp" # gem 'clamp'
+require "logstash/environment"
+require "logstash/errors"
+require "uri"
+require "net/http"
+LogStash::Environment.load_locale!
+
+class LogStash::Agent < Clamp::Command
+  option ["-f", "--config"], "CONFIG_PATH",
+    I18n.t("logstash.agent.flag.config"),
+    :attribute_name => :config_path
+
+  option "-e", "CONFIG_STRING",
+    I18n.t("logstash.agent.flag.config-string"),
+    :default => "", :attribute_name => :config_string
+
+  option ["-w", "--filterworkers"], "COUNT",
+    I18n.t("logstash.agent.flag.filterworkers"),
+    :attribute_name => :filter_workers, :default => 1, &:to_i
+
+  option "--watchdog-timeout", "SECONDS",
+    I18n.t("logstash.agent.flag.watchdog-timeout"),
+    :default => 10, &:to_f
+
+  option ["-l", "--log"], "FILE",
+    I18n.t("logstash.agent.flag.log"),
+    :attribute_name => :log_file
+
+  # Old support for the '-v' flag'
+  option "-v", :flag,
+    I18n.t("logstash.agent.flag.verbosity"),
+    :attribute_name => :verbosity, :multivalued => true
+
+  option "--quiet", :flag, I18n.t("logstash.agent.flag.quiet")
+  option "--verbose", :flag, I18n.t("logstash.agent.flag.verbose")
+  option "--debug", :flag, I18n.t("logstash.agent.flag.debug")
+
+  option ["-V", "--version"], :flag,
+    I18n.t("logstash.agent.flag.version")
+
+  option ["-p", "--pluginpath"] , "PATH",
+    I18n.t("logstash.agent.flag.pluginpath"),
+    :multivalued => true,
+    :attribute_name => :plugin_paths
+
+  option ["-t", "--configtest"], :flag,
+    I18n.t("logstash.agent.flag.configtest"),
+    :attribute_name => :config_test
+
+  # Emit a warning message.
+  def warn(message)
+    # For now, all warnings are fatal.
+    raise LogStash::ConfigurationError, message
+  end # def warn
+
+  # Emit a failure message and abort.
+  def fail(message)
+    raise LogStash::ConfigurationError, message
+  end # def fail
+
+  def report(message)
+    # Print to stdout just in case we're logging to a file
+    puts message
+    @logger.log(message) if log_file
+  end
+
+  # Run the agent. This method is invoked after clamp parses the
+  # flags given to this program.
+  def execute
+    require "logstash/pipeline"
+    require "cabin" # gem 'cabin'
+    require "logstash/plugin"
+    @logger = Cabin::Channel.get(LogStash)
+
+    if version?
+      show_version
+      return 0
+    end
+
+    # temporarily send logs to stdout as well if a --log is specified
+    # and stdout appears to be a tty
+    show_startup_errors = log_file && STDOUT.tty?
+
+    if show_startup_errors
+      stdout_logs = @logger.subscribe(STDOUT)
+    end
+    configure
+
+    # You must specify a config_string or config_path
+    if @config_string.nil? && @config_path.nil?
+      fail(help + "\n" + I18n.t("logstash.agent.missing-configuration"))
+    end
+
+    @config_string = @config_string.to_s
+
+    if @config_path
+      # Append the config string.
+      # This allows users to provide both -f and -e flags. The combination
+      # is rare, but useful for debugging.
+      @config_string = @config_string + load_config(@config_path)
+    else
+      # include a default stdin input if no inputs given
+      if @config_string !~ /input *{/
+        @config_string += "input { stdin { type => stdin } }"
+      end
+      # include a default stdout output if no outputs given
+      if @config_string !~ /output *{/
+        @config_string += "output { stdout { codec => rubydebug } }"
+      end
+    end
+
+    begin
+      pipeline = LogStash::Pipeline.new(@config_string)
+    rescue LoadError => e
+      fail("Configuration problem.")
+    end
+
+    # Make SIGINT shutdown the pipeline.
+    sigint_id = Stud::trap("INT") do
+      @logger.warn(I18n.t("logstash.agent.sigint"))
+      pipeline.shutdown
+    end
+
+    # Make SIGTERM shutdown the pipeline.
+    sigterm_id = Stud::trap("TERM") do
+      @logger.warn(I18n.t("logstash.agent.sigterm"))
+      pipeline.shutdown
+    end
+
+    Stud::trap("HUP") do
+      @logger.info(I18n.t("logstash.agent.sighup"))
+      configure_logging(log_file)
+    end
+
+    pipeline.configure("filter-workers", filter_workers)
+
+    # Stop now if we are only asking for a config test.
+    if config_test?
+      report "Configuration OK"
+      return
+    end
+
+    @logger.unsubscribe(stdout_logs) if show_startup_errors
+
+    # TODO(sissel): Get pipeline completion status.
+    pipeline.run
+    return 0
+  rescue LogStash::ConfigurationError => e
+    @logger.unsubscribe(stdout_logs) if show_startup_errors
+    report I18n.t("logstash.agent.error", :error => e)
+    if !config_test?
+      report I18n.t("logstash.agent.configtest-flag-information")
+    end
+    return 1
+  rescue => e
+    @logger.unsubscribe(stdout_logs) if show_startup_errors
+    report I18n.t("oops", :error => e)
+    report e.backtrace if @logger.debug? || $DEBUGLIST.include?("stacktrace")
+    return 1
+  ensure
+    @log_fd.close if @log_fd
+    Stud::untrap("INT", sigint_id) unless sigint_id.nil?
+    Stud::untrap("TERM", sigterm_id) unless sigterm_id.nil?
+  end # def execute
+
+  def show_version
+    show_version_logstash
+
+    if [:info, :debug].include?(verbosity?) || debug? || verbose?
+      show_version_ruby
+
+      if RUBY_PLATFORM == "java"
+        show_version_java
+        show_version_elasticsearch
+      end
+
+      if [:debug].include?(verbosity?) || debug?
+        show_gems
+      end
+    end
+  end # def show_version
+
+  def show_version_logstash
+    require "logstash/version"
+    puts "logstash #{LOGSTASH_VERSION}"
+  end # def show_version_logstash
+
+  def show_version_ruby
+    puts RUBY_DESCRIPTION
+  end # def show_version_ruby
+
+  def show_version_elasticsearch
+    LogStash::Environment.load_elasticsearch_jars!
+
+    $stdout.write("Elasticsearch: ");
+    org.elasticsearch.Version::main([])
+  end # def show_version_elasticsearch
+
+  def show_version_java
+    properties = java.lang.System.getProperties
+    puts "java #{properties["java.version"]} (#{properties["java.vendor"]})"
+    puts "jvm #{properties["java.vm.name"]} / #{properties["java.vm.version"]}"
+  end # def show_version_java
+
+  def show_gems
+    require "rubygems"
+    Gem::Specification.each do |spec|
+      puts "gem #{spec.name} #{spec.version}"
+    end
+  end # def show_gems
+
+  # Do any start-time configuration.
+  #
+  # Log file stuff, plugin path checking, etc.
+  def configure
+    configure_logging(log_file)
+    configure_plugin_path(plugin_paths) if !plugin_paths.nil?
+  end # def configure
+
+  # Point logging at a specific path.
+  def configure_logging(path)
+    # Set with the -v (or -vv...) flag
+    if quiet?
+      @logger.level = :error
+    elsif verbose?
+      @logger.level = :info
+    elsif debug?
+      @logger.level = :debug
+    else
+      # Old support for the -v and -vv stuff.
+      if verbosity? && verbosity?.any?
+        # this is an array with length of how many times the flag is given
+        if verbosity?.length == 1
+          @logger.warn("The -v flag is deprecated and will be removed in a future release. You should use --verbose instead.")
+          @logger.level = :info
+        else
+          @logger.warn("The -vv flag is deprecated and will be removed in a future release. You should use --debug instead.")
+          @logger.level = :debug
+        end
+      else
+        @logger.level = :warn
+      end
+
+    end
+
+    if log_file
+      # TODO(sissel): Implement file output/rotation in Cabin.
+      # TODO(sissel): Catch exceptions, report sane errors.
+      begin
+        @log_fd.close if @log_fd
+        @log_fd = File.new(path, "a")
+      rescue => e
+        fail(I18n.t("logstash.agent.configuration.log_file_failed",
+                    :path => path, :error => e))
+      end
+
+      puts "Sending logstash logs to #{path}."
+      @logger.unsubscribe(@logger_subscription) if @logger_subscription
+      @logger_subscription = @logger.subscribe(@log_fd)
+    else
+      @logger.subscribe(STDOUT)
+    end
+
+    # TODO(sissel): redirect stdout/stderr to the log as well
+    # http://jira.codehaus.org/browse/JRUBY-7003
+  end # def configure_logging
+
+  # Validate and add any paths to the list of locations
+  # logstash will look to find plugins.
+  def configure_plugin_path(paths)
+    # Append any plugin paths to the ruby search path
+    paths.each do |path|
+      # Verify the path exists
+      if !Dir.exists?(path)
+        warn(I18n.t("logstash.agent.configuration.plugin_path_missing",
+                    :path => path))
+
+      end
+
+      # TODO(sissel): Verify the path looks like the correct form.
+      # aka, there must be file in path/logstash/{inputs,codecs,filters,outputs}/*.rb
+      plugin_glob = File.join(path, "logstash", "{inputs,codecs,filters,outputs}", "*.rb")
+      if Dir.glob(plugin_glob).empty?
+        @logger.warn(I18n.t("logstash.agent.configuration.no_plugins_found",
+                    :path => path, :plugin_glob => plugin_glob))
+      end
+
+      # We push plugin paths to the front of the LOAD_PATH so that folks
+      # can override any core logstash plugins if they need to.
+      @logger.debug("Adding plugin path", :path => path)
+      $LOAD_PATH.unshift(path)
+    end
+  end # def configure_plugin_path
+
+  def load_config(path)
+    begin
+      uri = URI.parse(path)
+
+      case uri.scheme
+      when nil then
+        local_config(path)
+      when /http/ then
+        fetch_config(uri)
+      when "file" then
+        local_config(uri.path)
+      else
+        fail(I18n.t("logstash.agent.configuration.scheme-not-supported", :path => path))
+      end
+    rescue URI::InvalidURIError
+      # fallback for windows.
+      # if the parsing of the file failed we assume we can reach it locally.
+      # some relative path on windows arent parsed correctly (.\logstash.conf)
+      local_config(path)
+    end
+  end
+
+  def local_config(path)
+    path = File.expand_path(path)
+    path = File.join(path, "*") if File.directory?(path)
+
+    if Dir.glob(path).length == 0
+      fail(I18n.t("logstash.agent.configuration.file-not-found", :path => path))
+    end
+
+    config = ""
+    encoding_issue_files = []
+    Dir.glob(path).sort.each do |file|
+      next unless File.file?(file)
+      if file.match(/~$/)
+        @logger.debug("NOT reading config file because it is a temp file", :file => file)
+        next
+      end
+      @logger.debug("Reading config file", :file => file)
+      cfg = File.read(file)
+      if !cfg.ascii_only? && !cfg.valid_encoding?
+        encoding_issue_files << file
+      end
+      config << cfg + "\n"
+    end
+    if (encoding_issue_files.any?)
+      fail("The following config files contains non-ascii characters but are not UTF-8 encoded #{encoding_issue_files}")
+    end
+    return config
+  end # def load_config
+
+  def fetch_config(uri)
+    begin
+      Net::HTTP.get(uri) + "\n"
+    rescue Exception => e
+      fail(I18n.t("logstash.agent.configuration.fetch-failed", :path => uri.to_s, :message => e.message))
+    end
+  end
+
+end # class LogStash::Agent

data/lib/logstash/bundler.rb

@@ -0,0 +1,124 @@
+require "bundler"
+require "bundler/cli"
+
+module Bundler
+  # Patch bundler to write a .lock file specific to the version of ruby.
+  # This keeps MRI/JRuby/RBX from conflicting over the Gemfile.lock updates
+  module SharedHelpers
+    def default_lockfile
+      ruby = "#{LogStash::Environment.ruby_engine}-#{LogStash::Environment.ruby_abi_version}"
+      Pathname.new("#{default_gemfile}.#{ruby}.lock")
+    end
+  end
+
+  # Add the Bundler.reset! method which has been added in master but is not in 1.7.9.
+  class << self
+    unless self.method_defined?("reset!")
+      def reset!
+        @definition = nil
+      end
+    end
+
+    def reset_settings
+      # reset!
+      # clear_gemspec_cache
+
+      # remove_instance_variable(:@setup) if defined?(@setup)
+      # remove_instance_variable(:@locked_gems) if defined?(@locked_gems)
+      # remove_instance_variable(:@load) if defined?(@load)
+      remove_instance_variable(:@settings) if defined?(@settings)
+    end
+  end
+end
+
+module LogStash
+  module Bundler
+
+    # capture any $stdout from the passed block. also trap any exception in that block, in which case the trapped exception will be returned
+    # @param [Proc] the code block to execute
+    # @return [String, Exception] the captured $stdout string and any trapped exception or nil if none
+    def self.capture_stdout(&block)
+      old_stdout = $stdout
+      $stdout = StringIO.new("", "w")
+      begin
+        block.call
+      rescue => e
+        return [$stdout.string, e]
+      end
+
+      [$stdout.string, nil]
+    ensure
+      $stdout = old_stdout
+    end
+
+    # execute bundle install and capture any $stdout output. any raised exception in the process will be trapped
+    # and returned. logs errors to $stdout.
+    # @param options [Hash] invoke options with default values, :max_tries => 10, :clean => false, :install => false, :update => false
+    # @param   options[:update] must be either false or a String or an Array of String
+    # @return [String, Exception] the installation captured output and any raised exception or nil if none
+    def self.invoke_bundler!(options = {})
+      options = {:max_tries => 10, :clean => false, :install => false, :update => false, :without => [:development]}.merge(options)
+      options[:without] = Array(options[:without])
+      options[:update] = Array(options[:update]) if options[:update]
+
+      ENV["GEM_PATH"] = LogStash::Environment.logstash_gem_home
+      ENV["BUNDLE_PATH"] = LogStash::Environment::BUNDLE_DIR
+      ENV["BUNDLE_GEMFILE"] = LogStash::Environment::GEMFILE_PATH
+      ENV["BUNDLE_WITHOUT"] = options[:without].join(":")
+
+      try = 0
+
+      # capture_stdout also traps any raised exception and pass them back as the function return [output, exception]
+      capture_stdout do
+        loop do
+          begin
+            ::Bundler.reset!
+            ::Bundler::CLI.start(bundler_arguments(options))
+            break
+          rescue ::Bundler::VersionConflict => e
+            $stderr.puts("Plugin version conflict, aborting")
+            raise(e)
+          rescue ::Bundler::GemNotFound => e
+            $stderr.puts("Plugin not found, aborting")
+            raise(e)
+          rescue => e
+            if try >= options[:max_tries]
+              $stderr.puts("Too many retries, aborting, caused by #{e.class}")
+              $stderr.puts(e.message) if ENV["DEBUG"]
+              raise(e)
+            end
+
+            try += 1
+            $stderr.puts("Error #{e.class}, retrying #{try}/#{options[:max_tries]}")
+            $stderr.puts(e.message) if ENV["DEBUG"]
+            sleep(0.5)
+          end
+        end
+      end
+    end
+
+    # build Bundler::CLI.start arguments array from the given options hash
+    # @param option [Hash] the invoke_bundler! options hash
+    # @return [Array<String>] Bundler::CLI.start string arguments array
+    def self.bundler_arguments(options = {})
+      arguments = []
+
+      if options[:install]
+        arguments << "install"
+        arguments << "--gemfile=#{LogStash::Environment::GEMFILE_PATH}"
+        arguments << ["--path", LogStash::Environment::BUNDLE_DIR]
+        # note that generating "--without=" when options[:without] is empty is intended
+        arguments << "--without=#{options[:without].join(' ')}"
+      end
+
+      if options[:update]
+        arguments << "update"
+        arguments << options[:update]
+      end
+
+      arguments << "--clean" if options[:clean]
+
+      arguments.flatten
+    end
+  end
+end