logstash-core 2.4.1-java → 5.0.0.alpha1-java

data/lib/logstash/util.rb

@@ -184,6 +184,15 @@ module LogStash::Util
     end
   end
 
+  # Take a instance reference and return the name of the class
+  # stripping all the modules.
+  #
+  # @param [Object] The object to return the class)
+  # @return [String] The name of the class
+  def self.class_name(instance)
+    instance.class.name.split("::").last
+  end
+
   def self.deep_clone(o)
     case o
     when Hash

data/lib/logstash/util/duration_formatter.rb

@@ -0,0 +1,15 @@
+# encoding: utf-8
+require "chronic_duration"
+module LogStash::Util::DurationFormatter
+  CHRONIC_OPTIONS = { :format => :short }
+
+  # Take a duration in milliseconds and transform it into
+  # a format that a human can understand. This is currently used by
+  # the API.
+  #
+  # @param [Fixnum] Duration in milliseconds
+  # @return [String] Duration in human format
+  def self.human_format(duration)
+    ChronicDuration.output(duration / 1000, CHRONIC_OPTIONS)
+  end
+end

data/lib/logstash/util/java_version.rb

@@ -9,7 +9,7 @@ module LogStash::Util::JavaVersion
   # Print a warning if we're on a bad version of java
   def self.warn_on_bad_java_version
     if self.bad_java_version?(self.version)
-      msg = "!!! Please upgrade your java version, the current version '#{self.version}' may cause problems. We recommend a minimum version of 1.7.0_51"
+      msg = "!!! Please upgrade your java version, the current version '#{self.version}' is not supported. We recommend a minimum version of Java 8"
       STDERR.puts(msg)
       logger.warn(msg)
     end
@@ -55,9 +55,7 @@ module LogStash::Util::JavaVersion
     parsed = parse_java_version(version_string)
     return false unless parsed
 
-    if parsed[:major] == 1 && parsed[:minor] == 7 && parsed[:patch] == 0 && parsed[:update] < 51
-      true
-    elsif parsed[:major] == 1 && parsed[:minor] < 7
+    if parsed[:major] == 1 && parsed[:minor] < 8
       true
     else
       false

data/lib/logstash/util/loggable.rb

@@ -0,0 +1,29 @@
+# encoding: utf-8
+require "logstash/namespace"
+require "cabin"
+
+module LogStash module Util
+  module Loggable
+    class << self
+      def logger=(new_logger)
+        @logger = new_logger
+      end
+
+      def logger
+        @logger ||= Cabin::Channel.get(LogStash)
+      end
+    end
+
+    def self.included(base)
+      class << base
+        def logger
+          Loggable.logger
+        end
+      end
+    end
+
+    def logger
+      Loggable.logger
+    end
+  end
+end; end

data/lib/logstash/version.rb

@@ -11,4 +11,4 @@
 #       eventually this file should be in the root logstash lib fir and dependencies in logstash-core should be
 #       fixed.
 
-LOGSTASH_VERSION = "2.4.1"
+LOGSTASH_VERSION = "5.0.0-alpha1"

data/lib/logstash/webserver.rb

@@ -0,0 +1,98 @@
+# encoding: utf-8
+require "puma"
+require "puma/single"
+require "puma/binder"
+require "puma/configuration"
+require "puma/commonlogger"
+
+module LogStash 
+  class WebServer
+
+    extend Forwardable
+
+    attr_reader :logger, :status, :config, :options, :cli_options, :runner, :binder, :events
+
+    def_delegator :@runner, :stats
+
+    DEFAULT_HOST = "127.0.0.1".freeze
+    DEFAULT_PORT = 9600.freeze
+
+    def initialize(logger, options={})
+      @logger      = logger
+      http_host    = options[:http_host] || DEFAULT_HOST
+      http_port    = options[:http_port] || DEFAULT_PORT
+      @options     = {}
+      @cli_options = options.merge({ :rackup => ::File.join(::File.dirname(__FILE__), "api", "init.ru"),
+                                     :binds => ["tcp://#{http_host}:#{http_port}"],
+                                     :debug => logger.debug?,
+                                     # Prevent puma from queueing request when not able to properly handling them,
+                                     # fixed https://github.com/elastic/logstash/issues/4674. See
+                                     # https://github.com/puma/puma/pull/640 for mode internal details in PUMA.
+                                     :queue_requests => false
+      })
+      @status      = nil
+
+      parse_options
+
+      @runner  = nil
+      @events  = ::Puma::Events.strings
+      @binder  = ::Puma::Binder.new(@events)
+      @binder.import_from_env
+
+      set_environment
+    end
+
+    def run
+      log "=== puma start: #{Time.now} ==="
+
+      @runner = Puma::Single.new(self)
+      @status = :run
+      @runner.run
+      stop(:graceful => true)
+    end
+
+    def log(str)
+      logger.debug(str)
+    end
+
+    def error(str)
+      logger.error(str)
+    end
+
+    # Empty method, this method is required because of the puma usage we make through
+    # the Single interface, https://github.com/puma/puma/blob/master/lib/puma/single.rb#L82
+    # for more details. This can always be implemented when we want to keep track of this
+    # bit of data.
+    def write_state; end
+
+    def stop(options={})
+      graceful = options.fetch(:graceful, true)
+
+      if graceful
+        @runner.stop_blocked
+      else
+        @runner.stop
+      end rescue nil
+
+      @status = :stop
+      log "=== puma shutdown: #{Time.now} ==="
+    end
+
+    private
+
+    def env
+      @options[:debug] ? "development" : "production"
+    end
+
+    def set_environment
+      @options[:environment] = env
+      ENV['RACK_ENV']        = env
+    end
+
+    def parse_options
+      @config  = ::Puma::Configuration.new(cli_options)
+      @config.load
+      @options = @config.options
+    end
+  end
+end

data/locales/en.yml

@@ -6,6 +6,8 @@ en:
   oops: |-
     An unexpected error occurred!
   logstash:
+    error: >-
+      Error: %{error}
     environment:
       jruby-required:  >-
         JRuby is required
@@ -56,15 +58,6 @@ en:
     agent:
       sighup: >-
         SIGHUP received.
-      missing-configuration: >-
-        No configuration file was specified. Perhaps you forgot to provide
-        the '-f yourlogstash.conf' flag?
-      invalid-configuration: >-
-        The given configuration is invalid. Reason: %{error}
-      reload-without-config-path: >-
-        Configuration reloading also requires passing a configuration path with '-f yourlogstash.conf'
-      error: >-
-        Error: %{error}
       sigint: >-
         SIGINT received. Shutting down the agent.
       sigterm: >-
@@ -78,11 +71,38 @@ en:
       non_reloadable_config_reload: >-
         Unable to reload configuration because it does not support dynamic reloading
       non_reloadable_config_register: |-
-        Logstash is not able to start since configuration auto reloading was enabled but the configuration contains plugins that don't support it. Quitting...
+        Logstash was not able to load configuration since it does not support
+        dynamic reloading and -r or --auto-reload flag was enabled
+    web_api:
+      flag:
+        http_host: Web API binding host
+        http_port: Web API http port
+      hot_threads:
+        title: |-
+          ::: {%{hostname}}
+            Hot threads at %{time}, busiestThreads=%{top_count}:
+        thread_title: |-
+            %{percent_of_cpu_time} % of cpu usage by %{thread_state} thread named '%{thread_name}'
+    runner:
+      short-help: |-
+        usage:
+          bin/logstash -f CONFIG_PATH [-t] [-r] [--quiet|verbose|debug] [-w COUNT] [-l LOG]
+          bin/logstash -e CONFIG_STR [-t] [--quiet|verbose|debug] [-w COUNT] [-l LOG]
+          bin/logstash -i SHELL [--quiet|verbose|debug]
+          bin/logstash -V [--verbose|debug]
+          bin/logstash --help
+      invalid-configuration: >-
+        The given configuration is invalid. Reason: %{error}
+      missing-configuration: >-
+        No configuration file was specified. Perhaps you forgot to provide
+        the '-f yourlogstash.conf' flag?
+      reload-without-config-path: >-
+        Configuration reloading also requires passing a configuration path with '-f yourlogstash.conf'
+      invalid-shell: >-
+        Invalid option for interactive Ruby shell. Use either "irb" or "pry"
       configtest-flag-information: |-
-        You may be interested in the '--configtest' flag which you can
-        use to validate logstash's configuration before you choose
-        to restart a running system.
+        You may be interested in the '--configtest' flag which you can use to validate
+        logstash's configuration before you choose to restart a running system.
       configuration:
         obsolete: >-
           The setting `%{name}` in plugin `%{plugin}` is obsolete and is no
@@ -168,8 +188,6 @@ en:
           with the respective environment variable value named "VAR".
         pipeline-workers: |+
           Sets the number of pipeline workers to run.
-        filterworkers: |+
-          DEPRECATED. Now an alias for --pipeline-workers and -w
         pipeline-batch-size: |+
           Size of batches the pipeline is to work in.
         pipeline-batch-delay: |+
@@ -212,20 +230,20 @@ en:
         debug: |+
           Most verbose logging. This causes 'debug'
           level logs to be emitted.
-        debug-config: |+
-          Print the compiled config ruby code out as a debug log (you must also have --debug enabled).
-          WARNING: This will include any 'password' options passed to plugin configs as plaintext, and may result
-          in plaintext passwords appearing in your logs!
         unsafe_shutdown: |+
           Force logstash to exit during shutdown even
           if there are still inflight events in memory.
           By default, logstash will refuse to quit until all
           received events have been pushed to the outputs.
+        rubyshell: |+
+          Drop to shell instead of running as normal.
+          Valid shells are "irb" and "pry"
+        node_name: |+
+          Specify the name of this logstash instance, if no value is given
+          it will default to the current hostname.
+        agent: |+
+          Specify an alternate agent plugin name.
         debug_config: |+
           Print the compiled config ruby code out as a debug log (you must also have --debug enabled).
           WARNING: This will include any 'password' options passed to plugin configs as plaintext, and may result
-          in plaintext passwords appearing in your logs!
-        log-in-json: |+
-          Specify that Logstash should write its own logs in JSON form - one
-          event per line. If false, Logstash will log using Ruby's
-          Object#inspect (not easy to machine-parse)
+          in plaintext passwords appearing in your logs!

data/logstash-core.gemspec

@@ -11,14 +11,13 @@ Gem::Specification.new do |gem|
   gem.homepage      = "http://www.elastic.co/guide/en/logstash/current/index.html"
   gem.license       = "Apache License (2.0)"
 
-  gem.files         = Dir.glob(["logstash-core.gemspec", "lib/**/*.rb", "spec/**/*.rb", "locales/*"])
+  gem.files         = Dir.glob(["logstash-core.gemspec", "lib/**/*.rb", "spec/**/*.rb", "locales/*", "lib/logstash/api/init.ru"])
   gem.test_files    = gem.files.grep(%r{^(test|spec|features)/})
   gem.name          = "logstash-core"
   gem.require_paths = ["lib"]
-  gem.version       = LOGSTASH_CORE_VERSION
+  gem.version       = LOGSTASH_CORE_VERSION.gsub(/-/, '.')
 
-  gem.add_runtime_dependency "logstash-core-event", "2.4.1"
-  # gem.add_runtime_dependency "logstash-core-event-java", "2.4.0.dev"
+  gem.add_runtime_dependency "logstash-core-event-java", "~> 5.0.0.alpha1"
 
   gem.add_runtime_dependency "cabin", "~> 0.8.0" #(Apache 2.0 license)
   gem.add_runtime_dependency "pry", "~> 0.10.1"  #(Ruby license)
@@ -26,8 +25,12 @@ Gem::Specification.new do |gem|
   gem.add_runtime_dependency "clamp", "~> 0.6.5" #(MIT license) for command line args/flags
   gem.add_runtime_dependency "filesize", "0.0.4" #(MIT license) for :bytes config validator
   gem.add_runtime_dependency "gems", "~> 0.8.3"  #(MIT license)
-  gem.add_runtime_dependency "concurrent-ruby", "0.9.2"
-  gem.add_runtime_dependency "jruby-openssl", "0.9.16" # >= 0.9.13 Required to support TLSv1.2
+  gem.add_runtime_dependency "concurrent-ruby", "1.0.0"
+  gem.add_runtime_dependency "sinatra", '~> 1.4', '>= 1.4.6'
+  gem.add_runtime_dependency 'puma', '~> 2.16', '>= 2.16.0'
+  gem.add_runtime_dependency "jruby-openssl", "0.9.13" # Required to support TLSv1.2
+  gem.add_runtime_dependency "chronic_duration", "0.10.6"
+  gem.add_runtime_dependency "jruby-monitoring", '~> 0.3.1'
 
   # TODO(sissel): Treetop 1.5.x doesn't seem to work well, but I haven't
   # investigated what the cause might be. -Jordan

data/spec/api/lib/api/node_spec.rb

@@ -0,0 +1,64 @@
+# encoding: utf-8
+require_relative "../../spec_helper"
+require "sinatra"
+require "app/modules/node"
+require "logstash/json"
+
+describe LogStash::Api::Node do
+
+  include Rack::Test::Methods
+
+  def app()
+    described_class
+  end
+
+  describe "#hot threads" do
+
+    before(:all) do
+      do_request { get "/hot_threads" }
+    end
+
+    it "respond OK" do
+      expect(last_response).to be_ok
+    end
+
+    it "should return a JSON object" do
+      expect{ LogStash::Json.load(last_response.body) }.not_to raise_error
+    end
+
+    context "#threads count" do
+
+      before(:all) do
+        do_request { get "/hot_threads?threads=5" }
+      end
+
+      let(:payload) { LogStash::Json.load(last_response.body) }
+
+      it "should return a json payload content type" do
+        expect(last_response.content_type).to eq("application/json")
+      end
+
+      it "should return information for <= # requested threads" do
+        expect(payload["threads"].count).to be <= 5
+      end
+    end
+
+    context "when asking for human output" do
+
+      before(:all) do
+        do_request { get "/hot_threads?human" }
+      end
+
+      let(:payload) { last_response.body }
+
+      it "should return a text/plain content type" do
+        expect(last_response.content_type).to eq("text/plain;charset=utf-8")
+      end
+
+      it "should return a plain text payload" do
+        expect{ JSON.parse(payload) }.to raise_error
+      end
+    end
+
+  end
+end

data/spec/api/lib/api/node_stats_spec.rb

@@ -0,0 +1,68 @@
+# encoding: utf-8
+require_relative "../../spec_helper"
+require "sinatra"
+require "app/modules/node_stats"
+require "logstash/json"
+
+describe LogStash::Api::NodeStats do
+
+  include Rack::Test::Methods
+
+  def app()
+    described_class
+  end
+
+  let(:payload) { LogStash::Json.load(last_response.body) }
+
+  context "#root" do
+
+    before(:all) do
+      do_request { get "/" }
+    end
+
+    it "respond OK" do
+      expect(last_response).to be_ok
+    end
+
+    ["events", "jvm"].each do |key|
+      it "contains #{key} information" do
+        expect(payload).to include(key)
+      end
+    end
+  end
+
+  context "#events" do
+
+    let(:payload) { LogStash::Json.load(last_response.body) }
+
+    before(:all) do
+      do_request { get "/events" }
+    end
+
+    it "respond OK" do
+      expect(last_response).to be_ok
+    end
+
+    it "contains events information" do
+      expect(payload).to include("events")
+    end
+  end
+
+  context "#jvm" do
+
+    let(:payload) { LogStash::Json.load(last_response.body) }
+
+    before(:all) do
+      do_request { get "/jvm" }
+    end
+
+    it "respond OK" do
+      expect(last_response).to be_ok
+    end
+
+    it "contains memory information" do
+      expect(payload).to include("mem")
+    end
+  end
+
+end