logstash-core 5.0.0.alpha3.snapshot8-java → 5.0.0.alpha4.snapshot1-java

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 55ba0791d8ac74319ca218b52dc2e72ed670b15a
-  data.tar.gz: 22bf69cd7c1be717894980dce6a211d30b229a9a
+  metadata.gz: 34ea2c9c79b6a5828714ad66dc7ce558d7952280
+  data.tar.gz: 76dcab330cb125653bfe94ef514f82f80a08a974
 SHA512:
-  metadata.gz: d0e738d681e75a687f2060df0993c01c0c3ae04c8543b54d005e8a1ae39712e9a5e2bf221cfe946fc15b3c4df64f4403a3b344c6dc3a397948808ab4372182a0
-  data.tar.gz: 1d73274a7a405911f431a5624c7c6805fd64719f5a72c285e214440697f372f415aad7273de57fd92b6d8c37497f1a50a60ac8fd0db178929978ab231731e02f
+  metadata.gz: 7274fedd8a22a9518fec19299c53295243832c17a376c85f59c2ea6dfe85c139d2161b0f2d5e6854e348824c38b3427fa60ea4d0e7281a6232bfee628848e45f
+  data.tar.gz: f1432ba6f7d34a38c3fbf57a7ddc6f02fefc6916f1349775a3254f75d720735101ad24da81971f81edbf42cd091c783959b2e46e2e2502162e0c2f10d39742ec

data/lib/logstash-core/version.rb

@@ -5,4 +5,4 @@
 # Note to authors: this should not include dashes because 'gem' barfs if
 # you include a dash in the version string.
 
-LOGSTASH_CORE_VERSION = "5.0.0-alpha3.snapshot8"
+LOGSTASH_CORE_VERSION = "5.0.0-alpha4.snapshot1"

data/lib/logstash/agent.rb

@@ -38,6 +38,7 @@ class LogStash::Agent
     @node_name = setting("node.name")
     @http_host = setting("http.host")
     @http_port = setting("http.port")
+    @http_environment = setting("http.environment")
 
     @config_loader = LogStash::Config::Loader.new(@logger)
     @reload_interval = setting("config.reload.interval")
@@ -129,7 +130,7 @@ class LogStash::Agent
 
   private
   def start_webserver
-    options = {:http_host => @http_host, :http_port => @http_port }
+    options = {:http_host => @http_host, :http_port => @http_port, :http_environment => @http_environment }
     @webserver = LogStash::WebServer.new(@logger, options)
     Thread.new(@webserver) do |webserver|
       LogStash::Util.set_thread_name("Api Webserver")

data/lib/logstash/api/modules/base.rb

@@ -7,15 +7,16 @@ module LogStash
       class Base < ::Sinatra::Base
         helpers AppHelpers
 
+        # These options never change
+        # Sinatra isn't good at letting you change internal settings at runtime
+        # which is a requirement. We always propagate errors up and catch them
+        # in a custom rack handler in the RackApp class
         set :environment, :production
+        set :raise_errors, true
+        set :show_exceptions, false
 
         attr_reader :factory
 
-        if settings.environment != :production
-          set :raise_errors, true
-          set :show_exceptions, :after_handler
-        end
-
         include LogStash::Util::Loggable
 
         helpers AppHelpers
@@ -31,12 +32,6 @@ module LogStash
           text = as == :string ? "" : {}
           respond_with(text, :as => as)
         end
-
-        error do
-          e = env['sinatra.error']
-          logger.error(e.message, :url => request.url, :ip => request.ip, :params => request.params, :class => e.class.name, :backtrace => e.backtrace)
-        end
-
       end
     end
   end

data/lib/logstash/api/modules/node.rb

@@ -1,4 +1,6 @@
 # encoding: utf-8
+require "logstash/api/modules/base"
+
 module LogStash
   module Api
     module Modules

data/lib/logstash/api/rack_app.rb

@@ -1,3 +1,5 @@
+require "sinatra"
+require "rack"
 require "logstash/api/modules/base"
 require "logstash/api/modules/node"
 require "logstash/api/modules/node_stats"
@@ -8,9 +10,83 @@ require "logstash/api/modules/stats"
 module LogStash
   module Api
     module RackApp
-      def self.app
+      # Cabin is not compatible with CommonLogger, and this gives us more control anyway
+      METADATA_FIELDS = [:request_method, :path_info, :query_string, :http_version, :http_accept].freeze
+      def self.log_metadata(status, env)
+        METADATA_FIELDS.reduce({:status => status}) do |acc, field|
+          acc[field] = env[field.to_s.upcase]
+          acc
+        end        
+      end
+      
+      class ApiLogger
+        LOG_MESSAGE = "API HTTP Request".freeze
+        
+        def initialize(app, logger)
+          @app = app
+          @logger = logger
+        end
+
+        def call(env)
+          res = @app.call(env)
+          status, headers, body = res
+          
+          if fatal_error?(status)
+            @logger.warn? && @logger.warn(LOG_MESSAGE, RackApp.log_metadata(status, env))                      
+          else          
+            @logger.info? && @logger.info(LOG_MESSAGE, RackApp.log_metadata(status, env))                      
+          end
+
+          res          
+        end
+
+        def fatal_error?(status)
+          status >= 500 && status < 600
+        end
+      end
+      
+      class ApiErrorHandler
+        LOG_MESSAGE = "Internal API server error".freeze
+        
+        def initialize(app, logger)
+          @app = app
+          @logger = logger
+        end
+
+        def call(env)
+          @app.call(env)
+        rescue => e
+          body = RackApp.log_metadata(500, env).
+                   merge({
+                           :error => "Unexpected Internal Error",
+                           :class => e.class.name,
+                           :message => e.message,
+                           :backtrace => e.backtrace
+                         })
+
+          @logger.error(LOG_MESSAGE, body)
+          
+          [500,
+           {'Content-Type' => 'application/json'},
+           [LogStash::Json.dump(body)]
+          ]
+        end
+      end
+      
+      def self.app(logger, environment)
         namespaces = rack_namespaces 
         Rack::Builder.new do
+          # Custom logger object. Rack CommonLogger does not work with cabin
+          use ApiLogger, logger
+          
+          # In test env we want errors to propogate up the chain
+          # so we get easy to understand test failures.
+          # In production / dev we don't want a bad API endpoint
+          # to crash the process
+          if environment != "test"
+            use ApiErrorHandler, logger
+          end
+          
           run LogStash::Api::Modules::Root
           namespaces.each_pair do |namespace, app|
             map(namespace) do

data/lib/logstash/api/service.rb

@@ -26,7 +26,7 @@ module LogStash
       end
 
       def started?
-        !@snapshot.nil? && has_counters?
+        !@snapshot.nil? && has_counters?        
       end
 
       def update(snapshot)

data/lib/logstash/environment.rb

@@ -29,6 +29,7 @@ module LogStash
             Setting::String.new("log.format", "plain", true, ["json", "plain"]),
             Setting::String.new("http.host", "127.0.0.1"),
               Setting::Port.new("http.port", 9600),
+            Setting::String.new("http.environment", "production"),
   ].each {|setting| SETTINGS.register(setting) }
 
   module Environment

data/lib/logstash/runner.rb

@@ -138,7 +138,15 @@ class LogStash::Runner < Clamp::StrictCommand
 
     @settings.set("path.settings", settings_path) if settings_path
 
-    LogStash::SETTINGS.from_yaml(LogStash::SETTINGS.get("path.settings"))
+    begin
+      LogStash::SETTINGS.from_yaml(LogStash::SETTINGS.get("path.settings"))
+    rescue => e
+      @logger.subscribe(STDOUT)
+      @logger.warn("Logstash has a new settings file which defines start up time settings. This file is typically located in $LS_HOME/config or /etc/logstash. If you installed Logstash through a package and are starting it manually please specify the location to this settings file by passing in \"--path.settings=/path/..\" in the command line options")
+      @logger.fatal("Failed to load settings file from \"path.settings\". Aborting...", "path.settings" => LogStash::SETTINGS.get("path.settings"), "exception" => e.class, "message" => e.message)
+      exit(-1)
+    end
+
     super(*[args])
   end
 

data/lib/logstash/version.rb

@@ -11,4 +11,4 @@
 #       eventually this file should be in the root logstash lib fir and dependencies in logstash-core should be
 #       fixed.
 
-LOGSTASH_VERSION = "5.0.0-alpha3.snapshot8"
+LOGSTASH_VERSION = "5.0.0-alpha4.snapshot1"

data/lib/logstash/webserver.rb

@@ -1,25 +1,25 @@
 # encoding: utf-8
 require "puma"
 require "puma/server"
-require "sinatra"
-require "rack"
 require "logstash/api/rack_app"
 
 module LogStash 
   class WebServer
     extend Forwardable
 
-    attr_reader :logger, :status, :config, :options, :cli_options, :runner, :binder, :events, :http_host, :http_port
+    attr_reader :logger, :status, :config, :options, :cli_options, :runner, :binder, :events, :http_host, :http_port, :http_environment
 
     def_delegator :@runner, :stats
 
     DEFAULT_HOST = "127.0.0.1".freeze
     DEFAULT_PORT = 9600.freeze
+    DEFAULT_ENVIRONMENT = 'production'.freeze
 
     def initialize(logger, options={})
       @logger      = logger
       @http_host    = options[:http_host] || DEFAULT_HOST
       @http_port    = options[:http_port] || DEFAULT_PORT
+      @http_environment = options[:http_environment] || DEFAULT_ENVIRONMENT
       @options     = {}
       @cli_options = options.merge({ :rackup => ::File.join(::File.dirname(__FILE__), "api", "init.ru"),
                                      :binds => ["tcp://#{http_host}:#{http_port}"],
@@ -37,7 +37,8 @@ module LogStash
 
       stop # Just in case
 
-      @server = ::Puma::Server.new(LogStash::Api::RackApp.app)
+      app = LogStash::Api::RackApp.app(logger, http_environment)
+      @server = ::Puma::Server.new(app)
       @server.add_tcp_listener(http_host, http_port)
 
       @server.run.join

data/logstash-core.gemspec

@@ -17,7 +17,7 @@ Gem::Specification.new do |gem|
   gem.require_paths = ["lib"]
   gem.version       = LOGSTASH_CORE_VERSION.gsub(/-/, '.')
 
-  gem.add_runtime_dependency "logstash-core-event-java", "5.0.0.alpha3.snapshot8"
+  gem.add_runtime_dependency "logstash-core-event-java", "5.0.0.alpha4.snapshot1"
 
   gem.add_runtime_dependency "cabin", "~> 0.8.0" #(Apache 2.0 license)
   gem.add_runtime_dependency "pry", "~> 0.10.1"  #(Ruby license)

data/spec/api/lib/rack_app_spec.rb

@@ -0,0 +1,88 @@
+require "logstash/api/rack_app"
+require "rack/test"
+
+describe LogStash::Api::RackApp do
+  include Rack::Test::Methods
+
+  class DummyApp
+    class RaisedError < StandardError; end
+    
+    def call(env)
+      case env["PATH_INFO"]
+      when "/good-page"
+        [200, {}, ["200 OK"]]
+      when "/service-unavailable"
+        [503, {}, ["503 service unavailable"]]
+      when "/raise-error"
+        raise RaisedError, "Error raised"
+      else
+        [404, {}, ["404 Page not found"]]
+      end
+    end
+  end
+
+  let(:logger) { Cabin::Channel.get }
+
+  describe LogStash::Api::RackApp::ApiErrorHandler do
+    let(:app) do
+      # Scoping in rack builder is weird, these need to be locals
+      rack_class = described_class
+      rack_logger = logger
+      Rack::Builder.new do
+        use rack_class, rack_logger
+        run DummyApp.new
+      end
+    end
+
+    it "should let good requests through as normal" do
+      get "/good-page"
+      expect(last_response).to be_ok
+    end
+
+    it "should let through 5xx codes" do
+      get "/service-unavailable"
+      expect(last_response.status).to eql(503)
+    end
+
+    describe "raised exceptions" do
+      before do
+        allow(logger).to receive(:error).with(any_args)
+        get "/raise-error"
+      end
+      
+      it "should return a 500 error" do
+        expect(last_response.status).to eql(500)
+      end
+
+      it "should return valid JSON" do
+        expect { LogStash::Json.load(last_response.body) }.not_to raise_error
+      end
+
+      it "should log the error" do
+        expect(logger).to have_received(:error).with(LogStash::Api::RackApp::ApiErrorHandler::LOG_MESSAGE, anything).once
+      end
+    end
+  end
+
+  describe LogStash::Api::RackApp::ApiLogger do
+    let(:app) do
+      # Scoping in rack builder is weird, these need to be locals
+      rack_class = described_class
+      rack_logger = logger
+      Rack::Builder.new do
+        use rack_class, rack_logger
+        run DummyApp.new
+      end
+    end
+    
+    it "should log good requests as info" do
+      expect(logger).to receive(:info).with(LogStash::Api::RackApp::ApiLogger::LOG_MESSAGE, anything).once
+      get "/good-page"
+    end
+
+    it "should log 5xx requests as warnings" do
+      expect(logger).to receive(:warn).with(LogStash::Api::RackApp::ApiLogger::LOG_MESSAGE, anything).once
+      get "/service-unavailable"
+    end
+  end
+end

data/spec/api/spec_helper.rb

@@ -46,6 +46,7 @@ class LogStashRunner
       "log.level" => "debug",
       "node.name" => "test_agent",
       "http.port" => rand(9600..9700),
+      "http.environment" => "test",      
       "config.string" => @config_str,
       "pipeline.batch.size" => 1,
       "pipeline.workers" => 1
@@ -60,7 +61,8 @@ class LogStashRunner
     @runner = Thread.new(agent) do |_agent|
       _agent.execute
     end
-    wait_until_snapshot_received
+
+    wait_until_ready
   end
 
   def stop
@@ -71,8 +73,9 @@ class LogStashRunner
 
   private
 
-  def wait_until_snapshot_received
-    while !LogStash::Api::Service.instance.started? do
+  def wait_until_ready
+    # Wait until the service and pipeline have started
+    while !(LogStash::Api::Service.instance.started? && agent.pipelines["main"].running?) do
       sleep 0.5
     end
   end

data/spec/logstash/agent_spec.rb

@@ -373,7 +373,7 @@ describe LogStash::Agent do
       @t.join
     end
 
-    xit "resets the metric collector" do
+    it "resets the metric collector" do
       # We know that the store has more events coming in.
       sleep(0.01) while dummy_output.events.size < new_config_generator_counter
       snapshot = LogStash::Instrument::Collector.instance.snapshot_metric

metadata

@@ -1,21 +1,21 @@
 --- !ruby/object:Gem::Specification
 name: logstash-core
 version: !ruby/object:Gem::Version
-  version: 5.0.0.alpha3.snapshot8
+  version: 5.0.0.alpha4.snapshot1
 platform: java
 authors:
 - Elastic
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2016-05-28 00:00:00.000000000 Z
+date: 2016-06-03 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - '='
       - !ruby/object:Gem::Version
-        version: 5.0.0.alpha3.snapshot8
+        version: 5.0.0.alpha4.snapshot1
   name: logstash-core-event-java
   prerelease: false
   type: :runtime
@@ -23,7 +23,7 @@ dependencies:
     requirements:
     - - '='
       - !ruby/object:Gem::Version
-        version: 5.0.0.alpha3.snapshot8
+        version: 5.0.0.alpha4.snapshot1
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
     requirements:
@@ -388,6 +388,7 @@ files:
 - spec/api/lib/api/root_spec.rb
 - spec/api/lib/api/support/resource_dsl_methods.rb
 - spec/api/lib/commands/stats.rb
+- spec/api/lib/rack_app_spec.rb
 - spec/api/spec_helper.rb
 - spec/conditionals_spec.rb
 - spec/logstash/agent_spec.rb
@@ -464,6 +465,7 @@ test_files:
 - spec/api/lib/api/root_spec.rb
 - spec/api/lib/api/support/resource_dsl_methods.rb
 - spec/api/lib/commands/stats.rb
+- spec/api/lib/rack_app_spec.rb
 - spec/api/spec_helper.rb
 - spec/conditionals_spec.rb
 - spec/logstash/agent_spec.rb