logstash-core 5.0.0.alpha6.snapshot5-java → 5.0.0-java

data/lib/logstash/version.rb

@@ -11,4 +11,4 @@
 #       eventually this file should be in the root logstash lib fir and dependencies in logstash-core should be
 #       fixed.
 
-LOGSTASH_VERSION = "5.0.0.alpha6.snapshot5"
+LOGSTASH_VERSION = "5.0.0"

data/lib/logstash/webserver.rb

@@ -2,7 +2,9 @@
 require "logstash/api/rack_app"
 require "puma"
 require "puma/server"
+require "logstash/patches/puma"
 require "concurrent"
+require "thread"
 
 module LogStash
   class WebServer
@@ -71,12 +73,21 @@ module LogStash
     end
 
     def start_webserver(port)
+      # wrap any output that puma could generate into a wrapped logger
+      # use the puma namespace to override STDERR, STDOUT in that scope.
+      Puma::STDERR.logger = logger
+      Puma::STDOUT.logger = logger
+
+      io_wrapped_logger = LogStash::IOWrappedLogger.new(logger)
+
       app = LogStash::Api::RackApp.app(logger, agent, http_environment)
 
-      @server = ::Puma::Server.new(app)
+      events = ::Puma::Events.new(io_wrapped_logger, io_wrapped_logger)
+
+      @server = ::Puma::Server.new(app, events)
       @server.add_tcp_listener(http_host, port)
 
-      logger.info("Succesfully started Logstash API", :port => @port)
+      logger.info("Successfully started Logstash API endpoint", :port => @port)
 
       @server.run.join
     end

data/locales/en.yml

@@ -74,15 +74,18 @@ en:
         Logstash is not able to start since configuration auto reloading was enabled but the configuration contains plugins that don't support it. Quitting...
     web_api:
       cant_bind_to_port: |-
-        Logstash tried to bind to port %{port}, but the port is already in use. You can specify a new port by launching logtash with the --http-port option."
+        Logstash tried to bind to port %{port}, but the port is already in use. You can specify a new port by launching logtash with the --http.port option."
       cant_bind_to_port_in_range: |-
-        Logstash tried to bind to port range %{http_ports}, but all the ports are already in use. You can specify a new port by launching logtash with the --http-port option."
+        Logstash tried to bind to port range %{http_ports}, but all the ports are already in use. You can specify a new port by launching logtash with the --http.port option."
       hot_threads:
         title: |-
           ::: {%{hostname}}
           Hot threads at %{time}, busiestThreads=%{top_count}:
         thread_title: |-
           %{percent_of_cpu_time} % of cpu usage, state: %{thread_state}, thread name: '%{thread_name}'
+      logging:
+        unrecognized_option: |-
+          unrecognized option [%{option}]
     runner:
       short-help: |-
         usage:
@@ -194,6 +197,8 @@ en:
         path_settings: |+
           Directory containing logstash.yml file. This can also be
           set through the LS_SETTINGS_DIR environment variable.
+        path_logs: |+
+            Directory to Write Logstash internal logs to.
         auto_reload: |+
           Monitor configuration changes and reload
           whenever it is changed.

data/logstash-core.gemspec

@@ -17,7 +17,7 @@ Gem::Specification.new do |gem|
   gem.require_paths = ["lib", "vendor/jars"]
   gem.version       = LOGSTASH_CORE_VERSION
 
-  gem.add_runtime_dependency "logstash-core-event-java", "5.0.0.alpha6.snapshot5"
+  gem.add_runtime_dependency "logstash-core-event-java", "5.0.0"
 
   gem.add_runtime_dependency "pry", "~> 0.10.1"  #(Ruby license)
   gem.add_runtime_dependency "stud", "~> 0.0.19" #(Apache 2.0 license)

data/spec/api/lib/api/logging_spec.rb

@@ -0,0 +1,41 @@
+# encoding: utf-8
+require_relative "../../spec_helper"
+require "sinatra"
+require "logstash/api/modules/logging"
+require "logstash/json"
+
+describe LogStash::Api::Modules::Logging do
+  include_context "api setup"
+
+  describe "#logging" do
+
+    context "when setting a logger's log level" do
+      before(:all) do
+        @runner = LogStashRunner.new
+        @runner.start
+      end
+
+      after(:all) do
+        @runner.stop
+      end
+
+      it "should return a positive acknowledgement on success" do
+        put '/', '{"logger.logstash": "ERROR"}'
+        payload = LogStash::Json.load(last_response.body)
+        expect(payload['acknowledged']).to eq(true)
+      end
+
+      it "should throw error when level is invalid" do
+        put '/', '{"logger.logstash": "invalid"}'
+        payload = LogStash::Json.load(last_response.body)
+        expect(payload['error']).to eq("invalid level[invalid] for logger[logstash]")
+      end
+
+      it "should throw error when key logger is invalid" do
+        put '/', '{"invalid" : "ERROR"}'
+        payload = LogStash::Json.load(last_response.body)
+        expect(payload['error']).to eq("unrecognized option [invalid]")
+      end
+    end
+  end
+end

data/spec/api/lib/api/node_plugins_spec.rb

@@ -1,4 +1,5 @@
 # encoding: utf-8
+require_relative "../../../support/shared_examples"
 require_relative "../../spec_helper"
 require "sinatra"
 require "logstash/api/modules/plugins"
@@ -6,10 +7,11 @@ require "logstash/json"
 
 describe LogStash::Api::Modules::Plugins do
   include_context "api setup"
+  include_examples "not found"
 
   extend ResourceDSLMethods
 
-  before(:all) do
+  before(:each) do
     do_request { get "/" }
   end
 
@@ -20,13 +22,12 @@ describe LogStash::Api::Modules::Plugins do
       expect(last_response).to be_ok
     end
 
-    it "should return a list of plugins" do      
+    it "should return a list of plugins" do
       expect(payload["plugins"]).to be_a(Array)
     end
 
     it "should return the total number of plugins" do
       expect(payload["total"]).to be_a(Numeric)
     end
-    
   end
 end

data/spec/api/lib/api/node_spec.rb

@@ -1,11 +1,13 @@
 # encoding: utf-8
 require_relative "../../spec_helper"
+require_relative "../../../support/shared_examples"
 require "sinatra"
 require "logstash/api/modules/node"
 require "logstash/json"
 
 describe LogStash::Api::Modules::Node do
   include_context "api setup"
+  include_examples "not found"
 
   describe "#hot threads" do
 

data/spec/api/lib/api/node_stats_spec.rb

@@ -1,11 +1,13 @@
 # encoding: utf-8
 require_relative "../../spec_helper"
+require_relative "../../../support/shared_examples"
 require "sinatra"
 require "logstash/api/modules/node_stats"
 require "logstash/json"
 
 describe LogStash::Api::Modules::NodeStats do
   include_context "api setup"
+  include_examples "not found"
 
   extend ResourceDSLMethods
 

data/spec/api/lib/api/plugins_spec.rb

@@ -1,13 +1,15 @@
 # encoding: utf-8
 require_relative "../../spec_helper"
+require_relative "../../../support/shared_examples"
 require "sinatra"
 require "logstash/api/modules/plugins"
 require "logstash/json"
 
 describe LogStash::Api::Modules::Plugins do
   include_context "api setup"
+  include_examples "not found"
 
-  before(:all) do
+  before(:each) do
     get "/"
   end
 

data/spec/api/lib/api/root_spec.rb

@@ -1,5 +1,6 @@
 # encoding: utf-8
 require_relative "../../spec_helper"
+require_relative "../../../support/shared_examples"
 require "sinatra"
 require "logstash/api/modules/root"
 require "logstash/json"
@@ -11,5 +12,7 @@ describe LogStash::Api::Modules::Root do
     do_request { get "/" }
     expect(last_response).to be_ok
   end
+
+  include_examples "not found"
 end
 

data/spec/api/lib/errors_spec.rb

@@ -0,0 +1,27 @@
+# encoding: utf-8
+require_relative "../spec_helper"
+require "logstash/api/errors"
+
+describe LogStash::Api::ApiError do
+  subject { described_class.new }
+
+  it "#status_code returns 500" do
+    expect(subject.status_code).to eq(500)
+  end
+
+  it "#to_hash return the message of the exception" do
+    expect(subject.to_hash).to include(:message => "Api Error")
+  end
+end
+
+describe LogStash::Api::NotFoundError do
+  subject { described_class.new }
+
+  it "#status_code returns 404" do
+    expect(subject.status_code).to eq(404)
+  end
+
+  it "#to_hash return the message of the exception" do
+    expect(subject.to_hash).to include(:message => "Not Found")
+  end
+end

data/spec/api/lib/rack_app_spec.rb

@@ -76,14 +76,14 @@ describe LogStash::Api::RackApp do
     end
 
     it "should log good requests as info" do
-      expect(logger).to receive(:info?).and_return(true)
-      expect(logger).to receive(:info).with(LogStash::Api::RackApp::ApiLogger::LOG_MESSAGE, anything).once
+      expect(logger).to receive(:debug?).and_return(true)
+      expect(logger).to receive(:debug).with(LogStash::Api::RackApp::ApiLogger::LOG_MESSAGE, anything).once
       get "/good-page"
     end
 
     it "should log 5xx requests as warnings" do
-      expect(logger).to receive(:warn?).and_return(true)
-      expect(logger).to receive(:warn).with(LogStash::Api::RackApp::ApiLogger::LOG_MESSAGE, anything).once
+      expect(logger).to receive(:error?).and_return(true)
+      expect(logger).to receive(:error).with(LogStash::Api::RackApp::ApiLogger::LOG_MESSAGE, anything).once
       get "/service-unavailable"
     end
   end

data/spec/logstash/agent_spec.rb

@@ -47,11 +47,11 @@ describe LogStash::Agent do
     let(:pipeline_id) { "main" }
     let(:config_string) { "input { } filter { } output { }" }
     let(:agent_args) do
-      { 
+      {
         "config.string" => config_string,
         "config.reload.automatic" => true,
         "config.reload.interval" => 0.01,
-	"pipeline.workers" => 4,
+        "pipeline.workers" => 4,
       }
     end
 
@@ -331,8 +331,6 @@ describe LogStash::Agent do
 
   context "metrics after config reloading" do
     let(:config) { "input { generator { } } output { dummyoutput { } }" }
-    let(:new_config_generator_counter) { 500 }
-    let(:new_config) { "input { generator { count => #{new_config_generator_counter} } } output { dummyoutput2 {} }" }
     let(:config_path) do
       f = Stud::Temporary.file
       f.write(config)
@@ -351,7 +349,7 @@ describe LogStash::Agent do
       super.merge({ "config.reload.automatic" => true,
                     "config.reload.interval" => interval,
                     "metric.collect" => true })
-    end 
+    end
 
     # We need to create theses dummy classes to know how many
     # events where actually generated by the pipeline and successfully send to the output.
@@ -390,33 +388,121 @@ describe LogStash::Agent do
       end
     end
 
-    it "resets the metric collector" do
-      # We know that the store has more events coming in.
-      i = 0
-      while dummy_output.events.size <= new_config_generator_counter
-        i += 1
-        raise "Waiting too long!" if i > 20
-        sleep(0.1)
+    context "when reloading a good config" do
+      let(:new_config_generator_counter) { 500 }
+      let(:new_config) { "input { generator { count => #{new_config_generator_counter} } } output { dummyoutput2 {} }" }
+      before :each do
+        # We know that the store has more events coming in.
+        i = 0
+        while dummy_output.events.size <= new_config_generator_counter
+          i += 1
+          raise "Waiting too long!" if i > 20
+          sleep(0.1)
+        end
+
+
+        # Also force a flush to disk to make sure ruby reload it.
+        File.open(config_path, "w") do |f|
+          f.write(new_config)
+          f.fsync
+        end
+
+        sleep(interval * 3) # Give time to reload the config
+
+        # be eventually consistent.
+        sleep(0.01) while dummy_output2.events.size < new_config_generator_counter
       end
 
-      snapshot = subject.metric.collector.snapshot_metric
-      expect(snapshot.metric_store.get_with_path("/stats/events")[:stats][:events][:in].value).to be > new_config_generator_counter
+      it "resets the pipeline metric collector" do
+        snapshot = subject.metric.collector.snapshot_metric
+        value = snapshot.metric_store.get_with_path("/stats/pipelines")[:stats][:pipelines][:main][:events][:in].value
+        expect(value).to eq(new_config_generator_counter)
+      end
 
-      # update the configuration and give some time to logstash to pick it up and do the work
-      # Also force a flush to disk to make sure ruby reload it.
-      File.open(config_path, "w") do |f|
-        f.write(new_config)
-        f.fsync
+      it "does not reset the global event count" do
+        snapshot = subject.metric.collector.snapshot_metric
+        value = snapshot.metric_store.get_with_path("/stats/events")[:stats][:events][:in].value
+        expect(value).to be > new_config_generator_counter
       end
 
-      sleep(interval * 3) # Give time to reload the config
-      
-      # be eventually consistent.
-      sleep(0.01) while dummy_output2.events.size < new_config_generator_counter
+      it "increases the successful reload count" do
+        snapshot = subject.metric.collector.snapshot_metric
+        value = snapshot.metric_store.get_with_path("/stats/pipelines")[:stats][:pipelines][:main][:reloads][:successes].value
+        expect(value).to be(1)
+      end
 
-      snapshot = subject.metric.collector.snapshot_metric
-      value = snapshot.metric_store.get_with_path("/stats/events")[:stats][:events][:in].value
-      expect(value).to eq(new_config_generator_counter)
+      it "does not set the failure reload timestamp" do
+        snapshot = subject.metric.collector.snapshot_metric
+        value = snapshot.metric_store.get_with_path("/stats/pipelines")[:stats][:pipelines][:main][:reloads][:last_failure_timestamp].value
+        expect(value).to be(nil)
+      end
+
+      it "sets the success reload timestamp" do
+        snapshot = subject.metric.collector.snapshot_metric
+        value = snapshot.metric_store.get_with_path("/stats/pipelines")[:stats][:pipelines][:main][:reloads][:last_success_timestamp].value
+        expect(value).to be_a(LogStash::Timestamp)
+      end
+
+      it "does not set the last reload error" do
+        snapshot = subject.metric.collector.snapshot_metric
+        value = snapshot.metric_store.get_with_path("/stats/pipelines")[:stats][:pipelines][:main][:reloads][:last_error].value
+        expect(value).to be(nil)
+      end
+
+    end
+
+    context "when reloading a bad config" do
+      let(:new_config) { "input { generator { count => " }
+      let(:new_config_generator_counter) { 500 }
+      before :each do
+        # We know that the store has more events coming in.
+        i = 0
+        while dummy_output.events.size <= new_config_generator_counter
+          i += 1
+          raise "Waiting too long!" if i > 20
+          sleep(0.1)
+        end
+
+
+        # Also force a flush to disk to make sure ruby reload it.
+        File.open(config_path, "w") do |f|
+          f.write(new_config)
+          f.fsync
+        end
+
+        sleep(interval * 3) # Give time to reload the config
+      end
+
+      it "does not increase the successful reload count" do
+        snapshot = subject.metric.collector.snapshot_metric
+        value = snapshot.metric_store.get_with_path("/stats/pipelines")[:stats][:pipelines][:main][:reloads][:successes].value
+        expect(value).to be(0)
+      end
+
+      it "does not set the successful reload timestamp" do
+        snapshot = subject.metric.collector.snapshot_metric
+        value = snapshot.metric_store.get_with_path("/stats/pipelines")[:stats][:pipelines][:main][:reloads][:last_success_timestamp].value
+        expect(value).to be(nil)
+      end
+
+      it "sets the failure reload timestamp" do
+        snapshot = subject.metric.collector.snapshot_metric
+        value = snapshot.metric_store.get_with_path("/stats/pipelines")[:stats][:pipelines][:main][:reloads][:last_failure_timestamp].value
+        expect(value).to be_a(LogStash::Timestamp)
+      end
+
+      it "sets the last reload error" do
+        snapshot = subject.metric.collector.snapshot_metric
+        value = snapshot.metric_store.get_with_path("/stats/pipelines")[:stats][:pipelines][:main][:reloads][:last_error].value
+        expect(value).to be_a(Hash)
+        expect(value).to include(:message, :backtrace)
+      end
+
+      it "increases the failed reload count" do
+        snapshot = subject.metric.collector.snapshot_metric
+        value = snapshot.metric_store.get_with_path("/stats/pipelines")[:stats][:pipelines][:main][:reloads][:failures].value
+        expect(value).to be > 0
+      end
     end
   end
 end

data/spec/logstash/instrument/metric_store_spec.rb

@@ -203,6 +203,12 @@ describe LogStash::Instrument::MetricStore do
       end
     end
 
+    describe "#size" do
+      it "returns the number of unique metrics" do
+        expect(subject.size).to eq(metric_events.size)
+      end
+    end
+
     describe "#each" do
       it "retrieves all the metric" do
         expect(subject.each.size).to eq(metric_events.size)
@@ -221,4 +227,35 @@ describe LogStash::Instrument::MetricStore do
       end
     end
   end
+
+  describe "#prune" do
+    let(:metric_events) {
+      [
+        [[:node, :sashimi, :pipelines, :pipeline01, :plugins, :"logstash-output-elasticsearch"], :event_in, :increment],
+        [[:node, :sashimi, :pipelines, :pipeline01], :processed_events_in, :increment],
+        [[:node, :sashimi, :pipelines, :pipeline01], :processed_events_out, :increment],
+        [[:node, :sashimi, :pipelines, :pipeline02], :processed_events_out, :increment],
+      ]
+    }
+
+    before :each do
+      # Lets add a few metrics in the store before trying to find them
+      metric_events.each do |namespaces, metric_key, action|
+        metric = subject.fetch_or_store(namespaces, metric_key, LogStash::Instrument::MetricType::Counter.new(namespaces, metric_key))
+        metric.execute(action)
+      end
+    end
+
+    it "should remove all keys with the same starting path as the argument" do
+      expect(subject.get(:node, :sashimi, :pipelines, :pipeline01)).to be_a(Hash)
+      subject.prune("/node/sashimi/pipelines/pipeline01")
+      expect { subject.get(:node, :sashimi, :pipelines, :pipeline01) }.to raise_error LogStash::Instrument::MetricStore::MetricNotFound
+    end
+
+    it "should keep other metrics on different path branches" do
+      expect(subject.get(:node, :sashimi, :pipelines, :pipeline02)).to be_a(Hash)
+      subject.prune("/node/sashimi/pipelines/pipeline01")
+      expect { subject.get(:node, :sashimi, :pipelines, :pipeline02) }.to_not raise_error
+    end
+  end
 end