logstash-core 5.6.16-java → 6.0.0.alpha1-java

data/spec/api/lib/api/support/resource_dsl_methods.rb

@@ -0,0 +1,87 @@
+# Ruby doesn't have common class for boolean,
+# And to simplify the ResourceDSLMethods check it make sense to have it.
+module Boolean; end
+class TrueClass
+  include Boolean
+end
+class FalseClass
+  include Boolean
+end
+
+module ResourceDSLMethods
+  # Convert a nested hash to a mapping of key paths to expected classes
+  def hash_to_mapping(h, path=[], mapping={})
+    h.each do |k,v|
+      if v.is_a?(Hash)
+        hash_to_mapping(v, path + [k], mapping)
+      else
+        full_path = path + [k]
+        mapping[full_path] = v
+      end
+    end
+    mapping
+  end
+
+  def test_api(expected, path)
+    context "GET #{path}" do
+      let(:payload) { LogStash::Json.load(last_response.body) }
+
+      before(:all) do
+        get path
+      end
+
+      it "should respond OK" do
+        expect(last_response).to be_ok
+      end
+
+
+      describe "the default metadata" do
+        it "should include the host" do
+          expect(payload["host"]).to eql(Socket.gethostname)
+        end
+
+        it "should include the version" do
+          expect(payload["version"]).to eql(LOGSTASH_CORE_VERSION)
+        end
+
+        it "should include the http address" do
+          expect(payload["http_address"]).to eql("#{Socket.gethostname}:#{::LogStash::WebServer::DEFAULT_PORTS.first}")
+        end
+
+        it "should include the node name" do
+          expect(payload["name"]).to eql(@runner.agent.name)
+        end
+
+        it "should include the node id" do
+          expect(payload["id"]).to eql(@runner.agent.id)
+        end
+      end
+
+      hash_to_mapping(expected).each do |resource_path,klass|
+        dotted = resource_path.join(".")
+
+        it "should set '#{dotted}' at '#{path}' to be a '#{klass}'" do
+          expect(last_response).to be_ok # fail early if need be
+          resource_path_value = resource_path.reduce(payload) do |acc,v|
+            expect(acc.has_key?(v)).to eql(true), "Expected to find value '#{v}' in structure '#{acc}', but could not. Payload was '#{payload}'"
+            acc[v]
+          end
+          expect(resource_path_value).to be_a(klass), "could not find '#{dotted}' in #{payload}"
+        end
+      end
+    end
+
+    yield if block_given? # Add custom expectations
+  end
+
+  def test_api_and_resources(expected, xopts={})
+    xopts[:exclude_from_root] ||= []
+    root_expectation = expected.clone
+    xopts[:exclude_from_root].each {|k| root_expectation.delete(k)}
+    test_api(root_expectation, "/")
+
+    expected.keys.each do |key|
+      test_api({key => expected[key]}, "/#{key}")
+    end
+  end
+end

data/spec/{logstash/api/commands/stats_spec.rb → api/lib/commands/stats.rb}

@@ -1,15 +1,10 @@
 # encoding: utf-8
-require "spec_helper"
+require_relative "../../spec_helper"
 
 describe LogStash::Api::Commands::Stats do
-  include_context "api setup"
 
   let(:report_method) { :run }
-  subject(:report) do
-    factory = ::LogStash::Api::CommandFactory.new(LogStash::Api::Service.new(@agent))
-   
-    factory.build(:stats).send(report_method)
-  end
+  subject(:report) { report_class.new.send(report_method) }
 
   let(:report_class) { described_class }
 

data/spec/{logstash/api → api/lib}/errors_spec.rb

@@ -1,5 +1,5 @@
 # encoding: utf-8
-require "spec_helper"
+require_relative "../spec_helper"
 require "logstash/api/errors"
 
 describe LogStash::Api::ApiError do

data/spec/api/spec_helper.rb

@@ -0,0 +1,106 @@
+# encoding: utf-8
+API_ROOT = File.expand_path(File.join(File.dirname(__FILE__), "..", "..", "lib", "logstash", "api"))
+
+require "stud/task"
+require "logstash/devutils/rspec/spec_helper"
+$LOAD_PATH.unshift(File.expand_path(File.dirname(__FILE__)))
+require "lib/api/support/resource_dsl_methods"
+require_relative "../support/mocks_classes"
+require_relative "../support/helpers"
+require 'rspec/expectations'
+require "logstash/settings"
+require 'rack/test'
+require 'rspec'
+require "json"
+
+def read_fixture(name)
+  path = File.join(File.dirname(__FILE__), "fixtures", name)
+  File.read(path)
+end
+
+module LogStash
+  class DummyAgent < Agent
+    def start_webserver
+      http_address = "#{Socket.gethostname}:#{::LogStash::WebServer::DEFAULT_PORTS.first}"
+      @webserver = Struct.new(:address).new(http_address)
+      self.metric.gauge([], :http_address, http_address)
+    end
+
+    def stop_webserver; end
+  end
+end
+
+##
+# Class used to wrap and manage the execution of an agent for test,
+# this helps a lot in order to have a more integrated test for the
+# web api, could be also used for other use cases if generalized enough
+##
+class LogStashRunner
+
+  attr_reader :config_str, :agent, :pipeline_settings
+
+  def initialize
+
+    require "securerandom"
+    id = SecureRandom.uuid
+
+    @config_str   = "input { generator {id => 'api-generator-pipeline-#{id}' count => 100 } } output { dummyoutput {} }"
+
+    args = {
+      "config.reload.automatic" => false,
+      "metric.collect" => true,
+      "log.level" => "debug",
+      "node.name" => "test_agent",
+      "http.port" => rand(9600..9700),
+      "http.environment" => "test",
+      "config.string" => @config_str,
+      "pipeline.batch.size" => 1,
+      "pipeline.workers" => 1
+    }
+
+    @settings = ::LogStash::SETTINGS.clone.merge(args)
+    source_loader = LogStash::Config::SourceLoader.new
+    source_loader.configure_sources(LogStash::Config::Source::Local.new(@settings))
+    @agent = LogStash::DummyAgent.new(@settings, source_loader)
+  end
+
+  def start
+    # We start a pipeline that will generate a finite number of events
+    # before starting the expectations
+    @agent_task = Stud::Task.new { agent.execute }
+    @agent_task.wait
+  end
+
+  def stop
+    agent.shutdown
+  end
+end
+
+RSpec::Matchers.define :be_available? do
+  match do |plugin|
+    begin
+      Gem::Specification.find_by_name(plugin["name"])
+      true
+    rescue
+      false
+    end
+  end
+end
+
+shared_context "api setup" do
+  before :all do
+    clear_data_dir
+    @runner = LogStashRunner.new
+    @runner.start
+  end
+
+  after :all do
+    @runner.stop
+  end
+
+  include Rack::Test::Methods
+
+  def app()
+    described_class.new(nil, @runner.agent)
+  end
+end

data/spec/logstash/agent/converge_spec.rb

@@ -0,0 +1,286 @@
+# encoding: utf-8
+require "logstash/agent"
+require_relative "../../support/helpers"
+require_relative "../../support/matchers"
+require_relative "../../support/mocks_classes"
+require "spec_helper"
+
+describe LogStash::Agent do
+  # by default no tests uses the auto reload logic
+  let(:agent_settings) { mock_settings("config.reload.automatic" => false, "queue.type" => "persisted") }
+
+  subject { described_class.new(agent_settings, source_loader) }
+
+  before do
+    clear_data_dir
+
+    # until we decouple the webserver from the agent
+    allow(subject).to receive(:start_webserver).and_return(false)
+    allow(subject).to receive(:stop_webserver).and_return(false)
+  end
+
+  # Make sure that we close any running pipeline to release any pending locks
+  # on the queues
+  after do
+    converge_result = subject.shutdown
+    expect(converge_result).to be_a_successful_converge
+  end
+
+
+  describe "passing the agent to the pipeline" do
+    let(:source_loader) { TestSourceLoader.new(pipeline_config) }
+    let(:pipeline_config) { mock_pipeline_config(:main, "input { generator { count => 10 } } output { null {} }") }
+      
+    before { subject.execute }
+
+    it "execute the pipeline and stop execution" do
+      expect(subject.get_pipeline(:main).execution_context.agent).to eq(subject)
+    end
+  end
+
+  context "Agent execute options" do
+    let(:source_loader) do
+      TestSourceLoader.new(finite_pipeline_config)
+    end
+
+    context "when the pipeline execution is finite" do
+      let(:finite_pipeline_config) { mock_pipeline_config(:main, "input { generator { count => 1000 } } output { null {} }") }
+
+      it "execute the pipeline and stop execution" do
+        expect(subject.execute).to eq(0)
+      end
+    end
+
+    context "when the config is short lived (generator { count => 1 })" do
+      let(:finite_pipeline_config) { mock_pipeline_config(:main, "input { generator { count => 1 } } output { null {} }") }
+
+      it "execute the pipeline and stop execution" do
+        expect(subject.execute).to eq(0)
+      end
+    end
+
+    context "system pipeline" do
+      let(:finite_pipeline_config) { mock_pipeline_config(:main, "input { generator { count => 1000 } } output { null {} }") }
+      let(:system_pipeline_config) { mock_pipeline_config(:system_pipeline, "input { generator { } } output { null {} }", { "pipeline.system" => true }) }
+
+      let(:source_loader) do
+        TestSourceLoader.new(finite_pipeline_config, system_pipeline_config)
+      end
+
+      context "when we have a finite pipeline and a system pipeline running" do
+        it "execute the pipeline and stop execution" do
+          expect(subject.execute).to eq(0)
+        end
+      end
+
+      describe "#running_user_defined_pipelines" do
+        it "returns the user defined pipelines" do
+          agent_task = start_agent(subject)
+          expect(subject.running_user_defined_pipelines.keys).to include(:main)
+          expect(subject.running_user_defined_pipelines.keys).not_to include(:system_pipeline)
+          subject.shutdown
+        end
+      end
+
+      describe "#running_user_defined_pipelines?" do
+        it "returns true" do
+          agent_task = start_agent(subject)
+          expect(subject.running_user_defined_pipelines?).to be_truthy
+          subject.shutdown
+        end
+      end
+    end
+
+    context "when `config.reload.automatic`" do
+      let(:pipeline_config) { mock_pipeline_config(:main, "input { generator {} } output { null {} }") }
+
+      let(:source_loader) do
+        TestSourceLoader.new(pipeline_config)
+      end
+
+      context "is set to`FALSE`" do
+        context "and successfully load the config" do
+          let(:agent_settings) { mock_settings("config.reload.automatic" => false) }
+
+          it "converge only once" do
+            agent_task = start_agent(subject)
+
+            expect(source_loader.fetch_count).to eq(1)
+            expect(subject).to have_running_pipeline?(pipeline_config)
+
+            subject.shutdown
+            agent_task.stop!
+          end
+        end
+
+        context "and it fails to load the config" do
+          let(:source_loader) do
+            TestSourceLoader.new(TestSourceLoader::FailedFetch.new("can't load the file"))
+          end
+
+          it "doesn't execute any pipeline" do
+            expect { subject.execute }.not_to raise_error # errors is logged
+
+            expect(source_loader.fetch_count).to eq(1)
+            expect(subject.pipelines_count).to eq(0)
+
+            subject.shutdown
+          end
+        end
+      end
+
+      context "is set to `TRUE`" do
+        let(:interval) { 0.01 }
+        let(:agent_settings) do
+          mock_settings(
+            "config.reload.automatic" => true,
+            "config.reload.interval" =>  interval
+          )
+        end
+
+        context "and successfully load the config" do
+          it "converges periodically the pipelines from the configs source" do
+            agent_task = start_agent(subject)
+
+            sleep(2) # let the interval reload a few times
+            expect(subject).to have_running_pipeline?(pipeline_config)
+
+            # we rely on a periodic thread to call fetch count, we have seen unreliable run on
+            # travis, so lets add a few retries
+            try do
+              expect(source_loader.fetch_count).to be > 1
+            end
+
+            subject.shutdown
+            agent_task.stop!
+          end
+        end
+
+        context "and it fails to load the config" do
+          let(:source_loader) do
+            TestSourceLoader.new(TestSourceLoader::FailedFetch.new("can't load the file"))
+          end
+
+          it "it will keep trying to converge" do
+            agent_task = start_agent(subject)
+
+            sleep(interval * 20) # let the interval reload a few times
+            expect(subject.pipelines_count).to eq(0)
+            expect(source_loader.fetch_count).to be > 1
+
+            subject.shutdown
+            agent_task.stop!
+          end
+        end
+      end
+    end
+  end
+
+  context "when shutting down the agent" do
+    let(:pipeline_config) { mock_pipeline_config(:main, "input { generator {} } output { null {} }") }
+    let(:new_pipeline_config) { mock_pipeline_config(:new, "input { generator { id => 'new' } } output { null {} }") }
+
+    let(:source_loader) do
+      TestSourceLoader.new([pipeline_config, new_pipeline_config])
+    end
+
+    it "stops the running pipelines" do
+      expect(subject.converge_state_and_update).to be_a_successful_converge
+      expect { subject.shutdown }.to change { subject.running_pipelines.size }.from(2).to(0)
+    end
+  end
+
+  context "Configuration converge scenario" do
+    let(:pipeline_config) { mock_pipeline_config(:main, "input { generator {} } output { null {} }", { "pipeline.reloadable" => true }) }
+    let(:new_pipeline_config) { mock_pipeline_config(:new, "input { generator {} } output { null {} }", { "pipeline.reloadable" => true }) }
+
+    before do
+      # Set the Agent to an initial state of pipelines
+      expect(subject.converge_state_and_update).to be_a_successful_converge
+    end
+
+    context "no pipelines is running" do
+      let(:source_loader) do
+        TestSequenceSourceLoader.new([], pipeline_config)
+      end
+
+      it "creates and starts the new pipeline" do
+        expect {
+          expect(subject.converge_state_and_update).to be_a_successful_converge
+        }.to change { subject.running_pipelines.count }.from(0).to(1)
+        expect(subject).to have_running_pipeline?(pipeline_config)
+      end
+    end
+
+    context "when a pipeline is running" do
+      context "when the source returns the current pipeline and a new one" do
+        let(:source_loader) do
+          TestSequenceSourceLoader.new(
+            pipeline_config,
+            [pipeline_config, new_pipeline_config]
+          )
+        end
+
+        it "start a new pipeline and keep the original" do
+          expect {
+            expect(subject.converge_state_and_update).to be_a_successful_converge
+          }.to change { subject.running_pipelines.count }.from(1).to(2)
+          expect(subject).to have_running_pipeline?(pipeline_config)
+          expect(subject).to have_running_pipeline?(new_pipeline_config)
+        end
+      end
+
+      context "when the source returns a new pipeline but not the old one" do
+        let(:source_loader) do
+          TestSequenceSourceLoader.new(
+            pipeline_config,
+            new_pipeline_config
+          )
+        end
+
+        it "stops the missing pipeline and start the new one" do
+          expect {
+            expect(subject.converge_state_and_update).to be_a_successful_converge
+          }.not_to change { subject.running_pipelines.count }
+          expect(subject).not_to have_pipeline?(pipeline_config)
+          expect(subject).to have_running_pipeline?(new_pipeline_config)
+        end
+      end
+    end
+
+    context "when the source return a modified pipeline" do
+      let(:modified_pipeline_config) { mock_pipeline_config(:main, "input { generator { id => 'new-and-modified' } } output { null {} }", { "pipeline.reloadable" => true }) }
+
+      let(:source_loader) do
+        TestSequenceSourceLoader.new(
+          [pipeline_config],
+          [modified_pipeline_config]
+        )
+      end
+
+      it "reloads the modified pipeline" do
+        expect {
+          expect(subject.converge_state_and_update).to be_a_successful_converge
+        }.not_to change { subject.running_pipelines.count }
+        expect(subject).to have_running_pipeline?(modified_pipeline_config)
+        expect(subject).not_to have_pipeline?(pipeline_config)
+      end
+    end
+
+    context "when the source return no pipelines" do
+      let(:source_loader) do
+        TestSequenceSourceLoader.new(
+          [pipeline_config, new_pipeline_config],
+          []
+        )
+      end
+
+      it "stops all the pipelines" do
+        expect {
+          expect(subject.converge_state_and_update).to be_a_successful_converge
+        }.to change { subject.running_pipelines.count }.from(2).to(0)
+        expect(subject).not_to have_pipeline?(pipeline_config)
+      end
+    end
+  end
+end