logstash-core 2.4.1-java → 5.0.0.alpha1-java

data/spec/api/lib/api/plugins_spec.rb

@@ -0,0 +1,57 @@
+# encoding: utf-8
+require_relative "../../spec_helper"
+require "sinatra"
+require "app/modules/plugins"
+require "logstash/json"
+
+describe LogStash::Api::Plugins do
+
+  include Rack::Test::Methods
+
+  def app()
+    described_class
+  end
+
+  before(:all) do
+    get "/"
+  end
+
+  let(:payload) { LogStash::Json.load(last_response.body) }
+
+  it "respond to plugins resource" do
+    expect(last_response).to be_ok
+  end
+
+  it "return valid json content type" do
+    expect(last_response.content_type).to eq("application/json")
+  end
+
+  context "#schema" do
+    it "return the expected schema" do
+      expect(payload.keys).to include("plugins", "total")
+      payload["plugins"].each do |plugin|
+        expect(plugin.keys).to include("name", "version")
+      end
+    end
+  end
+
+  context "#values" do
+
+    it "return totals of plugins" do
+      expect(payload["total"]).to eq(payload["plugins"].count)
+    end
+
+    it "return a list of available plugins" do
+      payload["plugins"].each do |plugin|
+        expect(plugin).to be_available?
+      end
+    end
+
+    it "return non empty version values" do
+      payload["plugins"].each do |plugin|
+        expect(plugin["version"]).not_to be_empty
+      end
+    end
+
+  end
+end

data/spec/api/lib/api/root_spec.rb

@@ -0,0 +1,20 @@
+# encoding: utf-8
+require_relative "../../spec_helper"
+require "sinatra"
+require "app/root"
+require "logstash/json"
+
+describe LogStash::Api::Root do
+
+  include Rack::Test::Methods
+
+  def app()
+    described_class
+  end
+
+  it "should respond to root resource" do
+    do_request { get "/" }
+    expect(last_response).to be_ok
+  end
+
+end

data/spec/api/lib/api/stats_spec.rb

@@ -0,0 +1,19 @@
+# encoding: utf-8
+require_relative "../../spec_helper"
+require "sinatra"
+require "app/modules/stats"
+
+describe LogStash::Api::Stats do
+
+  include Rack::Test::Methods
+
+  def app()
+    described_class
+  end
+
+  it "respond to the jvm resource" do
+    do_request { get "/jvm" }
+    expect(last_response).to be_ok
+  end
+
+end

data/spec/api/lib/commands/events_spec.rb

@@ -0,0 +1,17 @@
+# encoding: utf-8
+require_relative "../../spec_helper"
+require "app/commands/stats/events_command"
+
+describe LogStash::Api::StatsEventsCommand do
+
+  context "#schema" do
+
+    let(:report) do
+      do_request { subject.run }
+    end
+
+    it "return events information" do
+      expect(report).to include("in", "filtered", "out")
+    end
+  end
+end

data/spec/api/lib/commands/jvm_spec.rb

@@ -0,0 +1,45 @@
+# encoding: utf-8
+require_relative "../../spec_helper"
+require "app/commands/stats/hotthreads_command"
+require "app/commands/stats/memory_command"
+
+describe "JVM stats" do
+
+  describe LogStash::Api::HotThreadsCommand do
+
+    let(:report) do
+      do_request { subject.run }
+    end
+
+    context "#schema" do
+      it "return hot threads information" do
+        report = do_request { subject.run }
+        expect(report.to_s).not_to be_empty
+      end
+
+    end
+  end
+
+  describe LogStash::Api::JvmMemoryCommand do
+
+    context "#schema" do
+
+      let(:report) do
+        do_request { subject.run }
+      end
+
+      it "return hot threads information" do
+        expect(report).not_to be_empty
+      end
+
+      it "return heap information" do
+        expect(report.keys).to include(:heap_used_in_bytes)
+      end
+
+      it "return non heap information" do
+        expect(report.keys).to include(:non_heap_used_in_bytes)
+      end
+
+    end
+  end
+end

data/spec/api/spec_helper.rb

@@ -0,0 +1,128 @@
+# encoding: utf-8
+API_ROOT = File.expand_path(File.join(File.dirname(__FILE__), "..", "..", "lib", "logstash", "api"))
+
+require "logstash/devutils/rspec/spec_helper"
+
+require 'rack/test'
+require 'rspec'
+require "json"
+
+ENV['RACK_ENV'] = 'test'
+
+Rack::Builder.parse_file(File.join(API_ROOT, 'init.ru'))
+
+def read_fixture(name)
+  path = File.join(File.dirname(__FILE__), "fixtures", name)
+  File.read(path)
+end
+
+module LogStash
+  class DummyAgent < Agent
+    def fetch_config(settings)
+      "input { generator {count => 0} } output { }"
+    end
+
+    def start_webserver; end
+    def stop_webserver; end
+  end
+end
+
+##
+# Class used to wrap and manage the execution of an agent for test,
+# this helps a lot in order to have a more integrated test for the
+# web api, could be also used for other use cases if generalized enought
+##
+class LogStashRunner
+
+  attr_reader :config_str, :agent, :pipeline_settings
+
+  def initialize
+    args = [
+      :logger => Cabin::Channel.get(LogStash),
+      :auto_reload => false,
+      :collect_metric => true,
+      :debug => false,
+      :node_name => "test_agent",
+      :web_api_http_port => rand(9600..9700)
+    ]
+
+    @config_str   = "input { generator {count => 0} } output { }"
+    @agent = LogStash::DummyAgent.new(*args)
+    @pipeline_settings ||= { :pipeline_id => "main",
+                             :config_str => config_str,
+                            :pipeline_batch_size => 1,
+                            :flush_interval => 1,
+                            :pipeline_workers => 1 }
+  end
+
+  def start
+    agent.register_pipeline("main", pipeline_settings)
+    @runner = Thread.new(agent) do |_agent|
+      _agent.execute
+    end
+    wait_until_snapshot_received
+  end
+
+  def stop
+    agent.shutdown
+    Thread.kill(@runner)
+    sleep 0.1 while !@runner.stop?
+  end
+
+  private
+
+  def wait_until_snapshot_received
+    while !LogStash::Api::Service.instance.started? do
+      sleep 0.5
+    end
+  end
+end
+
+
+##
+# Method used to wrap up a request in between of a running
+# pipeline, this makes the hole execution model easier and
+# more contained as some threads might go wild.
+##
+def do_request(&block)
+  runner = LogStashRunner.new
+  runner.start
+  ret_val = block.call
+  runner.stop
+  ret_val
+end
+
+##
+# Helper module that setups necessary mocks when doing the requests,
+# this could be just included in the test and the runner will be
+# started managed for all tests.
+##
+module LogStash; module RSpec; module RunnerConfig
+  def self.included(klass)
+    klass.before(:all) do
+      LogStashRunner.instance.start
+    end
+
+    klass.before(:each) do
+      runner = LogStashRunner.instance
+      allow(LogStash::Instrument::Collector.instance).to receive(:agent).and_return(runner.agent)
+    end
+
+    klass.after(:all) do
+      LogStashRunner.instance.stop
+    end
+  end
+end; end; end
+
+require 'rspec/expectations'
+
+RSpec::Matchers.define :be_available? do
+  match do |plugin|
+    begin
+      Gem::Specification.find_by_name(plugin["name"])
+      true
+    rescue
+      false
+    end
+  end
+end

data/spec/logstash/agent_spec.rb

@@ -1,26 +1,20 @@
 # encoding: utf-8
 require 'spec_helper'
 require 'stud/temporary'
-require 'stud/task'
 
 describe LogStash::Agent do
 
   let(:logger) { double("logger") }
-  let(:agent_args) { [] }
-  subject { LogStash::Agent.new("", "") }
+  let(:agent_args) { { :logger => logger } }
+  subject { LogStash::Agent.new(agent_args) }
 
   before :each do
-    [:log, :info, :warn, :error, :fatal, :debug, :terminal].each do |level|
+    [:info, :warn, :error, :fatal, :debug].each do |level|
       allow(logger).to receive(level)
     end
     [:info?, :warn?, :error?, :fatal?, :debug?].each do |level|
       allow(logger).to receive(level)
     end
-    allow(logger).to receive(:level=)
-    allow(logger).to receive(:subscribe)
-    subject.parse(agent_args)
-    subject.instance_variable_set("@reload_interval", 0.01)
-    subject.instance_variable_set("@logger", logger)
   end
 
   describe "register_pipeline" do
@@ -31,6 +25,12 @@ describe LogStash::Agent do
       :pipeline_workers => 4
     } }
 
+    let(:agent_args) { {
+      :logger => logger,
+      :auto_reload => false,
+      :reload_interval => 0.01
+    } }
+
     it "should delegate settings to new pipeline" do
       expect(LogStash::Pipeline).to receive(:new).with(settings[:config_string], hash_including(settings))
       subject.register_pipeline(pipeline_id, settings)
@@ -42,7 +42,9 @@ describe LogStash::Agent do
     let(:config_file) { Stud::Temporary.pathname }
 
     before :each do
-      File.open(config_file, "w") {|f| f.puts sample_config }
+      allow(subject).to receive(:start_webserver).and_return(false)
+      allow(subject).to receive(:stop_webserver).and_return(false)
+      File.open(config_file, "w") { |f| f.puts sample_config }
     end
 
     after :each do
@@ -50,7 +52,7 @@ describe LogStash::Agent do
     end
 
     context "when auto_reload is false" do
-      let(:agent_args) { [ "--config", config_file] } #reload_interval => 0.01, :config_path => } }
+      let(:agent_args) { { :logger => logger, :auto_reload => false } }
       let(:pipeline_id) { "main" }
       let(:pipeline_settings) { { :config_path => config_file } }
 
@@ -68,7 +70,6 @@ describe LogStash::Agent do
         it "should not reload_state!" do
           expect(subject).to_not receive(:reload_state!)
           t = Thread.new { subject.execute }
-          sleep 0.01 until subject.running_pipelines? && subject.pipelines.values.first.ready?
           sleep 0.1
           Stud.stop!(t)
           t.join
@@ -109,7 +110,7 @@ describe LogStash::Agent do
     end
 
     context "when auto_reload is true" do
-      let(:agent_args) { [ "--auto-reload", "--config", config_file] } #reload_interval => 0.01, :config_path => } }
+      let(:agent_args) { { :logger => logger, :auto_reload => true, :reload_interval => 0.01 } }
       let(:pipeline_id) { "main" }
       let(:pipeline_settings) { { :config_path => config_file } }
 
@@ -119,10 +120,10 @@ describe LogStash::Agent do
 
       context "if state is clean" do
         it "should periodically reload_state" do
+          allow(subject).to receive(:clean_state?).and_return(false)
           expect(subject).to receive(:reload_state!).at_least(3).times
-          t = Thread.new(subject) {|subject| subject.execute }
-          sleep 0.01 until (subject.running_pipelines? && subject.pipelines.values.first.ready?)
-          # now that the pipeline has started, give time for reload_state! to happen a few times
+          t = Thread.new { subject.execute }
+          sleep 0.01 until subject.running_pipelines? && subject.pipelines.values.first.ready?
           sleep 0.1
           Stud.stop!(t)
           t.join
@@ -190,6 +191,44 @@ describe LogStash::Agent do
     end
   end
 
+  describe "Environment Variables In Configs" do
+    let(:agent_args) { {
+      :logger => logger,
+      :auto_reload => false,
+      :reload_interval => 0.01
+    } }
+    let(:pipeline_id) { "main" }
+    let(:pipeline_config) { "input { generator { message => '${FOO}-bar' } } filter { } output { }" }
+    let(:pipeline_settings) { {
+      :config_string => pipeline_config,
+    } }
+
+    context "when allow_env is false" do
+      it "does not interpolate environment variables" do
+        expect(subject).to receive(:fetch_config).and_return(pipeline_config)
+        subject.register_pipeline(pipeline_id, pipeline_settings)
+        expect(subject.pipelines[pipeline_id].inputs.first.message).to eq("${FOO}-bar")
+      end
+    end
+
+    context "when allow_env is true" do
+      before :each do
+        @foo_content = ENV["FOO"]
+        ENV["FOO"] = "foo"
+        pipeline_settings.merge!(:allow_env => true)
+      end
+
+      after :each do
+        ENV["FOO"] = @foo_content
+      end
+
+      it "doesn't upgrade the state" do
+        expect(subject).to receive(:fetch_config).and_return(pipeline_config)
+        subject.register_pipeline(pipeline_id, pipeline_settings)
+        expect(subject.pipelines[pipeline_id].inputs.first.message).to eq("foo-bar")
+      end
+    end
+  end
 
   describe "#upgrade_pipeline" do
     let(:pipeline_id) { "main" }
@@ -242,34 +281,11 @@ describe LogStash::Agent do
     end
   end
 
-  context "--pluginpath" do
-    let(:single_path) { "/some/path" }
-    let(:multiple_paths) { ["/some/path1", "/some/path2"] }
-
-    it "should add single valid dir path to the environment" do
-      expect(File).to receive(:directory?).and_return(true)
-      expect(LogStash::Environment).to receive(:add_plugin_path).with(single_path)
-      subject.configure_plugin_paths(single_path)
-    end
-
-    it "should fail with single invalid dir path" do
-      expect(File).to receive(:directory?).and_return(false)
-      expect(LogStash::Environment).not_to receive(:add_plugin_path)
-      expect{subject.configure_plugin_paths(single_path)}.to raise_error(Clamp::UsageError)
-    end
-
-    it "should add multiple valid dir path to the environment" do
-      expect(File).to receive(:directory?).exactly(multiple_paths.size).times.and_return(true)
-      multiple_paths.each{|path| expect(LogStash::Environment).to receive(:add_plugin_path).with(path)}
-      subject.configure_plugin_paths(multiple_paths)
-    end
-  end
-
   describe "#fetch_config" do
     let(:file_config) { "input { generator { count => 100 } } output { }" }
     let(:cli_config) { "filter { drop { } } " }
     let(:tmp_config_path) { Stud::Temporary.pathname }
-    let(:agent_args) { [ "-e", "filter { drop { } } ", "-f", tmp_config_path ] }
+    let(:agent_args) { { :logger => logger, :config_string => "filter { drop { } } ", :config_path => tmp_config_path } }
 
     before :each do
       IO.write(tmp_config_path, file_config)
@@ -286,138 +302,15 @@ describe LogStash::Agent do
     end
   end
 
-  context "--pluginpath" do
-    let(:single_path) { "/some/path" }
-    let(:multiple_paths) { ["/some/path1", "/some/path2"] }
-
-    it "should fail with single invalid dir path" do
-      expect(File).to receive(:directory?).and_return(false)
-      expect(LogStash::Environment).not_to receive(:add_plugin_path)
-      expect{subject.configure_plugin_paths(single_path)}.to raise_error(Clamp::UsageError)
+  context "#started_at" do
+    it "return the start time when the agent is started" do
+      expect(described_class::STARTED_AT).to be_kind_of(Time)
     end
   end
 
-  describe "--config-test" do
-    let(:cli_args) { ["-t", "-e", pipeline_string] }
-    let(:pipeline_string) { "input { } filter { } output { }" }
-
-    context "with a good configuration" do
-      it "should exit successfuly" do
-        expect(subject.run(cli_args)).to eq(0)
-      end
-    end
-
-    context "with a bad configuration" do
-      let(:pipeline_string) { "rlwekjhrewlqrkjh" }
-      it "should fail by returning a bad exit code" do
-        expect(subject.run(cli_args)).to eq(1)
-      end
-    end
-
-    it "requests the config loader to format_config" do
-      expect(subject.config_loader).to receive(:format_config)
-      subject.run(cli_args)
-    end
-  end
-
-  describe "pipeline settings" do
-    let(:pipeline_string) { "input { stdin {} } output { stdout {} }" }
-    let(:main_pipeline_settings) { { :pipeline_id => "main" } }
-    let(:pipeline) { double("pipeline") }
-
-    before(:each) do
-      task = Stud::Task.new { 1 }
-      allow(pipeline).to receive(:run).and_return(task)
-      allow(pipeline).to receive(:shutdown)
-    end
-
-    context "when :pipeline_workers is not defined by the user" do
-      it "should not pass the value to the pipeline" do
-        expect(LogStash::Pipeline).to receive(:new).once.with(pipeline_string, hash_excluding(:pipeline_workers)).and_return(pipeline)
-        args = ["-e", pipeline_string]
-        subject.run(args)
-      end
-    end
-
-    context "when :pipeline_workers is defined by the user" do
-      it "should pass the value to the pipeline" do
-        main_pipeline_settings[:pipeline_workers] = 2
-        expect(LogStash::Pipeline).to receive(:new).with(pipeline_string, hash_including(main_pipeline_settings)).and_return(pipeline)
-        args = ["-w", "2", "-e", pipeline_string]
-        subject.run(args)
-      end
-    end
-  end
-
-  describe "debug_config" do
-    let(:pipeline_string) { "input {} output {}" }
-    let(:pipeline) { double("pipeline") }
-
-    it "should set 'debug_config' to false by default" do
-      expect(LogStash::Pipeline).to receive(:new).and_return(pipeline)
-      args = ["--debug", "-e", pipeline_string]
-      subject.run(args)
-
-      expect(subject.config_loader.debug_config).to be_falsey
-    end
-
-    it "should allow overriding debug_config" do
-      expect(LogStash::Pipeline).to receive(:new).and_return(pipeline)
-      args = ["--debug", "--debug-config",  "-e", pipeline_string]
-      subject.run(args)
-
-      expect(subject.config_loader.debug_config).to be_truthy
-    end
-  end
-
-  describe "allow_env param passing to pipeline" do
-    let(:pipeline_string) { "input {} output {}" }
-    let(:pipeline) { double("pipeline") }
-
-    it "should set 'allow_env' to false by default" do
-      args = ["-e", pipeline_string]
-      expect(LogStash::Pipeline).to receive(:new).with(pipeline_string, hash_including(:allow_env => false)).and_return(pipeline)
-      subject.run(args)
-    end
-
-    it "should support templating environment variables" do
-      args = ["-e", pipeline_string, "--allow-env"]
-      expect(LogStash::Pipeline).to receive(:new).with(pipeline_string, hash_including(:allow_env => true)).and_return(pipeline)
-      subject.run(args)
-    end
-  end
-
-  describe "Environment variables in config" do
-    let(:pipeline_id) { "main" }
-    let(:pipeline_config) { "input { generator { message => '${FOO}-bar' } } filter { } output { }" }
-    let(:pipeline_settings) { { :config_string => pipeline_config } }
-    let(:pipeline) { double("pipeline") }
-
-    context "when allow_env is false" do
-      it "does not interpolate environment variables" do
-        expect(subject).to receive(:fetch_config).and_return(pipeline_config)
-        subject.register_pipeline(pipeline_id, pipeline_settings)
-        expect(subject.pipelines[pipeline_id].inputs.first.message).to eq("${FOO}-bar")
-      end
-    end
-
-    context "when allow_env is true" do
-      before :each do
-        @foo_content = ENV["FOO"]
-        ENV["FOO"] = "foo"
-        pipeline_settings.merge!(:allow_env => true)
-      end
-
-      after :each do
-        ENV["FOO"] = @foo_content
-      end
-
-      it "doesn't upgrade the state" do
-        expect(subject).to receive(:fetch_config).and_return(pipeline_config)
-        subject.register_pipeline(pipeline_id, pipeline_settings)
-        expect(subject.pipelines[pipeline_id].inputs.first.message).to eq("foo-bar")
-      end
+  context "#uptime" do
+    it "return the number of milliseconds since start time" do
+      expect(subject.uptime).to be >= 0
     end
   end
 end
-