logstash-core 5.0.0.alpha4.snapshot1-java → 5.0.0.alpha4.snapshot2-java

data/lib/logstash/pipeline.rb

@@ -32,8 +32,8 @@ module LogStash; class Pipeline
     :started_at,
     :thread,
     :config_str,
-    :settings
-  attr_accessor :metric
+    :settings,
+    :metric
 
   MAX_INFLIGHT_WARN_THRESHOLD = 10_000
 
@@ -41,7 +41,7 @@ module LogStash; class Pipeline
     "LogStash::Inputs::Stdin"
   ]
 
-  def initialize(config_str, settings = LogStash::SETTINGS)
+  def initialize(config_str, settings = LogStash::SETTINGS, namespaced_metric = nil)
     @config_str = config_str
     @logger = Cabin::Channel.get(LogStash)
     @settings = settings
@@ -56,7 +56,7 @@ module LogStash; class Pipeline
 
     # This needs to be configured before we evaluate the code to make
     # sure the metric instance is correctly send to the plugins to make the namespace scoping work
-    @metric = settings.get_value("metric.collect") ? Instrument::Metric.new : Instrument::NullMetric.new 
+    @metric = namespaced_metric.nil? ? LogStash::Instrument::NullMetric.new : namespaced_metric
 
     grammar = LogStashConfigParser.new
     @config = grammar.parse(config_str)
@@ -188,6 +188,12 @@ module LogStash; class Pipeline
       batch_size = @settings.get("pipeline.batch.size")
       batch_delay = @settings.get("pipeline.batch.delay")
       max_inflight = batch_size * pipeline_workers
+
+      config_metric = metric.namespace([:stats, :pipelines, pipeline_id.to_s.to_sym, :config])   
+      config_metric.gauge(:workers, pipeline_workers)
+      config_metric.gauge(:batch_size, batch_size)
+      config_metric.gauge(:batch_delay, batch_delay)
+      
       @logger.info("Starting pipeline",
                    "id" => self.pipeline_id,
                    "pipeline.workers" => pipeline_workers,

data/lib/logstash/runner.rb

@@ -18,10 +18,10 @@ require "logstash/patches/clamp"
 require "logstash/settings"
 
 class LogStash::Runner < Clamp::StrictCommand
-  # The `path.settings` need to be defined in the runner instead of the `logstash-core/lib/logstash/environment.r`
+  # The `path.settings` need to be defined in the runner instead of the `logstash-core/lib/logstash/environment.rb`
   # because the `Environment::LOGSTASH_HOME` doesn't exist in the context of the `logstash-core` gem.
   # 
-  # See issues https://github.com/elastic/logstash/issues/5361
+  # See issue https://github.com/elastic/logstash/issues/5361
   LogStash::SETTINGS.register(LogStash::Setting::String.new("path.settings", ::File.join(LogStash::Environment::LOGSTASH_HOME, "config")))
 
   # Node Settings

data/lib/logstash/util/safe_uri.rb

@@ -0,0 +1,48 @@
+# encoding: utf-8
+require "logstash/namespace"
+require "logstash/util"
+
+# This class exists to quietly wrap a password string so that, when printed or
+# logged, you don't accidentally print the password itself.
+class LogStash::Util::SafeURI
+  PASS_PLACEHOLDER = "xxxxxx".freeze
+  
+  extend Forwardable
+  
+  def_delegators :@uri, :coerce, :query=, :route_from, :port=, :default_port, :select, :normalize!, :absolute?, :registry=, :path, :password, :hostname, :merge, :normalize, :host, :component_ary, :userinfo=, :query, :set_opaque, :+, :merge!, :-, :password=, :parser, :port, :set_host, :set_path, :opaque=, :scheme, :fragment=, :set_query, :set_fragment, :userinfo, :hostname=, :set_port, :path=, :registry, :opaque, :route_to, :set_password, :hierarchical?, :set_user, :set_registry, :set_userinfo, :fragment, :component, :user=, :set_scheme, :absolute, :host=, :relative?, :scheme=, :user
+  
+  attr_reader :uri
+  
+  public
+  def initialize(arg)    
+    @uri = case arg
+           when String
+             URI.parse(arg)
+           when URI
+             arg
+           else
+             raise ArgumentError, "Expected a string or URI, got a #{arg.class} creating a URL"
+           end
+  end
+
+  def to_s
+    sanitized.to_s
+  end
+
+  def inspect
+    sanitized.to_s
+  end
+
+  def sanitized
+    return uri unless uri.password # nothing to sanitize here!
+    
+    safe = uri.clone
+    safe.password = PASS_PLACEHOLDER
+    safe
+  end
+
+  def ==(other)
+    other.is_a?(::LogStash::Util::SafeURI) ? @uri == other.uri : false
+  end
+end
+

data/lib/logstash/version.rb

@@ -11,4 +11,4 @@
 #       eventually this file should be in the root logstash lib fir and dependencies in logstash-core should be
 #       fixed.
 
-LOGSTASH_VERSION = "5.0.0-alpha4.snapshot1"
+LOGSTASH_VERSION = "5.0.0-alpha4.snapshot2"

data/lib/logstash/webserver.rb

@@ -7,7 +7,7 @@ module LogStash
   class WebServer
     extend Forwardable
 
-    attr_reader :logger, :status, :config, :options, :cli_options, :runner, :binder, :events, :http_host, :http_port, :http_environment
+    attr_reader :logger, :status, :config, :options, :cli_options, :runner, :binder, :events, :http_host, :http_port, :http_environment, :agent
 
     def_delegator :@runner, :stats
 
@@ -15,12 +15,13 @@ module LogStash
     DEFAULT_PORT = 9600.freeze
     DEFAULT_ENVIRONMENT = 'production'.freeze
 
-    def initialize(logger, options={})
-      @logger      = logger
-      @http_host    = options[:http_host] || DEFAULT_HOST
-      @http_port    = options[:http_port] || DEFAULT_PORT
+    def initialize(logger, agent, options={})
+      @logger = logger
+      @agent = agent
+      @http_host = options[:http_host] || DEFAULT_HOST
+      @http_port = options[:http_port] || DEFAULT_PORT
       @http_environment = options[:http_environment] || DEFAULT_ENVIRONMENT
-      @options     = {}
+      @options = {}
       @cli_options = options.merge({ :rackup => ::File.join(::File.dirname(__FILE__), "api", "init.ru"),
                                      :binds => ["tcp://#{http_host}:#{http_port}"],
                                      :debug => logger.debug?,
@@ -37,7 +38,7 @@ module LogStash
 
       stop # Just in case
 
-      app = LogStash::Api::RackApp.app(logger, http_environment)
+      app = LogStash::Api::RackApp.app(logger, agent, http_environment)
       @server = ::Puma::Server.new(app)
       @server.add_tcp_listener(http_host, http_port)
 

data/logstash-core.gemspec

@@ -17,7 +17,7 @@ Gem::Specification.new do |gem|
   gem.require_paths = ["lib"]
   gem.version       = LOGSTASH_CORE_VERSION.gsub(/-/, '.')
 
-  gem.add_runtime_dependency "logstash-core-event-java", "5.0.0.alpha4.snapshot1"
+  gem.add_runtime_dependency "logstash-core-event-java", "5.0.0.alpha4.snapshot2"
 
   gem.add_runtime_dependency "cabin", "~> 0.8.0" #(Apache 2.0 license)
   gem.add_runtime_dependency "pry", "~> 0.10.1"  #(Ruby license)

data/spec/api/lib/api/node_plugins_spec.rb

@@ -0,0 +1,32 @@
+# encoding: utf-8
+require_relative "../../spec_helper"
+require "sinatra"
+require "logstash/api/modules/plugins"
+require "logstash/json"
+
+describe LogStash::Api::Modules::Plugins do
+  include_context "api setup"
+
+  extend ResourceDSLMethods
+
+  before(:all) do
+    do_request { get "/" }
+  end
+
+  let(:payload) { LogStash::Json.load(last_response.body) }
+
+  describe "retrieving plugins" do
+    it "should return OK" do
+      expect(last_response).to be_ok
+    end
+
+    it "should return a list of plugins" do      
+      expect(payload["plugins"]).to be_a(Array)
+    end
+
+    it "should return the total number of plugins" do
+      expect(payload["total"]).to be_a(Numeric)
+    end
+    
+  end
+end

data/spec/api/lib/api/node_spec.rb

@@ -5,12 +5,7 @@ require "logstash/api/modules/node"
 require "logstash/json"
 
 describe LogStash::Api::Modules::Node do
-
-  include Rack::Test::Methods
-
-  def app()
-    described_class
-  end
+  include_context "api setup"
 
   describe "#hot threads" do
 
@@ -39,7 +34,7 @@ describe LogStash::Api::Modules::Node do
       end
 
       it "should return information for <= # requested threads" do
-        expect(payload["threads"].count).to be <= 5
+        expect(payload["hot_threads"]["threads"].count).to be <= 5
       end
     end
 
@@ -60,5 +55,44 @@ describe LogStash::Api::Modules::Node do
       end
     end
 
+    describe "Generic JSON testing" do
+      extend ResourceDSLMethods
+      
+      root_structure = {
+        "pipeline" => {
+          "workers" => Numeric,
+          "batch_size" => Numeric,
+          "batch_delay" => Numeric
+        },
+        "os" => {
+          "name" => String,
+          "arch" => String,
+          "version" => String,
+          "available_processors" => Numeric
+        },
+        "jvm" => {
+          "pid" => Numeric,
+          "version" => String,
+          "vm_name" => String,
+          "vm_version" => String,
+          "vm_vendor" => String,
+          "start_time_in_millis" => Numeric,
+          "mem" => {
+            "heap_init_in_bytes" => Numeric,
+            "heap_max_in_bytes" => Numeric,
+            "non_heap_init_in_bytes" => Numeric,
+            "non_heap_max_in_bytes" => Numeric
+          }
+        },
+        "hot_threads"=> {
+          "hostname" => String,
+          "time" => String,
+          "busiest_threads" => Numeric,
+          "threads" => Array
+        }
+      }
+      
+      test_api_and_resources(root_structure, :exclude_from_root => ["hot_threads"])
+    end   
   end
 end

data/spec/api/lib/api/node_stats_spec.rb

@@ -5,12 +5,9 @@ require "logstash/api/modules/node_stats"
 require "logstash/json"
 
 describe LogStash::Api::Modules::NodeStats do
-  include Rack::Test::Methods
-  extend ResourceDSLMethods
+  include_context "api setup"
 
-  def app() # Used by Rack::Test::Methods
-    described_class
-  end
+  extend ResourceDSLMethods
 
   # DSL describing response structure
   root_structure = {
@@ -36,8 +33,36 @@ describe LogStash::Api::Modules::NodeStats do
         "total_in_millis"=>Numeric,
         "percent"=>Numeric
       }
+    },    
+    "mem" => {
+      "heap_used_in_bytes" => Numeric,
+      "heap_used_percent" => Numeric,
+      "heap_committed_in_bytes" => Numeric,
+      "heap_max_in_bytes" => Numeric,
+      "non_heap_used_in_bytes" => Numeric,
+      "non_heap_committed_in_bytes" => Numeric,
+      "pools" => {
+        "survivor" => {
+          "peak_used_in_bytes" => Numeric,
+          "used_in_bytes" => Numeric,
+          "peak_max_in_bytes" => Numeric,
+          "max_in_bytes" => Numeric
+        },
+        "old" => {
+          "peak_used_in_bytes" => Numeric,
+          "used_in_bytes" => Numeric,
+          "peak_max_in_bytes" => Numeric,
+          "max_in_bytes" => Numeric
+        },
+        "young" => {
+          "peak_used_in_bytes" => Numeric,
+          "used_in_bytes" => Numeric,
+          "peak_max_in_bytes" => Numeric,
+          "max_in_bytes" => Numeric
+        }
+      }
     }
   }
-
+  
   test_api_and_resources(root_structure)
 end

data/spec/api/lib/api/plugins_spec.rb

@@ -5,12 +5,7 @@ require "logstash/api/modules/plugins"
 require "logstash/json"
 
 describe LogStash::Api::Modules::Plugins do
-
-  include Rack::Test::Methods
-
-  def app()
-    described_class
-  end
+  include_context "api setup"
 
   before(:all) do
     get "/"
@@ -52,6 +47,5 @@ describe LogStash::Api::Modules::Plugins do
         expect(plugin["version"]).not_to be_empty
       end
     end
-
   end
 end

data/spec/api/lib/api/root_spec.rb

@@ -5,16 +5,11 @@ require "logstash/api/modules/root"
 require "logstash/json"
 
 describe LogStash::Api::Modules::Root do
-
-  include Rack::Test::Methods
-
-  def app()
-    described_class
-  end
+  include_context "api setup"
 
   it "should respond to root resource" do
     do_request { get "/" }
     expect(last_response).to be_ok
   end
-
 end
+

data/spec/api/lib/api/support/resource_dsl_methods.rb

@@ -24,12 +24,16 @@ module ResourceDSLMethods
         expect(last_response).to be_ok
       end
       
-      hash_to_mapping(expected).each do |path,klass|
-        dotted = path.join(".")
+      hash_to_mapping(expected).each do |resource_path,klass|
+        dotted = resource_path.join(".")
         
-        it "should set '#{dotted}' to be a '#{klass}'" do
-          path_value = path.reduce(payload) {|acc,v| acc[v]}
-          expect(path_value).to be_a(klass), "could not find '#{dotted}' in #{payload}"
+        it "should set '#{dotted}' at '#{path}' to be a '#{klass}'" do
+          expect(last_response).to be_ok # fail early if need be
+          resource_path_value = resource_path.reduce(payload) do |acc,v|
+            expect(acc.has_key?(v)).to eql(true), "Expected to find value '#{v}' in structure '#{acc}', but could not. Payload was '#{payload}'"
+            acc[v]
+          end
+          expect(resource_path_value).to be_a(klass), "could not find '#{dotted}' in #{payload}"
         end
       end
     end
@@ -37,8 +41,11 @@ module ResourceDSLMethods
     yield if block_given? # Add custom expectations
   end
 
-  def test_api_and_resources(expected)
-    test_api(expected, "/")
+  def test_api_and_resources(expected, xopts={})
+    xopts[:exclude_from_root] ||= []
+    root_expectation = expected.clone
+    xopts[:exclude_from_root].each {|k| root_expectation.delete(k)}
+    test_api(root_expectation, "/")
 
     expected.keys.each do |key|
       test_api({key => expected[key]}, "/#{key}")

data/spec/api/spec_helper.rb

@@ -1,13 +1,10 @@
 # encoding: utf-8
 API_ROOT = File.expand_path(File.join(File.dirname(__FILE__), "..", "..", "lib", "logstash", "api"))
 
-
-
 require "logstash/devutils/rspec/spec_helper"
-
 $LOAD_PATH.unshift(File.expand_path(File.dirname(__FILE__)))
 require "lib/api/support/resource_dsl_methods"
-
+require 'rspec/expectations'
 require "logstash/settings"
 require 'rack/test'
 require 'rspec'
@@ -20,10 +17,6 @@ end
 
 module LogStash
   class DummyAgent < Agent
-    def fetch_config(settings)
-      "input { generator {count => 0} } output { }"
-    end
-
     def start_webserver; end
     def stop_webserver; end
   end
@@ -39,7 +32,8 @@ class LogStashRunner
   attr_reader :config_str, :agent, :pipeline_settings
 
   def initialize
-    @config_str   = "input { generator {count => 0} } output { }"
+    @config_str   = "input { generator {count => 100 } } output { dummyoutput {} }"
+
     args = {
       "config.reload.automatic" => false,
       "metric.collect" => true,
@@ -57,31 +51,18 @@ class LogStashRunner
   end
 
   def start
+    # We start a pipeline that will generate a finite number of events
+    # before starting the expectations
     agent.register_pipeline("main", @settings)
-    @runner = Thread.new(agent) do |_agent|
-      _agent.execute
-    end
-
-    wait_until_ready
+    @agent_task = Stud::Task.new { agent.execute }
+    @agent_task.wait
   end
 
   def stop
     agent.shutdown
-    Thread.kill(@runner)
-    sleep 0.1 while !@runner.stop?
-  end
-
-  private
-
-  def wait_until_ready
-    # Wait until the service and pipeline have started
-    while !(LogStash::Api::Service.instance.started? && agent.pipelines["main"].running?) do
-      sleep 0.5
-    end
   end
 end
 
-
 ##
 # Method used to wrap up a request in between of a running
 # pipeline, this makes the whole execution model easier and
@@ -95,30 +76,6 @@ def do_request(&block)
   ret_val
 end
 
-##
-# Helper module that setups necessary mocks when doing the requests,
-# this could be just included in the test and the runner will be
-# started managed for all tests.
-##
-module LogStash; module RSpec; module RunnerConfig
-  def self.included(klass)
-    klass.before(:all) do
-      LogStashRunner.instance.start
-    end
-
-    klass.before(:each) do
-      runner = LogStashRunner.instance
-      allow(LogStash::Instrument::Collector.instance).to receive(:agent).and_return(runner.agent)
-    end
-
-    klass.after(:all) do
-      LogStashRunner.instance.stop
-    end
-  end
-end; end; end
-
-require 'rspec/expectations'
-
 RSpec::Matchers.define :be_available? do
   match do |plugin|
     begin
@@ -129,3 +86,20 @@ RSpec::Matchers.define :be_available? do
     end
   end
 end
+
+shared_context "api setup" do
+  before :all do
+    @runner = LogStashRunner.new
+    @runner.start
+  end
+
+  after :all do
+    @runner.stop
+  end
+
+  include Rack::Test::Methods
+
+  def app()
+    described_class.new(nil, @runner.agent)
+  end
+end