logstash-core 5.0.0.alpha2.snapshot2-java → 5.0.0.alpha3-java

data/locales/en.yml

@@ -1,6 +1,6 @@
 # YAML notes
 #   |- means 'scalar block' useful for formatted text
-#   > means 'scalar block' but it chomps all newlines. Useful 
+#   > means 'scalar block' but it chomps all newlines. Useful
 #     for unformatted text.
 en:
   oops: |-
@@ -71,16 +71,12 @@ en:
       non_reloadable_config_reload: >-
         Unable to reload configuration because it does not support dynamic reloading
       non_reloadable_config_register: |-
-        Logstash was not able to load configuration since it does not support
-        dynamic reloading and -r or --auto-reload flag was enabled
+        Logstash is not able to start since configuration auto reloading was enabled but the configuration contains plugins that don't support it. Quitting...
     web_api:
-      flag:
-        http_host: Web API binding host
-        http_port: Web API http port
       hot_threads:
         title: |-
           ::: {%{hostname}}
-            Hot threads at %{time}, busiestThreads=%{top_count}:
+          Hot threads at %{time}, busiestThreads=%{top_count}:
         thread_title: |-
             %{percent_of_cpu_time} % of cpu usage by %{thread_state} thread named '%{thread_name}'
     runner:
@@ -145,18 +141,18 @@ en:
           after %{after}
         invalid_plugin_register: >-
           Cannot register %{plugin} %{type} plugin.
-          The error reported is: 
+          The error reported is:
             %{error}
         plugin_path_missing: >-
           You specified a plugin path that does not exist: %{path}
         no_plugins_found: |-
           Could not find any plugins in "%{path}"
-          I tried to find files matching the following, but found none: 
+          I tried to find files matching the following, but found none:
             %{plugin_glob}
         log_file_failed: |-
           Failed to open %{path} for writing: %{error}
 
-          This is often a permissions issue, or the wrong 
+          This is often a permissions issue, or the wrong
           path was specified?
       flag:
         # Note: Wrap these at 55 chars so they display nicely when clamp emits
@@ -182,10 +178,8 @@ en:
           the empty string for the '-e' flag.
         configtest: |+
           Check configuration for valid syntax and then exit.
-        allow-env: |+
-          EXPERIMENTAL. Enables templating of environment variable
-          values. Instances of "${VAR}" in strings will be replaced
-          with the respective environment variable value named "VAR".
+        http_host: Web API binding host
+        http_port: Web API http port
         pipeline-workers: |+
           Sets the number of pipeline workers to run.
         pipeline-batch-size: |+
@@ -193,6 +187,9 @@ en:
         pipeline-batch-delay: |+
           When creating pipeline batches, how long to wait while polling
           for the next event.
+        path_settings: |+
+          Directory containing logstash.yml file. This can also be
+          set through the LS_SETTINGS_DIR environment variable.
         auto_reload: |+
           Monitor configuration changes and reload
           whenever it is changed.
@@ -221,15 +218,12 @@ en:
           'PATH/logstash/TYPE/NAME.rb' where TYPE is
           'inputs' 'filters', 'outputs' or 'codecs'
           and NAME is the name of the plugin.
-        quiet: |+
-          Quieter logstash logging. This causes only 
-          errors to be emitted.
-        verbose: |+
-          More verbose logging. This causes 'info' 
-          level logs to be emitted.
-        debug: |+
-          Most verbose logging. This causes 'debug'
-          level logs to be emitted.
+        log_level: |+
+          Set the log level for logstash. Possible values are:
+            - quiet => :error
+            - verbose => :info
+            - debug => :debug
+            - warn => :warn
         unsafe_shutdown: |+
           Force logstash to exit during shutdown even
           if there are still inflight events in memory.
@@ -243,7 +237,10 @@ en:
           it will default to the current hostname.
         agent: |+
           Specify an alternate agent plugin name.
-        debug_config: |+
+        config_debug: |+
           Print the compiled config ruby code out as a debug log (you must also have --debug enabled).
           WARNING: This will include any 'password' options passed to plugin configs as plaintext, and may result
-          in plaintext passwords appearing in your logs!
+          in plaintext passwords appearing in your logs!
+        log_format: |+
+          Specify if Logstash should write its own logs in JSON form (one
+          event per line) or in plain text (using Ruby's Object#inspect)

data/logstash-core.gemspec

@@ -17,7 +17,7 @@ Gem::Specification.new do |gem|
   gem.require_paths = ["lib"]
   gem.version       = LOGSTASH_CORE_VERSION.gsub(/-/, '.')
 
-  gem.add_runtime_dependency "logstash-core-event-java", "~> 5.0.0.alpha2.snapshot2"
+  gem.add_runtime_dependency "logstash-core-event-java", "5.0.0.alpha3"
 
   gem.add_runtime_dependency "cabin", "~> 0.8.0" #(Apache 2.0 license)
   gem.add_runtime_dependency "pry", "~> 0.10.1"  #(Ruby license)
@@ -27,10 +27,10 @@ Gem::Specification.new do |gem|
   gem.add_runtime_dependency "gems", "~> 0.8.3"  #(MIT license)
   gem.add_runtime_dependency "concurrent-ruby", "1.0.0"
   gem.add_runtime_dependency "sinatra", '~> 1.4', '>= 1.4.6'
-  gem.add_runtime_dependency 'puma', '~> 2.16', '>= 2.16.0'
+  gem.add_runtime_dependency 'puma', '~> 2.16'
   gem.add_runtime_dependency "jruby-openssl", "0.9.16" # >= 0.9.13 Required to support TLSv1.2
   gem.add_runtime_dependency "chronic_duration", "0.10.6"
-  gem.add_runtime_dependency "jruby-monitoring", '~> 0.3.1'
+  gem.add_runtime_dependency "jrmonitor", '~> 0.4.2'
 
   # TODO(sissel): Treetop 1.5.x doesn't seem to work well, but I haven't
   # investigated what the cause might be. -Jordan

data/spec/api/lib/api/node_spec.rb

@@ -1,10 +1,10 @@
 # encoding: utf-8
 require_relative "../../spec_helper"
 require "sinatra"
-require "app/modules/node"
+require "logstash/api/modules/node"
 require "logstash/json"
 
-describe LogStash::Api::Node do
+describe LogStash::Api::Modules::Node do
 
   include Rack::Test::Methods
 

data/spec/api/lib/api/node_stats_spec.rb

@@ -1,68 +1,43 @@
 # encoding: utf-8
 require_relative "../../spec_helper"
 require "sinatra"
-require "app/modules/node_stats"
+require "logstash/api/modules/node_stats"
 require "logstash/json"
 
-describe LogStash::Api::NodeStats do
-
+describe LogStash::Api::Modules::NodeStats do
   include Rack::Test::Methods
+  extend ResourceDSLMethods
 
-  def app()
+  def app() # Used by Rack::Test::Methods
     described_class
   end
 
-  let(:payload) { LogStash::Json.load(last_response.body) }
-
-  context "#root" do
-
-    before(:all) do
-      do_request { get "/" }
-    end
-
-    it "respond OK" do
-      expect(last_response).to be_ok
-    end
-
-    ["events", "jvm"].each do |key|
-      it "contains #{key} information" do
-        expect(payload).to include(key)
-      end
-    end
-  end
-
-  context "#events" do
-
-    let(:payload) { LogStash::Json.load(last_response.body) }
-
-    before(:all) do
-      do_request { get "/events" }
-    end
-
-    it "respond OK" do
-      expect(last_response).to be_ok
-    end
-
-    it "contains events information" do
-      expect(payload).to include("events")
-    end
-  end
-
-  context "#jvm" do
-
-    let(:payload) { LogStash::Json.load(last_response.body) }
-
-    before(:all) do
-      do_request { get "/jvm" }
-    end
-
-    it "respond OK" do
-      expect(last_response).to be_ok
-    end
-
-    it "contains memory information" do
-      expect(payload).to include("mem")
-    end
-  end
-
+  # DSL describing response structure
+  root_structure = {
+    "events"=>{
+      "in"=>Numeric,
+      "filtered"=>Numeric,
+      "out"=>Numeric
+    },
+    "jvm"=>{
+      "threads"=>{
+        "count"=>Numeric,
+        "peak_count"=>Numeric
+      }
+    },
+    "process"=>{
+      "peak_open_file_descriptors"=>Numeric,
+      "max_file_descriptors"=>Numeric,
+      "open_file_descriptors"=>Numeric,
+      "mem"=>{
+        "total_virtual_in_bytes"=>Numeric
+      },
+      "cpu"=>{
+        "total_in_millis"=>Numeric,
+        "percent"=>Numeric
+      }
+    }
+  }
+
+  test_api_and_resources(root_structure)
 end

data/spec/api/lib/api/plugins_spec.rb

@@ -1,10 +1,10 @@
 # encoding: utf-8
 require_relative "../../spec_helper"
 require "sinatra"
-require "app/modules/plugins"
+require "logstash/api/modules/plugins"
 require "logstash/json"
 
-describe LogStash::Api::Plugins do
+describe LogStash::Api::Modules::Plugins do
 
   include Rack::Test::Methods
 
@@ -23,7 +23,7 @@ describe LogStash::Api::Plugins do
   end
 
   it "return valid json content type" do
-    expect(last_response.content_type).to eq("application/json")
+    expect(last_response.content_type).to eq("application/json"), "Did not get json, got #{last_response.content_type} / #{last_response.body}"
   end
 
   context "#schema" do

data/spec/api/lib/api/root_spec.rb

@@ -1,10 +1,10 @@
 # encoding: utf-8
 require_relative "../../spec_helper"
 require "sinatra"
-require "app/root"
+require "logstash/api/modules/root"
 require "logstash/json"
 
-describe LogStash::Api::Root do
+describe LogStash::Api::Modules::Root do
 
   include Rack::Test::Methods
 

data/spec/api/lib/api/support/resource_dsl_methods.rb

@@ -0,0 +1,47 @@
+module ResourceDSLMethods
+  # Convert a nested hash to a mapping of key paths to expected classes
+  def hash_to_mapping(h, path=[], mapping={})
+    h.each do |k,v|
+      if v.is_a?(Hash)
+        hash_to_mapping(v, path + [k], mapping)
+      else
+        full_path = path + [k]
+        mapping[full_path] = v
+      end
+    end
+    mapping
+  end
+
+  def test_api(expected, path)
+    context "GET #{path}" do
+      let(:payload) { LogStash::Json.load(last_response.body) }
+      
+      before(:all) do
+        do_request { get path }
+      end      
+      
+      it "should respond OK" do
+        expect(last_response).to be_ok
+      end
+      
+      hash_to_mapping(expected).each do |path,klass|
+        dotted = path.join(".")
+        
+        it "should set '#{dotted}' to be a '#{klass}'" do
+          path_value = path.reduce(payload) {|acc,v| acc[v]}
+          expect(path_value).to be_a(klass), "could not find '#{dotted}' in #{payload}"
+        end
+      end
+    end
+
+    yield if block_given? # Add custom expectations
+  end
+
+  def test_api_and_resources(expected)
+    test_api(expected, "/")
+
+    expected.keys.each do |key|
+      test_api({key => expected[key]}, "/#{key}")
+    end
+  end
+end

data/spec/api/lib/commands/stats.rb

@@ -0,0 +1,47 @@
+# encoding: utf-8
+require_relative "../../spec_helper"
+
+describe LogStash::Api::Commands::Stats do
+
+  let(:report_method) { :run }
+  subject(:report) { do_request { report_class.new.send(report_method) } }
+
+  let(:report_class) { described_class }
+
+  describe "#events" do
+    let(:report_method) { :events }
+
+    it "return events information" do
+      expect(report.keys).to include(:in, :filtered, :out)
+    end
+  end
+  
+  describe "#hot_threads" do
+    let(:report_method) { :hot_threads }
+    
+    it "should return hot threads information as a string" do
+      expect(report.to_s).to be_a(String)
+    end
+
+    it "should return hot threads info as a hash" do
+      expect(report.to_hash).to be_a(Hash)
+    end
+  end
+
+  describe "memory stats" do
+    let(:report_method) { :memory }
+      
+    it "return hot threads information" do
+      expect(report).not_to be_empty
+    end
+
+    it "return heap information" do
+      expect(report.keys).to include(:heap_used_in_bytes)
+    end
+
+    it "return non heap information" do
+      expect(report.keys).to include(:non_heap_used_in_bytes)
+    end
+
+  end
+end

data/spec/api/spec_helper.rb

@@ -1,16 +1,18 @@
 # encoding: utf-8
 API_ROOT = File.expand_path(File.join(File.dirname(__FILE__), "..", "..", "lib", "logstash", "api"))
 
+
+
 require "logstash/devutils/rspec/spec_helper"
 
+$LOAD_PATH.unshift(File.expand_path(File.dirname(__FILE__)))
+require "lib/api/support/resource_dsl_methods"
+
+require "logstash/settings"
 require 'rack/test'
 require 'rspec'
 require "json"
 
-ENV['RACK_ENV'] = 'test'
-
-Rack::Builder.parse_file(File.join(API_ROOT, 'init.ru'))
-
 def read_fixture(name)
   path = File.join(File.dirname(__FILE__), "fixtures", name)
   File.read(path)
@@ -37,26 +39,24 @@ class LogStashRunner
   attr_reader :config_str, :agent, :pipeline_settings
 
   def initialize
-    args = [
-      :logger => Cabin::Channel.get(LogStash),
-      :auto_reload => false,
-      :collect_metric => true,
-      :debug => false,
-      :node_name => "test_agent",
-      :web_api_http_port => rand(9600..9700)
-    ]
-
     @config_str   = "input { generator {count => 0} } output { }"
-    @agent = LogStash::DummyAgent.new(*args)
-    @pipeline_settings ||= { :pipeline_id => "main",
-                             :config_str => config_str,
-                            :pipeline_batch_size => 1,
-                            :flush_interval => 1,
-                            :pipeline_workers => 1 }
+    args = {
+      "config.reload.automatic" => false,
+      "metric.collect" => true,
+      "log.level" => "debug",
+      "node.name" => "test_agent",
+      "http.port" => rand(9600..9700),
+      "config.string" => @config_str,
+      "pipeline.batch.size" => 1,
+      "pipeline.workers" => 1
+    }
+    @settings = ::LogStash::SETTINGS.clone.merge(args)
+
+    @agent = LogStash::DummyAgent.new(@settings)
   end
 
   def start
-    agent.register_pipeline("main", pipeline_settings)
+    agent.register_pipeline("main", @settings)
     @runner = Thread.new(agent) do |_agent|
       _agent.execute
     end
@@ -81,7 +81,7 @@ end
 
 ##
 # Method used to wrap up a request in between of a running
-# pipeline, this makes the hole execution model easier and
+# pipeline, this makes the whole execution model easier and
 # more contained as some threads might go wild.
 ##
 def do_request(&block)