logstash-core 5.4.3-java → 5.5.0-java

data/spec/logstash/timestamp_spec.rb

@@ -20,6 +20,12 @@ describe LogStash::Timestamp do
       expect(t.time.to_i).to eq(now.to_i)
     end
 
+    it "should have consistent behaviour across == and .eql?" do
+      its_xmas = Time.utc(2015, 12, 25, 0, 0, 0)
+      expect(LogStash::Timestamp.new(its_xmas)).to eql(LogStash::Timestamp.new(its_xmas))
+      expect(LogStash::Timestamp.new(its_xmas)).to be ==(LogStash::Timestamp.new(its_xmas))
+    end
+
     it "should raise exception on invalid format" do
       expect{LogStash::Timestamp.new("foobar")}.to raise_error
     end

data/spec/support/helpers.rb

@@ -8,9 +8,88 @@ ensure
 end
 
 def clear_data_dir
-    data_path = agent_settings.get("path.data")
+    if defined?(agent_settings)
+      data_path = agent_settings.get("path.data")
+    else
+      data_path = LogStash::SETTINGS.get("path.data")
+    end
+
     Dir.foreach(data_path) do |f|
     next if f == "." || f == ".." || f == ".gitkeep"
     FileUtils.rm_rf(File.join(data_path, f))
   end
 end
+
+def mock_settings(settings_values={})
+  settings = LogStash::SETTINGS.clone
+
+  settings_values.each do |key, value|
+    settings.set(key, value)
+  end
+
+  settings
+end
+
+def make_test_agent(settings=mock_settings)
+    ::LogStash::Agent.new(settings)
+end
+
+def mock_pipeline(pipeline_id, reloadable = true, config_hash = nil)
+  config_string = "input { stdin { id => '#{pipeline_id}' }}"
+  settings = mock_settings("pipeline.id" => pipeline_id.to_s,
+                           "config.string" => config_string,
+                           "config.reload.automatic" => reloadable)
+  pipeline = LogStash::Pipeline.new(config_string, settings)
+  pipeline
+end
+
+def mock_pipeline_config(pipeline_id, config_string = nil, settings = {})
+  config_string = "input { stdin { id => '#{pipeline_id}' }}" if config_string.nil?
+
+  # This is for older tests when we already have a config
+  unless settings.is_a?(LogStash::Settings)
+    settings.merge!({ "pipeline.id" => pipeline_id.to_s })
+    settings = mock_settings(settings)
+  end
+
+  config_part = org.logstash.common.SourceWithMetadata.new("config_string", "config_string", config_string)
+
+  LogStash::Config::PipelineConfig.new(LogStash::Config::Source::Local, pipeline_id, config_part, settings)
+end
+
+def start_agent(agent)
+  agent_task = Stud::Task.new do
+    begin
+      agent.execute
+    rescue => e
+      raise "Start Agent exception: #{e}"
+    end
+  end
+
+  sleep(0.1) unless subject.running?
+  agent_task
+end
+
+def temporary_file(content, file_name = Time.now.to_i.to_s, directory = Stud::Temporary.pathname)
+  FileUtils.mkdir_p(directory)
+  target = ::File.join(directory, file_name)
+
+  File.open(target, "w+") do |f|
+    f.write(content)
+  end
+  target
+end
+
+RSpec::Matchers.define :ir_eql do |expected|
+  match do |actual|
+    next unless expected.java_kind_of?(org.logstash.config.ir.SourceComponent) && actual.java_kind_of?(org.logstash.config.ir.SourceComponent)
+
+    expected.sourceComponentEquals(actual)
+  end
+
+  failure_message do |actual|
+    "actual value \n#{actual.to_s}\nis not .sourceComponentEquals to the expected value: \n#{expected.to_s}\n"
+  end
+end
+
+SUPPORT_DIR = Pathname.new(::File.join(::File.dirname(__FILE__), "support"))

data/spec/support/matchers.rb

@@ -28,3 +28,16 @@ RSpec::Matchers.define :implement_interface_of do |type, key, value|
     "Expecting `#{expected}` to implements instance methods of `#{actual}`, missing methods: #{missing_methods.join(",")}"
   end
 end
+
+RSpec::Matchers.define :be_a_config_loading_error_hash do |regex|
+  match do |hash|
+    expect(hash).to include(:error)
+    error = hash[:error]
+    expect(error).to be_a(LogStash::ConfigLoadingError)
+    expect(error.message).to match(regex)
+  end
+
+  match_when_negated do
+    raise "Not implemented"
+  end
+end

data/spec/support/shared_contexts.rb

@@ -0,0 +1,38 @@
+shared_context "execution_context" do
+  let(:pipeline) { double("pipeline") }
+  let(:pipeline_id) { :main }
+  let(:agent) { double("agent") }
+  let(:plugin_id) { :plugin_id }
+  let(:plugin_type) { :plugin_type }
+  let(:dlq_writer) { double("dlq_writer") }
+  let(:execution_context) do
+    ::LogStash::ExecutionContext.new(pipeline, agent, plugin_id, plugin_type, dlq_writer)
+  end
+
+  before do
+    allow(pipeline).to receive(:pipeline_id).and_return(pipeline_id)
+    allow(pipeline).to receive(:agent).and_return(agent)
+  end
+end
+
+shared_context "api setup" do
+  before :all do
+    clear_data_dir
+    settings = mock_settings
+    config_string = "input { generator {id => 'api-generator-pipeline' count => 100 } } output { dummyoutput {} }"
+    settings.set("config.string", config_string)
+    @agent = make_test_agent(settings)
+    @agent.register_pipeline(settings)
+    @agent.execute
+  end
+
+  after :all do
+    @agent.shutdown
+  end
+
+  include Rack::Test::Methods
+
+  def app()
+    described_class.new(nil, @agent)
+  end
+end

data/spec/support/shared_examples.rb

@@ -97,7 +97,7 @@ end
 
 shared_examples "not found" do
   it "should return a 404 to unknown request" do
-    do_request { get "/i_want_to_believe-#{Time.now.to_i}" }
+    get "/i_want_to_believe-#{Time.now.to_i}"
     expect(last_response.content_type).to eq("application/json")
     expect(last_response).not_to be_ok
     expect(last_response.status).to eq(404)

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: logstash-core
 version: !ruby/object:Gem::Version
-  version: 5.4.3
+  version: 5.5.0
 platform: java
 authors:
 - Elastic
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2017-06-22 00:00:00.000000000 Z
+date: 2017-06-30 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
@@ -100,6 +100,20 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: 1.0.5
+- !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 1.6.6
+  name: rack
+  prerelease: false
+  type: :runtime
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 1.6.6
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
     requirements:
@@ -139,7 +153,7 @@ dependencies:
     requirements:
     - - '='
       - !ruby/object:Gem::Version
-        version: 0.9.16
+        version: 0.9.19
   name: jruby-openssl
   prerelease: false
   type: :runtime
@@ -147,7 +161,7 @@ dependencies:
     requirements:
     - - '='
       - !ruby/object:Gem::Version
-        version: 0.9.16
+        version: 0.9.19
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
     requirements:
@@ -162,20 +176,6 @@ dependencies:
     - - '='
       - !ruby/object:Gem::Version
         version: 0.10.6
-- !ruby/object:Gem::Dependency
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: 0.4.2
-  name: jrmonitor
-  prerelease: false
-  type: :runtime
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: 0.4.2
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
     requirements:
@@ -288,6 +288,46 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: 3.3.9
+- !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '5.0'
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 5.0.4
+  name: elasticsearch
+  prerelease: false
+  type: :runtime
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '5.0'
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 5.0.4
+- !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.5.4
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: 1.0.0
+  name: manticore
+  prerelease: false
+  type: :runtime
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.5.4
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: 1.0.0
 description: The core components of logstash, the scalable log and event management tool
 email:
 - info@elastic.co
@@ -330,6 +370,8 @@ files:
 - lib/logstash/config/grammar.rb
 - lib/logstash/config/loader.rb
 - lib/logstash/config/mixin.rb
+- lib/logstash/config/modules_common.rb
+- lib/logstash/elasticsearch_client.rb
 - lib/logstash/environment.rb
 - lib/logstash/errors.rb
 - lib/logstash/event.rb
@@ -340,6 +382,7 @@ files:
 - lib/logstash/inputs/base.rb
 - lib/logstash/inputs/threadable.rb
 - lib/logstash/instrument/collector.rb
+- lib/logstash/instrument/global_metrics.rb
 - lib/logstash/instrument/metric.rb
 - lib/logstash/instrument/metric_store.rb
 - lib/logstash/instrument/metric_type.rb
@@ -364,6 +407,17 @@ files:
 - lib/logstash/logging.rb
 - lib/logstash/logging/json.rb
 - lib/logstash/logging/logger.rb
+- lib/logstash/modules/cli_parser.rb
+- lib/logstash/modules/elasticsearch_config.rb
+- lib/logstash/modules/elasticsearch_resource.rb
+- lib/logstash/modules/file_reader.rb
+- lib/logstash/modules/importer.rb
+- lib/logstash/modules/kibana_base_resource.rb
+- lib/logstash/modules/kibana_config.rb
+- lib/logstash/modules/kibana_resource.rb
+- lib/logstash/modules/logstash_config.rb
+- lib/logstash/modules/resource_base.rb
+- lib/logstash/modules/scaffold.rb
 - lib/logstash/namespace.rb
 - lib/logstash/output_delegator.rb
 - lib/logstash/output_delegator_strategies/legacy.rb
@@ -397,6 +451,7 @@ files:
 - lib/logstash/util/buftok.rb
 - lib/logstash/util/byte_value.rb
 - lib/logstash/util/charset.rb
+- lib/logstash/util/dead_letter_queue_manager.rb
 - lib/logstash/util/decorators.rb
 - lib/logstash/util/duration_formatter.rb
 - lib/logstash/util/environment_variables.rb
@@ -419,20 +474,18 @@ files:
 - lib/logstash/webserver.rb
 - locales/en.yml
 - logstash-core.gemspec
-- spec/api/lib/api/logging_spec.rb
-- spec/api/lib/api/node_plugins_spec.rb
-- spec/api/lib/api/node_spec.rb
-- spec/api/lib/api/node_stats_spec.rb
-- spec/api/lib/api/plugins_spec.rb
-- spec/api/lib/api/root_spec.rb
-- spec/api/lib/api/support/resource_dsl_methods.rb
-- spec/api/lib/commands/stats.rb
-- spec/api/lib/errors_spec.rb
-- spec/api/lib/rack_app_spec.rb
-- spec/api/spec_helper.rb
 - spec/conditionals_spec.rb
 - spec/logstash/acked_queue_concurrent_stress_spec.rb
 - spec/logstash/agent_spec.rb
+- spec/logstash/api/commands/stats_spec.rb
+- spec/logstash/api/errors_spec.rb
+- spec/logstash/api/modules/logging_spec.rb
+- spec/logstash/api/modules/node_plugins_spec.rb
+- spec/logstash/api/modules/node_spec.rb
+- spec/logstash/api/modules/node_stats_spec.rb
+- spec/logstash/api/modules/plugins_spec.rb
+- spec/logstash/api/modules/root_spec.rb
+- spec/logstash/api/rack_app_spec.rb
 - spec/logstash/codecs/base_spec.rb
 - spec/logstash/config/config_ast_spec.rb
 - spec/logstash/config/cpu_core_strategy_spec.rb
@@ -467,6 +520,7 @@ files:
 - spec/logstash/output_delegator_spec.rb
 - spec/logstash/outputs/base_spec.rb
 - spec/logstash/patches_spec.rb
+- spec/logstash/pipeline_dlq_commit_spec.rb
 - spec/logstash/pipeline_pq_file_spec.rb
 - spec/logstash/pipeline_reporter_spec.rb
 - spec/logstash/pipeline_spec.rb
@@ -504,6 +558,7 @@ files:
 - spec/support/helpers.rb
 - spec/support/matchers.rb
 - spec/support/mocks_classes.rb
+- spec/support/shared_contexts.rb
 - spec/support/shared_examples.rb
 homepage: http://www.elastic.co/guide/en/logstash/current/index.html
 licenses:
@@ -536,20 +591,18 @@ signing_key:
 specification_version: 4
 summary: logstash-core - The core components of logstash
 test_files:
-- spec/api/lib/api/logging_spec.rb
-- spec/api/lib/api/node_plugins_spec.rb
-- spec/api/lib/api/node_spec.rb
-- spec/api/lib/api/node_stats_spec.rb
-- spec/api/lib/api/plugins_spec.rb
-- spec/api/lib/api/root_spec.rb
-- spec/api/lib/api/support/resource_dsl_methods.rb
-- spec/api/lib/commands/stats.rb
-- spec/api/lib/errors_spec.rb
-- spec/api/lib/rack_app_spec.rb
-- spec/api/spec_helper.rb
 - spec/conditionals_spec.rb
 - spec/logstash/acked_queue_concurrent_stress_spec.rb
 - spec/logstash/agent_spec.rb
+- spec/logstash/api/commands/stats_spec.rb
+- spec/logstash/api/errors_spec.rb
+- spec/logstash/api/modules/logging_spec.rb
+- spec/logstash/api/modules/node_plugins_spec.rb
+- spec/logstash/api/modules/node_spec.rb
+- spec/logstash/api/modules/node_stats_spec.rb
+- spec/logstash/api/modules/plugins_spec.rb
+- spec/logstash/api/modules/root_spec.rb
+- spec/logstash/api/rack_app_spec.rb
 - spec/logstash/codecs/base_spec.rb
 - spec/logstash/config/config_ast_spec.rb
 - spec/logstash/config/cpu_core_strategy_spec.rb
@@ -584,6 +637,7 @@ test_files:
 - spec/logstash/output_delegator_spec.rb
 - spec/logstash/outputs/base_spec.rb
 - spec/logstash/patches_spec.rb
+- spec/logstash/pipeline_dlq_commit_spec.rb
 - spec/logstash/pipeline_pq_file_spec.rb
 - spec/logstash/pipeline_reporter_spec.rb
 - spec/logstash/pipeline_spec.rb
@@ -621,4 +675,5 @@ test_files:
 - spec/support/helpers.rb
 - spec/support/matchers.rb
 - spec/support/mocks_classes.rb
+- spec/support/shared_contexts.rb
 - spec/support/shared_examples.rb

data/spec/api/lib/api/support/resource_dsl_methods.rb

@@ -1,87 +0,0 @@
-# Ruby doesn't have common class for boolean,
-# And to simplify the ResourceDSLMethods check it make sense to have it.
-module Boolean; end
-class TrueClass
-  include Boolean
-end
-class FalseClass
-  include Boolean
-end
-
-module ResourceDSLMethods
-  # Convert a nested hash to a mapping of key paths to expected classes
-  def hash_to_mapping(h, path=[], mapping={})
-    h.each do |k,v|
-      if v.is_a?(Hash)
-        hash_to_mapping(v, path + [k], mapping)
-      else
-        full_path = path + [k]
-        mapping[full_path] = v
-      end
-    end
-    mapping
-  end
-
-  def test_api(expected, path)
-    context "GET #{path}" do
-      let(:payload) { LogStash::Json.load(last_response.body) }
-
-      before(:all) do
-        do_request { get path }
-      end
-
-      it "should respond OK" do
-        expect(last_response).to be_ok
-      end
-
-
-      describe "the default metadata" do
-        it "should include the host" do
-          expect(payload["host"]).to eql(Socket.gethostname)
-        end
-
-        it "should include the version" do
-          expect(payload["version"]).to eql(LOGSTASH_CORE_VERSION)
-        end
-
-        it "should include the http address" do
-          expect(payload["http_address"]).to eql("#{Socket.gethostname}:#{::LogStash::WebServer::DEFAULT_PORTS.first}")
-        end
-
-        it "should include the node name" do
-          expect(payload["name"]).to eql(@runner.agent.name)
-        end
-
-        it "should include the node id" do
-          expect(payload["id"]).to eql(@runner.agent.id)
-        end
-      end
-
-      hash_to_mapping(expected).each do |resource_path,klass|
-        dotted = resource_path.join(".")
-
-        it "should set '#{dotted}' at '#{path}' to be a '#{klass}'" do
-          expect(last_response).to be_ok # fail early if need be
-          resource_path_value = resource_path.reduce(payload) do |acc,v|
-            expect(acc.has_key?(v)).to eql(true), "Expected to find value '#{v}' in structure '#{acc}', but could not. Payload was '#{payload}'"
-            acc[v]
-          end
-          expect(resource_path_value).to be_a(klass), "could not find '#{dotted}' in #{payload}"
-        end
-      end
-    end
-
-    yield if block_given? # Add custom expectations
-  end
-
-  def test_api_and_resources(expected, xopts={})
-    xopts[:exclude_from_root] ||= []
-    root_expectation = expected.clone
-    xopts[:exclude_from_root].each {|k| root_expectation.delete(k)}
-    test_api(root_expectation, "/")
-
-    expected.keys.each do |key|
-      test_api({key => expected[key]}, "/#{key}")
-    end
-  end
-end