logstash-core 5.6.16-java → 6.0.0.alpha1-java

data/lib/logstash/util/wrapped_synchronous_queue.rb

@@ -18,6 +18,21 @@ module LogStash; module Util
     end
     alias_method(:<<, :push)
 
+    # Offer an object to the queue, wait for the specified amount of time.
+    # If adding to the queue was successful it wil return true, false otherwise.
+    #
+    # @param [Object] Object to add to the queue
+    # @param [Integer] Time in milliseconds to wait before giving up
+    # @return [Boolean] True if adding was successful if not it return false
+    def offer(obj, timeout_ms)
+      @queue.offer(obj, timeout_ms, TimeUnit::MILLISECONDS)
+    end
+
+    # Blocking
+    def take
+      @queue.take
+    end
+
     # Block for X millis
     def poll(millis)
       @queue.poll(millis, TimeUnit::MILLISECONDS)
@@ -68,17 +83,11 @@ module LogStash; module Util
 
       def set_events_metric(metric)
         @event_metric = metric
-        @event_metric_out = @event_metric.counter(:out)
-        @event_metric_filtered = @event_metric.counter(:filtered)
-        @event_metric_time = @event_metric.counter(:duration_in_millis)
         define_initial_metrics_values(@event_metric)
       end
 
       def set_pipeline_metric(metric)
         @pipeline_metric = metric
-        @pipeline_metric_out = @pipeline_metric.counter(:out)
-        @pipeline_metric_filtered = @pipeline_metric.counter(:filtered)
-        @pipeline_metric_time = @pipeline_metric.counter(:duration_in_millis)
         define_initial_metrics_values(@pipeline_metric)
       end
 
@@ -89,11 +98,8 @@ module LogStash; module Util
       end
 
       def inflight_batches
-        @mutex.lock
-        begin
+        @mutex.synchronize do
           yield(@inflight_batches)
-        ensure
-          @mutex.unlock
         end
       end
 
@@ -109,24 +115,16 @@ module LogStash; module Util
 
       def read_batch
         batch = new_batch
-        @mutex.lock
-        begin
-          batch.read_next
-        ensure
-          @mutex.unlock
-        end
+        @mutex.synchronize { batch.read_next }
         start_metrics(batch)
         batch
       end
 
       def start_metrics(batch)
-        @mutex.lock
-        # there seems to be concurrency issues with metrics, keep it in the mutex
-        begin
+        @mutex.synchronize do
+          # there seems to be concurrency issues with metrics, keep it in the mutex
           set_current_thread_inflight_batch(batch)
           start_clock
-        ensure
-          @mutex.unlock
         end
       end
 
@@ -135,18 +133,18 @@ module LogStash; module Util
       end
 
       def close_batch(batch)
-        @mutex.lock
-        begin
+        @mutex.synchronize do
           # there seems to be concurrency issues with metrics, keep it in the mutex
           @inflight_batches.delete(Thread.current)
           stop_clock(batch)
-        ensure
-          @mutex.unlock
         end
       end
 
       def start_clock
-        @inflight_clocks[Thread.current] = java.lang.System.current_time_millis
+        @inflight_clocks[Thread.current] = [
+          @event_metric.time(:duration_in_millis),
+          @pipeline_metric.time(:duration_in_millis)
+        ]
       end
 
       def stop_clock(batch)
@@ -155,22 +153,20 @@ module LogStash; module Util
             # only stop (which also records) the metrics if the batch is non-empty.
             # start_clock is now called at empty batch creation and an empty batch could
             # stay empty all the way down to the close_batch call.
-            time_taken = java.lang.System.current_time_millis - @inflight_clocks[Thread.current]
-            @event_metric_time.increment(time_taken)
-            @pipeline_metric_time.increment(time_taken)
+            @inflight_clocks[Thread.current].each(&:stop)
           end
           @inflight_clocks.delete(Thread.current)
         end
       end
 
       def add_filtered_metrics(batch)
-        @event_metric_filtered.increment(batch.filtered_size)
-        @pipeline_metric_filtered.increment(batch.filtered_size)
+        @event_metric.increment(:filtered, batch.filtered_size)
+        @pipeline_metric.increment(:filtered, batch.filtered_size)
       end
 
       def add_output_metrics(batch)
-        @event_metric_out.increment(batch.filtered_size)
-        @pipeline_metric_out.increment(batch.filtered_size)
+        @event_metric.increment(:out, batch.filtered_size)
+        @pipeline_metric.increment(:out, batch.filtered_size)
       end
     end
 
@@ -218,12 +214,6 @@ module LogStash; module Util
         # @cancelled[event] = true
       end
 
-      def to_a
-        events = []
-        each {|e| events << e}
-        events
-      end
-
       def each(&blk)
         # take care not to cause @originals or @generated to change during iteration
         @iterating = true

data/lib/logstash/version.rb

@@ -2,11 +2,13 @@
 
 # The version of the logstash package (not the logstash-core gem version).
 #
-# sourced from a copy of the master versions.yml file, see logstash-core/logstash-core.gemspec
-if !defined?(ALL_VERSIONS)
-  require 'yaml'
-  ALL_VERSIONS = YAML.load_file(File.expand_path("../../versions-gem-copy.yml", File.dirname(__FILE__)))
-end
-if !defined?(LOGSTASH_VERSION)
-  LOGSTASH_VERSION = ALL_VERSIONS.fetch("logstash")
-end
+# Note to authors: this should not include dashes because 'gem' barfs if
+# you include a dash in the version string.
+
+# TODO: (colin) the logstash-core gem uses it's own version number in logstash-core/lib/logstash-core/version.rb
+#       there are some dependencies in logstash-core on the LOGSTASH_VERSION constant this is why
+#       the logstash version is currently defined here in logstash-core/lib/logstash/version.rb but
+#       eventually this file should be in the root logstash lib fir and dependencies in logstash-core should be
+#       fixed.
+
+LOGSTASH_VERSION = "6.0.0-alpha1"

data/locales/en.yml

@@ -16,10 +16,12 @@ en:
     pipeline:
       worker-error: |-
         A plugin had an unrecoverable error. Will restart this plugin.
+          Pipeline_id:%{pipeline_id}
           Plugin: %{plugin}
           Error: %{error}
       worker-error-debug: |-
         A plugin had an unrecoverable error. Will restart this plugin.
+          Pipeline_id:%{pipeline_id}
           Plugin: %{plugin}
           Error: %{error}
           Exception: %{exception}
@@ -79,37 +81,14 @@ en:
           ::: {%{hostname}}
           Hot threads at %{time}, busiestThreads=%{top_count}:
         thread_title: |-
-          %{percent_of_cpu_time} % of cpu usage, state: %{thread_state}, thread name: '%{thread_name}', thread id: %{thread_id}
+          %{percent_of_cpu_time} % of cpu usage, state: %{thread_state}, thread name: '%{thread_name}'
       logging:
         unrecognized_option: |-
           unrecognized option [%{option}]
-    modules:
-      configuration:
-        parse-failed: |-
-          Failed to parse the module configuration: [%{error}]
-        modules-must-be-unique: >-
-          Only a single instance of any module can be run at a time. Duplicate
-          modules: %{duplicate_modules}
-        modules-invalid-name: >-
-          Invalid module name: %{module_name}
-        modules-variables-malformed: >-
-          Failed to parse module variable %{rawvar}.  Must be in -M
-          "MODULE_NAME.KEY.SUBKEY=VALUE" format
-        modules-unavailable: >-
-          The modules specified are not available yet.
-          Specified modules: %{specified_modules}
-          Available modules: %{available_modules}
-        elasticsearch_connection_failed: >-
-          Failed to import module configurations to Elasticsearch and/or Kibana.
-          Module: %{module_name} has Elasticsearch hosts: %{elasticsearch_hosts} and Kibana hosts: %{kibana_hosts}
-        modules-too-many-specified: >-
-          Too many modules specified. Maximum allowed: %{max}, specified: %{specified_modules}
-
     runner:
       short-help: |-
         usage:
           bin/logstash -f CONFIG_PATH [-t] [-r] [] [-w COUNT] [-l LOG]
-          bin/logstash --modules MODULE_NAME [-M "MODULE_NAME.var.PLUGIN_TYPE.PLUGIN_NAME.VARIABLE_NAME=VALUE"] [-t] [-w COUNT] [-l LOG]
           bin/logstash -e CONFIG_STR [-t] [--log.level fatal|error|warn|info|debug|trace] [-w COUNT] [-l LOG]
           bin/logstash -i SHELL [--log.level fatal|error|warn|info|debug|trace]
           bin/logstash -V [--log.level fatal|error|warn|info|debug|trace]
@@ -119,14 +98,6 @@ en:
       missing-configuration: >-
         No configuration file was specified. Perhaps you forgot to provide
         the '-f yourlogstash.conf' flag?
-      config-module-exclusive: >-
-        Settings 'path.config' (-f) or 'config.string' (-e) can't be used in conjunction with
-        (--modules) or the "modules:" block in the logstash.yml file.
-      reload-with-modules: >-
-        Configuration reloading can't be used with command-line or logstash.yml specified modules.
-      cli-module-override: >-
-        Both command-line and logstash.yml modules configurations detected.
-        Using command-line module configuration to override logstash.yml module configuration.
       reload-without-config-path: >-
         Configuration reloading also requires passing a configuration path with '-f yourlogstash.conf'
       locked-data-path: >-
@@ -212,28 +183,6 @@ en:
           "%{default_output}"
           If you wish to use both defaults, please use
           the empty string for the '-e' flag.
-        modules: |+
-          Load Logstash modules.
-          Modules can be defined using multiple instances
-          '--modules module1 --modules module2',
-             or comma-separated syntax
-          '--modules=module1,module2'
-          Cannot be used in conjunction with '-e' or '-f'
-          Use of '--modules' will override modules declared
-          in the 'logstash.yml' file.
-        modules_variable: |+
-          Load variables for module template.
-          Multiple instances of '-M' or
-          '--modules.variable' are supported.
-          Ignored if '--modules' flag is not used.
-          Should be in the format of
-          '-M "MODULE_NAME.var.PLUGIN_TYPE.PLUGIN_NAME.VARIABLE_NAME=VALUE"'
-          as in
-          '-M "example.var.filter.mutate.fieldname=fieldvalue"'
-        modules_setup: |+
-          Load index template into Elasticsearch, and saved searches, 
-          index-pattern, visualizations, and dashboards into Kibana when
-          running modules.
         configtest: |+
           Check configuration for valid syntax and then exit.
         http_host: Web API binding host

data/logstash-core.gemspec

@@ -1,25 +1,6 @@
 # -*- encoding: utf-8 -*-
 lib = File.expand_path('../lib', __FILE__)
 $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
-
-project_versions_yaml_path = File.expand_path("../versions.yml", File.dirname(__FILE__))
-if File.exist?(project_versions_yaml_path)
-  # we need to copy the project level versions.yml into the gem root
-  # to be able to package it into the gems file structure
-  # as the require 'logstash-core/version' loads the yaml file from within the gem root.
-  #
-  # we ignore the copy in git and we overwrite an existing file
-  # each time we build the logstash-core gem
-  original_lines = IO.readlines(project_versions_yaml_path)
-  original_lines << ""
-  original_lines << "# This is a copy the project level versions.yml into this gem's root and it is created when the gemspec is evaluated."
-  gem_versions_yaml_path = File.expand_path("./versions-gem-copy.yml", File.dirname(__FILE__))
-  File.open(gem_versions_yaml_path, 'w') do |new_file|
-    # create or overwrite
-    new_file.puts(original_lines)
-  end
-end
-
 require 'logstash-core/version'
 
 Gem::Specification.new do |gem|
@@ -30,14 +11,11 @@ Gem::Specification.new do |gem|
   gem.homepage      = "http://www.elastic.co/guide/en/logstash/current/index.html"
   gem.license       = "Apache License (2.0)"
 
-  gem.files         = Dir.glob(
-    %w(versions-gem-copy.yml logstash-core.gemspec gemspec_jars.rb lib/**/*.rb spec/**/*.rb locales/*
-    lib/logstash/api/init.ru lib/logstash-core/logstash-core.jar)
-  )
+  gem.files         = Dir.glob(["logstash-core.gemspec", "gemspec_jars.rb", "lib/**/*.rb", "spec/**/*.rb", "locales/*", "lib/logstash/api/init.ru", "lib/logstash-core/logstash-core.jar"])
   gem.test_files    = gem.files.grep(%r{^(test|spec|features)/})
   gem.name          = "logstash-core"
   gem.require_paths = ["lib"]
-  gem.version       = LOGSTASH_CORE_VERSION
+  gem.version       = LOGSTASH_CORE_VERSION.gsub("-", ".")
 
   gem.platform = "java"
 
@@ -46,15 +24,12 @@ Gem::Specification.new do |gem|
   gem.add_runtime_dependency "clamp", "~> 0.6.5" #(MIT license) for command line args/flags
   gem.add_runtime_dependency "filesize", "0.0.4" #(MIT license) for :bytes config validator
   gem.add_runtime_dependency "gems", "~> 0.8.3"  #(MIT license)
-  gem.add_runtime_dependency "concurrent-ruby", "~> 1.0", ">= 1.0.5"
-
-  # Later versions are ruby 2.0 only. We should remove the rack dep once we support 9k
-  gem.add_runtime_dependency "rack", '1.6.6'
-
+  gem.add_runtime_dependency "concurrent-ruby", "1.0.0"
   gem.add_runtime_dependency "sinatra", '~> 1.4', '>= 1.4.6'
   gem.add_runtime_dependency 'puma', '~> 2.16'
-  gem.add_runtime_dependency "jruby-openssl", "~> 0.10.1" # >= 0.9.13 Required to support TLSv1.2
+  gem.add_runtime_dependency "jruby-openssl", "0.9.16" # >= 0.9.13 Required to support TLSv1.2
   gem.add_runtime_dependency "chronic_duration", "0.10.6"
+  gem.add_runtime_dependency "jrmonitor", '~> 0.4.2'
 
   # TODO(sissel): Treetop 1.5.x doesn't seem to work well, but I haven't
   # investigated what the cause might be. -Jordan
@@ -64,19 +39,17 @@ Gem::Specification.new do |gem|
   gem.add_runtime_dependency "i18n", "= 0.6.9" #(MIT license)
 
   # filetools and rakelib
-  gem.add_runtime_dependency "minitar", "~> 0.6.1"
-  gem.add_runtime_dependency "rubyzip", "~> 1.2.1"
+  gem.add_runtime_dependency "minitar", "~> 0.5.4"
+  gem.add_runtime_dependency "rubyzip", "~> 1.1.7"
   gem.add_runtime_dependency "thread_safe", "~> 0.3.5" #(Apache 2.0 license)
 
-  gem.add_runtime_dependency "jrjackson", "= #{ALL_VERSIONS.fetch('jrjackson')}" #(Apache 2.0 license)
+  gem.add_runtime_dependency "jrjackson", "~> 0.4.2" #(Apache 2.0 license)
 
   gem.add_runtime_dependency "jar-dependencies"
   # as of Feb 3rd 2016, the ruby-maven gem is resolved to version 3.3.3 and that version
   # has an rdoc problem that causes a bundler exception. 3.3.9 is the current latest version
   # which does not have this problem.
   gem.add_runtime_dependency "ruby-maven", "~> 3.3.9"
-  gem.add_runtime_dependency "elasticsearch", "~> 5.0", ">= 5.0.4" # Ruby client for ES (Apache 2.0 license)
-  gem.add_runtime_dependency "manticore", '>= 0.5.4', '< 1.0.0'
 
   eval(File.read(File.expand_path("../gemspec_jars.rb", __FILE__)))
 end

data/spec/{logstash/api/modules → api/lib/api}/logging_spec.rb

@@ -1,5 +1,5 @@
 # encoding: utf-8
-require "spec_helper"
+require_relative "../../spec_helper"
 require "sinatra"
 require "logstash/api/modules/logging"
 require "logstash/json"
@@ -10,6 +10,15 @@ describe LogStash::Api::Modules::Logging do
   describe "#logging" do
 
     context "when setting a logger's log level" do
+      before(:all) do
+        @runner = LogStashRunner.new
+        @runner.start
+      end
+
+      after(:all) do
+        @runner.stop
+      end
+
       it "should return a positive acknowledgement on success" do
         put '/', '{"logger.logstash": "ERROR"}'
         payload = LogStash::Json.load(last_response.body)

data/spec/{logstash/api/modules → api/lib/api}/node_plugins_spec.rb

@@ -1,5 +1,6 @@
 # encoding: utf-8
-require "spec_helper"
+require_relative "../../../support/shared_examples"
+require_relative "../../spec_helper"
 require "sinatra"
 require "logstash/api/modules/plugins"
 require "logstash/json"

data/spec/{logstash/api/modules → api/lib/api}/node_spec.rb

@@ -1,5 +1,6 @@
 # encoding: utf-8
-require "spec_helper"
+require_relative "../../spec_helper"
+require_relative "../../../support/shared_examples"
 require "sinatra"
 require "logstash/api/modules/node"
 require "logstash/json"
@@ -114,8 +115,7 @@ describe LogStash::Api::Modules::Node do
           "batch_size" => Numeric,
           "batch_delay" => Numeric,
           "config_reload_automatic" => Boolean,
-          "config_reload_interval" => Numeric,
-          "dead_letter_queue_enabled" => Boolean
+          "config_reload_interval" => Numeric
         },
         "os" => {
           "name" => String,

data/spec/{logstash/api/modules → api/lib/api}/node_stats_spec.rb

@@ -1,6 +1,6 @@
 # encoding: utf-8
-require "spec_helper"
-
+require_relative "../../spec_helper"
+require_relative "../../../support/shared_examples"
 require "sinatra"
 require "logstash/api/modules/node_stats"
 require "logstash/json"
@@ -70,7 +70,7 @@ describe LogStash::Api::Modules::NodeStats do
       "cpu"=>{
         "total_in_millis"=>Numeric,
         "percent"=>Numeric,
-        # load_average is not supported on Windows, set it below
+        "load_average" => { "1m" => Numeric }
       }
     },
    "pipeline" => {
@@ -88,9 +88,5 @@ describe LogStash::Api::Modules::NodeStats do
    }
   }
 
-  unless LogStash::Environment.windows?
-    root_structure["process"]["cpu"]["load_average"] = { "1m" => Numeric }
-  end
-
   test_api_and_resources(root_structure)
 end

data/spec/{logstash/api/modules → api/lib/api}/plugins_spec.rb

@@ -1,5 +1,6 @@
 # encoding: utf-8
-require "spec_helper"
+require_relative "../../spec_helper"
+require_relative "../../../support/shared_examples"
 require "sinatra"
 require "logstash/api/modules/plugins"
 require "logstash/json"
@@ -39,9 +40,7 @@ describe LogStash::Api::Modules::Plugins do
 
     it "return a list of available plugins" do
       payload["plugins"].each do |plugin|
-        expect do 
-          Gem::Specification.find_by_name(plugin["name"])
-        end.not_to raise_error
+        expect(plugin).to be_available?
       end
     end
 

data/spec/{logstash/api/modules → api/lib/api}/root_spec.rb

@@ -1,6 +1,6 @@
 # encoding: utf-8
-require "spec_helper"
-
+require_relative "../../spec_helper"
+require_relative "../../../support/shared_examples"
 require "sinatra"
 require "logstash/api/modules/root"
 require "logstash/json"