logstash-core 5.0.0.alpha4.snapshot3-java → 5.0.0.alpha5.snapshot1-java

data/lib/logstash/util/wrapped_synchronous_queue.rb

@@ -37,5 +37,225 @@ module LogStash; module Util
     def poll(millis)
       @queue.poll(millis, TimeUnit::MILLISECONDS)
     end
+
+    def write_client
+      WriteClient.new(self)
+    end
+
+    def read_client()
+      ReadClient.new(self)
+    end
+
+    class ReadClient
+      # We generally only want one thread at a time able to access pop/take/poll operations
+      # from this queue. We also depend on this to be able to block consumers while we snapshot
+      # in-flight buffers
+
+      def initialize(queue, batch_size = 125, wait_for = 5)
+        @queue = queue
+        @mutex = Mutex.new
+        # Note that @infilght_batches as a central mechanism for tracking inflight
+        # batches will fail if we have multiple read clients in the pipeline.
+        @inflight_batches = {}
+        @batch_size = batch_size
+        @wait_for = wait_for
+      end
+
+      def set_batch_dimensions(batch_size, wait_for)
+        @batch_size = batch_size
+        @wait_for = wait_for
+      end
+
+      def set_events_metric(metric)
+        @event_metric = metric
+      end
+
+      def set_pipeline_metric(metric)
+        @pipeline_metric = metric
+      end
+
+      def inflight_batches
+        @mutex.synchronize do
+          yield(@inflight_batches)
+        end
+      end
+
+      def current_inflight_batch
+        @inflight_batches.fetch(Thread.current, [])
+      end
+
+      def take_batch
+        @mutex.synchronize do
+          batch = ReadBatch.new(@queue, @batch_size, @wait_for)
+          add_starting_metrics(batch)
+          set_current_thread_inflight_batch(batch)
+          batch
+        end
+      end
+
+      def set_current_thread_inflight_batch(batch)
+        @inflight_batches[Thread.current] = batch
+      end
+
+      def close_batch(batch)
+        @mutex.synchronize do
+          @inflight_batches.delete(Thread.current)
+        end
+      end
+
+      def add_starting_metrics(batch)
+        return if @event_metric.nil? || @pipeline_metric.nil?
+        @event_metric.increment(:in, batch.starting_size)
+        @pipeline_metric.increment(:in, batch.starting_size)
+      end
+
+      def add_filtered_metrics(batch)
+        @event_metric.increment(:filtered, batch.filtered_size)
+        @pipeline_metric.increment(:filtered, batch.filtered_size)
+      end
+
+      def add_output_metrics(batch)
+        @event_metric.increment(:out, batch.filtered_size)
+        @pipeline_metric.increment(:out, batch.filtered_size)
+      end
+    end
+
+    class ReadBatch
+      def initialize(queue, size, wait)
+        @shutdown_signal_received = false
+        @flush_signal_received = false
+        @originals = Hash.new
+        @cancelled = Hash.new
+        @generated = Hash.new
+        @iterating_temp = Hash.new
+        @iterating = false # Atomic Boolean maybe? Although batches are not shared across threads
+        take_originals_from_queue(queue, size, wait)
+      end
+
+      def merge(event)
+        return if event.nil? || @originals.key?(event)
+        # take care not to cause @generated to change during iteration
+        # @iterating_temp is merged after the iteration
+        if iterating?
+          @iterating_temp[event] = true
+        else
+          # the periodic flush could generate events outside of an each iteration
+          @generated[event] = true
+        end
+      end
+
+      def cancel(event)
+        @cancelled[event] = true
+      end
+
+      def each(&blk)
+        # take care not to cause @originals or @generated to change during iteration
+        @iterating = true
+        @originals.each do |e, _|
+          blk.call(e) unless @cancelled.include?(e)
+        end
+        @generated.each do |e, _|
+          blk.call(e) unless @cancelled.include?(e)
+        end
+        @iterating = false
+        update_generated
+      end
+
+      def size
+        filtered_size
+      end
+
+      def starting_size
+        @originals.size
+      end
+
+      def filtered_size
+        @originals.size + @generated.size
+      end
+
+      def cancelled_size
+        @cancelled.size
+      end
+
+      def shutdown_signal_received?
+        @shutdown_signal_received
+      end
+
+      def flush_signal_received?
+        @flush_signal_received
+      end
+
+      private
+
+      def iterating?
+        @iterating
+      end
+
+      def update_generated
+        @generated.update(@iterating_temp)
+        @iterating_temp.clear
+      end
+
+      def take_originals_from_queue(queue, size, wait)
+        size.times do |t|
+          event = (t == 0) ? queue.take : queue.poll(wait)
+          if event.nil?
+            # queue poll timed out
+            next
+          elsif event == LogStash::SHUTDOWN
+            # We MUST break here. If a batch consumes two SHUTDOWN events
+            # then another worker may have its SHUTDOWN 'stolen', thus blocking
+            # the pipeline.
+            @shutdown_signal_received = true
+            break
+          elsif event == LogStash::FLUSH
+            # See comment above
+            # We should stop doing work after flush as well.
+            @flush_signal_received = true
+            break
+          else
+            @originals[event] = true
+          end
+        end
+      end
+    end
+
+    class WriteClient
+      def initialize(queue)
+        @queue = queue
+      end
+
+      def get_new_batch
+        WriteBatch.new
+      end
+
+      def push(event)
+        @queue.push(event)
+      end
+      alias_method(:<<, :push)
+
+      def push_batch(batch)
+        batch.each do |event|
+          push(event)
+        end
+      end
+    end
+
+    class WriteBatch
+      def initialize
+        @events = []
+      end
+
+      def push(event)
+        @events.push(event)
+      end
+      alias_method(:<<, :push)
+
+      def each(&blk)
+        @events.each do |e|
+          blk.call(e)
+        end
+      end
+    end
   end
 end end

data/lib/logstash/version.rb

@@ -11,4 +11,4 @@
 #       eventually this file should be in the root logstash lib fir and dependencies in logstash-core should be
 #       fixed.
 
-LOGSTASH_VERSION = "5.0.0-alpha4.snapshot3"
+LOGSTASH_VERSION = "5.0.0.alpha5.snapshot1"

data/lib/logstash/webserver.rb

@@ -55,6 +55,10 @@ module LogStash
     def error(str)
       logger.error(str)
     end
+
+    def address
+      "#{http_host}:#{http_port}"
+    end
     
     # Empty method, this method is required because of the puma usage we make through
     # the Single interface, https://github.com/puma/puma/blob/master/lib/puma/single.rb#L82

data/lib/logstash-core/version.rb

@@ -5,4 +5,4 @@
 # Note to authors: this should not include dashes because 'gem' barfs if
 # you include a dash in the version string.
 
-LOGSTASH_CORE_VERSION = "5.0.0-alpha4.snapshot3"
+LOGSTASH_CORE_VERSION = "5.0.0.alpha5.snapshot1"

data/locales/en.yml

@@ -210,6 +210,10 @@ en:
         version: |+
           Emit the version of logstash and its friends,
           then exit.
+        datapath: |+
+          This should point to a writable directory. Logstash
+          will use this directory whenever it needs to store
+          data. Plugins will also have access to this path.
         pluginpath: |+
           A path of where to find plugins. This flag
           can be given multiple times to include

data/logstash-core.gemspec

@@ -15,9 +15,9 @@ Gem::Specification.new do |gem|
   gem.test_files    = gem.files.grep(%r{^(test|spec|features)/})
   gem.name          = "logstash-core"
   gem.require_paths = ["lib"]
-  gem.version       = LOGSTASH_CORE_VERSION.gsub(/-/, '.')
+  gem.version       = LOGSTASH_CORE_VERSION
 
-  gem.add_runtime_dependency "logstash-core-event-java", "5.0.0.alpha4.snapshot3"
+  gem.add_runtime_dependency "logstash-core-event-java", "5.0.0.alpha5.snapshot1"
 
   gem.add_runtime_dependency "cabin", "~> 0.8.0" #(Apache 2.0 license)
   gem.add_runtime_dependency "pry", "~> 0.10.1"  #(Ruby license)

data/spec/api/lib/api/node_spec.rb

@@ -85,7 +85,6 @@ describe LogStash::Api::Modules::Node do
           }
         },
         "hot_threads"=> {
-          "hostname" => String,
           "time" => String,
           "busiest_threads" => Numeric,
           "threads" => Array

data/spec/api/lib/api/node_stats_spec.rb

@@ -11,15 +11,38 @@ describe LogStash::Api::Modules::NodeStats do
 
   # DSL describing response structure
   root_structure = {
-    "events"=>{
-      "in"=>Numeric,
-      "filtered"=>Numeric,
-      "out"=>Numeric
-    },
     "jvm"=>{
       "threads"=>{
         "count"=>Numeric,
         "peak_count"=>Numeric
+      },
+      "mem" => {
+        "heap_used_in_bytes" => Numeric,
+        "heap_used_percent" => Numeric,
+        "heap_committed_in_bytes" => Numeric,
+        "heap_max_in_bytes" => Numeric,
+        "non_heap_used_in_bytes" => Numeric,
+        "non_heap_committed_in_bytes" => Numeric,
+        "pools" => {
+          "survivor" => {
+            "peak_used_in_bytes" => Numeric,
+            "used_in_bytes" => Numeric,
+            "peak_max_in_bytes" => Numeric,
+            "max_in_bytes" => Numeric
+          },
+          "old" => {
+            "peak_used_in_bytes" => Numeric,
+            "used_in_bytes" => Numeric,
+            "peak_max_in_bytes" => Numeric,
+            "max_in_bytes" => Numeric
+          },
+          "young" => {
+            "peak_used_in_bytes" => Numeric,
+            "used_in_bytes" => Numeric,
+            "peak_max_in_bytes" => Numeric,
+            "max_in_bytes" => Numeric
+          }
+        }
       }
     },
     "process"=>{
@@ -33,35 +56,14 @@ describe LogStash::Api::Modules::NodeStats do
         "total_in_millis"=>Numeric,
         "percent"=>Numeric
       }
-    },    
-    "mem" => {
-      "heap_used_in_bytes" => Numeric,
-      "heap_used_percent" => Numeric,
-      "heap_committed_in_bytes" => Numeric,
-      "heap_max_in_bytes" => Numeric,
-      "non_heap_used_in_bytes" => Numeric,
-      "non_heap_committed_in_bytes" => Numeric,
-      "pools" => {
-        "survivor" => {
-          "peak_used_in_bytes" => Numeric,
-          "used_in_bytes" => Numeric,
-          "peak_max_in_bytes" => Numeric,
-          "max_in_bytes" => Numeric
-        },
-        "old" => {
-          "peak_used_in_bytes" => Numeric,
-          "used_in_bytes" => Numeric,
-          "peak_max_in_bytes" => Numeric,
-          "max_in_bytes" => Numeric
-        },
-        "young" => {
-          "peak_used_in_bytes" => Numeric,
-          "used_in_bytes" => Numeric,
-          "peak_max_in_bytes" => Numeric,
-          "max_in_bytes" => Numeric
-        }
-      }
-    }
+    },
+   "pipeline" => {
+     "events" => {
+        "in" => Numeric,
+        "filtered" => Numeric,
+        "out" => Numeric
+     } 
+    } 
   }
   
   test_api_and_resources(root_structure)

data/spec/api/lib/api/support/resource_dsl_methods.rb

@@ -23,6 +23,21 @@ module ResourceDSLMethods
       it "should respond OK" do
         expect(last_response).to be_ok
       end
+
+      
+      describe "the default metadata" do
+        it "should include the host" do
+          expect(payload["host"]).to eql(Socket.gethostname)
+        end
+
+        it "should include the version" do
+          expect(payload["version"]).to eql(LOGSTASH_CORE_VERSION)
+        end
+
+        it "should include the http address" do
+          expect(payload["http_address"]).to eql("#{Socket.gethostname}:#{::LogStash::WebServer::DEFAULT_PORT}")
+        end
+      end
       
       hash_to_mapping(expected).each do |resource_path,klass|
         dotted = resource_path.join(".")

data/spec/api/spec_helper.rb

@@ -1,6 +1,7 @@
 # encoding: utf-8
 API_ROOT = File.expand_path(File.join(File.dirname(__FILE__), "..", "..", "lib", "logstash", "api"))
 
+require "stud/task"
 require "logstash/devutils/rspec/spec_helper"
 $LOAD_PATH.unshift(File.expand_path(File.dirname(__FILE__)))
 require "lib/api/support/resource_dsl_methods"
@@ -17,7 +18,9 @@ end
 
 module LogStash
   class DummyAgent < Agent
-    def start_webserver; end
+    def start_webserver
+      @webserver = Struct.new(:address).new("#{Socket.gethostname}:#{::LogStash::WebServer::DEFAULT_PORT}")
+    end
     def stop_webserver; end
   end
 end
@@ -92,7 +95,7 @@ shared_context "api setup" do
     @runner = LogStashRunner.new
     @runner.start
   end
-
+  
   after :all do
     @runner.stop
   end

data/spec/logstash/inputs/metrics_spec.rb

@@ -8,7 +8,7 @@ describe LogStash::Inputs::Metrics do
   let(:queue) { [] }
 
   before :each do
-    allow(subject).to receive(:metric).and_return(metric)
+    subject.metric = metric
   end
 
   describe "#run" do

data/spec/logstash/instrument/metric_type/counter_spec.rb

@@ -28,12 +28,7 @@ describe LogStash::Instrument::MetricType::Counter do
 
   context "When creating a hash " do
     it "creates the hash from all the values" do
-      metric_hash = {
-        "key" => key,
-        "namespaces" => namespaces,
-        "value" => 0,
-        "type" => "counter"
-      }
+      metric_hash = { key => 0 }
       expect(subject.to_hash).to match(metric_hash)
     end
   end

data/spec/logstash/instrument/metric_type/gauge_spec.rb

@@ -29,10 +29,7 @@ describe LogStash::Instrument::MetricType::Gauge do
   context "When creating a hash " do
     it "creates the hash from all the values" do
       metric_hash = {
-        "key" => key,
-        "namespaces" => namespaces,
-        "value" => value,
-        "type" => "gauge"
+        key => value
       }
       expect(subject.to_hash).to match(metric_hash)
     end

data/spec/logstash/instrument/namespaced_metric_spec.rb

@@ -2,10 +2,11 @@
 require "logstash/instrument/namespaced_metric"
 require "logstash/instrument/metric"
 require_relative "../../support/matchers"
+require_relative "../../support/shared_examples"
 require "spec_helper"
 
 describe LogStash::Instrument::NamespacedMetric do
-  let(:namespace) { :stats }
+  let(:namespace) { :root }
   let(:collector) { [] }
   let(:metric) { LogStash::Instrument::Metric.new(collector) }
 
@@ -27,6 +28,64 @@ describe LogStash::Instrument::NamespacedMetric do
     new_namespace = subject.namespace(:wally)
 
     expect(subject.namespace_name).to eq([namespace])
-    expect(new_namespace.namespace_name).to eq([:stats, :wally])
+    expect(new_namespace.namespace_name).to eq([:root, :wally])
   end
+
+  context "#increment" do
+    it "a counter by 1" do
+      metric = subject.increment(:error_rate)
+      expect(collector).to be_a_metric_event([:root, :error_rate], :counter, :increment, 1)
+    end
+
+    it "a counter by a provided value" do
+      metric = subject.increment(:error_rate, 20)
+      expect(collector).to be_a_metric_event([:root, :error_rate], :counter, :increment, 20)
+    end
+  end
+
+  context "#decrement" do
+    it "a counter by 1" do
+      metric = subject.decrement(:error_rate)
+      expect(collector).to be_a_metric_event([:root, :error_rate], :counter, :decrement, 1)
+    end
+
+    it "a counter by a provided value" do
+      metric = subject.decrement(:error_rate, 20)
+      expect(collector).to be_a_metric_event([:root, :error_rate], :counter, :decrement, 20)
+    end
+  end
+
+  context "#gauge" do
+    it "set the value of a key" do
+      metric = subject.gauge(:size_queue, 20)
+      expect(collector).to be_a_metric_event([:root, :size_queue], :gauge, :set, 20)
+    end
+  end
+
+  context "#time" do
+    let(:sleep_time) { 2 }
+    let(:sleep_time_ms) { sleep_time * 1_000 }
+
+    it "records the duration" do
+      subject.time(:duration_ms) { sleep(sleep_time) }
+
+      expect(collector.last).to be_within(sleep_time_ms).of(sleep_time_ms + 5)
+      expect(collector[0]).to match([:root])
+      expect(collector[1]).to be(:duration_ms)
+      expect(collector[2]).to be(:counter)
+    end
+
+    it "return a TimedExecution" do
+      execution = subject.time(:duration_ms)
+      sleep(sleep_time)
+      execution.stop
+
+      expect(collector.last).to be_within(sleep_time_ms).of(sleep_time_ms + 0.1)
+      expect(collector[0]).to match([:root])
+      expect(collector[1]).to be(:duration_ms)
+      expect(collector[2]).to be(:counter)
+    end
+  end
+
+  include_examples "metrics commons operations"
 end

data/spec/logstash/instrument/null_metric_spec.rb

@@ -1,21 +1,19 @@
 # encoding: utf-8
 require "logstash/instrument/null_metric"
 require "logstash/instrument/namespaced_metric"
-require_relative "../../support/matchers"
+require_relative "../../support/shared_examples"
 
 describe LogStash::Instrument::NullMetric do
+  # This is defined in the `namespaced_metric_spec`
+  include_examples "metrics commons operations"
+
   it "defines the same interface as `Metric`" do
     expect(described_class).to implement_interface_of(LogStash::Instrument::NamespacedMetric)
   end
 
-  describe "#time" do
-    it "returns the value of the block without recording any metrics" do
-      expect(subject.time(:execution_time) { "hello" }).to eq("hello")
-    end
-
-    it "return a TimedExecution" do
-      execution = subject.time(:do_something)
-      expect { execution.stop }.not_to raise_error
+  describe "#namespace" do
+    it "return a NullMetric" do
+      expect(subject.namespace(key)).to be_kind_of LogStash::Instrument::NullMetric
     end
   end
 end

data/spec/logstash/pipeline_spec.rb

@@ -360,7 +360,6 @@ describe LogStash::Pipeline do
   end
 
   context "compiled filter funtions" do
-
     context "new events should propagate down the filters" do
       config <<-CONFIG
         filter {
@@ -466,11 +465,12 @@ describe LogStash::Pipeline do
       pipeline = LogStash::Pipeline.new(config, pipeline_settings_obj)
       Thread.new { pipeline.run }
       sleep 0.1 while !pipeline.ready?
-      # give us a bit of time to flush the events
       wait(5).for do
-        next unless output && output.events && !(event = output.events.pop).nil?
-        event.get("message").split("\n").count
-      end.to eq(number_of_events)
+        # give us a bit of time to flush the events
+        output.events.empty?
+      end.to be_falsey
+      event = output.events.pop
+      expect(event.get("message").count("\n")).to eq(99)
       pipeline.shutdown
     end
   end
@@ -604,7 +604,7 @@ describe LogStash::Pipeline do
 
       Thread.new { subject.run }
       # make sure we have received all the generated events
-      sleep 1 while dummyoutput.events.size < number_of_events
+      sleep 0.25 while dummyoutput.events.size < number_of_events
     end
 
     after :each do
@@ -614,7 +614,7 @@ describe LogStash::Pipeline do
     context "global metric" do
       let(:collected_metric) { metric_store.get_with_path("stats/events") }
 
-      it "populates the differents" do
+      it "populates the different metrics" do
         expect(collected_metric[:stats][:events][:in].value).to eq(number_of_events)
         expect(collected_metric[:stats][:events][:filtered].value).to eq(number_of_events)
         expect(collected_metric[:stats][:events][:out].value).to eq(number_of_events)

data/spec/logstash/plugin_spec.rb

@@ -270,4 +270,77 @@ describe LogStash::Plugin do
       end
     end
   end
+
+
+  context "When the plugin record a metric" do
+    let(:config) { {} }
+
+    [LogStash::Inputs::Base, LogStash::Filters::Base, LogStash::Outputs::Base].each do |base|
+      let(:plugin) do
+        Class.new(base) do
+          config_name "testing"
+
+          def register
+            metric.gauge("power_level", 9000)
+          end
+        end
+      end
+
+      subject { plugin.new(config) } 
+
+      context "when no metric is set to the plugin" do
+        context "when `enable_metric` is TRUE" do
+          it "recording metric should not raise an exception" do
+            expect { subject.register }.not_to raise_error
+          end
+
+          it "should use a `NullMetric`" do
+            expect(subject.metric).to be_kind_of(LogStash::Instrument::NullMetric)
+          end
+        end
+
+        context "when `enable_metric` is FALSE" do
+          let(:config) { { "enable_metric" => false } }
+
+          it "recording metric should not raise an exception" do
+            expect { subject.register }.not_to raise_error
+          end
+
+          it "should use a `NullMetric`" do
+            expect(subject.metric).to be_kind_of(LogStash::Instrument::NullMetric)
+          end
+        end
+      end
+
+      context "When a specific metric collector is configured" do
+        context "when `enable_metric` is TRUE" do
+          let(:metric) { LogStash::Instrument::Metric.new(LogStash::Instrument::Collector.new).namespace("dbz") }
+
+          before :each do
+            subject.metric = metric
+          end
+
+          it "recording metric should not raise an exception" do
+            expect { subject.register }.not_to raise_error
+          end
+
+          it "should use the configured metric" do
+            expect(subject.metric).to eq(metric)
+          end
+        end
+
+        context "when `enable_metric` is FALSE" do
+          let(:config) { { "enable_metric" => false } }
+
+          it "recording metric should not raise an exception" do
+            expect { subject.register }.not_to raise_error
+          end
+
+          it "should use a `NullMetric`" do
+            expect(subject.metric).to be_kind_of(LogStash::Instrument::NullMetric)
+          end
+        end
+      end
+    end
+  end
 end