logstash-devutils 2.0.0-java → 2.4.0-java

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 9529292d4be6bf7ae845758206433d9fff38a8fdf068d4eee49581a595022563
-  data.tar.gz: '09bac2f78a40a15e156f893929a0113bb815234f5e24dacd0b806d94a7db7784'
+  metadata.gz: f6acb50f9d6e040af22acda0163ef38f8bce77506f023dd8a58a4ee925d9df3c
+  data.tar.gz: 51c47c93c93335ed8cba0e0349dc712808051e77ca69ca8f79617eaa9442731a
 SHA512:
-  metadata.gz: 2c1f07ee224a06900492e5421031b1f8e4a15705edf0feb91bfe4279853cb30a3bd248933209e20e16d91e19c65f7b8dd6e29b5a75ee293b80c5e3c5032aa4b9
-  data.tar.gz: 1b151df23dd77671425a0874a4b0903de5460d10f93a8b521c8019ff4f71965b8ff52089d2978eebe3dfd0b91e1342dac9ecaffbde6a3fa3735a7a938482b81e
+  metadata.gz: 8fa7d3f493f223e497c7877bfc9c6a9440a269367a02d7c6f468580f0ac6d4e6156600ba30fc2fcc831a27cef74a6e65cbfa7daebd29623042557b524a3423ff
+  data.tar.gz: a6530c6d9bd097ea6f081224a7ff6254e38f3d6abc400f3b4bbaca5cea7b354e4e37b60071755eef6d05bac2afce89601b53b72b4339ba15b97696dceb541cb1

data/CHANGELOG.md

@@ -1,17 +1,45 @@
+## 2.4.0
+ - Feat: shared test (spec) task for all! [#96](https://github.com/elastic/logstash-devutils/pull/96)
+
+## 2.3.0
+ - Introduce `be_a_logstash_timestamp_equivalent_to` RSpec matcher to compare LogStash::Timestamp [#99](https://github.com/elastic/logstash-devutils/pull/99)
+
+## 2.2.1
+ - Fixed `LogStashHelpers#sample` to work with pipelines whose filters add, clone, and cancel events.
+
+## 2.2.0
+ - Add `allowed_lag` config for shared input interruptiblity spec
+
+## 2.1.0
+ - Remove ruby pipeline dependency
+
+## 2.0.4
+ - Fix: avoid double registering filters on `sample` spec helper
+
+## 2.0.3
+ - Fix: add missing `events` method to QueuedBatchDelegator, which was causing test failures
+ after https://github.com/elastic/logstash/pull/11737 was committed.
+
+## 2.0.2
+ - Fix: add plain codec as runtime dependency for TestPipeline helper
+
+## 2.0.1
+ - Fix: unwrap output and refactor test sink (#82)
+
 ## 2.0.0
  - Reinvented helpers using Java pipeline, only LS >= 6.x (JRuby >= 9.1) is supported.
  - [BREAKING] changes:
    * `plugin_input` helper no longer works - simply fails with a not implemented error
    * `type` and `tags` helpers have no effect - they will print a deprecation warning
-   * using gem **insist** is discouraged and has to be pulled in manually 
+   * using gem **insist** is discouraged and has to be pulled in manually
      (in *plugin.gemspec* `add_development_dependency 'insist'` and `require "insist"`)
    * shared examples need to be explicitly required, as they are not re-used that much
      (in spec_helper.rb `require "logstash/devutils/rspec/shared_examples"'`)
    * `input` helper now yields a Queue-like collection (with `Queue#pop` blocking semantics)
-     with a default timeout polling mechanism to guard against potential dead-locks 
+     with a default timeout polling mechanism to guard against potential dead-locks
 
 ## 1.3.6
- - Revert the removal (e.g. add back) of the log4j spec helper. It is still needed for 5.x builds. 
+ - Revert the removal (e.g. add back) of the log4j spec helper. It is still needed for 5.x builds.
 
 ## 1.3.5
  - Fix spec helper method `input` generating an invalid `output_func` that returned `nil` instead of an array

data/Gemfile

@@ -3,4 +3,5 @@ gemspec
 
 if Dir.exist?(logstash_path = ENV["LOGSTASH_PATH"])
   gem 'logstash-core', path: "#{logstash_path}/logstash-core"
+  gem 'logstash-core-plugin-api', path: "#{logstash_path}/logstash-core-plugin-api"
 end

data/Rakefile

@@ -1,2 +1,3 @@
 raise "only JRuby is supported" unless defined? JRUBY_VERSION
 require 'bundler/gem_tasks'
+require 'logstash/devutils/rake'

data/lib/logstash/devutils/rake/test.rake

@@ -0,0 +1,17 @@
+begin
+  require 'rspec/core/rake_task'
+  RSpec::Core::RakeTask.new(:spec) # default glob: 'spec/**{,/*/**}/*_spec.rb'
+rescue LoadError
+end
+
+desc "Run tests including Java tests (if any)"
+task :test => [ 'test:java', 'test:ruby' ]
+namespace :test do
+  task :java do
+    gradlew = File.join(Dir.pwd, 'gradlew')
+    sh "#{gradlew} --no-daemon test" if File.exist?(gradlew)
+  end
+  task :ruby => :vendor do
+    Rake::Task[:spec].invoke
+  end
+end

data/lib/logstash/devutils/rake.rb

@@ -1,3 +1,4 @@
 raise "Only JRuby is supported at this time." unless RUBY_PLATFORM == "java"
 load "logstash/devutils/rake/publish.rake"
 load "logstash/devutils/rake/vendor.rake"
+load "logstash/devutils/rake/test.rake"

data/lib/logstash/devutils/rspec/logstash_helpers.rb

@@ -1,6 +1,7 @@
 require "logstash/agent"
 require "logstash/event"
 require "logstash/test_pipeline"
+require "logstash/outputs/test_sink"
 
 require "stud/try"
 require "rspec/expectations"
@@ -52,11 +53,30 @@ module LogStashHelper
     deprecated "tags(#{tags.inspect}) - let(:default_tags) are not used"
   end
 
-  def sample(sample_event, &block)
+  ##
+  # Creates a single-example example group in the current rspec context to validate the results of filter operations.
+  # The current group is expected to use `LogStashHelpers#config` to provide a pipeline configuration.
+  #
+  # The provided block has access to an [Array] `results`, and also sets an unnamed `subject` that is either a
+  # single `LogStash::Event` or an `[Array[LogStash::Event]]` containing at least two entries.
+  #
+  # @param sample_event [String, #to_json]
+  # @param validation_source [:output, :queue]
+  # @yield creates an example with expectations provided by the block
+  # @return [void]
+  def sample(sample_event, validation_source: :output, &block)
+    validation_sources = [:output, :queue].freeze
+    fail(ArgumentError, "Unexpected source `#{validation_source.inspect}`, expected one of #{validation_sources.inspect}") unless validation_sources.include?(validation_source)
+
     name = sample_event.is_a?(String) ? sample_event : LogStash::Json.dump(sample_event)
     name = name[0..50] + "..." if name.length > 50
 
-    describe "\"#{name}\"" do
+    describe name.inspect do
+      if validation_source == :output
+        let(:test_sink) { LogStash::Outputs::TestSink.new("release_on_close" => false, "store_events" => true) }
+        before { expect(LogStash::Outputs::TestSink).to receive(:new).and_return(test_sink) }
+      end
+
       let(:pipeline) { new_pipeline_from_string(config) }
       let(:event) do
         sample_event = [sample_event] unless sample_event.is_a?(Array)
@@ -67,14 +87,17 @@ module LogStashHelper
       end
 
       let(:results) do
-        pipeline.filters.each(&:register)
-
+        # Java pipeline (since 6.1) registers filters from start_workers
         pipeline.run_with(event)
 
         # flush makes sure to empty any buffered events in the filter
         pipeline.flush_filters(:final => true) { |flushed_event| results << flushed_event }
 
-        pipeline.filter_queue_client.processed_events
+        case validation_source
+        when :queue  then pipeline.filter_queue_client.processed_events # legacy?
+        when :output then test_sink.event_store.to_a
+        else              fail NotImplementedError("validation source `#{validation_source}` not implemented.")
+        end
       end
 
       # starting at logstash-core 5.3 an initialized pipeline need to be closed
@@ -84,20 +107,25 @@ module LogStashHelper
 
       subject { results.length > 1 ? results : results.first }
 
-      it("when processed", &block)
+      it("processes events as specified", &block)
     end
   end # def sample
 
+  org.logstash.config.ir.compiler.OutputStrategyExt::SimpleAbstractOutputStrategyExt.class_eval do
+    field_reader :output # available since LS 6.3
+  end
+
   def input(config_string, test_sink: {}, &block); require 'logstash/outputs/test_sink'
     config_parts = [ config_source(config_string), test_sink_output_source(**test_sink) ]
 
-    # TODO unwrapping output from LogStash::OutputDelegator is cumbersome
-    instances = LogStash::Outputs::TestSink::TRACKER.keys.to_a
     pipeline = new_pipeline(config_parts)
 
-    start_thread = pipeline.start_and_wait
+    output_delegator = pipeline.outputs.last # LogStash::OutputDelegator
+    fail('test_sink output expected') unless output_delegator.config_name.eql?('test_sink')
+    test_sink_output = output_delegator.strategy.to_java.output
+    queue = test_sink_output.init_event_store
 
-    queue = (LogStash::Outputs::TestSink::TRACKER.keys.to_a - instances).first.event_store
+    start_thread = pipeline.start_and_wait
 
     # NOTE: we used to pass a Queue here, now its a Java List/Queue collection
     result = block.call(pipeline, queue)

data/lib/logstash/devutils/rspec/shared_examples.rb

@@ -1,6 +1,10 @@
 require 'rspec/wait'
 
 RSpec.shared_examples "an interruptible input plugin" do
+  # why 3? 2 is not enough, 4 is too much..
+  # a plugin that is known to be slower can override
+  let(:allowed_lag) { 3 }
+
   describe "#stop" do
     let(:queue) { SizedQueue.new(20) }
     subject { described_class.new(config) }
@@ -14,8 +18,7 @@ RSpec.shared_examples "an interruptible input plugin" do
       expect(plugin_thread).to be_alive
       # now let's actually stop the plugin
       subject.do_stop
-      # why 3? 2 is not enough, 4 is too much..
-      wait(3).for { plugin_thread }.to_not be_alive
+      wait(allowed_lag).for { plugin_thread }.to_not be_alive
     end
   end
 end

data/lib/logstash/devutils/rspec/spec_helper.rb

@@ -29,7 +29,7 @@ if RUBY_VERSION < "2.3"
   raise LoadError.new("Ruby >= 2.3.0 or later is required. (You are running: " + RUBY_VERSION + ")")
 end
 
-if level = (ENV["TEST_DEBUG"] || ENV['LOGGER_LEVEL'])
+if level = (ENV['LOG_LEVEL'] || ENV['LOGGER_LEVEL'] || ENV["TEST_DEBUG"])
   logger, level = level.split('=') # 'logstash.filters.grok=DEBUG'
   level, logger = logger, nil if level.nil? # only level given e.g. 'DEBUG'
   level = org.apache.logging.log4j.Level.toLevel(level, org.apache.logging.log4j.Level::WARN)
@@ -38,6 +38,16 @@ else
   LogStash::Logging::Logger::configure_logging('ERROR')
 end
 
+RSpec::Matchers.define :be_a_logstash_timestamp_equivalent_to do |expected|
+  # use the Timestamp compare to avoid suffering of precision loss of time format
+  expected = LogStash::Timestamp.new(expected) unless expected.kind_of?(LogStash::Timestamp)
+  description { "be a LogStash::Timestamp equivalent to #{expected}" }
+
+  match do |actual|
+    actual.kind_of?(LogStash::Timestamp) && actual == expected
+  end
+end
+
 RSpec.configure do |config|
   # for now both include and extend are required because the newly refactored "input" helper method need to be visible in a "it" block
   # and this is only possible by calling include on LogStashHelper
@@ -57,4 +67,3 @@ RSpec.configure do |config|
   #     --seed 1234
   config.order = :random
 end
-

data/lib/logstash/outputs/test_sink.rb

@@ -22,7 +22,7 @@ class LogStash::Outputs::TestSink < LogStash::Outputs::Base
 
   # @override plugin hook
   def register
-    TRACKER[self] = Queue.new(@event_poll_timeout.to_f * 1000)
+    TRACKER[self] = @_event_store || init_event_store
   end
 
   # @override plugin impl
@@ -32,7 +32,8 @@ class LogStash::Outputs::TestSink < LogStash::Outputs::Base
 
   # @override plugin hook
   def close
-    TRACKER.delete(self) if release_on_close?
+    TRACKER.delete(self)
+    @_event_store = false if release_on_close?
   end
 
   def store_events?
@@ -50,7 +51,24 @@ class LogStash::Outputs::TestSink < LogStash::Outputs::Base
 
   # @return [Queue] (enumerable) event store
   def event_store
-    TRACKER[self] || raise("#{self} not registered; please call plugin.register before use")
+    if @_event_store.nil?
+      warn("#{self} event store not initialized (call plugin.register to initialize)", caller_locations(2))
+      return init_event_store
+    elsif @_event_store.eql?(false)
+      warn("#{self} closed - event store no longer available (release_on_close => false if you need to retain events)", caller_locations(2))
+      return nil
+    end
+    @_event_store
+  end
+
+  def init_event_store
+    @_event_store = Queue.new(@event_poll_timeout.to_f * 1000)
+  end
+
+  private
+
+  def warn(msg, callstack = nil)
+    Kernel.warn("#{msg}#{callstack ? " called from #{callstack.first}" : nil}")
   end
 
   # TODO refactor to java.util.concurrent.ConcurrentLinkedQueue

data/lib/logstash/test_pipeline.rb

@@ -1,4 +1,3 @@
-require "logstash/pipeline"
 require "logstash/java_pipeline"
 
 module LogStash
@@ -151,6 +150,13 @@ module LogStash
         @delegate.to_java.filtered_size
       end
 
+      def events
+        @delegate.events.tap do |events|
+          # filters out rogue (cancelled) events
+          @event_tracker.filtered_events events
+        end
+      end
+
       # @override void merge(IRubyObject event);
       def merge(event)
         @delegate.merge(event)

data/logstash-devutils.gemspec

@@ -4,7 +4,7 @@ Gem::Specification.new do |spec|
   files = %x{git ls-files}.split("\n")
 
   spec.name = "logstash-devutils"
-  spec.version = "2.0.0"
+  spec.version = "2.4.0"
   spec.license = "Apache-2.0"
   spec.authors = ["Elastic"]
   spec.email = "info@elastic.co"
@@ -45,4 +45,5 @@ Gem::Specification.new do |spec|
   spec.add_runtime_dependency "kramdown", '1.14.0'
   spec.add_runtime_dependency "stud", " >= 0.0.20"
   spec.add_runtime_dependency "fivemat"
+  spec.add_runtime_dependency "logstash-codec-plain"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: logstash-devutils
 version: !ruby/object:Gem::Version
-  version: 2.0.0
+  version: 2.4.0
 platform: java
 authors:
 - Elastic
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2020-02-11 00:00:00.000000000 Z
+date: 2022-03-14 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
@@ -136,6 +136,20 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  name: logstash-codec-plain
+  prerelease: false
+  type: :runtime
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 description: logstash-devutils gem is meant to be used as a development dependency
   from other plugins/gems.
 email: info@elastic.co
@@ -149,6 +163,7 @@ files:
 - Rakefile
 - lib/logstash/devutils/rake.rb
 - lib/logstash/devutils/rake/publish.rake
+- lib/logstash/devutils/rake/test.rake
 - lib/logstash/devutils/rake/vendor.rake
 - lib/logstash/devutils/rspec/log4j2.properties
 - lib/logstash/devutils/rspec/logstash_helpers.rb
@@ -177,8 +192,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubyforge_project:
-rubygems_version: 2.6.13
+rubygems_version: 3.1.6
 signing_key:
 specification_version: 4
 summary: An assortment of tooling/libraries to make Logstash plugin development and