logstash-core 6.0.1-java → 6.1.0-java

data/lib/logstash/util/wrapped_synchronous_queue.rb

@@ -2,33 +2,22 @@
 
 module LogStash; module Util
   class WrappedSynchronousQueue
-    java_import java.util.concurrent.SynchronousQueue
+    java_import java.util.concurrent.ArrayBlockingQueue
     java_import java.util.concurrent.TimeUnit
+    java_import org.logstash.common.LsQueueUtils
 
-    def initialize
-      @queue = java.util.concurrent.SynchronousQueue.new
+    def initialize (size)
+      @queue = ArrayBlockingQueue.new(size)
     end
 
-    # Push an object to the queue if the queue is full
-    # it will block until the object can be added to the queue.
-    #
-    # @param [Object] Object to add to the queue
-    def push(obj)
-      @queue.put(obj)
-    end
-    alias_method(:<<, :push)
-
-    # Block for X millis
-    def poll(millis)
-      @queue.poll(millis, TimeUnit::MILLISECONDS)
-    end
+    attr_reader :queue
 
     def write_client
-      WriteClient.new(self)
+      WriteClient.new(@queue)
     end
 
     def read_client
-      ReadClient.new(self)
+      ReadClient.new(@queue)
     end
 
     def close
@@ -42,15 +31,14 @@ module LogStash; module Util
 
       def initialize(queue, batch_size = 125, wait_for = 250)
         @queue = queue
-        @mutex = Mutex.new
         # Note that @inflight_batches as a central mechanism for tracking inflight
         # batches will fail if we have multiple read clients in the pipeline.
-        @inflight_batches = {}
+        @inflight_batches = Concurrent::Map.new
 
         # allow the worker thread to report the execution time of the filter + output
-        @inflight_clocks = {}
+        @inflight_clocks = Concurrent::Map.new
         @batch_size = batch_size
-        @wait_for = wait_for
+        @wait_for = TimeUnit::NANOSECONDS.convert(wait_for, TimeUnit::MILLISECONDS)
       end
 
       def close
@@ -58,12 +46,12 @@ module LogStash; module Util
       end
 
       def empty?
-        true # synchronous queue is alway empty
+        @queue.isEmpty
       end
 
       def set_batch_dimensions(batch_size, wait_for)
         @batch_size = batch_size
-        @wait_for = wait_for
+        @wait_for = TimeUnit::NANOSECONDS.convert(wait_for, TimeUnit::MILLISECONDS)
       end
 
       def set_events_metric(metric)
@@ -89,185 +77,84 @@ module LogStash; module Util
       end
 
       def inflight_batches
-        @mutex.lock
-        begin
-          yield(@inflight_batches)
-        ensure
-          @mutex.unlock
-        end
-      end
-
-      def current_inflight_batch
-        @inflight_batches.fetch(Thread.current, [])
+        yield(@inflight_batches)
       end
 
       # create a new empty batch
       # @return [ReadBatch] a new empty read batch
       def new_batch
-        ReadBatch.new(@queue, @batch_size, @wait_for)
+        ReadBatch.new(@queue, 0, 0)
       end
 
       def read_batch
-        batch = new_batch
-        @mutex.lock
-        begin
-          batch.read_next
-        ensure
-          @mutex.unlock
-        end
+        batch = ReadBatch.new(@queue, @batch_size, @wait_for)
         start_metrics(batch)
         batch
       end
 
       def start_metrics(batch)
-        @mutex.lock
-        # there seems to be concurrency issues with metrics, keep it in the mutex
-        begin
-          set_current_thread_inflight_batch(batch)
-          start_clock
-        ensure
-          @mutex.unlock
-        end
-      end
-
-      def set_current_thread_inflight_batch(batch)
-        @inflight_batches[Thread.current] = batch
+        thread = Thread.current
+        @inflight_batches[thread] = batch
+        @inflight_clocks[thread] = java.lang.System.nano_time
       end
 
       def close_batch(batch)
-        @mutex.lock
-        begin
-          # there seems to be concurrency issues with metrics, keep it in the mutex
-          @inflight_batches.delete(Thread.current)
-          stop_clock(batch)
-        ensure
-          @mutex.unlock
-        end
-      end
-
-      def start_clock
-        @inflight_clocks[Thread.current] = java.lang.System.nano_time
-      end
-
-      def stop_clock(batch)
-        unless @inflight_clocks[Thread.current].nil?
+        thread = Thread.current
+        @inflight_batches.delete(thread)
+        start_time = @inflight_clocks.get_and_set(thread, nil)
+        unless start_time.nil?
           if batch.size > 0
             # only stop (which also records) the metrics if the batch is non-empty.
             # start_clock is now called at empty batch creation and an empty batch could
             # stay empty all the way down to the close_batch call.
-            time_taken = (java.lang.System.nano_time - @inflight_clocks[Thread.current]) / 1_000_000
+            time_taken = (java.lang.System.nano_time - start_time) / 1_000_000
             @event_metric_time.increment(time_taken)
             @pipeline_metric_time.increment(time_taken)
           end
-          @inflight_clocks.delete(Thread.current)
         end
       end
 
-      def add_filtered_metrics(batch)
-        @event_metric_filtered.increment(batch.filtered_size)
-        @pipeline_metric_filtered.increment(batch.filtered_size)
+      def add_filtered_metrics(filtered_size)
+        @event_metric_filtered.increment(filtered_size)
+        @pipeline_metric_filtered.increment(filtered_size)
       end
 
-      def add_output_metrics(batch)
-        @event_metric_out.increment(batch.filtered_size)
-        @pipeline_metric_out.increment(batch.filtered_size)
+      def add_output_metrics(filtered_size)
+        @event_metric_out.increment(filtered_size)
+        @pipeline_metric_out.increment(filtered_size)
       end
     end
 
     class ReadBatch
       def initialize(queue, size, wait)
-        @queue = queue
-        @size = size
-        @wait = wait
-
-        @originals = Hash.new
-
         # TODO: disabled for https://github.com/elastic/logstash/issues/6055 - will have to properly refactor
         # @cancelled = Hash.new
 
-        @generated = Hash.new
-        @iterating_temp = Hash.new
-        @iterating = false # Atomic Boolean maybe? Although batches are not shared across threads
-        @acked_batch = nil
-      end
-
-      def read_next
-        @size.times do |t|
-          event = @queue.poll(@wait)
-          return if event.nil? # queue poll timed out
-
-          @originals[event] = true
-        end
+        @originals = LsQueueUtils.drain(queue, size, wait)
       end
 
       def merge(event)
-        return if event.nil? || @originals.key?(event)
-        # take care not to cause @generated to change during iteration
-        # @iterating_temp is merged after the iteration
-        if iterating?
-          @iterating_temp[event] = true
-        else
-          # the periodic flush could generate events outside of an each iteration
-          @generated[event] = true
-        end
-      end
-
-      def cancel(event)
-        # TODO: disabled for https://github.com/elastic/logstash/issues/6055 - will have to properly refactor
-        raise("cancel is unsupported")
-        # @cancelled[event] = true
+        return if event.nil?
+        @originals.add(event)
       end
 
       def to_a
         events = []
-        each {|e| events << e}
+        @originals.each {|e| events << e unless e.cancelled?}
         events
       end
 
       def each(&blk)
-        # take care not to cause @originals or @generated to change during iteration
-        @iterating = true
-
         # below the checks for @cancelled.include?(e) have been replaced by e.cancelled?
         # TODO: for https://github.com/elastic/logstash/issues/6055 = will have to properly refactor
-        @originals.each do |e, _|
-          blk.call(e) unless e.cancelled?
-        end
-        @generated.each do |e, _|
-          blk.call(e) unless e.cancelled?
-        end
-        @iterating = false
-        update_generated
-      end
-
-      def size
-        filtered_size
-      end
-
-      def starting_size
-        @originals.size
+        @originals.each {|e| blk.call(e) unless e.cancelled?}
       end
 
       def filtered_size
-        @originals.size + @generated.size
-      end
-
-      def cancelled_size
-      # TODO: disabled for https://github.com/elastic/logstash/issues/6055 = will have to properly refactor
-      raise("cancelled_size is unsupported ")
-        # @cancelled.size
-      end
-
-      private
-
-      def iterating?
-        @iterating
+        @originals.size
       end
 
-      def update_generated
-        @generated.update(@iterating_temp)
-        @iterating_temp.clear
-      end
+      alias_method(:size, :filtered_size)
     end
 
     class WriteClient
@@ -275,40 +162,13 @@ module LogStash; module Util
         @queue = queue
       end
 
-      def get_new_batch
-        WriteBatch.new
-      end
-
       def push(event)
-        @queue.push(event)
+        @queue.put(event)
       end
       alias_method(:<<, :push)
 
       def push_batch(batch)
-        batch.each do |event|
-          push(event)
-        end
-      end
-    end
-
-    class WriteBatch
-      def initialize
-        @events = []
-      end
-
-      def size
-        @events.size
-      end
-
-      def push(event)
-        @events.push(event)
-      end
-      alias_method(:<<, :push)
-
-      def each(&blk)
-        @events.each do |e|
-          blk.call(e)
-        end
+        LsQueueUtils.addAll(@queue, batch)
       end
     end
   end

data/locales/en.yml

@@ -264,6 +264,8 @@ en:
         http_port: Web API http port
         pipeline-workers: |+
           Sets the number of pipeline workers to run.
+        experimental-java-execution: |+
+          (Experimental) Use new Java execution engine.
         pipeline-batch-size: |+
           Size of batches the pipeline is to work in.
         pipeline-batch-delay: |+

data/spec/conditionals_spec.rb

@@ -1,7 +1,9 @@
 # encoding: utf-8
 require 'spec_helper'
+require 'support/pipeline/pipeline_helpers'
 
 module ConditionalFanciness
+  include PipelineHelpers
   def description
     return self.metadata[:description]
   end
@@ -76,21 +78,21 @@ describe "conditionals in filter" do
       }
     CONFIG
 
-    sample({"foo" => "bar"}) do
+    sample_one({"foo" => "bar"}) do
       expect(subject.get("always")).to eq("awesome")
       expect(subject.get("hello")).to eq("world")
       expect(subject.get("fancy")).to be_nil
       expect(subject.get("free")).to be_nil
     end
 
-    sample({"notfoo" => "bar"}) do
+    sample_one({"notfoo" => "bar"}) do
       expect(subject.get("always")).to eq("awesome")
       expect(subject.get("hello")).to be_nil
       expect(subject.get("fancy")).to be_nil
       expect(subject.get("free")).to eq("hugs")
     end
 
-    sample({"bar" => "baz"}) do
+    sample_one({"bar" => "baz"}) do
       expect(subject.get("always")).to eq("awesome")
       expect(subject.get("hello")).to be_nil
       expect(subject.get("fancy")).to eq("pants")
@@ -114,28 +116,28 @@ describe "conditionals in filter" do
       }
     CONFIG
 
-    sample("foo" => "bar", "nest" => 124) do
+    sample_one("foo" => "bar", "nest" => 124) do
       expect(subject.get("always")).to be_nil
       expect(subject.get("hello")).to be_nil
       expect(subject.get("fancy")).to be_nil
       expect(subject.get("free")).to be_nil
     end
 
-    sample("foo" => "bar", "nest" => 123) do
+    sample_one("foo" => "bar", "nest" => 123) do
       expect(subject.get("always")).to eq("awesome")
       expect(subject.get("hello")).to eq("world")
       expect(subject.get("fancy")).to be_nil
       expect(subject.get("free")).to be_nil
     end
 
-    sample("notfoo" => "bar", "nest" => 123) do
+    sample_one("notfoo" => "bar", "nest" => 123) do
       expect(subject.get("always")).to eq("awesome")
       expect(subject.get("hello")).to be_nil
       expect(subject.get("fancy")).to be_nil
       expect(subject.get("free")).to eq("hugs")
     end
 
-    sample("bar" => "baz", "nest" => 123) do
+    sample_one("bar" => "baz", "nest" => 123) do
       expect(subject.get("always")).to eq("awesome")
       expect(subject.get("hello")).to be_nil
       expect(subject.get("fancy")).to eq("pants")
@@ -152,7 +154,7 @@ describe "conditionals in filter" do
       }
     CONFIG
 
-    sample("foo" => 123, "bar" => 123) do
+    sample_one("foo" => 123, "bar" => 123) do
       expect(subject.get("tags") ).to include("woot")
     end
   end
@@ -181,7 +183,7 @@ describe "conditionals in filter" do
       }
     CONFIG
 
-    sample("foo" => "foo", "foobar" => "foobar", "greeting" => "hello world") do
+    sample_one("foo" => "foo", "foobar" => "foobar", "greeting" => "hello world") do
       expect(subject.get("tags")).to include("field in field")
       expect(subject.get("tags")).to include("field in string")
       expect(subject.get("tags")).to include("string in field")
@@ -203,7 +205,7 @@ describe "conditionals in filter" do
       }
     CONFIG
 
-    sample("foo" => "foo", "somelist" => [ "one", "two" ], "foobar" => "foobar", "greeting" => "hello world", "tags" => [ "fancypantsy" ]) do
+    sample_one("foo" => "foo", "somelist" => [ "one", "two" ], "foobar" => "foobar", "greeting" => "hello world", "tags" => [ "fancypantsy" ]) do
       # verify the original exists
       expect(subject.get("tags")).to include("fancypantsy")
 
@@ -218,94 +220,156 @@ describe "conditionals in filter" do
 
   describe "operators" do
     conditional "[message] == 'sample'" do
-      sample("sample") { expect(subject.get("tags") ).to include("success") }
-      sample("different") { expect(subject.get("tags") ).to include("failure") }
+      sample_one("sample") { expect(subject.get("tags") ).to include("success") }
+      sample_one("different") { expect(subject.get("tags") ).to include("failure") }
+    end
+
+    conditional "'sample' == [message]" do
+      sample_one("sample") {expect(subject.get("tags")).to include("success")}
+      sample_one("different") {expect(subject.get("tags")).to include("failure")}
+    end
+
+    conditional "'value' == 'value'" do
+      sample_one("sample") {expect(subject.get("tags")).to include("success")}
+    end
+
+    conditional "'value' == 'other'" do
+      sample_one("sample") {expect(subject.get("tags")).to include("failure")}
     end
 
     conditional "[message] != 'sample'" do
-      sample("sample") { expect(subject.get("tags") ).to include("failure") }
-      sample("different") { expect(subject.get("tags") ).to include("success") }
+      sample_one("sample") { expect(subject.get("tags") ).to include("failure") }
+      sample_one("different") { expect(subject.get("tags") ).to include("success") }
     end
 
     conditional "[message] < 'sample'" do
-      sample("apple") { expect(subject.get("tags") ).to include("success") }
-      sample("zebra") { expect(subject.get("tags") ).to include("failure") }
+      sample_one("apple") { expect(subject.get("tags") ).to include("success") }
+      sample_one("zebra") { expect(subject.get("tags") ).to include("failure") }
     end
 
     conditional "[message] > 'sample'" do
-      sample("zebra") { expect(subject.get("tags") ).to include("success") }
-      sample("apple") { expect(subject.get("tags") ).to include("failure") }
+      sample_one("zebra") { expect(subject.get("tags") ).to include("success") }
+      sample_one("apple") { expect(subject.get("tags") ).to include("failure") }
     end
 
     conditional "[message] <= 'sample'" do
-      sample("apple") { expect(subject.get("tags") ).to include("success") }
-      sample("zebra") { expect(subject.get("tags") ).to include("failure") }
-      sample("sample") { expect(subject.get("tags") ).to include("success") }
+      sample_one("apple") { expect(subject.get("tags") ).to include("success") }
+      sample_one("zebra") { expect(subject.get("tags") ).to include("failure") }
+      sample_one("sample") { expect(subject.get("tags") ).to include("success") }
     end
 
     conditional "[message] >= 'sample'" do
-      sample("zebra") { expect(subject.get("tags") ).to include("success") }
-      sample("sample") { expect(subject.get("tags") ).to include("success") }
-      sample("apple") { expect(subject.get("tags") ).to include("failure") }
+      sample_one("zebra") { expect(subject.get("tags") ).to include("success") }
+      sample_one("sample") { expect(subject.get("tags") ).to include("success") }
+      sample_one("apple") { expect(subject.get("tags") ).to include("failure") }
+    end
+
+    conditional "[message] == 5" do
+      sample_one("message" => 5) {expect(subject.get("tags")).to include("success")}
+      sample_one("message" => 3) {expect(subject.get("tags")).to include("failure")}
+    end
+
+    conditional "5 == [message]" do
+      sample_one("message" => 5) {expect(subject.get("tags")).to include("success")}
+      sample_one("message" => 3) {expect(subject.get("tags")).to include("failure")}
+    end
+
+    conditional "7 == 7" do
+      sample_one("message" => 7) {expect(subject.get("tags")).to include("success")}
+      sample_one("message" => 3) {expect(subject.get("tags")).to include("success")}
+    end
+
+    conditional "5 == 7" do
+      sample_one("message" => 3) {expect(subject.get("tags")).to include("failure")}
+      sample_one("message" => 2) {expect(subject.get("tags")).to include("failure")}
+    end
+
+    conditional "[message] != 5" do
+      sample_one("message" => 5) {expect(subject.get("tags")).to include("failure")}
+      sample_one("message" => 3) {expect(subject.get("tags")).to include("success")}
+    end
+
+    conditional "[message] < 5" do
+      sample_one("message" => 3) {expect(subject.get("tags")).to include("success")}
+      sample_one("message" => 5) {expect(subject.get("tags")).to include("failure")}
+      sample_one("message" => 9) {expect(subject.get("tags")).to include("failure")}
+    end
+
+    conditional "[message] > 5" do
+      sample_one("message" => 9) {expect(subject.get("tags")).to include("success")}
+      sample_one("message" => 5) {expect(subject.get("tags")).to include("failure")}
+      sample_one("message" => 4) {expect(subject.get("tags")).to include("failure")}
+    end
+
+    conditional "[message] <= 5" do
+      sample_one("message" => 9) {expect(subject.get("tags")).to include("failure")}
+      sample_one("message" => 5) {expect(subject.get("tags")).to include("success")}
+      sample_one("message" => 3) {expect(subject.get("tags")).to include("success")}
+    end
+
+    conditional "[message] >= 5" do
+      sample_one("message" => 5) {expect(subject.get("tags")).to include("success")}
+      sample_one("message" => 7) {expect(subject.get("tags")).to include("success")}
+      sample_one("message" => 3) {expect(subject.get("tags")).to include("failure")}
     end
 
     conditional "[message] =~ /sample/" do
-      sample("apple") { expect(subject.get("tags") ).to include("failure") }
-      sample("sample") { expect(subject.get("tags") ).to include("success") }
-      sample("some sample") { expect(subject.get("tags") ).to include("success") }
+      sample_one("apple") { expect(subject.get("tags") ).to include("failure") }
+      sample_one("sample") { expect(subject.get("tags") ).to include("success") }
+      sample_one("some sample") { expect(subject.get("tags") ).to include("success") }
     end
 
     conditional "[message] !~ /sample/" do
-      sample("apple") { expect(subject.get("tags")).to include("success") }
-      sample("sample") { expect(subject.get("tags")).to include("failure") }
-      sample("some sample") { expect(subject.get("tags")).to include("failure") }
+      sample_one("apple") { expect(subject.get("tags")).to include("success") }
+      sample_one("sample") { expect(subject.get("tags")).to include("failure") }
+      sample_one("some sample") { expect(subject.get("tags")).to include("failure") }
     end
 
   end
 
   describe "negated expressions" do
     conditional "!([message] == 'sample')" do
-      sample("sample") { expect(subject.get("tags")).not_to include("success") }
-      sample("different") { expect(subject.get("tags")).not_to include("failure") }
+      sample_one("sample") { expect(subject.get("tags")).not_to include("success") }
+      sample_one("different") { expect(subject.get("tags")).not_to include("failure") }
     end
 
     conditional "!([message] != 'sample')" do
-      sample("sample") { expect(subject.get("tags")).not_to include("failure") }
-      sample("different") { expect(subject.get("tags")).not_to include("success") }
+      sample_one("sample") { expect(subject.get("tags")).not_to include("failure") }
+      sample_one("different") { expect(subject.get("tags")).not_to include("success") }
     end
 
     conditional "!([message] < 'sample')" do
-      sample("apple") { expect(subject.get("tags")).not_to include("success") }
-      sample("zebra") { expect(subject.get("tags")).not_to include("failure") }
+      sample_one("apple") { expect(subject.get("tags")).not_to include("success") }
+      sample_one("zebra") { expect(subject.get("tags")).not_to include("failure") }
     end
 
     conditional "!([message] > 'sample')" do
-      sample("zebra") { expect(subject.get("tags")).not_to include("success") }
-      sample("apple") { expect(subject.get("tags")).not_to include("failure") }
+      sample_one("zebra") { expect(subject.get("tags")).not_to include("success") }
+      sample_one("apple") { expect(subject.get("tags")).not_to include("failure") }
     end
 
     conditional "!([message] <= 'sample')" do
-      sample("apple") { expect(subject.get("tags")).not_to include("success") }
-      sample("zebra") { expect(subject.get("tags")).not_to include("failure") }
-      sample("sample") { expect(subject.get("tags")).not_to include("success")}
+      sample_one("apple") { expect(subject.get("tags")).not_to include("success") }
+      sample_one("zebra") { expect(subject.get("tags")).not_to include("failure") }
+      sample_one("sample") { expect(subject.get("tags")).not_to include("success")}
     end
 
     conditional "!([message] >= 'sample')" do
-      sample("zebra") { expect(subject.get("tags")).not_to include("success") }
-      sample("sample") { expect(subject.get("tags")).not_to include("success") }
-      sample("apple") { expect(subject.get("tags")).not_to include("failure") }
+      sample_one("zebra") { expect(subject.get("tags")).not_to include("success") }
+      sample_one("sample") { expect(subject.get("tags")).not_to include("success") }
+      sample_one("apple") { expect(subject.get("tags")).not_to include("failure") }
     end
 
     conditional "!([message] =~ /sample/)" do
-      sample("apple") { expect(subject.get("tags")).not_to include("failure") }
-      sample("sample") { expect(subject.get("tags")).not_to include("success") }
-      sample("some sample") { expect(subject.get("tags")).not_to include("success") }
+      sample_one("apple") { expect(subject.get("tags")).not_to include("failure") }
+      sample_one("sample") { expect(subject.get("tags")).not_to include("success") }
+      sample_one("some sample") { expect(subject.get("tags")).not_to include("success") }
     end
 
     conditional "!([message] !~ /sample/)" do
-      sample("apple") { expect(subject.get("tags")).not_to include("success") }
-      sample("sample") { expect(subject.get("tags")).not_to include("failure") }
-      sample("some sample") { expect(subject.get("tags")).not_to include("failure") }
+      sample_one("apple") { expect(subject.get("tags")).not_to include("success") }
+      sample_one("sample") { expect(subject.get("tags")).not_to include("failure") }
+      sample_one("some sample") { expect(subject.get("tags")).not_to include("failure") }
     end
 
   end
@@ -313,66 +377,96 @@ describe "conditionals in filter" do
   describe "value as an expression" do
     # testing that a field has a value should be true.
     conditional "[message]" do
-      sample("apple") { expect(subject.get("tags")).to include("success") }
-      sample("sample") { expect(subject.get("tags")).to include("success") }
-      sample("some sample") { expect(subject.get("tags")).to include("success") }
+      sample_one("apple") { expect(subject.get("tags")).to include("success") }
+      sample_one("sample") { expect(subject.get("tags")).to include("success") }
+      sample_one("some sample") { expect(subject.get("tags")).to include("success") }
     end
 
     # testing that a missing field has a value should be false.
     conditional "[missing]" do
-      sample("apple") { expect(subject.get("tags")).to include("failure") }
-      sample("sample") { expect(subject.get("tags")).to include("failure") }
-      sample("some sample") { expect(subject.get("tags")).to include("failure") }
+      sample_one("apple") { expect(subject.get("tags")).to include("failure") }
+      sample_one("sample") { expect(subject.get("tags")).to include("failure") }
+      sample_one("some sample") { expect(subject.get("tags")).to include("failure") }
     end
   end
 
   describe "logic operators" do
     describe "and" do
       conditional "[message] and [message]" do
-        sample("whatever") { expect(subject.get("tags")).to include("success") }
+        sample_one("whatever") { expect(subject.get("tags")).to include("success") }
       end
       conditional "[message] and ![message]" do
-        sample("whatever") { expect(subject.get("tags")).to include("failure") }
+        sample_one("whatever") { expect(subject.get("tags")).to include("failure") }
       end
       conditional "![message] and [message]" do
-        sample("whatever") { expect(subject.get("tags")).to include("failure") }
+        sample_one("whatever") { expect(subject.get("tags")).to include("failure") }
       end
       conditional "![message] and ![message]" do
-        sample("whatever") { expect(subject.get("tags")).to include("failure") }
+        sample_one("whatever") { expect(subject.get("tags")).to include("failure") }
+      end
+    end
+
+    describe "nand" do
+      conditional "[message] nand [message]" do
+        sample_one("whatever") { expect(subject.get("tags")).to include("failure") }
+      end
+      conditional "[message] nand ![message]" do
+        sample_one("whatever") { expect(subject.get("tags")).to include("success") }
+      end
+      conditional "![message] nand [message]" do
+        sample_one("whatever") { expect(subject.get("tags")).to include("success") }
+      end
+      conditional "![message] nand ![message]" do
+        sample_one("whatever") { expect(subject.get("tags")).to include("success") }
+      end
+    end
+
+    describe "xor" do
+      conditional "[message] xor [message]" do
+        sample_one("whatever") { expect(subject.get("tags")).to include("failure") }
+      end
+      conditional "[message] xor ![message]" do
+        sample_one("whatever") { expect(subject.get("tags")).to include("success") }
+      end
+      conditional "![message] xor [message]" do
+        sample_one("whatever") { expect(subject.get("tags")).to include("success") }
+      end
+      conditional "![message] xor ![message]" do
+        sample_one("whatever") { expect(subject.get("tags")).to include("failure") }
       end
     end
 
     describe "or" do
       conditional "[message] or [message]" do
-        sample("whatever") { expect(subject.get("tags")).to include("success") }
+        sample_one("whatever") { expect(subject.get("tags")).to include("success") }
       end
       conditional "[message] or ![message]" do
-        sample("whatever") { expect(subject.get("tags")).to include("success") }
+        sample_one("whatever") { expect(subject.get("tags")).to include("success") }
       end
       conditional "![message] or [message]" do
-        sample("whatever") { expect(subject.get("tags")).to include("success") }
+        sample_one("whatever") { expect(subject.get("tags")).to include("success") }
       end
       conditional "![message] or ![message]" do
-        sample("whatever") { expect(subject.get("tags")).to include("failure") }
+        sample_one("whatever") { expect(subject.get("tags")).to include("failure") }
       end
     end
   end
 
   describe "field references" do
     conditional "[field with space]" do
-      sample("field with space" => "hurray") do
+      sample_one("field with space" => "hurray") do
         expect(subject.get("tags")).to include("success")
       end
     end
 
     conditional "[field with space] == 'hurray'" do
-      sample("field with space" => "hurray") do
+      sample_one("field with space" => "hurray") do
         expect(subject.get("tags")).to include("success")
       end
     end
 
     conditional "[nested field][reference with][some spaces] == 'hurray'" do
-      sample({"nested field" => { "reference with" => { "some spaces" => "hurray" } } }) do
+      sample_one({"nested field" => { "reference with" => { "some spaces" => "hurray" } } }) do
         expect(subject.get("tags")).to include("success")
       end
     end
@@ -394,15 +488,16 @@ describe "conditionals in filter" do
       }
     CONFIG
 
-    sample({"type" => "original"}) do
+    sample_one({"type" => "original"}) do
       expect(subject).to be_an(Array)
       expect(subject.length).to eq(2)
+      subject.sort! {|a, b| a.get("type") <=> b.get("type")}
 
-      expect(subject[0].get("type")).to eq("original")
-      expect(subject[0].get("cond1")).to eq("true")
-      expect(subject[0].get("cond2")).to eq(nil)
+      expect(subject[1].get("type")).to eq("original")
+      expect(subject[1].get("cond1")).to eq("true")
+      expect(subject[1].get("cond2")).to eq(nil)
 
-      expect(subject[1].get("type")).to eq("clone")
+      expect(subject[0].get("type")).to eq("clone")
       # expect(subject[1].get("cond1")).to eq(nil)
       # expect(subject[1].get("cond2")).to eq("true")
     end
@@ -424,18 +519,78 @@ describe "conditionals in filter" do
       }
     CONFIG
 
-    sample({"type" => "original"}) do
-      # puts subject.inspect
-      expect(subject[0].get("cond1")).to eq(nil)
+    sample_one({"type" => "original"}) do
+      expect(subject.length).to eq(3)
+      subject.sort! {|a, b| a.get("type") <=> b.get("type")}
+
+      expect(subject[0].get("type")).to eq("clone1")
+      expect(subject[0].get("cond1")).to eq("true")
       expect(subject[0].get("cond2")).to eq(nil)
 
-      expect(subject[1].get("type")).to eq("clone1")
-      expect(subject[1].get("cond1")).to eq("true")
-      expect(subject[1].get("cond2")).to eq(nil)
+      expect(subject[1].get("type")).to eq("clone2")
+      expect(subject[1].get("cond1")).to eq(nil)
+      expect(subject[1].get("cond2")).to eq("true")
 
-      expect(subject[2].get("type")).to eq("clone2")
+      expect(subject[2].get("type")).to eq("original")
       expect(subject[2].get("cond1")).to eq(nil)
-      expect(subject[2].get("cond2")).to eq("true")
+      expect(subject[2].get("cond2")).to eq(nil)
+    end
+  end
+
+  describe "complex case" do
+    config <<-CONFIG
+      filter {
+        if ("foo" in [tags]) {
+          mutate { id => addbar add_tag => bar }
+      
+          if ("bar" in [tags]) {
+            mutate { id => addbaz  add_tag => baz }
+          }
+      
+          if ("baz" in [tags]) {
+            mutate { id => addbot add_tag => bot }
+      
+            if ("bot" in [tags]) {
+              mutate { id => addbonk add_tag => bonk }
+            }
+          }
+        }
+      
+        if ("bot" in [tags]) {
+          mutate { id => addwat add_tag => wat }
+        }
+      
+        mutate { id => addprev add_tag => prev }
+      
+        mutate { id => addfinal add_tag => final }
+      
+      }
+    CONFIG
+
+    sample_one("tags" => ["bot"]) do
+      tags = subject.get("tags")
+      expect(tags[0]).to eq("bot")
+      expect(tags[1]).to eq("wat")
+      expect(tags[2]).to eq("prev")
+      expect(tags[3]).to eq("final")
+    end
+
+    sample_one("tags" => ["foo"]) do
+      tags = subject.get("tags")
+      expect(tags[0]).to eq("foo")
+      expect(tags[1]).to eq("bar")
+      expect(tags[2]).to eq("baz")
+      expect(tags[3]).to eq("bot")
+      expect(tags[4]).to eq("bonk")
+      expect(tags[5]).to eq("wat")
+      expect(tags[6]).to eq("prev")
+      expect(tags[7]).to eq("final")
+    end
+    
+    sample_one("type" => "original") do
+      tags = subject.get("tags")
+      expect(tags[0]).to eq("prev")
+      expect(tags[1]).to eq("final")
     end
   end
 end