logstash-core 5.3.3-java → 5.4.0-java

data/lib/logstash/version.rb

@@ -11,4 +11,4 @@
 #       eventually this file should be in the root logstash lib fir and dependencies in logstash-core should be
 #       fixed.
 
-LOGSTASH_VERSION = "5.3.3"
+LOGSTASH_VERSION = "5.4.0"

data/locales/en.yml

@@ -71,9 +71,9 @@ en:
         Logstash is not able to start since configuration auto reloading was enabled but the configuration contains plugins that don't support it. Quitting...
     web_api:
       cant_bind_to_port: |-
-        Logstash tried to bind to port %{port}, but the port is already in use. You can specify a new port by launching logtash with the --http.port option."
+        Logstash tried to bind to port %{port}, but the port is already in use. You can specify a new port by launching logstash with the --http.port option."
       cant_bind_to_port_in_range: |-
-        Logstash tried to bind to port range %{http_ports}, but all the ports are already in use. You can specify a new port by launching logtash with the --http.port option."
+        Logstash tried to bind to port range %{http_ports}, but all the ports are already in use. You can specify a new port by launching logstash with the --http.port option."
       hot_threads:
         title: |-
           ::: {%{hostname}}
@@ -98,6 +98,8 @@ en:
         the '-f yourlogstash.conf' flag?
       reload-without-config-path: >-
         Configuration reloading also requires passing a configuration path with '-f yourlogstash.conf'
+      locked-data-path: >-
+        Logstash could not be started because there is already another instance using the configured data directory.  If you wish to run multiple instances, you must change the "path.data" setting.
       invalid-shell: >-
         Invalid option for interactive Ruby shell. Use either "irb" or "pry"
       configtest-flag-information: |-

data/logstash-core.gemspec

@@ -19,9 +19,6 @@ Gem::Specification.new do |gem|
 
   gem.platform = "java"
 
-  gem.add_runtime_dependency "logstash-core-event-java", LOGSTASH_CORE_VERSION
-  gem.add_runtime_dependency "logstash-core-queue-jruby", LOGSTASH_CORE_VERSION
-
   gem.add_runtime_dependency "pry", "~> 0.10.1"  #(Ruby license)
   gem.add_runtime_dependency "stud", "~> 0.0.19" #(Apache 2.0 license)
   gem.add_runtime_dependency "clamp", "~> 0.6.5" #(MIT license) for command line args/flags

data/spec/api/lib/api/node_stats_spec.rb

@@ -78,7 +78,8 @@ describe LogStash::Api::Modules::NodeStats do
         "duration_in_millis" => Numeric,
         "in" => Numeric,
         "filtered" => Numeric,
-        "out" => Numeric
+        "out" => Numeric,
+        "queue_push_duration_in_millis" => Numeric
      }
    },
    "reloads" => {

data/spec/api/spec_helper.rb

@@ -31,7 +31,7 @@ end
 ##
 # Class used to wrap and manage the execution of an agent for test,
 # this helps a lot in order to have a more integrated test for the
-# web api, could be also used for other use cases if generalized enought
+# web api, could be also used for other use cases if generalized enough
 ##
 class LogStashRunner
 

data/spec/logstash/acked_queue_concurrent_stress_spec.rb

@@ -0,0 +1,291 @@
+# encoding: utf-8
+require "logstash/util/wrapped_acked_queue"
+require "logstash/event"
+require "logstash/instrument/namespaced_metric"
+
+describe LogStash::Util::WrappedAckedQueue, :stress_test => true do
+  let(:path) { Stud::Temporary.directory }
+
+  context "with multiple writers" do
+    let(:items) { expected_count / writers }
+    let(:page_capacity) { 1 << page_capacity_multiplier }
+    let(:queue_capacity) { page_capacity * queue_capacity_multiplier }
+
+    let(:output_strings) { [] }
+    let(:reject_memo_keys) { [:reject_memo_keys, :path, :queue, :writer_threads, :collector, :metric, :reader_threads, :output_strings] }
+
+    let(:queue) do
+      described_class.create_file_based(path, page_capacity, 0, queue_checkpoint_acks, queue_checkpoint_writes, queue_checkpoint_interval, queue_capacity)
+    end
+
+    let(:writer_threads) do
+      writer = queue.write_client
+      writers.times.map do |i|
+        Thread.new(i, items, writer) do |_i, _items, _writer|
+          publisher(_items, _writer)
+        end
+      end
+    end
+
+    let(:writers_finished) { Concurrent::AtomicBoolean.new(false) }
+
+    let(:reader_threads) do
+      reader = queue.read_client
+      reader.set_batch_dimensions(batch_size, batch_wait)
+      reader.set_events_metric(metric.namespace([:stats, :events]))
+      reader.set_pipeline_metric(metric.namespace([:stats, :pipelines, :main, :events]))
+
+      readers.times.map do |i|
+        Thread.new(i, reader, counts) do |_i, _reader, _counts|
+          begin
+            tally = 0
+            while true
+              batch = _reader.read_batch
+              break if batch.size.zero? && writers_finished.value == true && queue.queue.is_fully_acked?
+              sleep(rand * 0.01) if simulate_work
+              tally += batch.size
+              batch.close
+            end
+            _counts[_i] = tally
+            # puts("reader #{_i}, tally=#{tally}, _counts=#{_counts.inspect}")
+          rescue => e
+            p :reader_error => e
+          end
+        end
+      end
+    end
+
+    def publisher(items, writer)
+      items.times.each do |i|
+        event = LogStash::Event.new("sequence" => "#{i}".ljust(string_size))
+        writer.push(event)
+      end
+    rescue => e
+      p :publisher_error => e
+    end
+
+    let(:collector) { LogStash::Instrument::Collector.new }
+    let(:metric) { LogStash::Instrument::Metric.new(collector) }
+
+    shared_examples "a well behaved queue" do
+      it "writes, reads, closes and reopens" do
+        Thread.abort_on_exception = true
+
+        # force lazy initialization to avoid concurency issues within threads
+        counts
+        queue
+
+        # Start the threads
+        writer_threads
+        reader_threads
+
+        writer_threads.each(&:join)
+        writers_finished.make_true
+
+        reader_threads.each(&:join)
+
+        enqueued = queue.queue.unread_count
+
+        if enqueued != 0
+          output_strings << "unread events in queue: #{enqueued}"
+        end
+
+        got = counts.reduce(&:+)
+
+        if got != expected_count
+          # puts("count=#{counts.inspect}")
+          output_strings << "events read: #{got}"
+        end
+
+        sleep 0.1
+        expect { queue.close }.not_to raise_error
+        sleep 0.1
+        files = Dir.glob(path + '/*').map{|f| f.sub("#{path}/", '')}
+        if files.count != 2
+          output_strings << "File count after close mismatch expected: 2 got: #{files.count}"
+          output_strings.concat files
+        end
+
+        begin
+          queue.queue.open
+        rescue Exception => e
+          output_strings << e.message
+        end
+
+        queue.queue.close
+
+        if output_strings.any?
+          output_strings << __memoized.reject{|k,v| reject_memo_keys.include?(k)}.inspect
+        end
+
+        expect(output_strings).to eq([])
+      end
+    end
+
+    let(:writers) { 3 }
+    let(:readers) { 3 }
+    let(:simulate_work) { true }
+    let(:counts) { Concurrent::Array.new([0, 0, 0, 0, 0, 0, 0, 0]) }
+    let(:page_capacity_multiplier) { 20 }
+    let(:queue_capacity_multiplier) { 128 }
+    let(:queue_checkpoint_acks) { 1024 }
+    let(:queue_checkpoint_writes) { 1024 }
+    let(:queue_checkpoint_interval) { 1000 }
+    let(:batch_size) { 500 }
+    let(:batch_wait) { 1000 }
+    let(:expected_count) { 60000 }
+    let(:string_size) { 256 }
+
+    describe "with simulate_work ON" do
+      let(:simulate_work) { true }
+
+      context "> more writers than readers <" do
+        let(:writers) { 4 }
+        let(:readers) { 2 }
+        it_behaves_like "a well behaved queue"
+      end
+
+      context "> less writers than readers <" do
+        let(:writers) { 2 }
+        let(:readers) { 4 }
+        it_behaves_like "a well behaved queue"
+      end
+
+      context "> larger checkpoint acks <" do
+        let(:queue_checkpoint_acks) { 3000 }
+        it_behaves_like "a well behaved queue"
+      end
+
+      context "> smaller checkpoint acks <" do
+        let(:queue_checkpoint_acks) { 500 }
+        it_behaves_like "a well behaved queue"
+      end
+
+      context "> larger checkpoint writes <" do
+        let(:queue_checkpoint_writes) { 3000 }
+        it_behaves_like "a well behaved queue"
+      end
+
+      context "> smaller checkpoint writes <" do
+        let(:queue_checkpoint_writes) { 500 }
+        it_behaves_like "a well behaved queue"
+      end
+
+      context "> larger checkpoint interval <" do
+        let(:queue_checkpoint_interval) { 3000 }
+        it_behaves_like "a well behaved queue"
+      end
+
+      context "> smaller checkpoint interval <" do
+        let(:queue_checkpoint_interval) { 500 }
+        it_behaves_like "a well behaved queue"
+      end
+
+      context "> smaller batch wait <" do
+        let(:batch_wait) { 125 }
+        it_behaves_like "a well behaved queue"
+      end
+
+      context "> larger batch wait <" do
+        let(:batch_wait) { 5000 }
+        it_behaves_like "a well behaved queue"
+      end
+
+      context "> smaller event size <" do
+        let(:string_size) { 8 }
+        it_behaves_like "a well behaved queue"
+      end
+
+      context "> larger event size <" do
+        let(:string_size) { 8192 }
+        it_behaves_like "a well behaved queue"
+      end
+
+      context "> small queue size limit <" do
+        let(:queue_capacity_multiplier) { 10 }
+        it_behaves_like "a well behaved queue"
+      end
+
+      context "> very large queue size limit <" do
+        let(:queue_capacity_multiplier) { 512 }
+        it_behaves_like "a well behaved queue"
+      end
+    end
+
+    describe "with simulate_work OFF" do
+      let(:simulate_work) { false }
+
+      context "> more writers than readers <" do
+        let(:writers) { 4 }
+        let(:readers) { 2 }
+        it_behaves_like "a well behaved queue"
+      end
+
+      context "> less writers than readers <" do
+        let(:writers) { 2 }
+        let(:readers) { 4 }
+        it_behaves_like "a well behaved queue"
+      end
+
+      context "> larger checkpoint acks <" do
+        let(:queue_checkpoint_acks) { 3000 }
+        it_behaves_like "a well behaved queue"
+      end
+
+      context "> smaller checkpoint acks <" do
+        let(:queue_checkpoint_acks) { 500 }
+        it_behaves_like "a well behaved queue"
+      end
+
+      context "> larger checkpoint writes <" do
+        let(:queue_checkpoint_writes) { 3000 }
+        it_behaves_like "a well behaved queue"
+      end
+
+      context "> smaller checkpoint writes <" do
+        let(:queue_checkpoint_writes) { 500 }
+        it_behaves_like "a well behaved queue"
+      end
+
+      context "> larger checkpoint interval <" do
+        let(:queue_checkpoint_interval) { 3000 }
+        it_behaves_like "a well behaved queue"
+      end
+
+      context "> smaller checkpoint interval <" do
+        let(:queue_checkpoint_interval) { 500 }
+        it_behaves_like "a well behaved queue"
+      end
+
+      context "> smaller batch wait <" do
+        let(:batch_wait) { 125 }
+        it_behaves_like "a well behaved queue"
+      end
+
+      context "> larger batch wait <" do
+        let(:batch_wait) { 5000 }
+        it_behaves_like "a well behaved queue"
+      end
+
+      context "> smaller event size <" do
+        let(:string_size) { 8 }
+        it_behaves_like "a well behaved queue"
+      end
+
+      context "> larger event size <" do
+        let(:string_size) { 8192 }
+        it_behaves_like "a well behaved queue"
+      end
+
+      context "> small queue size limit <" do
+        let(:queue_capacity_multiplier) { 10 }
+        it_behaves_like "a well behaved queue"
+      end
+
+      context "> very large queue size limit <" do
+        let(:queue_capacity_multiplier) { 512 }
+        it_behaves_like "a well behaved queue"
+      end
+    end
+  end
+end

data/spec/logstash/agent_spec.rb

@@ -99,6 +99,30 @@ describe LogStash::Agent do
         subject.register_pipeline(pipeline_settings)
       end
 
+      context "when a system pipeline is running" do
+        context "when one pipeline is finite" do
+          let(:pipeline_args) {
+            {
+              "path.config" => "a",
+              "config.string" => "input { generator { count => 1000 }} output { null {} }"
+            }
+          }
+          let(:system_pipeline_settings) do
+            s = agent_settings.clone
+            s.set("path.config", "")
+            s.set("config.string", "input { generator {}} output { null {} }")
+            s.set("pipeline.id", ".monitoring")
+            s.set("pipeline.system", true)
+            s
+          end
+
+          it "stops logstash at the end of the execution of the finite pipeline" do
+            subject.register_pipeline(system_pipeline_settings)
+            expect(subject.execute).to be_nil
+          end
+        end
+      end
+
       context "if state is clean" do
         before :each do
           allow(subject).to receive(:running_pipelines?).and_return(true)

data/spec/logstash/config/mixin_spec.rb

@@ -192,7 +192,7 @@ describe LogStash::Config::Mixin do
         expect(clone.uri.to_s).to eql(uri_hidden)
       end
 
-      it "should make the real URI object availale under #uri" do
+      it "should make the real URI object available under #uri" do
         expect(subject.uri.uri).to be_a(::URI)
       end
 
@@ -322,6 +322,9 @@ describe LogStash::Config::Mixin do
         config :oneNumber, :validate => :number, :required => false
         config :oneArray, :validate => :array, :required => false
         config :oneHash, :validate => :hash, :required => false
+        config :nestedHash, :validate => :hash, :required => false
+        config :nestedArray, :validate => :hash, :required => false
+        config :deepHash, :validate => :hash, :required => false
 
         def initialize(params)
           super(params)
@@ -378,7 +381,10 @@ describe LogStash::Config::Mixin do
           "oneString" => "${FunString:foo}",
           "oneBoolean" => "${FunBool:false}",
           "oneArray" => [ "first array value", "${FunString:foo}" ],
-          "oneHash" => { "key1" => "${FunString:foo}", "key2" => "${FunString} is ${FunBool}", "key3" => "${FunBool:false} or ${funbool:false}" }
+          "oneHash" => { "key1" => "${FunString:foo}", "key2" => "${FunString} is ${FunBool}", "key3" => "${FunBool:false} or ${funbool:false}" },
+          "nestedHash" => { "level1" => { "key1" => "http://${FunString}:8080/blah.txt" } },
+          "nestedArray" => { "level1" => [{ "key1" => "http://${FunString}:8080/blah.txt" }, { "key2" => "http://${FunString}:8080/foo.txt" }] },
+          "deepHash" => { "level1" => { "level2" => {"level3" => { "key1" => "http://${FunString}:8080/blah.txt" } } } }
         )
       end
 
@@ -387,6 +393,9 @@ describe LogStash::Config::Mixin do
         expect(subject.oneBoolean).to(be_truthy)
         expect(subject.oneArray).to(be == [ "first array value", "fancy" ])
         expect(subject.oneHash).to(be == { "key1" => "fancy", "key2" => "fancy is true", "key3" => "true or false" })
+        expect(subject.nestedHash).to(be == { "level1" => { "key1" => "http://fancy:8080/blah.txt" } })
+        expect(subject.nestedArray).to(be == { "level1" => [{ "key1" => "http://fancy:8080/blah.txt" }, { "key2" => "http://fancy:8080/foo.txt" }] })
+        expect(subject.deepHash).to(be == { "level1" => { "level2" => { "level3" => { "key1" => "http://fancy:8080/blah.txt" } } } })
       end
     end
 

data/spec/logstash/event_dispatcher_spec.rb

@@ -34,6 +34,13 @@ describe LogStash::EventDispatcher do
   let(:listener) { CustomSpy }
   subject(:emitter) { DummyEmitter.new }
 
+  it "ignores duplicate listener" do
+    emitter.dispatcher.add_listener(listener)
+    emitter.dispatcher.add_listener(listener)
+    expect(listener).to receive(:method_exists).with(emitter).once
+    emitter.method_exists
+  end
+
   describe "Emits events" do
     before do
       emitter.dispatcher.add_listener(listener)
@@ -65,7 +72,7 @@ describe LogStash::EventDispatcher do
       emitter.method_exists
     end
 
-    it "allows to remove a listner to an emitter" do
+    it "allows to remove a listener to an emitter" do
       expect(listener).to receive(:method_exists).with(emitter).once
       emitter.dispatcher.add_listener(listener)
       emitter.method_exists

data/spec/logstash/event_spec.rb

@@ -0,0 +1,346 @@
+# encoding: utf-8
+
+require "spec_helper"
+require "logstash/util"
+require "logstash/event"
+require "json"
+require "java"
+
+TIMESTAMP = "@timestamp"
+
+describe LogStash::Event do
+  context "to_json" do
+    it "should serialize simple values" do
+      e = LogStash::Event.new({"foo" => "bar", "bar" => 1, "baz" => 1.0, TIMESTAMP => "2015-05-28T23:02:05.350Z"})
+      expect(JSON.parse(e.to_json)).to eq(JSON.parse("{\"foo\":\"bar\",\"bar\":1,\"baz\":1.0,\"@timestamp\":\"2015-05-28T23:02:05.350Z\",\"@version\":\"1\"}"))
+    end
+
+    it "should serialize deep hash values" do
+      e = LogStash::Event.new({"foo" => {"bar" => 1, "baz" => 1.0, "biz" => "boz"}, TIMESTAMP => "2015-05-28T23:02:05.350Z"})
+      expect(JSON.parse(e.to_json)).to eq(JSON.parse("{\"foo\":{\"bar\":1,\"baz\":1.0,\"biz\":\"boz\"},\"@timestamp\":\"2015-05-28T23:02:05.350Z\",\"@version\":\"1\"}"))
+    end
+
+    it "should serialize deep array values" do
+      e = LogStash::Event.new({"foo" => ["bar", 1, 1.0], TIMESTAMP => "2015-05-28T23:02:05.350Z"})
+      expect(JSON.parse(e.to_json)).to eq(JSON.parse("{\"foo\":[\"bar\",1,1.0],\"@timestamp\":\"2015-05-28T23:02:05.350Z\",\"@version\":\"1\"}"))
+    end
+
+    it "should serialize deep hash from field reference assignments" do
+      e = LogStash::Event.new({TIMESTAMP => "2015-05-28T23:02:05.350Z"})
+      e.set("foo", "bar")
+      e.set("bar", 1)
+      e.set("baz", 1.0)
+      e.set("[fancy][pants][socks]", "shoes")
+      expect(JSON.parse(e.to_json)).to eq(JSON.parse("{\"@timestamp\":\"2015-05-28T23:02:05.350Z\",\"@version\":\"1\",\"foo\":\"bar\",\"bar\":1,\"baz\":1.0,\"fancy\":{\"pants\":{\"socks\":\"shoes\"}}}"))
+    end
+  end
+
+  context "#get" do
+    it "should get simple values" do
+      e = LogStash::Event.new({"foo" => "bar", "bar" => 1, "baz" => 1.0, TIMESTAMP => "2015-05-28T23:02:05.350Z"})
+      expect(e.get("foo")).to eq("bar")
+      expect(e.get("[foo]")).to eq("bar")
+      expect(e.get("bar")).to eq(1)
+      expect(e.get("[bar]")).to eq(1)
+      expect(e.get("baz")).to eq(1.0)
+      expect(e.get("[baz]")).to eq(1.0)
+      expect(e.get(TIMESTAMP).to_s).to eq("2015-05-28T23:02:05.350Z")
+      expect(e.get("[#{TIMESTAMP}]").to_s).to eq("2015-05-28T23:02:05.350Z")
+    end
+
+    it "should get deep hash values" do
+      e = LogStash::Event.new({"foo" => {"bar" => 1, "baz" => 1.0}})
+      expect(e.get("[foo][bar]")).to eq(1)
+      expect(e.get("[foo][baz]")).to eq(1.0)
+    end
+
+    it "should get deep array values" do
+      e = LogStash::Event.new({"foo" => ["bar", 1, 1.0]})
+      expect(e.get("[foo][0]")).to eq("bar")
+      expect(e.get("[foo][1]")).to eq(1)
+      expect(e.get("[foo][2]")).to eq(1.0)
+      expect(e.get("[foo][3]")).to be_nil
+    end
+
+    context "negative array values" do
+      it "should index from the end of the array" do
+        list = ["bar", 1, 1.0]
+        e = LogStash::Event.new({"foo" => list})
+        expect(e.get("[foo][-3]")).to eq(list[-3])
+        expect(e.get("[foo][-2]")).to eq(list[-2])
+        expect(e.get("[foo][-1]")).to eq(list[-1])
+      end
+    end
+  end
+
+  context "#set" do
+    it "should set simple values" do
+      e = LogStash::Event.new()
+      expect(e.set("foo", "bar")).to eq("bar")
+      expect(e.get("foo")).to eq("bar")
+
+      e = LogStash::Event.new({"foo" => "test"})
+      expect(e.set("foo", "bar")).to eq("bar")
+      expect(e.get("foo")).to eq("bar")
+    end
+
+    it "should set deep hash values" do
+      e = LogStash::Event.new()
+      expect(e.set("[foo][bar]", "baz")).to eq("baz")
+      expect(e.get("[foo][bar]")).to eq("baz")
+      expect(e.get("[foo][baz]")).to be_nil
+    end
+
+    it "should set deep array values" do
+      e = LogStash::Event.new()
+      expect(e.set("[foo][0]", "bar")).to eq("bar")
+      expect(e.get("[foo][0]")).to eq("bar")
+      expect(e.set("[foo][1]", 1)).to eq(1)
+      expect(e.get("[foo][1]")).to eq(1)
+      expect(e.set("[foo][2]", 1.0)).to eq(1.0)
+      expect(e.get("[foo][2]")).to eq(1.0)
+      expect(e.get("[foo][3]")).to be_nil
+    end
+
+    it "should add key when setting nil value" do
+      e = LogStash::Event.new()
+      e.set("[foo]", nil)
+      expect(e.to_hash).to include("foo" => nil)
+    end
+
+    # BigDecimal is now natively converted by JRuby, see https://github.com/elastic/logstash/pull/4838
+    it "should set BigDecimal" do
+      e = LogStash::Event.new()
+      e.set("[foo]", BigDecimal.new(1))
+      expect(e.get("foo")).to be_kind_of(BigDecimal)
+      expect(e.get("foo")).to eq(BigDecimal.new(1))
+    end
+
+    it "should set RubyBignum" do
+      e = LogStash::Event.new()
+      e.set("[foo]", -9223372036854776000)
+      expect(e.get("foo")).to be_kind_of(Bignum)
+      expect(e.get("foo")).to eq(-9223372036854776000)
+    end
+
+    it "should convert Time to Timestamp" do
+      e = LogStash::Event.new()
+      time = Time.now
+      e.set("[foo]", Time.at(time.to_f))
+      expect(e.get("foo")).to be_kind_of(LogStash::Timestamp)
+      expect(e.get("foo").to_f).to be_within(0.1).of(time.to_f)
+    end
+
+    it "should set XXJavaProxy Jackson crafted" do
+      proxy = org.logstash.Util.getMapFixtureJackson()
+      # proxy is {"string": "foo", "int": 42, "float": 42.42, "array": ["bar","baz"], "hash": {"string":"quux"} }
+      e = LogStash::Event.new()
+      e.set("[proxy]", proxy)
+      expect(e.get("[proxy][string]")).to eql("foo")
+      expect(e.get("[proxy][int]")).to eql(42)
+      expect(e.get("[proxy][float]")).to eql(42.42)
+      expect(e.get("[proxy][array][0]")).to eql("bar")
+      expect(e.get("[proxy][array][1]")).to eql("baz")
+      expect(e.get("[proxy][hash][string]")).to eql("quux")
+    end
+
+    it "should set XXJavaProxy hand crafted" do
+      proxy = org.logstash.Util.getMapFixtureHandcrafted()
+      # proxy is {"string": "foo", "int": 42, "float": 42.42, "array": ["bar","baz"], "hash": {"string":"quux"} }
+      e = LogStash::Event.new()
+      e.set("[proxy]", proxy)
+      expect(e.get("[proxy][string]")).to eql("foo")
+      expect(e.get("[proxy][int]")).to eql(42)
+      expect(e.get("[proxy][float]")).to eql(42.42)
+      expect(e.get("[proxy][array][0]")).to eql("bar")
+      expect(e.get("[proxy][array][1]")).to eql("baz")
+      expect(e.get("[proxy][hash][string]")).to eql("quux")
+    end
+
+    it "should fail on non UTF-8 encoding" do
+      # e = LogStash::Event.new
+      # s1 = "\xE0 Montr\xE9al".force_encoding("ISO-8859-1")
+      # expect(s1.encoding.name).to eq("ISO-8859-1")
+      # expect(s1.valid_encoding?).to eq(true)
+      # e.set("test", s1)
+      # s2 = e.get("test")
+      # expect(s2.encoding.name).to eq("UTF-8")
+      # expect(s2.valid_encoding?).to eq(true)
+    end
+  end
+
+  context "timestamp" do
+    it "getters should present a Ruby LogStash::Timestamp" do
+      e = LogStash::Event.new()
+      expect(e.timestamp.class).to eq(LogStash::Timestamp)
+      expect(e.get(TIMESTAMP).class).to eq(LogStash::Timestamp)
+    end
+
+    it "to_hash should inject a Ruby LogStash::Timestamp" do
+      e = LogStash::Event.new()
+
+      expect(e.to_java).to be_kind_of(Java::OrgLogstash::Event)
+      expect(e.to_java.get_field(TIMESTAMP)).to be_kind_of(Java::OrgLogstash::Timestamp)
+
+      expect(e.to_hash[TIMESTAMP]).to be_kind_of(LogStash::Timestamp)
+      # now make sure the original map was not touched
+      expect(e.to_java.get_field(TIMESTAMP)).to be_kind_of(Java::OrgLogstash::Timestamp)
+    end
+
+    it "should set timestamp" do
+      e = LogStash::Event.new
+      now = Time.now
+      e.set("@timestamp", LogStash::Timestamp.at(now.to_i))
+      expect(e.timestamp.to_i).to eq(now.to_i)
+      expect(e.get("@timestamp").to_i).to eq(now.to_i)
+    end
+  end
+
+  context "append" do
+    it "should append" do
+      event = LogStash::Event.new("message" => "hello world")
+      event.append(LogStash::Event.new("message" => "another thing"))
+      expect(event.get("message")).to eq(["hello world", "another thing"])
+    end
+  end
+
+  context "tags" do
+    it "should tag" do
+      event = LogStash::Event.new("message" => "hello world")
+      expect(event.get("tags")).to be_nil
+      event.tag("foo")
+      expect(event.get("tags")).to eq(["foo"])
+    end
+  end
+
+
+  # TODO(talevy): migrate tests to Java. no reason to test logging logic in ruby when it is being
+  #               done in java land.
+
+  # context "logger" do
+
+  #   let(:logger) { double("Logger") }
+
+  #   before(:each) do
+  #     allow(LogStash::Event).to receive(:logger).and_return(logger)
+  #   end
+
+  #   it "should set logger using a module" do
+  #     expect(logger).to receive(:warn).once
+  #     LogStash::Event.new(TIMESTAMP => "invalid timestamp")
+  #   end
+
+  #   it "should warn on invalid timestamp object" do
+  #     expect(logger).to receive(:warn).once.with(/^Unrecognized/)
+  #     LogStash::Event.new(TIMESTAMP => Array.new)
+  #   end
+  # end
+
+  context "to_hash" do
+    let (:source_hash) {  {"a" => 1, "b" => [1, 2, 3, {"h" => 1, "i" => "baz"}], "c" => {"d" => "foo", "e" => "bar", "f" => [4, 5, "six"]}} }
+    let (:source_hash_with_metadata) {  source_hash.merge({"@metadata" => {"a" => 1, "b" => 2}}) }
+    subject { LogStash::Event.new(source_hash_with_metadata) }
+
+    it "should include @timestamp and @version" do
+      h = subject.to_hash
+      expect(h).to include("@timestamp")
+      expect(h).to include("@version")
+      expect(h).not_to include("@metadata")
+    end
+
+    it "should include @timestamp and @version and @metadata" do
+      h = subject.to_hash_with_metadata
+      expect(h).to include("@timestamp")
+      expect(h).to include("@version")
+      expect(h).to include("@metadata")
+    end
+
+    it "should produce valid deep Ruby hash without metadata" do
+      h = subject.to_hash
+      h.delete("@timestamp")
+      h.delete("@version")
+      expect(h).to eq(source_hash)
+    end
+
+    it "should produce valid deep Ruby hash with metadata" do
+      h = subject.to_hash_with_metadata
+      h.delete("@timestamp")
+      h.delete("@version")
+      expect(h).to eq(source_hash_with_metadata)
+    end
+  end
+
+  context "from_json" do
+    let (:source_json) { "{\"foo\":1, \"bar\":\"baz\"}" }
+    let (:blank_strings) {["", "  ",  "   "]}
+    let (:bare_strings) {["aa", "  aa", "aa  "]}
+
+    it "should produce a new event from json" do
+      expect(LogStash::Event.from_json(source_json).size).to eq(1)
+
+      event = LogStash::Event.from_json(source_json)[0]
+      expect(event.get("[foo]")).to eq(1)
+      expect(event.get("[bar]")).to eq("baz")
+    end
+
+    it "should ignore blank strings" do
+      blank_strings.each do |s|
+        expect(LogStash::Event.from_json(s).size).to eq(0)
+      end
+    end
+
+    it "should raise TypeError on nil string" do
+      expect{LogStash::Event.from_json(nil)}.to raise_error TypeError
+    end
+
+    it "should consistently handle nil" do
+      blank_strings.each do |s|
+        expect{LogStash::Event.from_json(nil)}.to raise_error
+        expect{LogStash::Event.new(LogStash::Json.load(nil))}.to raise_error
+      end
+    end
+
+    it "should consistently handle bare string" do
+      bare_strings.each do |s|
+        expect{LogStash::Event.from_json(s)}.to raise_error LogStash::Json::ParserError
+        expect{LogStash::Event.new(LogStash::Json.load(s))}.to raise_error LogStash::Json::ParserError
+       end
+    end
+  end
+
+  context "initialize" do
+
+    it "should accept Ruby Hash" do
+      e = LogStash::Event.new({"foo" => 1, TIMESTAMP => "2015-05-28T23:02:05.350Z"})
+      expect(e.get("foo")).to eq(1)
+      expect(e.timestamp.to_iso8601).to eq("2015-05-28T23:02:05.350Z")
+    end
+
+    it "should accept Java Map" do
+      h = Java::JavaUtil::HashMap.new
+      h.put("foo", 2);
+      h.put(TIMESTAMP, "2016-05-28T23:02:05.350Z");
+      e = LogStash::Event.new(h)
+
+      expect(e.get("foo")).to eq(2)
+      expect(e.timestamp.to_iso8601).to eq("2016-05-28T23:02:05.350Z")
+    end
+
+  end
+
+  context "method missing exception messages" do
+    subject { LogStash::Event.new({"foo" => "bar"}) }
+
+    it "#[] method raises a better exception message" do
+      expect { subject["foo"] }.to raise_error(NoMethodError, /Direct event field references \(i\.e\. event\['field'\]\)/)
+    end
+
+    it "#[]= method raises a better exception message" do
+      expect { subject["foo"] = "baz" }.to raise_error(NoMethodError, /Direct event field references \(i\.e\. event\['field'\] = 'value'\)/)
+    end
+
+    it "other missing method raises normal exception message" do
+      expect { subject.baz() }.to raise_error(NoMethodError, /undefined method `baz' for/)
+    end
+  end
+end