logstash-core-event 2.2.4.snapshot1

data/lib/logstash/timestamp.rb

@@ -0,0 +1,103 @@
+# encoding: utf-8
+require "logstash/environment"
+require "logstash/json"
+require "forwardable"
+require "date"
+require "time"
+
+module LogStash
+  class TimestampParserError < StandardError; end
+
+  class Timestamp
+    extend Forwardable
+    include Comparable
+
+    def_delegators :@time, :tv_usec, :usec, :year, :iso8601, :to_i, :tv_sec, :to_f, :to_edn, :<=>, :+
+
+    attr_reader :time
+
+    ISO8601_STRFTIME = "%04d-%02d-%02dT%02d:%02d:%02d.%06d%+03d:00".freeze
+    ISO8601_PRECISION = 3
+
+    def initialize(time = Time.new)
+      @time = time.utc
+    end
+
+    def self.at(*args)
+      epoch = args.first
+      if epoch.is_a?(BigDecimal)
+        # bug in JRuby prevents correcly parsing a BigDecimal fractional part, see https://github.com/elastic/logstash/issues/4565
+        Timestamp.new(::Time.at(epoch.to_i, epoch.frac.to_f * 1000000))
+      else
+        Timestamp.new(::Time.at(*args))
+      end
+    end
+
+    def self.parse(*args)
+      Timestamp.new(::Time.parse(*args))
+    end
+
+    def self.now
+      Timestamp.new(::Time.now)
+    end
+
+    # coerce tries different strategies based on the time object class to convert into a Timestamp.
+    # @param [String, Time, Timestamp] time the time object to try coerce
+    # @return [Timestamp, nil] Timestamp will be returned if successful otherwise nil
+    # @raise [TimestampParserError] on String with invalid format
+    def self.coerce(time)
+      case time
+      when String
+        LogStash::Timestamp.parse_iso8601(time)
+      when LogStash::Timestamp
+        time
+      when Time
+        LogStash::Timestamp.new(time)
+      else
+        nil
+      end
+    end
+
+    if LogStash::Environment.jruby?
+      JODA_ISO8601_PARSER = org.joda.time.format.ISODateTimeFormat.dateTimeParser
+      UTC = org.joda.time.DateTimeZone.forID("UTC")
+
+      def self.parse_iso8601(t)
+        millis = JODA_ISO8601_PARSER.parseMillis(t)
+        LogStash::Timestamp.at(millis / 1000, (millis % 1000) * 1000)
+      rescue => e
+        raise(TimestampParserError, "invalid timestamp string #{t.inspect}, error=#{e.inspect}")
+      end
+
+    else
+
+      def self.parse_iso8601(t)
+        # warning, ruby's Time.parse is *really* terrible and slow.
+        LogStash::Timestamp.new(::Time.parse(t))
+      rescue => e
+        raise(TimestampParserError, "invalid timestamp string #{t.inspect}, error=#{e.inspect}")
+      end
+    end
+
+    def utc
+      @time.utc # modifies the receiver
+      self
+    end
+    alias_method :gmtime, :utc
+
+    def to_json(*args)
+      # ignore arguments to respect accepted to_json method signature
+      "\"" + to_iso8601 + "\""
+    end
+    alias_method :inspect, :to_json
+
+    def to_iso8601
+      @iso8601 ||= @time.iso8601(ISO8601_PRECISION)
+    end
+    alias_method :to_s, :to_iso8601
+
+    def -(value)
+      @time - (value.is_a?(Timestamp) ? value.time : value)
+    end
+  end
+end

data/lib/logstash/util/accessors.rb

@@ -0,0 +1,123 @@
+# encoding: utf-8
+require "logstash/namespace"
+require "logstash/util"
+require "thread_safe"
+
+module LogStash::Util
+
+  # PathCache is a singleton which globally caches the relation between a field reference and its
+  # decomposition into a [key, path array] tuple. For example the field reference [foo][bar][baz]
+  # is decomposed into ["baz", ["foo", "bar"]].
+  module PathCache
+    extend self
+
+    # requiring libraries and defining constants is thread safe in JRuby so
+    # PathCache::CACHE will be corretly initialized, once, when accessors.rb
+    # will be first required
+    CACHE = ThreadSafe::Cache.new
+
+    def get(field_reference)
+      # the "get_or_default(x, nil) || put(x, parse(x))" is ~2x faster than "get || put" because the get call is
+      # proxied through the JRuby JavaProxy op_aref method. the correct idiom here would be to use
+      # "compute_if_absent(x){parse(x)}" but because of the closure creation, it is ~1.5x slower than
+      # "get_or_default || put".
+      # this "get_or_default || put" is obviously non-atomic which is not really important here
+      # since all threads will set the same value and this cache will stabilize very quickly after the first
+      # few events.
+      CACHE.get_or_default(field_reference, nil) || CACHE.put(field_reference, parse(field_reference))
+    end
+
+    def parse(field_reference)
+      path = field_reference.split(/[\[\]]/).select{|s| !s.empty?}
+      [path.pop, path]
+    end
+  end
+
+  # Accessors uses a lookup table to speedup access of a field reference of the form
+  # "[hello][world]" to the underlying store hash into {"hello" => {"world" => "foo"}}
+  class Accessors
+
+    # @param store [Hash] the backing data store field refereces point to
+    def initialize(store)
+      @store = store
+
+      # @lut is a lookup table between a field reference and a [target, key] tuple
+      # where target is the containing Hash or Array for key in @store.
+      # this allows us to directly access the containing object for key instead of
+      # walking the field reference path into the inner @store objects
+      @lut = {}
+    end
+
+    # @param field_reference [String] the field reference
+    # @return [Object] the value in @store for this field reference
+    def get(field_reference)
+      target, key = lookup(field_reference)
+      return nil unless target
+      target.is_a?(Array) ? target[key.to_i] : target[key]
+    end
+
+    # @param field_reference [String] the field reference
+    # @param value [Object] the value to set in @store for this field reference
+    # @return [Object] the value set
+    def set(field_reference, value)
+      target, key = lookup_or_create(field_reference)
+      target[target.is_a?(Array) ? key.to_i : key] = value
+    end
+
+    # @param field_reference [String] the field reference to remove
+    # @return [Object] the removed value in @store for this field reference
+    def del(field_reference)
+      target, key = lookup(field_reference)
+      return nil unless target
+      target.is_a?(Array) ? target.delete_at(key.to_i) : target.delete(key)
+    end
+
+    # @param field_reference [String] the field reference to test for inclusion in the store
+    # @return [Boolean] true if the store contains a value for this field reference
+    def include?(field_reference)
+      target, key = lookup(field_reference)
+      return false unless target
+
+      target.is_a?(Array) ? !target[key.to_i].nil? : target.include?(key)
+    end
+
+    private
+
+    # retrieve the [target, key] tuple associated with this field reference
+    # @param field_reference [String] the field referece
+    # @return [[Object, String]] the  [target, key] tuple associated with this field reference
+    def lookup(field_reference)
+      @lut[field_reference] ||= find_target(field_reference)
+    end
+
+    # retrieve the [target, key] tuple associated with this field reference and create inner
+    # container objects if they do not exists
+    # @param field_reference [String] the field referece
+    # @return [[Object, String]] the  [target, key] tuple associated with this field reference
+    def lookup_or_create(field_reference)
+      @lut[field_reference] ||= find_or_create_target(field_reference)
+    end
+
+    # find the target container object in store for this field reference
+    # @param field_reference [String] the field referece
+    # @return [Object] the target container object in store associated with this field reference
+    def find_target(field_reference)
+      key, path = PathCache.get(field_reference)
+      target = path.inject(@store) do |r, k|
+        return nil unless r
+        r[r.is_a?(Array) ? k.to_i : k]
+      end
+      target ? [target, key] : nil
+    end
+
+    # find the target container object in store for this field reference and create inner
+    # container objects if they do not exists
+    # @param field_reference [String] the field referece
+    # @return [Object] the target container object in store associated with this field reference
+    def find_or_create_target(accessor)
+      key, path = PathCache.get(accessor)
+      target = path.inject(@store) {|r, k| r[r.is_a?(Array) ? k.to_i : k] ||= {}}
+      [target, key]
+    end
+  end # class Accessors
+end # module LogStash::Util

data/logstash-core-event.gemspec

@@ -0,0 +1,23 @@
+# -*- encoding: utf-8 -*-
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require 'logstash-core-event/version'
+
+Gem::Specification.new do |gem|
+  gem.authors       = ["Elastic"]
+  gem.email         = ["info@elastic.co"]
+  gem.description   = %q{The core event component of logstash, the scalable log and event management tool}
+  gem.summary       = %q{logstash-core-event - The core event component of logstash}
+  gem.homepage      = "http://www.elastic.co/guide/en/logstash/current/index.html"
+  gem.license       = "Apache License (2.0)"
+
+  gem.files         = Dir.glob(["logstash-core-event.gemspec", "lib/**/*.rb", "spec/**/*.rb"])
+  gem.test_files    = gem.files.grep(%r{^(test|spec|features)/})
+  gem.name          = "logstash-core-event"
+  gem.require_paths = ["lib"]
+  gem.version       = LOGSTASH_CORE_EVENT_VERSION
+
+  if RUBY_PLATFORM == 'java'
+    gem.platform = RUBY_PLATFORM
+  end
+end

data/spec/logstash/event_spec.rb

@@ -0,0 +1,534 @@
+# encoding: utf-8
+require "spec_helper"
+require "json"
+
+describe LogStash::Event do
+
+  shared_examples "all event tests" do
+    context "[]=" do
+      it "should raise an exception if you attempt to set @timestamp to a value type other than a Time object" do
+        expect{subject["@timestamp"] = "crash!"}.to raise_error(TypeError)
+      end
+
+      it "should assign simple fields" do
+        expect(subject["foo"]).to be_nil
+        expect(subject["foo"] = "bar").to eq("bar")
+        expect(subject["foo"]).to eq("bar")
+      end
+
+      it "should overwrite simple fields" do
+        expect(subject["foo"]).to be_nil
+        expect(subject["foo"] = "bar").to eq("bar")
+        expect(subject["foo"]).to eq("bar")
+
+        expect(subject["foo"] = "baz").to eq("baz")
+        expect(subject["foo"]).to eq("baz")
+      end
+
+      it "should assign deep fields" do
+        expect(subject["[foo][bar]"]).to be_nil
+        expect(subject["[foo][bar]"] = "baz").to eq("baz")
+        expect(subject["[foo][bar]"]).to eq("baz")
+      end
+
+      it "should overwrite deep fields" do
+        expect(subject["[foo][bar]"]).to be_nil
+        expect(subject["[foo][bar]"] = "baz").to eq("baz")
+        expect(subject["[foo][bar]"]).to eq("baz")
+
+        expect(subject["[foo][bar]"] = "zab").to eq("zab")
+        expect(subject["[foo][bar]"]).to eq("zab")
+      end
+
+      it "allow to set the @metadata key to a hash" do
+        subject["@metadata"] = { "action" => "index" }
+        expect(subject["[@metadata][action]"]).to eq("index")
+      end
+
+      it "should add key when setting nil value" do
+        subject["[baz]"] = nil
+        expect(subject.to_hash).to include("baz" => nil)
+      end
+    end
+
+    context "#sprintf" do
+      it "should report a unix timestamp for %{+%s}" do
+        expect(subject.sprintf("%{+%s}")).to eq("1356998400")
+      end
+
+      it "should work if there is no fieldref in the string" do
+        expect(subject.sprintf("bonjour")).to eq("bonjour")
+      end
+
+      it "should raise error when formatting %{+%s} when @timestamp field is missing" do
+        str = "hello-%{+%s}"
+        subj = subject.clone
+        subj.remove("[@timestamp]")
+        expect{ subj.sprintf(str) }.to raise_error(LogStash::Error)
+      end
+
+      it "should report a time with %{+format} syntax", :if => RUBY_ENGINE == "jruby" do
+        expect(subject.sprintf("%{+YYYY}")).to eq("2013")
+        expect(subject.sprintf("%{+MM}")).to eq("01")
+        expect(subject.sprintf("%{+HH}")).to eq("00")
+      end
+
+      it "should support mixed string" do
+        expect(subject.sprintf("foo %{+YYYY-MM-dd} %{type}")).to eq("foo 2013-01-01 sprintf")
+      end
+
+      it "should raise error with %{+format} syntax when @timestamp field is missing", :if => RUBY_ENGINE == "jruby" do
+        str = "logstash-%{+YYYY}"
+        subj = subject.clone
+        subj.remove("[@timestamp]")
+        expect{ subj.sprintf(str) }.to raise_error(LogStash::Error)
+      end
+
+      it "should report fields with %{field} syntax" do
+        expect(subject.sprintf("%{type}")).to eq("sprintf")
+        expect(subject.sprintf("%{message}")).to eq(subject["message"])
+      end
+
+      it "should print deep fields" do
+        expect(subject.sprintf("%{[j][k1]}")).to eq("v")
+        expect(subject.sprintf("%{[j][k2][0]}")).to eq("w")
+      end
+
+      it "should be able to take a non-string for the format" do
+        expect(subject.sprintf(2)).to eq("2")
+      end
+
+      it "should allow to use the metadata when calling #sprintf" do
+        expect(subject.sprintf("super-%{[@metadata][fancy]}")).to eq("super-pants")
+      end
+
+      it "should allow to use nested hash from the metadata field" do
+        expect(subject.sprintf("%{[@metadata][have-to-go][deeper]}")).to eq("inception")
+      end
+
+      it "should return a json string if the key is a hash" do
+        expect(subject.sprintf("%{[j][k3]}")).to eq("{\"4\":\"m\"}")
+      end
+
+      it "should not strip last character" do
+        expect(subject.sprintf("%{type}%{message}|")).to eq("sprintfhello world|")
+      end
+
+      context "#encoding" do
+        it "should return known patterns as UTF-8" do
+          expect(subject.sprintf("%{message}").encoding).to eq(Encoding::UTF_8)
+        end
+
+        it "should return unknown patterns as UTF-8" do
+          expect(subject.sprintf("%{unkown_pattern}").encoding).to eq(Encoding::UTF_8)
+        end
+      end
+    end
+
+    context "#[]" do
+      it "should fetch data" do
+        expect(subject["type"]).to eq("sprintf")
+      end
+      it "should fetch fields" do
+        expect(subject["a"]).to eq("b")
+        expect(subject['c']['d']).to eq("f")
+      end
+      it "should fetch deep fields" do
+        expect(subject["[j][k1]"]).to eq("v")
+        expect(subject["[c][d]"]).to eq("f")
+        expect(subject['[f][g][h]']).to eq("i")
+        expect(subject['[j][k3][4]']).to eq("m")
+        expect(subject['[j][5]']).to eq(7)
+
+      end
+
+      it "should be fast?", :performance => true do
+        count = 1000000
+        2.times do
+          start = Time.now
+          count.times { subject["[j][k1]"] }
+          duration = Time.now - start
+          puts "event #[] rate: #{"%02.0f/sec" % (count / duration)}, elapsed: #{duration}s"
+        end
+      end
+    end
+
+    context "#include?" do
+      it "should include existing fields" do
+        expect(subject.include?("c")).to eq(true)
+        expect(subject.include?("[c][d]")).to eq(true)
+        expect(subject.include?("[j][k4][0][nested]")).to eq(true)
+      end
+
+      it "should include field with nil value" do
+        expect(subject.include?("nilfield")).to eq(true)
+      end
+
+      it "should include @metadata field" do
+        expect(subject.include?("@metadata")).to eq(true)
+      end
+
+      it "should include field within @metadata" do
+        expect(subject.include?("[@metadata][fancy]")).to eq(true)
+      end
+
+      it "should not include non-existing fields" do
+        expect(subject.include?("doesnotexist")).to eq(false)
+        expect(subject.include?("[j][doesnotexist]")).to eq(false)
+        expect(subject.include?("[tag][0][hello][yes]")).to eq(false)
+      end
+
+      it "should include within arrays" do
+        expect(subject.include?("[tags][0]")).to eq(true)
+        expect(subject.include?("[tags][1]")).to eq(false)
+      end
+    end
+
+    context "#overwrite" do
+      it "should swap data with new content" do
+        new_event = LogStash::Event.new(
+          "type" => "new",
+          "message" => "foo bar",
+        )
+        subject.overwrite(new_event)
+
+        expect(subject["message"]).to eq("foo bar")
+        expect(subject["type"]).to eq("new")
+
+        ["tags", "source", "a", "c", "f", "j"].each do |field|
+          expect(subject[field]).to be_nil
+        end
+      end
+    end
+
+    context "#append" do
+      it "should append strings to an array" do
+        subject.append(LogStash::Event.new("message" => "another thing"))
+        expect(subject["message"]).to eq([ "hello world", "another thing" ])
+      end
+
+      it "should concatenate tags" do
+        subject.append(LogStash::Event.new("tags" => [ "tag2" ]))
+        # added to_a for when array is a Java Collection when produced from json input
+        # TODO: we have to find a better way to handle this in tests. maybe override
+        # rspec eq or == to do an explicit to_a when comparing arrays?
+        expect(subject["tags"].to_a).to eq([ "tag1", "tag2" ])
+      end
+
+      context "when event field is nil" do
+        it "should add single value as string" do
+          subject.append(LogStash::Event.new({"field1" => "append1"}))
+          expect(subject[ "field1" ]).to eq("append1")
+        end
+        it "should add multi values as array" do
+          subject.append(LogStash::Event.new({"field1" => [ "append1","append2" ]}))
+          expect(subject[ "field1" ]).to eq([ "append1","append2" ])
+        end
+      end
+
+      context "when event field is a string" do
+        before { subject[ "field1" ] = "original1" }
+
+        it "should append string to values, if different from current" do
+          subject.append(LogStash::Event.new({"field1" => "append1"}))
+          expect(subject[ "field1" ]).to eq([ "original1", "append1" ])
+        end
+        it "should not change value, if appended value is equal current" do
+          subject.append(LogStash::Event.new({"field1" => "original1"}))
+          expect(subject[ "field1" ]).to eq("original1")
+        end
+        it "should concatenate values in an array" do
+          subject.append(LogStash::Event.new({"field1" => [ "append1" ]}))
+          expect(subject[ "field1" ]).to eq([ "original1", "append1" ])
+        end
+        it "should join array, removing duplicates" do
+          subject.append(LogStash::Event.new({"field1" => [ "append1","original1" ]}))
+          expect(subject[ "field1" ]).to eq([ "original1", "append1" ])
+        end
+      end
+      context "when event field is an array" do
+        before { subject[ "field1" ] = [ "original1", "original2" ] }
+
+        it "should append string values to array, if not present in array" do
+          subject.append(LogStash::Event.new({"field1" => "append1"}))
+          expect(subject[ "field1" ]).to eq([ "original1", "original2", "append1" ])
+        end
+        it "should not append string values, if the array already contains it" do
+          subject.append(LogStash::Event.new({"field1" => "original1"}))
+          expect(subject[ "field1" ]).to eq([ "original1", "original2" ])
+        end
+        it "should join array, removing duplicates" do
+          subject.append(LogStash::Event.new({"field1" => [ "append1","original1" ]}))
+          expect(subject[ "field1" ]).to eq([ "original1", "original2", "append1" ])
+        end
+      end
+
+    end
+
+    it "timestamp parsing speed", :performance => true do
+      warmup = 10000
+      count = 1000000
+
+      data = { "@timestamp" => "2013-12-21T07:25:06.605Z" }
+      event = LogStash::Event.new(data)
+      expect(event["@timestamp"]).to be_a(LogStash::Timestamp)
+
+      duration = 0
+      [warmup, count].each do |i|
+        start = Time.now
+        i.times do
+          data = { "@timestamp" => "2013-12-21T07:25:06.605Z" }
+          LogStash::Event.new(data.clone)
+        end
+        duration = Time.now - start
+      end
+      puts "event @timestamp parse rate: #{"%02.0f/sec" % (count / duration)}, elapsed: #{duration}s"
+    end
+
+    context "acceptable @timestamp formats" do
+      subject { LogStash::Event.new }
+
+      formats = [
+        "YYYY-MM-dd'T'HH:mm:ss.SSSZ",
+        "YYYY-MM-dd'T'HH:mm:ss.SSSSSSZ",
+        "YYYY-MM-dd'T'HH:mm:ss.SSS",
+        "YYYY-MM-dd'T'HH:mm:ss",
+        "YYYY-MM-dd'T'HH:mm:ssZ",
+      ]
+      formats.each do |format|
+        it "includes #{format}" do
+          time = subject.sprintf("%{+#{format}}")
+          begin
+            LogStash::Event.new("@timestamp" => time)
+          rescue => e
+            raise StandardError, "Time '#{time}' was rejected. #{e.class}: #{e.to_s}"
+          end
+        end
+      end
+
+      context "from LOGSTASH-1738" do
+        it "does not error" do
+          LogStash::Event.new("@timestamp" => "2013-12-29T23:12:52.371240+02:00")
+        end
+      end
+
+      context "from LOGSTASH-1732" do
+        it "does not error" do
+          LogStash::Event.new("@timestamp" => "2013-12-27T11:07:25+00:00")
+        end
+      end
+    end
+
+    context "timestamp initialization" do
+      let(:logger_mock) { double("logger") }
+
+      after(:each) do
+        LogStash::Event.logger = LogStash::Event::DEFAULT_LOGGER
+      end
+
+      it "should coerce timestamp" do
+        t = Time.iso8601("2014-06-12T00:12:17.114Z")
+        expect(LogStash::Event.new("@timestamp" => t).timestamp.to_i).to eq(t.to_i)
+        expect(LogStash::Event.new("@timestamp" => LogStash::Timestamp.new(t)).timestamp.to_i).to eq(t.to_i)
+        expect(LogStash::Event.new("@timestamp" => "2014-06-12T00:12:17.114Z").timestamp.to_i).to eq(t.to_i)
+      end
+
+      it "should assign current time when no timestamp" do
+        expect(LogStash::Event.new({}).timestamp.to_i).to be_within(1).of (Time.now.to_i)
+      end
+
+      it "should tag for invalid value" do
+        event = LogStash::Event.new("@timestamp" => "foo")
+        expect(event.timestamp.to_i).to be_within(1).of Time.now.to_i
+        expect(event["tags"]).to eq([LogStash::Event::TIMESTAMP_FAILURE_TAG])
+        expect(event[LogStash::Event::TIMESTAMP_FAILURE_FIELD]).to eq("foo")
+
+        event = LogStash::Event.new("@timestamp" => 666)
+        expect(event.timestamp.to_i).to be_within(1).of Time.now.to_i
+        expect(event["tags"]).to eq([LogStash::Event::TIMESTAMP_FAILURE_TAG])
+        expect(event[LogStash::Event::TIMESTAMP_FAILURE_FIELD]).to eq(666)
+      end
+
+      it "should warn for invalid value" do
+        LogStash::Event.logger = logger_mock
+
+        expect(logger_mock).to receive(:warn).twice
+
+        LogStash::Event.new("@timestamp" => :foo)
+        LogStash::Event.new("@timestamp" => 666)
+      end
+
+      it "should tag for invalid string format" do
+        event = LogStash::Event.new("@timestamp" => "foo")
+        expect(event.timestamp.to_i).to be_within(1).of Time.now.to_i
+        expect(event["tags"]).to eq([LogStash::Event::TIMESTAMP_FAILURE_TAG])
+        expect(event[LogStash::Event::TIMESTAMP_FAILURE_FIELD]).to eq("foo")
+      end
+
+      it "should warn for invalid string format" do
+        LogStash::Event.logger = logger_mock
+
+        expect(logger_mock).to receive(:warn)
+        LogStash::Event.new("@timestamp" => "foo")
+      end
+    end
+
+    context "to_json" do
+      it "should support to_json" do
+        new_event = LogStash::Event.new(
+          "@timestamp" => Time.iso8601("2014-09-23T19:26:15.832Z"),
+          "message" => "foo bar",
+        )
+        json = new_event.to_json
+
+        expect(JSON.parse(json)).to eq( JSON.parse("{\"@timestamp\":\"2014-09-23T19:26:15.832Z\",\"message\":\"foo bar\",\"@version\":\"1\"}"))
+      end
+
+      it "should support to_json and ignore arguments" do
+        new_event = LogStash::Event.new(
+          "@timestamp" => Time.iso8601("2014-09-23T19:26:15.832Z"),
+          "message" => "foo bar",
+        )
+        json = new_event.to_json(:foo => 1, :bar => "baz")
+
+        expect(JSON.parse(json)).to eq( JSON.parse("{\"@timestamp\":\"2014-09-23T19:26:15.832Z\",\"message\":\"foo bar\",\"@version\":\"1\"}"))
+      end
+    end
+
+    context "metadata" do
+      context "with existing metadata" do
+        subject { LogStash::Event.new("hello" => "world", "@metadata" => { "fancy" => "pants" }) }
+
+        it "should not include metadata in to_hash" do
+          expect(subject.to_hash.keys).not_to include("@metadata")
+
+          # 'hello', '@timestamp', and '@version'
+          expect(subject.to_hash.keys.count).to eq(3)
+        end
+
+        it "should still allow normal field access" do
+          expect(subject["hello"]).to eq("world")
+        end
+      end
+
+      context "with set metadata" do
+        let(:fieldref) { "[@metadata][foo][bar]" }
+        let(:value) { "bar" }
+        subject { LogStash::Event.new("normal" => "normal") }
+        before do
+          # Verify the test is configured correctly.
+          expect(fieldref).to start_with("[@metadata]")
+
+          # Set it.
+          subject[fieldref] = value
+        end
+
+        it "should still allow normal field access" do
+          expect(subject["normal"]).to eq("normal")
+        end
+
+        it "should allow getting" do
+          expect(subject[fieldref]).to eq(value)
+        end
+
+        it "should be hidden from .to_json" do
+          require "json"
+          obj = JSON.parse(subject.to_json)
+          expect(obj).not_to include("@metadata")
+        end
+
+        it "should be hidden from .to_hash" do
+          expect(subject.to_hash).not_to include("@metadata")
+        end
+
+        it "should be accessible through #to_hash_with_metadata" do
+          obj = subject.to_hash_with_metadata
+          expect(obj).to include("@metadata")
+          expect(obj["@metadata"]["foo"]["bar"]).to eq(value)
+        end
+      end
+
+      context "with no metadata" do
+        subject { LogStash::Event.new("foo" => "bar") }
+        it "should have no metadata" do
+          expect(subject["@metadata"]).to be_empty
+        end
+        it "should still allow normal field access" do
+          expect(subject["foo"]).to eq("bar")
+        end
+
+        it "should not include the @metadata key" do
+          expect(subject.to_hash_with_metadata).not_to include("@metadata")
+        end
+      end
+    end
+
+    context "signal events" do
+      it "should define the shutdown event" do
+        # the SHUTDOWN and FLUSH constants are part of the plugin API contract
+        # if they are changed, all plugins must be updated
+        expect(LogStash::SHUTDOWN).to be_a(LogStash::ShutdownEvent)
+        expect(LogStash::FLUSH).to be_a(LogStash::FlushEvent)
+      end
+    end
+  end
+
+  let(:event_hash) do
+    {
+      "@timestamp" => "2013-01-01T00:00:00.000Z",
+      "type" => "sprintf",
+      "message" => "hello world",
+      "tags" => [ "tag1" ],
+      "source" => "/home/foo",
+      "a" => "b",
+      "c" => {
+        "d" => "f",
+        "e" => {"f" => "g"}
+      },
+      "f" => { "g" => { "h" => "i" } },
+      "j" => {
+          "k1" => "v",
+          "k2" => [ "w", "x" ],
+          "k3" => {"4" => "m"},
+          "k4" => [ {"nested" => "cool"} ],
+          5 => 6,
+          "5" => 7
+      },
+      "nilfield" => nil,
+      "@metadata" => { "fancy" => "pants", "have-to-go" => { "deeper" => "inception" } }
+    }
+  end
+
+  describe "using normal hash input" do
+    it_behaves_like "all event tests" do
+      subject{LogStash::Event.new(event_hash)}
+    end
+  end
+
+  describe "using hash input from deserialized json" do
+    # this is to test the case when JrJackson deserialises Json and produces
+    # native Java Collections objects for efficiency
+    it_behaves_like "all event tests" do
+      subject{LogStash::Event.new(LogStash::Json.load(LogStash::Json.dump(event_hash)))}
+    end
+  end
+
+
+  describe "#to_s" do
+    let(:timestamp) { LogStash::Timestamp.new }
+    let(:event1) { LogStash::Event.new({ "@timestamp" => timestamp, "host" => "foo", "message" => "bar"}) }
+    let(:event2) { LogStash::Event.new({ "host" => "bar", "message" => "foo"}) }
+
+    it "should cache only one template" do
+      LogStash::StringInterpolation.clear_cache
+      expect {
+        event1.to_s
+        event2.to_s
+      }.to change { LogStash::StringInterpolation.cache_size }.by(1)
+    end
+
+    it "return the string containing the timestamp, the host and the message" do
+      expect(event1.to_s).to eq("#{timestamp.to_iso8601} #{event1["host"]} #{event1["message"]}")
+    end
+  end
+end