logstash-codec-netflow 2.0.5 → 2.1.0

data/lib/logstash/codecs/netflow/netflow.yaml

@@ -164,10 +164,10 @@
 - :dst_tos
 56:
 - :mac_addr
-- :in_src_max
+- :in_src_mac
 57:
 - :mac_addr
-- :out_dst_max
+- :out_dst_mac
 58:
 - :uint16
 - :src_vlan
@@ -213,3 +213,87 @@
 83:
 - :string
 - :if_desc
+84:
+- :string
+- :sampler_name
+85:
+- :uint32
+- :in_permanent_bytes
+86:
+- :uint32
+- :in_permanent_pkts
+148:
+- :uint32
+- :conn_id
+176:
+- :uint8
+- :icmp_type
+177:
+- :uint8
+- :icmp_code
+178:
+- :uint8
+- :icmp_type_ipv6
+179:
+- :uint8
+- :icmp_code_ipv6
+225:
+- :ip4_addr
+- :xlate_src_addr_ipv4
+226:
+- :ip4_addr
+- :xlate_dst_addr_ipv4
+227:
+- :uint16
+- :xlate_src_port
+228:
+- :uint16
+- :xlate_dst_port
+233:
+- :uint8
+- :fw_event
+281:
+- :ip6_addr
+- :xlate_src_addr_ipv6
+282:
+- :ip6_addr
+- :xlate_dst_addr_ipv6
+33002:
+- :uint16
+- :fw_ext_event
+323:
+- 8
+- :event_time_msec
+152:
+- 8
+- :flow_start_msec
+231:
+- :uint32 
+- :fwd_flow_delta_bytes
+232:
+- :uint32
+- :rev_flow_delta_bytes
+33000:
+- :acl_id_asa
+- :ingress_acl_id
+33001:
+- :acl_id_asa
+- egress_acl_id
+40000:
+- :string
+- :username
+40001:
+- :ip4_addr
+- :xlate_src_addr_ipv4
+40002:
+- :ip4_addr
+- :xlate_dst_addr_ipv4
+40003:
+- :uint16
+- :xlate_src_port
+40004:
+- :uint16
+- :xlate_dst_port
+40005:
+- :uint8
+- :fw_event

data/lib/logstash/codecs/netflow/util.rb

@@ -47,7 +47,20 @@ class MacAddr < BinData::Primitive
   end
 
   def get
-    self.bytes.collect { |byte| byte.to_s(16) }.join(":")
+    self.bytes.collect { |byte| byte.value.to_s(16).rjust(2,'0') }.join(":")
+  end
+end
+
+class ACLIdASA < BinData::Primitive
+  array :bytes, :type => :uint8, :initial_length => 12
+
+  def set(val)
+    self.bytes = val.split("-").collect { |aclid| aclid.scan(/../).collect { |hex| hex.to_i(16)} }.flatten
+  end
+
+  def get
+    hexstring = self.bytes.collect { |byte| byte.value.to_s(16).rjust(2,'0') }.join
+    hexstring.scan(/......../).collect { |aclid| aclid }.join("-")
   end
 end
 
@@ -59,7 +72,7 @@ end
 class Netflow5PDU < BinData::Record
   endian :big
   uint16 :version
-  uint16 :flow_records
+  uint16 :flow_records, :assert => lambda { flow_records.value.between?(1,30) }
   uint32 :uptime
   uint32 :unix_sec
   uint32 :unix_nsec
@@ -68,7 +81,7 @@ class Netflow5PDU < BinData::Record
   uint8  :engine_id
   bit2   :sampling_algorithm
   bit14  :sampling_interval
-  array  :records, :initial_length => :flow_records do
+  array :records, :initial_length => :flow_records do
     ip4_addr :ipv4_src_addr
     ip4_addr :ipv4_dst_addr
     ip4_addr :ipv4_next_hop
@@ -92,7 +105,7 @@ class Netflow5PDU < BinData::Record
   end
 end
 
-class TemplateFlowset < BinData::Record
+class NetflowTemplateFlowset < BinData::Record
   endian :big
   array  :templates, :read_until => lambda { array.num_bytes == flowset_length - 4 } do
     uint16 :template_id
@@ -104,7 +117,7 @@ class TemplateFlowset < BinData::Record
   end
 end
 
-class OptionFlowset < BinData::Record
+class NetflowOptionFlowset < BinData::Record
   endian :big
   array  :templates, :read_until => lambda { flowset_length - 4 - array.num_bytes <= 2 } do
     uint16 :template_id
@@ -131,12 +144,66 @@ class Netflow9PDU < BinData::Record
   uint32 :flow_seq_num
   uint32 :source_id
   array  :records, :read_until => :eof do
-    uint16 :flowset_id
-    uint16 :flowset_length
+    uint16 :flowset_id, :assert => lambda { [0, 1, *(256..65535)].include?(flowset_id) }
+    uint16 :flowset_length, :assert => lambda { flowset_length > 4 }
+    choice :flowset_data, :selection => :flowset_id do
+      netflow_template_flowset 0
+      netflow_option_flowset   1
+      string                   :default, :read_length => lambda { flowset_length - 4 }
+    end
+  end
+end
+
+class IpfixTemplateFlowset < BinData::Record
+  endian :big
+  array  :templates, :read_until => lambda { flowset_length - 4 - array.num_bytes <= 2 } do
+    uint16 :template_id
+    uint16 :field_count
+    array  :fields, :initial_length => :field_count do
+      bit1   :enterprise
+      bit15  :field_type
+      uint16 :field_length
+      uint32 :enterprise_id, :onlyif => lambda { enterprise != 0 }
+    end
+  end
+  # skip :length => lambda { flowset_length - 4 - set.num_bytes } ?
+end
+
+class IpfixOptionFlowset < BinData::Record
+  endian :big
+  array  :templates, :read_until => lambda { flowset_length - 4 - array.num_bytes <= 2 } do
+    uint16 :template_id
+    uint16 :field_count
+    uint16 :scope_count, :assert => lambda { scope_count > 0 }
+    array  :scope_fields, :initial_length => lambda { scope_count } do
+      bit1   :enterprise
+      bit15  :field_type
+      uint16 :field_length
+      uint32 :enterprise_id, :onlyif => lambda { enterprise != 0 }
+    end
+    array  :option_fields, :initial_length => lambda { field_count - scope_count } do
+      bit1   :enterprise
+      bit15  :field_type
+      uint16 :field_length
+      uint32 :enterprise_id, :onlyif => lambda { enterprise != 0 }
+    end
+  end
+end
+
+class IpfixPDU < BinData::Record
+  endian :big
+  uint16 :version
+  uint16 :pdu_length
+  uint32 :unix_sec
+  uint32 :flow_seq_num
+  uint32 :observation_domain_id
+  array  :records, :read_until => lambda { array.num_bytes == pdu_length - 16 } do
+    uint16 :flowset_id, :assert => lambda { [2, 3, *(256..65535)].include?(flowset_id) }
+    uint16 :flowset_length, :assert => lambda { flowset_length > 4 }
     choice :flowset_data, :selection => :flowset_id do
-      template_flowset 0
-      option_flowset   1
-      string           :default, :read_length => lambda { flowset_length - 4 }
+      ipfix_template_flowset 2
+      ipfix_option_flowset   3
+      string                 :default, :read_length => lambda { flowset_length - 4 }
     end
   end
 end

data/logstash-codec-netflow.gemspec

@@ -1,10 +1,10 @@
 Gem::Specification.new do |s|
 
   s.name            = 'logstash-codec-netflow'
-  s.version         = '2.0.5'
+  s.version         = '2.1.0'
   s.licenses        = ['Apache License (2.0)']
-  s.summary         = "The netflow codec is for decoding Netflow v5/v9 flows."
-  s.description     = "This gem is a logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/plugin install gemname. This gem is not a stand-alone program"
+  s.summary         = "The netflow codec is for decoding Netflow v5/v9/IPFIX flows."
+  s.description     = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program"
   s.authors         = ["Elastic"]
   s.email           = 'info@elastic.co'
   s.homepage        = "http://www.elastic.co/guide/en/logstash/current/index.html"
@@ -25,3 +25,4 @@ Gem::Specification.new do |s|
   s.add_development_dependency 'logstash-devutils'
 end
 
+

data/spec/codecs/netflow_spec.rb

@@ -13,16 +13,19 @@ describe LogStash::Codecs::Netflow do
 
   let(:decode) do
     [].tap do |events|
-      subject.decode(data){|event| events << event}
+      data.each { |packet| subject.decode(packet){|event| events << event}}
     end
   end
 
-  context "Netflow 5" do
+  ### NETFLOW v5
+ 
+  context "Netflow 5 valid 01" do
     let(:data) do
       # this netflow raw data was produced with softflowd and captured with netcat
       # softflowd -D -i eth0 -v 5 -t maxlife=1 -n 127.0.01:8765
       # nc -k -4 -u -l 127.0.0.1 8765 > netflow5.dat
-      IO.read(File.join(File.dirname(__FILE__), "netflow5.dat"), :mode => "rb")
+      data = []
+      data << IO.read(File.join(File.dirname(__FILE__), "netflow5.dat"), :mode => "rb")
     end
 
     let(:json_events) do
@@ -123,61 +126,162 @@ describe LogStash::Codecs::Netflow do
     end
   end
 
-  context "Netflow 9" do
+  context "Netflow 5 invalid 01 " do
+    let(:data) do
+      data = []
+      data << IO.read(File.join(File.dirname(__FILE__), "netflow5_test_invalid01.dat"), :mode => "rb")
+    end
+
+    it "should not raise_error " do
+      expect{decode.size}.not_to raise_error
+    end
+  end
+
+  context "Netflow 5 invalid 02 " do
+    let(:data) do
+      data = []
+      data << IO.read(File.join(File.dirname(__FILE__), "netflow5_test_invalid02.dat"), :mode => "rb")
+    end
+
+    it "should not raise_error" do
+      expect{decode.size}.not_to raise_error
+    end
+  end
+
+  ### NETFLOW v9
+
+  context "Netflow 9 valid 01" do
     let(:data) do
       # this netflow raw data was produced with softflowd and captured with netcat
-      # softflowd -D -i eth0 -v 9 -t maxlife=1 -n 127.0.01:8765
-      # nc -k -4 -u -l 127.0.0.1 8765 > netflow9.dat
-      IO.read(File.join(File.dirname(__FILE__), "netflow9.dat"), :mode => "rb")
+      # softflowd -v 9 -n 172.16.32.202:2055
+      # nc -4 -u -l 172.16.32.202 8765 > netflow9_test_valid01.dat
+      data = []
+      data << IO.read(File.join(File.dirname(__FILE__), "netflow9_test_valid01.dat"), :mode => "rb")
     end
 
     let(:json_events) do
       events = []
       events << <<-END
         {
-          "@timestamp": "2015-05-02T22:10:07.000Z",
+          "@timestamp": "2015-10-08T19:04:30.000Z",
           "netflow": {
             "version": 9,
-            "flow_seq_num": 0,
-            "flowset_id": 1024,
-            "ipv4_src_addr": "10.0.2.2",
-            "ipv4_dst_addr": "10.0.2.15",
-            "last_switched": "2015-05-02T22:10:07.999Z",
-            "first_switched": "2015-06-21T15:12:49.999Z",
-            "in_bytes": 230,
-            "in_pkts": 5,
-            "input_snmp": 0,
-            "output_snmp": 0,
-            "l4_src_port": 57369,
-            "l4_dst_port": 22,
-            "protocol": 6,
-            "tcp_flags": 16,
-            "ip_protocol_version": 4
+            "flow_seq_num":1,
+            "flowset_id":1024,
+            "ipv4_src_addr": "172.16.32.100",
+            "ipv4_dst_addr":"172.16.32.248",
+            "last_switched":"2015-10-08T19:03:47.999Z",
+            "first_switched":"2015-10-08T19:03:47.999Z",
+            "in_bytes":76,
+            "in_pkts":1,
+            "input_snmp":0,
+            "output_snmp":0,
+            "l4_src_port":123,
+            "l4_dst_port":123,
+            "protocol":17, 
+            "tcp_flags":0,
+            "ip_protocol_version":4,
+            "src_tos":0
           },
           "@version": "1"
         }
       END
 
+      events.map{|event| event.gsub(/\s+/, "")}
+
+    end
+
+    it "should decode raw data" do
+      expect(decode.size).to eq(7)
+      expect(decode[0]["[netflow][version]"]).to eq(9)
+    end
+
+    it "should serialize to json" do
+      # generated json order can change with different implementation, convert back to hash to compare.
+      expect(JSON.parse(decode[0].to_json)).to eq(JSON.parse(json_events[0]))
+    end
+
+  end
+
+  context "Netflow 9 macaddress" do
+    let(:data) do
+      data = []
+      data << IO.read(File.join(File.dirname(__FILE__), "netflow9_test_macaddr_tpl.dat"), :mode => "rb")
+      data << IO.read(File.join(File.dirname(__FILE__), "netflow9_test_macaddr_data.dat"), :mode => "rb")
+    end
+
+    let(:json_events) do
+      events = []
+      events << <<-END
+        {
+          "@timestamp":"2015-10-10T08:47:01.000Z",
+          "netflow":{
+            "version":9,
+            "flow_seq_num":2,
+            "flowset_id":257,
+            "protocol":6,
+            "l4_src_port":65058,
+            "ipv4_src_addr":"172.16.32.1",
+            "l4_dst_port":22,
+            "ipv4_dst_addr":"172.16.32.201",
+            "in_src_mac":"00:50:56:c0:00:01",
+            "in_dst_mac":"00:0c:29:70:86:09"
+          },
+          "@version":"1"
+        }
+      END
+
+      events.map{|event| event.gsub(/\s+/, "")}
+    end
+
+    it "should decode the mac address" do
+      expect(decode[1]["[netflow][in_src_mac]"]).to eq("00:50:56:c0:00:01")
+      expect(decode[1]["[netflow][in_dst_mac]"]).to eq("00:0c:29:70:86:09")
+    end
+
+    it "should serialize to json" do
+      expect(JSON.parse(decode[1].to_json)).to eq(JSON.parse(json_events[0]))
+    end
+  end
+
+  context "Netflow 9 Cisco ASA" do
+    let(:data) do
+      packets = []
+      packets << IO.read(File.join(File.dirname(__FILE__), "netflow9_test_valid_cisco_asa_tpl.dat"), :mode => "rb")
+      packets << IO.read(File.join(File.dirname(__FILE__), "netflow9_test_valid_cisco_asa_data.dat"), :mode => "rb")
+    end
+
+    let(:json_events) do
+      events = []
       events << <<-END
         {
-          "@timestamp": "2015-05-02T22:10:07.000Z",
+          "@timestamp": "2015-10-09T09:47:51.000Z",
           "netflow": {
             "version": 9,
-            "flow_seq_num": 0,
-            "flowset_id": 1024,
-            "ipv4_src_addr": "10.0.2.15",
-            "ipv4_dst_addr": "10.0.2.2",
-            "last_switched": "2015-05-02T22:10:07.999Z",
-            "first_switched": "2015-06-21T15:12:49.999Z",
-            "in_bytes": 352,
-            "in_pkts": 4,
-            "input_snmp": 0,
-            "output_snmp": 0,
-            "l4_src_port": 22,
-            "l4_dst_port": 57369,
-            "protocol": 6,
-            "tcp_flags": 24,
-            "ip_protocol_version": 4
+            "flow_seq_num": 662,
+            "flowset_id": 265,
+            "conn_id": 8501,
+            "ipv4_src_addr": "192.168.23.22",
+            "l4_src_port": 17549,
+            "input_snmp": 2,
+            "ipv4_dst_addr": "164.164.37.11",
+            "l4_dst_port": 0,
+            "output_snmp": 3,
+            "protocol": 1,
+            "icmp_type": 8,
+            "icmp_code": 0,
+            "xlate_src_addr_ipv4": "192.168.23.22",
+            "xlate_dst_addr_ipv4": "164.164.37.11",
+            "xlate_src_port": 17549,
+            "xlate_dst_port": 0,
+            "fw_event": 2,
+            "fw_ext_event": 2025,
+            "event_time_msec": 1444384070179,
+            "in_permanent_bytes": 56,
+            "flow_start_msec": 1444384068169,
+            "ingress_acl_id": "0f8e7ff3-fc1a030f-00000000",
+            "egress_acl_id": "00000000-00000000-00000000",
+            "username": ""
           },
           "@version": "1"
         }
@@ -187,27 +291,394 @@ describe LogStash::Codecs::Netflow do
     end
 
     it "should decode raw data" do
-      expect(decode.size).to eq(2)
+      expect(decode.size).to eq(14)
+      expect(decode[1]["[netflow][version]"]).to eq(9)
+    end
 
-      expect(decode[0]["[netflow][version]"]).to eq(9)
-      expect(decode[0]["[netflow][ipv4_src_addr]"]).to eq("10.0.2.2")
-      expect(decode[0]["[netflow][ipv4_dst_addr]"]).to eq("10.0.2.15")
-      expect(decode[0]["[netflow][l4_src_port]"]).to eq(57369)
-      expect(decode[0]["[netflow][l4_dst_port]"]).to eq(22)
-      expect(decode[0]["[netflow][tcp_flags]"]).to eq(16)
+    it "should serialize to json" do
+      expect(JSON.parse(decode[1].to_json)).to eq(JSON.parse(json_events[0]))
+    end
+  end
 
-      expect(decode[1]["[netflow][version]"]).to eq(9)
-      expect(decode[1]["[netflow][ipv4_src_addr]"]).to eq("10.0.2.15")
-      expect(decode[1]["[netflow][ipv4_dst_addr]"]).to eq("10.0.2.2")
-      expect(decode[1]["[netflow][l4_src_port]"]).to eq(22)
-      expect(decode[1]["[netflow][l4_dst_port]"]).to eq(57369)
-      expect(decode[1]["[netflow][tcp_flags]"]).to eq(24)
+  context "Netflow 9 multple netflow exporters" do
+    let(:data) do
+      # This tests whether a template from a 2nd netflow exporter overwrites the template sent from the first.
+      # In this test the 3rd packet (from nprobe) should still decode succesfully.
+      # Note that in this case the SourceID from exporter 1 is different from exporter 2, otherwise we hit issue #9
+      data = []
+      data << IO.read(File.join(File.dirname(__FILE__), "netflow9_test_nprobe_tpl.dat"), :mode => "rb")
+      data << IO.read(File.join(File.dirname(__FILE__), "netflow9_test_softflowd_tpl_data.dat"), :mode => "rb")
+      data << IO.read(File.join(File.dirname(__FILE__), "netflow9_test_nprobe_data.dat"), :mode => "rb")
+    end
+
+    let(:json_events) do
+      events = []
+      events << <<-END
+        {
+          "@timestamp": "2015-10-08T19:04:30.000Z",
+          "netflow": {
+            "version":9,
+            "flow_seq_num":1,
+            "flowset_id":1024,
+            "ipv4_src_addr":"172.16.32.100",
+            "ipv4_dst_addr":"172.16.32.248",
+            "last_switched":"2015-10-08T19:03:47.999Z",
+            "first_switched":"2015-10-08T19:03:47.999Z",
+            "in_bytes":76,
+            "in_pkts":1,
+            "input_snmp":0,
+            "output_snmp":0,
+            "l4_src_port":123,
+            "l4_dst_port":123,
+            "protocol":17,
+            "tcp_flags":0,
+            "ip_protocol_version":4,
+            "src_tos":0
+          },
+          "@version":"1"
+        }
+      END
+
+      events << <<-END
+        {
+          "@timestamp":"2015-10-08T19:06:29.000Z",
+          "netflow": {
+            "version":9,
+            "flow_seq_num":1,
+            "flowset_id":257,
+            "in_bytes":200,
+            "in_pkts":2,
+            "protocol":6,
+            "src_tos":16,
+            "tcp_flags":24,
+            "l4_src_port":22,
+            "ipv4_src_addr":"172.16.32.201",
+            "src_mask":0,
+            "input_snmp":0,
+            "l4_dst_port":65058,
+            "ipv4_dst_addr":"172.16.32.1",
+            "dst_mask":0,
+            "output_snmp":0,
+            "ipv4_next_hop":"0.0.0.0",
+            "src_as":0,
+            "dst_as":0,
+            "last_switched":"2015-10-08T19:05:56.999Z",
+            "first_switched":"2015-10-08T19:05:56.999Z"
+          },
+          "@version":"1"
+        }
+      END
+
+      events.map{|event| event.gsub(/\s+/, "")}
+    end
+
+    # These tests will start to fail whenever options template decoding is added.
+    # Nprobe includes options templates, which this test included a sample from.
+    # Currently it is not decoded, but if it is, decode.size will be 9, and 
+    # the packet currently identified with decode[7] will be decode[8]
+
+    it "should decode raw data" do
+      expect(decode.size).to eq(9)
+      expect(decode[1]["[netflow][l4_src_port]"]).to eq(123)
+      expect(decode[8]["[netflow][l4_src_port]"]).to eq(22)
+    end
+
+    it "should serialize to json" do
+      expect(JSON.parse(decode[1].to_json)).to eq(JSON.parse(json_events[0]))
+      expect(JSON.parse(decode[8].to_json)).to eq(JSON.parse(json_events[1]))
+    end
+  end
+
+  context "Netflow 9 invalid 01 " do
+    let(:data) do
+      data = []
+      data << IO.read(File.join(File.dirname(__FILE__), "netflow9_test_invalid01.dat"), :mode => "rb")
+    end
+
+    it "should not raise_error" do
+      expect{decode.size}.not_to raise_error
+    end
+  end
+
+  context "Netflow 9 options template with scope fields" do
+    let(:data) do
+      data = []
+      data << IO.read(File.join(File.dirname(__FILE__), "netflow9_test_nprobe_tpl.dat"), :mode => "rb")
+    end
+
+    let(:json_events) do
+      events = []
+      events << <<-END
+        {
+          "@timestamp":"2015-10-08T19:06:29.000Z",
+          "netflow": {
+              "version":9,
+              "flow_seq_num":0,
+              "flowset_id":259,
+              "scope_system":0,
+              "total_flows_exp":1,
+              "total_pkts_exp":0
+           },
+           "@version":"1"
+         }
+      END
+
+      events.map{|event| event.gsub(/\s+/, "")}
+    end
+
+    it "should serialize to json" do
+      expect(JSON.parse(decode[0].to_json)).to eq(JSON.parse(json_events[0]))
+    end
+
+    it "should decode raw data" do
+      expect(decode[0]["[netflow][scope_system]"]).to eq(0)
+      expect(decode[0]["[netflow][total_flows_exp]"]).to eq(1)
+    end
+  end
+
+  context "IPFIX" do
+    let(:data) do
+      # this netflow raw data was produced with softflowd and captured with netcat
+      # softflowd -D -i eth0 -v 10 -t maxlife=1 -n 127.0.01:8765
+      # nc -k -4 -u -l 127.0.0.1 8765 > ipfix.dat
+      data = []
+      data << IO.read(File.join(File.dirname(__FILE__), "ipfix.dat"), :mode => "rb")
+    end
+
+    let(:json_events) do
+      events = []
+      events << <<-END
+        {
+          "@timestamp": "2015-05-13T11:20:26.000Z",
+          "netflow": {
+            "version": 10,
+            "meteringProcessId": 2679,
+            "systemInitTimeMilliseconds": 1431516013506,
+            "selectorAlgorithm": 1,
+            "samplingPacketInterval": 1,
+            "samplingPacketSpace": 0
+          },
+          "@version": "1"
+        }
+      END
+
+      events << <<-END
+        {
+          "@timestamp": "2015-05-13T11:20:26.000Z",
+          "netflow": {
+            "version": 10,
+            "sourceIPv4Address": "192.168.253.1",
+            "destinationIPv4Address": "192.168.253.128",
+            "octetDeltaCount": 260,
+            "packetDeltaCount": 5,
+            "ingressInterface": 0,
+            "egressInterface": 0,
+            "sourceTransportPort": 60560,
+            "destinationTransportPort": 22,
+            "protocolIdentifier": 6,
+            "tcpControlBits": 16,
+            "ipVersion": 4,
+            "ipClassOfService": 0,
+            "icmpTypeCodeIPv4": 0,
+            "vlanId": 0,
+            "flowStartSysUpTime": 0,
+            "flowEndSysUpTime": 12726
+          },
+          "@version": "1"
+        }
+      END
+
+      events << <<-END
+        {
+          "@timestamp": "2015-05-13T11:20:26.000Z",
+          "netflow": {
+            "version": 10,
+            "sourceIPv4Address": "192.168.253.128",
+            "destinationIPv4Address": "192.168.253.1",
+            "octetDeltaCount": 1000,
+            "packetDeltaCount": 6,
+            "ingressInterface": 0,
+            "egressInterface": 0,
+            "sourceTransportPort": 22,
+            "destinationTransportPort": 60560,
+            "protocolIdentifier": 6,
+            "tcpControlBits": 24,
+            "ipVersion": 4,
+            "ipClassOfService": 0,
+            "icmpTypeCodeIPv4": 0,
+            "vlanId": 0,
+            "flowStartSysUpTime": 0,
+            "flowEndSysUpTime": 12726
+          },
+          "@version": "1"
+        }
+      END
+
+      events << <<-END
+        {
+          "@timestamp": "2015-05-13T11:20:26.000Z",
+          "netflow": {
+            "version": 10,
+            "sourceIPv4Address": "192.168.253.2",
+            "destinationIPv4Address": "192.168.253.132",
+            "octetDeltaCount": 601,
+            "packetDeltaCount": 2,
+            "ingressInterface": 0,
+            "egressInterface": 0,
+            "sourceTransportPort": 53,
+            "destinationTransportPort": 35262,
+            "protocolIdentifier": 17,
+            "tcpControlBits": 0,
+            "ipVersion": 4,
+            "ipClassOfService": 0,
+            "icmpTypeCodeIPv4": 0,
+            "vlanId": 0,
+            "flowStartSysUpTime": 1104,
+            "flowEndSysUpTime": 1142
+          },
+          "@version": "1"
+        }
+      END
+
+      events << <<-END
+        {
+          "@timestamp": "2015-05-13T11:20:26.000Z",
+          "netflow": {
+            "version": 10,
+            "sourceIPv4Address": "192.168.253.132",
+            "destinationIPv4Address": "192.168.253.2",
+            "octetDeltaCount": 148,
+            "packetDeltaCount": 2,
+            "ingressInterface": 0,
+            "egressInterface": 0,
+            "sourceTransportPort": 35262,
+            "destinationTransportPort": 53,
+            "protocolIdentifier": 17,
+            "tcpControlBits": 0,
+            "ipVersion": 4,
+            "ipClassOfService": 0,
+            "icmpTypeCodeIPv4": 0,
+            "vlanId": 0,
+            "flowStartSysUpTime": 1104,
+            "flowEndSysUpTime": 1142
+          },
+          "@version": "1"
+        }
+      END
+
+      events << <<-END
+        {
+          "@timestamp": "2015-05-13T11:20:26.000Z",
+          "netflow": {
+            "version": 10,
+            "sourceIPv4Address": "54.214.9.161",
+            "destinationIPv4Address": "192.168.253.132",
+            "octetDeltaCount": 5946,
+            "packetDeltaCount": 14,
+            "ingressInterface": 0,
+            "egressInterface": 0,
+            "sourceTransportPort": 443,
+            "destinationTransportPort": 49935,
+            "protocolIdentifier": 6,
+            "tcpControlBits": 26,
+            "ipVersion": 4,
+            "ipClassOfService": 0,
+            "icmpTypeCodeIPv4": 0,
+            "vlanId": 0,
+            "flowStartSysUpTime": 1142,
+            "flowEndSysUpTime": 2392
+          },
+          "@version": "1"
+        }
+      END
+
+      events << <<-END
+        {
+          "@timestamp": "2015-05-13T11:20:26.000Z",
+          "netflow": {
+            "version": 10,
+            "sourceIPv4Address": "192.168.253.132",
+            "destinationIPv4Address": "54.214.9.161",
+            "octetDeltaCount": 2608,
+            "packetDeltaCount": 13,
+            "ingressInterface": 0,
+            "egressInterface": 0,
+            "sourceTransportPort": 49935,
+            "destinationTransportPort": 443,
+            "protocolIdentifier": 6,
+            "tcpControlBits": 26,
+            "ipVersion": 4,
+            "ipClassOfService": 0,
+            "icmpTypeCodeIPv4": 0,
+            "vlanId": 0,
+            "flowStartSysUpTime": 1142,
+            "flowEndSysUpTime": 2392
+          },
+          "@version": "1"
+        }
+      END
+
+      events.map{|event| event.gsub(/\s+/, "")}
+    end
+
+    it "should decode raw data" do
+      expect(decode.size).to eq(7)
+
+      expect(decode[0]["[netflow][version]"]).to eq(10)
+      expect(decode[0]["[netflow][systemInitTimeMilliseconds]"]).to eq(1431516013506)
+
+      expect(decode[1]["[netflow][version]"]).to eq(10)
+      expect(decode[1]["[netflow][sourceIPv4Address]"]).to eq("192.168.253.1")
+      expect(decode[1]["[netflow][destinationIPv4Address]"]).to eq("192.168.253.128")
+      expect(decode[1]["[netflow][sourceTransportPort]"]).to eq(60560)
+      expect(decode[1]["[netflow][destinationTransportPort]"]).to eq(22)
+      expect(decode[1]["[netflow][protocolIdentifier]"]).to eq(6)
+      expect(decode[1]["[netflow][tcpControlBits]"]).to eq(16)
+
+      expect(decode[2]["[netflow][version]"]).to eq(10)
+      expect(decode[2]["[netflow][sourceIPv4Address]"]).to eq("192.168.253.128")
+      expect(decode[2]["[netflow][destinationIPv4Address]"]).to eq("192.168.253.1")
+      expect(decode[2]["[netflow][sourceTransportPort]"]).to eq(22)
+      expect(decode[2]["[netflow][destinationTransportPort]"]).to eq(60560)
+      expect(decode[2]["[netflow][protocolIdentifier]"]).to eq(6)
+      expect(decode[2]["[netflow][tcpControlBits]"]).to eq(24)
+
+      expect(decode[3]["[netflow][sourceIPv4Address]"]).to eq("192.168.253.2")
+      expect(decode[3]["[netflow][destinationIPv4Address]"]).to eq("192.168.253.132")
+      expect(decode[3]["[netflow][sourceTransportPort]"]).to eq(53)
+      expect(decode[3]["[netflow][destinationTransportPort]"]).to eq(35262)
+      expect(decode[3]["[netflow][protocolIdentifier]"]).to eq(17)
+
+      expect(decode[4]["[netflow][sourceIPv4Address]"]).to eq("192.168.253.132")
+      expect(decode[4]["[netflow][destinationIPv4Address]"]).to eq("192.168.253.2")
+      expect(decode[4]["[netflow][sourceTransportPort]"]).to eq(35262)
+      expect(decode[4]["[netflow][destinationTransportPort]"]).to eq(53)
+      expect(decode[4]["[netflow][protocolIdentifier]"]).to eq(17)
+
+      expect(decode[5]["[netflow][sourceIPv4Address]"]).to eq("54.214.9.161")
+      expect(decode[5]["[netflow][destinationIPv4Address]"]).to eq("192.168.253.132")
+      expect(decode[5]["[netflow][sourceTransportPort]"]).to eq(443)
+      expect(decode[5]["[netflow][destinationTransportPort]"]).to eq(49935)
+      expect(decode[5]["[netflow][protocolIdentifier]"]).to eq(6)
+      expect(decode[5]["[netflow][tcpControlBits]"]).to eq(26)
+
+      expect(decode[6]["[netflow][sourceIPv4Address]"]).to eq("192.168.253.132")
+      expect(decode[6]["[netflow][destinationIPv4Address]"]).to eq("54.214.9.161")
+      expect(decode[6]["[netflow][sourceTransportPort]"]).to eq(49935)
+      expect(decode[6]["[netflow][destinationTransportPort]"]).to eq(443)
+      expect(decode[6]["[netflow][protocolIdentifier]"]).to eq(6)
+      expect(decode[6]["[netflow][tcpControlBits]"]).to eq(26)
     end
 
     it "should serialize to json" do
-      # generated json order can change with different implementation, convert back to hash to compare.
       expect(JSON.parse(decode[0].to_json)).to eq(JSON.parse(json_events[0]))
       expect(JSON.parse(decode[1].to_json)).to eq(JSON.parse(json_events[1]))
+      expect(JSON.parse(decode[2].to_json)).to eq(JSON.parse(json_events[2]))
+      expect(JSON.parse(decode[3].to_json)).to eq(JSON.parse(json_events[3]))
+      expect(JSON.parse(decode[4].to_json)).to eq(JSON.parse(json_events[4]))
+      expect(JSON.parse(decode[5].to_json)).to eq(JSON.parse(json_events[5]))
+      expect(JSON.parse(decode[6].to_json)).to eq(JSON.parse(json_events[6]))
     end
   end
+
 end