logstash-codec-netflow 3.1.2 → 3.1.4

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 99ce8b242b817bf92aa4c140a56d43db630a26dc
-  data.tar.gz: c02fcefa562dec4a4f818af944fb32db523edd64
+  metadata.gz: 64fcf9c71a5f61b3d08657aa9330fe84c509130f
+  data.tar.gz: 7b3d28e45b1049e3164fc6e1bec4302aef0390c4
 SHA512:
-  metadata.gz: 3b26e57e5c3e600593041b1fdd3c3b89c77ec899efc78293f9363d13088b51a8f91a77869687aede41c74e01988fce798e2e3368f4486d1aa7fc4ce120436cde
-  data.tar.gz: da2d32f24ddca86e7aa32f01f9a588a52834abe0f1139842641878b6108275824ad9d539512e94eff5ad67c1a3f253d9d3b34dc15941d6487acf881ac36bf88a
+  metadata.gz: 721e9b3d27195d9fbcb97101fb710b4830544d9d7300e00e1fa69feca51eb0a45b3416370b94131ff38bd7edb2a751a33ea017e46d3ebe81d57f929d21d9edc1
+  data.tar.gz: 2596a72bf9fc4a8d738e51ffbdf0da7325207192390e1f21770808bd7e0c487ff333d735b89d559502b2b56c3d024f1ce5a95a24fdcdceaadd0feb50046e9d83

data/CHANGELOG.md

@@ -1,4 +1,14 @@
+## 3.1.4
+
+  - Added support for MPLS labels
+  - Added support for decoding forwarded status field (Netflow 9)
+
+## 3.1.3
+
+  - Confirmed support and tests added for 4 Netflow/IPFIX exporters
+
 ## 3.1.2
+
   - Relax constraint on logstash-core-plugin-api to >= 1.60 <= 2.99
 
 ## 3.1.1

data/CONTRIBUTORS

@@ -5,6 +5,8 @@ Contributors:
 * Aaron Mildenstein (untergeek)
 * Adam Kaminski (thimslugga)
 * Colin Surprenant (colinsurprenant)
+* Daniel Nägele (analogbyte)
+* Evgeniy Sudyr (ejectck)
 * Jordan Sissel (jordansissel)
 * Jorrit Folmer (jorritfolmer)
 * Matt Dainty (bodgit)
@@ -17,6 +19,8 @@ Contributors:
 * Rojuinex
 * debadair
 * hkshirish
+* hhindlem
+* niempy
 * jstopinsek
 
 Maintainer:

data/lib/logstash/codecs/netflow.rb

@@ -1,5 +1,5 @@
 # encoding: utf-8
-require "logstash/filters/base"
+require "logstash/codecs/base"
 require "logstash/namespace"
 require "logstash/timestamp"
 
@@ -11,13 +11,17 @@ require "logstash/timestamp"
 #
 # [cols="6,^2,^2,^2,12",options="header"]
 # |===========================================================================================
-# |Netflow exporter | v5 | v9 | IPFIX | Remarks
-# |Softflowd        |  y | y  |   y   | IPFIX supported in https://github.com/djmdjm/softflowd
-# |nProbe           |  y | y  |   y   |  
-# |ipt_NETFLOW      |  y | y  |   y   |
-# |Cisco ASA        |    | y  |       |  
-# |Cisco IOS 12.x   |    | y  |       |  
-# |fprobe           |  y |    |       |
+# |Netflow exporter      | v5 | v9 | IPFIX | Remarks
+# |Softflowd             |  y | y  |   y   | IPFIX supported in https://github.com/djmdjm/softflowd
+# |nProbe                |  y | y  |   y   |  
+# |ipt_NETFLOW           |  y | y  |   y   |
+# |Cisco ASA             |    | y  |       |  
+# |Cisco IOS 12.x        |    | y  |       |  
+# |fprobe                |  y |    |       |
+# |Juniper MX80          |  y |    |       | SW > 12.3R8
+# |OpenBSD pflow         |  y | n  |   y   | http://man.openbsd.org/OpenBSD-current/man4/pflow.4
+# |Mikrotik 6.35.4       |  y |    |   n   | http://wiki.mikrotik.com/wiki/Manual:IP/Traffic_Flow
+# |Ubiquiti Edgerouter X |    | y  |       | With MPLS labels
 # |===========================================================================================
 #
 # ==== Usage

data/lib/logstash/codecs/netflow/netflow.yaml

@@ -222,6 +222,15 @@
 86:
 - :uint32
 - :in_permanent_pkts
+89:
+- :forwarding_status
+- :forwarding_status
+95:
+- :uint32
+- :application_id
+136:
+- :uint8
+- :flow_end_reason
 148:
 - :uint32
 - :conn_id
@@ -237,6 +246,9 @@
 179:
 - :uint8
 - :icmp_code_ipv6
+201:
+- mpls_label_stack_octets
+- mpls_label_stack_octets
 225:
 - :ip4_addr
 - :xlate_src_addr_ipv4
@@ -297,3 +309,18 @@
 40005:
 - :uint8
 - :fw_event
+95:
+- :uint32
+- :application_id
+180:
+- :uint16
+- :udp_src_port
+181:
+- :uint16
+- :udp_dst_port
+182:
+- :uint16
+- :tcp_src_port
+183:
+- :uint16
+- :tcp_dst_port

data/lib/logstash/codecs/netflow/util.rb

@@ -64,6 +64,20 @@ class ACLIdASA < BinData::Primitive
   end
 end
 
+class MPLSLabelStackOctets < BinData::Record
+  endian :big
+  bit20  :label
+  bit3   :experimental
+  bit1   :bottom_of_stack
+  uint8  :ttl
+end
+
+class Forwarding_Status < BinData::Record
+  endian :big
+  bit2   :status
+  bit6   :reason
+end
+
 class Header < BinData::Record
   endian :big
   uint16 :version

data/logstash-codec-netflow.gemspec

@@ -1,7 +1,7 @@
 Gem::Specification.new do |s|
 
   s.name            = 'logstash-codec-netflow'
-  s.version         = '3.1.2'
+  s.version         = '3.1.4'
   s.licenses        = ['Apache License (2.0)']
   s.summary         = "The netflow codec is for decoding Netflow v5/v9/v10 (IPFIX) flows."
   s.description     = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program"

data/spec/codecs/netflow_spec.rb

@@ -247,8 +247,8 @@ describe LogStash::Codecs::Netflow do
   context "Netflow 9 Cisco ASA" do
     let(:data) do
       packets = []
-      packets << IO.read(File.join(File.dirname(__FILE__), "netflow9_test_valid_cisco_asa_tpl.dat"), :mode => "rb")
-      packets << IO.read(File.join(File.dirname(__FILE__), "netflow9_test_valid_cisco_asa_data.dat"), :mode => "rb")
+      packets << IO.read(File.join(File.dirname(__FILE__), "netflow9_test_cisco_asa_1_tpl.dat"), :mode => "rb")
+      packets << IO.read(File.join(File.dirname(__FILE__), "netflow9_test_cisco_asa_1_data.dat"), :mode => "rb")
     end
 
     let(:json_events) do
@@ -679,6 +679,292 @@ describe LogStash::Codecs::Netflow do
       expect(JSON.parse(decode[5].to_json)).to eq(JSON.parse(json_events[5]))
       expect(JSON.parse(decode[6].to_json)).to eq(JSON.parse(json_events[6]))
     end
+
+  end
+
+  context "Netflow 9 Cisco ASA #2" do
+    let(:data) do
+      # The ASA sent 2 packets with templates, 260-270, and 270-280
+      packets = []
+      packets << IO.read(File.join(File.dirname(__FILE__), "netflow9_test_cisco_asa_2_tpl_26x.dat"), :mode => "rb")
+      packets << IO.read(File.join(File.dirname(__FILE__), "netflow9_test_cisco_asa_2_tpl_27x.dat"), :mode => "rb")
+      packets << IO.read(File.join(File.dirname(__FILE__), "netflow9_test_cisco_asa_2_data.dat"), :mode => "rb")
+    end
+
+    let(:json_events) do
+      events = []
+      events << <<-END
+        {
+          "@timestamp": "2016-07-21T13:50:37.000Z",
+          "netflow": {
+            "version": 9,
+            "flow_seq_num": 31,
+            "flowset_id": 263,
+            "conn_id": 742820223,
+            "ipv4_src_addr": "192.168.0.1",
+            "l4_src_port":56651,
+            "input_snmp":3,
+            "ipv4_dst_addr":"192.168.0.18",
+            "l4_dst_port":80,
+            "output_snmp":4,
+            "protocol":6,
+            "icmp_type":0,
+            "icmp_code":0,
+            "xlate_src_addr_ipv4":"192.168.0.1",
+            "xlate_dst_addr_ipv4":"192.168.0.18",
+            "xlate_src_port":56651,
+            "xlate_dst_port":80,
+            "fw_event":2,
+            "fw_ext_event":2030,
+            "event_time_msec":1469109036495,
+            "fwd_flow_delta_bytes":69,
+            "rev_flow_delta_bytes":14178,
+            "flow_start_msec":1469109036395
+          },
+          "@version": "1"
+        }
+        END
+      events.map{|event| event.gsub(/\s+/, "")}
+    end
+
+    it "should decode raw data" do
+      expect(decode.size).to eq(19)
+      expect(decode[18].get("[netflow][ipv4_src_addr]")).to eq("192.168.0.1")
+      expect(decode[18].get("[netflow][ipv4_dst_addr]")).to eq("192.168.0.18")
+      expect(decode[18].get("[netflow][fwd_flow_delta_bytes]")).to eq(69)
+      expect(decode[18].get("[netflow][conn_id]")).to eq(742820223)
+    end
+
+    it "should serialize to json" do
+      expect(JSON.parse(decode[18].to_json)).to eq(JSON.parse(json_events[0]))
+    end
+
+  end
+
+  context "IPFIX OpenBSD pflow" do
+    let(:data) do
+      packets = []
+      packets << IO.read(File.join(File.dirname(__FILE__), "ipfix_test_openbsd_pflow_tpl.dat"), :mode => "rb")
+      packets << IO.read(File.join(File.dirname(__FILE__), "ipfix_test_openbsd_pflow_data.dat"), :mode => "rb")
+    end
+
+    let(:json_events) do
+      events = []
+      events << <<-END
+        {
+          "@timestamp": "2016-07-21T13:30:37.000Z",
+          "netflow": {
+            "version": 10,
+            "sourceIPv4Address": "192.168.0.1",
+            "destinationIPv4Address": "192.168.0.17",
+            "ingressInterface": 1,
+            "egressInterface": 1,
+            "packetDeltaCount": 8,
+            "octetDeltaCount": 6425,
+            "flowStartMilliseconds": "2016-07-21T13:29:59.000Z",
+            "flowEndMilliseconds": "2016-07-21T13:30:01.000Z",
+            "sourceTransportPort": 80,
+            "destinationTransportPort": 64026,
+            "ipClassOfService": 0,
+            "protocolIdentifier": 6
+          },
+          "@version": "1"
+        }
+        END
+      events.map{|event| event.gsub(/\s+/, "")}
+    end
+
+    it "should decode raw data" do
+      expect(decode.size).to eq(26)
+      expect(decode[25].get("[netflow][sourceIPv4Address]")).to eq("192.168.0.1")
+      expect(decode[25].get("[netflow][destinationIPv4Address]")).to eq("192.168.0.17")
+      expect(decode[25].get("[netflow][octetDeltaCount]")).to eq(6425)
+      expect(decode[25].get("[netflow][destinationTransportPort]")).to eq(64026)
+    end
+
+    it "should serialize to json" do
+      expect(JSON.parse(decode[25].to_json)).to eq(JSON.parse(json_events[0]))
+    end
+
+  end
+
+  context "Netflow5 microtik" do
+    let(:data) do
+      packets = []
+      packets << IO.read(File.join(File.dirname(__FILE__), "netflow5_test_microtik.dat"), :mode => "rb")
+    end
+
+    let(:json_events) do
+      events = []
+      events << <<-END
+        {
+          "@timestamp": "2016-07-21T13:51:57.514Z",
+          "netflow": {
+            "version": 5,
+            "flow_seq_num": 8140050,
+            "engine_type": 0,
+            "engine_id": 0,
+            "sampling_algorithm": 0,
+            "sampling_interval": 0,
+            "flow_records": 30,
+            "ipv4_src_addr": "10.0.8.1",
+            "ipv4_dst_addr": "192.168.0.1",
+            "ipv4_next_hop": "192.168.0.1",
+            "input_snmp": 13,
+            "output_snmp": 46,
+            "in_pkts": 13,
+            "in_bytes": 11442,
+            "first_switched": "2016-07-21T13:51:42.514Z",
+            "last_switched": "2016-07-21T13:51:42.514Z",
+            "l4_src_port": 80,
+            "l4_dst_port": 51826,
+            "tcp_flags": 82,
+            "protocol": 6,
+            "src_tos": 40,
+            "src_as": 0,
+            "dst_as": 0,
+            "src_mask": 0,
+            "dst_mask": 0
+          },
+          "@version": "1"
+        }
+        END
+      events.map{|event| event.gsub(/\s+/, "")}
+    end
+
+    it "should decode raw data" do
+      expect(decode.size).to eq(30)
+      expect(decode[29].get("[netflow][ipv4_src_addr]")).to eq("10.0.8.1")
+      expect(decode[29].get("[netflow][ipv4_dst_addr]")).to eq("192.168.0.1")
+      expect(decode[29].get("[netflow][l4_dst_port]")).to eq(51826)
+      expect(decode[29].get("[netflow][src_tos]")).to eq(40)
+    end
+
+    it "should serialize to json" do
+      expect(JSON.parse(decode[29].to_json)).to eq(JSON.parse(json_events[0]))
+    end
+
   end
 
+  context "Netflow5 Juniper MX80" do
+    let(:data) do
+      packets = []
+      packets << IO.read(File.join(File.dirname(__FILE__), "netflow5_test_juniper_mx80.dat"), :mode => "rb")
+    end
+
+    let(:json_events) do
+      events = []
+      events << <<-END
+        {
+          "@timestamp": "2016-07-21T13:52:52.000Z",
+          "netflow": {
+            "version": 5,
+            "flow_seq_num": 528678,
+            "engine_type": 0,
+            "engine_id": 0,
+            "sampling_algorithm": 0,
+            "sampling_interval": 1000,
+            "flow_records": 29,
+            "ipv4_src_addr": "66.249.92.75",
+            "ipv4_dst_addr": "192.168.0.1",
+            "ipv4_next_hop": "192.168.0.1",
+            "input_snmp": 542,
+            "output_snmp": 536,
+            "in_pkts": 2,
+            "in_bytes": 104,
+            "first_switched": "2016-07-21T13:52:34.999Z",
+            "last_switched": "2016-07-21T13:52:34.999Z",
+            "l4_src_port": 37387,
+            "l4_dst_port": 80,
+            "tcp_flags": 16,
+            "protocol": 6,
+            "src_tos": 0,
+            "src_as": 15169,
+            "dst_as": 64496,
+            "src_mask": 19,
+            "dst_mask": 24
+          },
+          "@version": "1"
+        }
+        END
+      events.map{|event| event.gsub(/\s+/, "")}
+    end
+
+    it "should decode raw data" do
+      expect(decode.size).to eq(29)
+      expect(decode[28].get("[netflow][ipv4_src_addr]")).to eq("66.249.92.75")
+      expect(decode[28].get("[netflow][ipv4_dst_addr]")).to eq("192.168.0.1")
+      expect(decode[28].get("[netflow][l4_dst_port]")).to eq(80)
+      expect(decode[28].get("[netflow][src_as]")).to eq(15169)
+      expect(decode[28].get("[netflow][dst_as]")).to eq(64496)
+    end
+
+    it "should serialize to json" do
+      expect(JSON.parse(decode[28].to_json)).to eq(JSON.parse(json_events[0]))
+    end
+
+  end
+
+  context "Netflow 9 Ubiquiti Edgerouter with MPLS labels" do
+    let(:data) do
+      packets = []
+      packets << IO.read(File.join(File.dirname(__FILE__), "netflow9_test_ubnt_edgerouter_tpl.dat"), :mode => "rb")
+      packets << IO.read(File.join(File.dirname(__FILE__), "netflow9_test_ubnt_edgerouter_data1024.dat"), :mode => "rb")
+      packets << IO.read(File.join(File.dirname(__FILE__), "netflow9_test_ubnt_edgerouter_data1025.dat"), :mode => "rb")
+    end
+
+    let(:json_events) do
+      events = []
+      events << <<-END
+        {
+          "@timestamp": "2016-09-10T16:24:08.000Z", 
+          "netflow": {
+            "output_snmp": 4, 
+            "out_src_mac": "06:be:ef:be:ef:b9", 
+            "in_pkts": 21, 
+            "ip_protocol_version": 4, 
+            "ipv4_dst_addr": "10.2.0.95",
+            "src_tos": 0, 
+            "first_switched": "2016-09-10T15:02:54.999Z", 
+            "flowset_id": 1025, 
+            "l4_src_port": 47690, 
+            "out_dst_mac": "44:d9:e7:be:ef:8e", 
+            "version": 9, 
+            "flow_seq_num": 31664, 
+            "ipv4_src_addr": "192.168.1.102", 
+            "in_bytes": 3668, 
+            "protocol": 6, 
+            "mpls_label_stack_octets": { 
+              "bottom_of_stack": 0, 
+              "experimental": 0, 
+              "label": 0, 
+              "ttl": 4
+            }, 
+            "last_switched": "2016-09-10T15:23:45.999Z", 
+            "input_snmp": 2, 
+            "flows": 0, 
+            "tcp_flags": 27, 
+            "dst_vlan": 0, 
+            "l4_dst_port": 443, 
+            "direction": 1
+          },
+          "@version": "1"
+        }
+        END
+      events.map{|event| event.gsub(/\s+/, "")}
+    end
+
+    it "should decode raw data" do
+      expect(decode.size).to eq(16)
+      expect(decode[0].get("[netflow][ipv4_src_addr]")).to eq("10.1.0.135")
+      expect(decode[15].get("[netflow][ipv4_src_addr]")).to eq("192.168.1.102")
+    end
+
+    it "should serialize to json" do
+      expect(JSON.parse(decode[15].to_json)).to eq(JSON.parse(json_events[0]))
+    end
+
+  end
+
+
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: logstash-codec-netflow
 version: !ruby/object:Gem::Version
-  version: 3.1.2
+  version: 3.1.4
 platform: ruby
 authors:
 - Elastic
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2016-07-14 00:00:00.000000000 Z
+date: 2016-12-19 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
@@ -77,18 +77,28 @@ files:
 - lib/logstash/codecs/netflow/util.rb
 - logstash-codec-netflow.gemspec
 - spec/codecs/ipfix.dat
+- spec/codecs/ipfix_test_openbsd_pflow_data.dat
+- spec/codecs/ipfix_test_openbsd_pflow_tpl.dat
 - spec/codecs/netflow5.dat
 - spec/codecs/netflow5_test_invalid01.dat
 - spec/codecs/netflow5_test_invalid02.dat
+- spec/codecs/netflow5_test_juniper_mx80.dat
+- spec/codecs/netflow5_test_microtik.dat
+- spec/codecs/netflow9_test_cisco_asa_1_data.dat
+- spec/codecs/netflow9_test_cisco_asa_1_tpl.dat
+- spec/codecs/netflow9_test_cisco_asa_2_data.dat
+- spec/codecs/netflow9_test_cisco_asa_2_tpl_26x.dat
+- spec/codecs/netflow9_test_cisco_asa_2_tpl_27x.dat
 - spec/codecs/netflow9_test_invalid01.dat
 - spec/codecs/netflow9_test_macaddr_data.dat
 - spec/codecs/netflow9_test_macaddr_tpl.dat
 - spec/codecs/netflow9_test_nprobe_data.dat
 - spec/codecs/netflow9_test_nprobe_tpl.dat
 - spec/codecs/netflow9_test_softflowd_tpl_data.dat
+- spec/codecs/netflow9_test_ubnt_edgerouter_data1024.dat
+- spec/codecs/netflow9_test_ubnt_edgerouter_data1025.dat
+- spec/codecs/netflow9_test_ubnt_edgerouter_tpl.dat
 - spec/codecs/netflow9_test_valid01.dat
-- spec/codecs/netflow9_test_valid_cisco_asa_data.dat
-- spec/codecs/netflow9_test_valid_cisco_asa_tpl.dat
 - spec/codecs/netflow_spec.rb
 homepage: http://www.elastic.co/guide/en/logstash/current/index.html
 licenses:
@@ -112,22 +122,32 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project:
-rubygems_version: 2.6.3
+rubygems_version: 2.4.8
 signing_key:
 specification_version: 4
 summary: The netflow codec is for decoding Netflow v5/v9/v10 (IPFIX) flows.
 test_files:
 - spec/codecs/ipfix.dat
+- spec/codecs/ipfix_test_openbsd_pflow_data.dat
+- spec/codecs/ipfix_test_openbsd_pflow_tpl.dat
 - spec/codecs/netflow5.dat
 - spec/codecs/netflow5_test_invalid01.dat
 - spec/codecs/netflow5_test_invalid02.dat
+- spec/codecs/netflow5_test_juniper_mx80.dat
+- spec/codecs/netflow5_test_microtik.dat
+- spec/codecs/netflow9_test_cisco_asa_1_data.dat
+- spec/codecs/netflow9_test_cisco_asa_1_tpl.dat
+- spec/codecs/netflow9_test_cisco_asa_2_data.dat
+- spec/codecs/netflow9_test_cisco_asa_2_tpl_26x.dat
+- spec/codecs/netflow9_test_cisco_asa_2_tpl_27x.dat
 - spec/codecs/netflow9_test_invalid01.dat
 - spec/codecs/netflow9_test_macaddr_data.dat
 - spec/codecs/netflow9_test_macaddr_tpl.dat
 - spec/codecs/netflow9_test_nprobe_data.dat
 - spec/codecs/netflow9_test_nprobe_tpl.dat
 - spec/codecs/netflow9_test_softflowd_tpl_data.dat
+- spec/codecs/netflow9_test_ubnt_edgerouter_data1024.dat
+- spec/codecs/netflow9_test_ubnt_edgerouter_data1025.dat
+- spec/codecs/netflow9_test_ubnt_edgerouter_tpl.dat
 - spec/codecs/netflow9_test_valid01.dat
-- spec/codecs/netflow9_test_valid_cisco_asa_data.dat
-- spec/codecs/netflow9_test_valid_cisco_asa_tpl.dat
 - spec/codecs/netflow_spec.rb