logstash-codec-netflow 3.12.0 → 3.13.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 23d7c5f0b11a7d4e1f1ec188639527e182eb0361
-  data.tar.gz: 40ff5c0a2e481c785649b7cac4df8f94f7bf2aff
+  metadata.gz: 2e7dc24e899f3afdf8980d5815727b8394cf9b69
+  data.tar.gz: 3fa3f494f7ad39af68d3b99e6c918b995cd8cae7
 SHA512:
-  metadata.gz: c07ea4ed3c53ff4147ac122cd065c002ce6e4361817de65122b129a6eee159754322e93b4b4f3bd109435a049d59221a58757f225c621cfebe1db5744b101d90
-  data.tar.gz: f2ea2d25ef1f77e7ad3e29ae4f343630124ec209f5adb7e1cdc676f6a6583fea172ccfc3157611915a8e3f4d85c2edc2ac14e96bba5180e09420a6bd88683a39
+  metadata.gz: d81ead3c2fe61b83376d7a97fa7c9ded2039d42b2f95d7d80e39727bb60dd305a80367775d75f060accd7ec611e06f3c355ca75a3d8a76ae8535d72b4f24dddf
+  data.tar.gz: fe71e6a1e05f666a8b43c3de974b98e59f7a4663f12dcad0879cf5ca7b41f02740d5fe30bbfd3476c3c5bad76866ca06961afabcd5f14b4de6620b703f696b9d

data/CHANGELOG.md

@@ -1,3 +1,8 @@
+## 3.13.0
+
+  - Added support for Netflow 9 reduced-size encoding support
+  - Added support for Barracuda IPFIX Extended Uniflow
+
 ## 3.12.0
 
   - Added support for IPFIX from Procera/NetIntact/Sandvine 15.1

data/CONTRIBUTORS

@@ -19,6 +19,7 @@ Contributors:
 * Jordan Sissel (jordansissel)
 * Jorrit Folmer (jorritfolmer)
 * Keenan Tims (ktims)
+* Magnus Kessler (kesslerm)
 * Marian Craciunescu (marian-craciunescu)
 * Matt Dainty (bodgit)
 * Paul Warren (pwarren)
@@ -28,6 +29,7 @@ Contributors:
 * Pulkit Agrawal (propulkit)
 * Raju Nair (rajutech76)
 * Richard Pijnenburg (electrical)
+* Rob Cowart (robcowart)
 * Salvador Ferrer (salva-ferrer)
 * Vishal Solanki
 * Will Rigby (wrigby)

data/lib/logstash/codecs/netflow.rb

@@ -480,9 +480,10 @@ class LogStash::Codecs::Netflow < LogStash::Codecs::Base
 
         # Small bit of fixup for:
         # - skip or string field types where the length is dynamic
-	# - uint(8|16|24|32} where we use the length as specified by the
+	# - uint(8|16|24|32|64} where we use the length as specified by the
 	#   template instead of the YAML (e.g. ipv6_flow_label is 3 bytes in
-	#   the YAML and Cisco doc, but Cisco ASR9k sends 4 bytes)
+	#   the YAML and Cisco doc, but Cisco ASR9k sends 4 bytes).
+	#   Another usecase is supporting reduced-size encoding as per RFC7011 6.2
 	# - application_id where we use the length as specified by the 
 	#   template and map it to custom types for handling.
 	#   
@@ -490,10 +491,21 @@ class LogStash::Codecs::Netflow < LogStash::Codecs::Base
         when :uint8
           field[0] = uint_field(length, field[0])
         when :uint16
+          if length>2
+            @logger.warn("Reduced-size encoding for uint16 is larger than uint16", :field => field, :length => length)
+          end
           field[0] = uint_field(length, field[0])
         when :uint24
           field[0] = uint_field(length, field[0])
         when :uint32
+          if length>4
+            @logger.warn("Reduced-size encoding for uint32 is larger than uint32", :field => field, :length => length)
+          end
+          field[0] = uint_field(length, field[0])
+        when :uint64
+          if length>8
+            @logger.warn("Reduced-size encoding for uint64 is larger than uint64", :field => field, :length => length)
+          end
           field[0] = uint_field(length, field[0])
         when :application_id
           case length

data/lib/logstash/codecs/netflow/ipfix.yaml

@@ -3770,4 +3770,77 @@
   47:
   - :string
   - :proceraTemplateName
-
+10704:
+  1:
+  - :uint32
+  - :Timestamp
+  2:
+  - :uint8
+  - :LogOp
+  3:
+  - :uint8
+  - TrafficType
+  4:
+  - :string
+  - :FW_Rule
+  5:
+  - :string
+  - :ServiceName
+  6:
+  - :uint32
+  - :Reason
+  7:
+  - :string
+  - :ReasonText
+  8:
+  - :ip4_addr
+  - :BindIPv4Address
+  9:
+  - :uint16
+  - :BindTransportPort
+  10:
+  - :ip4_addr
+  - :ConnIPv4Address
+  11:
+  - :uint16
+  - :ConnTransportPort
+  12:
+  - :uint32
+  - :AuditCounter
+12326:
+  1:
+  - :uint32
+  - :Timestamp
+  2:
+  - :uint8
+  - :LogOp
+  3:
+  - :uint8
+  - TrafficType
+  4:
+  - :string
+  - :FW_Rule
+  5:
+  - :string
+  - :ServiceName
+  6:
+  - :uint32
+  - :Reason
+  7:
+  - :string
+  - :ReasonText
+  8:
+  - :ip4_addr
+  - :BindIPv4Address
+  9:
+  - :uint16
+  - :BindTransportPort
+  10:
+  - :ip4_addr
+  - :ConnIPv4Address
+  11:
+  - :uint16
+  - :ConnTransportPort
+  12:
+  - :uint32
+  - :AuditCounter

data/logstash-codec-netflow.gemspec

@@ -1,7 +1,7 @@
 Gem::Specification.new do |s|
 
   s.name            = 'logstash-codec-netflow'
-  s.version         = '3.12.0'
+  s.version         = '3.13.0'
   s.licenses        = ['Apache License (2.0)']
   s.summary         = "Reads Netflow v5, Netflow v9 and IPFIX data"
   s.description     = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program"

data/spec/codecs/netflow_spec.rb

@@ -905,6 +905,62 @@ describe LogStash::Codecs::Netflow do
 
   end
 
+  context "Netflow 9 ipt_netflow reduced size encoding" do
+    let(:data) do
+      packets = []
+      packets << IO.read(File.join(File.dirname(__FILE__), "netflow9_test_iptnetflow_reduced_size_encoding_tpldata260.dat"), :mode => "rb")
+    end
+
+    let(:json_events) do
+      events = []
+      events << <<-END
+      {
+        "netflow": {
+          "l4_src_port": 443,
+          "last_switched": "2018-02-18T05:46:54.999Z",
+          "ingressPhysicalInterface": 7,
+          "in_bytes": 187,
+          "tcpOptions": 2164260864,
+          "in_dst_mac": "00:1b:21:bc:24:dd",
+          "protocol": 6,
+          "output_snmp": 8,
+          "ethernetType": 2048,
+          "src_tos": 0,
+          "l4_dst_port": 38164,
+          "input_snmp": 7,
+          "version": 9,
+          "in_pkts": 3,
+          "flow_seq_num": 344481,
+          "ipv4_next_hop": "10.232.5.1",
+          "flowset_id": 260,
+          "first_switched": "2018-02-18T05:46:54.999Z",
+          "tcp_flags": 25,
+          "ipv4_dst_addr": "10.233.150.21",
+          "ipv4_src_addr": "2.17.140.47",
+          "in_src_mac": "90:e2:ba:23:09:fc",
+          "egressPhysicalInterface": 8
+        },
+        "@timestamp": "2018-02-18T05:47:09.000Z",
+        "@version": "1"
+      }
+      END
+
+      events.map{|event| event.gsub(/\s+/, "")}
+    end
+
+    it "should decode raw data" do
+      expect(decode.size).to eq(12)
+      expect(decode[11].get("[netflow][in_dst_mac]")).to eq("00:1b:21:bc:24:dd")
+      expect(decode[11].get("[netflow][ipv4_src_addr]")).to eq("2.17.140.47")
+    end
+
+    it "should serialize to json" do
+      expect(JSON.parse(decode[11].to_json)).to eq(JSON.parse(json_events[0]))
+    end
+
+  end
+
+
   context "Netflow 9 IE150 IE151" do
     let(:data) do
       packets = []
@@ -1085,6 +1141,70 @@ describe LogStash::Codecs::Netflow do
 
   end
 
+  context "IPFIX Barracuda extended uniflow template 256" do
+    let(:data) do
+      packets = []
+      packets << IO.read(File.join(File.dirname(__FILE__), "ipfix_test_barracuda_extended_uniflow_tpl256.dat"), :mode => "rb")
+      packets << IO.read(File.join(File.dirname(__FILE__), "ipfix_test_barracuda_extended_uniflow_data256.dat"), :mode => "rb")
+    end
+
+    let(:json_events) do
+      events = []
+      events << <<-END
+      {
+        "netflow": {
+        "FW_Rule": "MTH:MTH-MC-to-Inet",
+        "AuditCounter": 4157725,
+        "sourceIPv4Address": "64.235.151.76",
+        "version": 10,
+        "sourceTransportPort": 443,
+        "sourceMacAddress": "00:00:00:00:00:00",
+        "ingressInterface": 3689,
+        "flowEndSysUpTime": 1957197969,
+        "octetTotalCount": 0,
+        "ConnTransportPort": 443,
+        "ConnIPv4Address": "64.235.151.76",
+        "firewallEvent": 1,
+        "protocolIdentifier": 6,
+        "flowStartSysUpTime": 1957197969,
+        "TrafficType": 0,
+        "destinationTransportPort": 51917,
+        "packetTotalCount": 0,
+        "BindIPv4Address": "213.208.150.99",
+        "Timestamp": 1524039407,
+        "flowDurationMilliseconds": 0,
+        "ServiceName": "https",
+        "BindTransportPort": 64238,
+        "octetDeltaCount": 0,
+        "packetDeltaCount": 0,
+        "destinationIPv4Address": "10.236.5.4",
+        "LogOp": 1,
+        "Reason": 0,
+        "egressInterface": 35233,
+        "ReasonText": "Normal Operation"
+        },
+        "@version": "1",
+        "@timestamp": "2018-04-18T08:16:47.000Z"
+      } 
+      END
+
+      events.map{|event| event.gsub(/\s+/, "")}
+      events.map{|event| event.gsub(/NormalOperation/, "Normal Operation")}
+    end
+
+    it "should decode raw data" do
+      expect(decode.size).to eq(2)
+      expect(decode[1].get("[netflow][FW_Rule]")).to eq("MTH:MTH-MC-to-Inet")
+      expect(decode[1].get("[netflow][ReasonText]")).to eq("Normal Operation")
+      expect(decode[1].get("[netflow][BindIPv4Address]")).to eq("213.208.150.99")
+    end
+
+    it "should serialize to json" do
+      expect(JSON.parse(decode[1].to_json)).to eq(JSON.parse(json_events[0]))
+    end
+
+  end
+
 
 
   context "Netflow 9 Ubiquiti Edgerouter with MPLS labels" do

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: logstash-codec-netflow
 version: !ruby/object:Gem::Version
-  version: 3.12.0
+  version: 3.13.0
 platform: ruby
 authors:
 - Elastic
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2018-04-15 00:00:00.000000000 Z
+date: 2018-04-28 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
@@ -81,6 +81,8 @@ files:
 - spec/codecs/benchmarks/netflow_bench_cisco_asr.py
 - spec/codecs/ipfix.dat
 - spec/codecs/ipfix_test_barracuda_data256.dat
+- spec/codecs/ipfix_test_barracuda_extended_uniflow_data256.dat
+- spec/codecs/ipfix_test_barracuda_extended_uniflow_tpl256.dat
 - spec/codecs/ipfix_test_barracuda_tpl.dat
 - spec/codecs/ipfix_test_mikrotik_data258.dat
 - spec/codecs/ipfix_test_mikrotik_data259.dat
@@ -138,6 +140,7 @@ files:
 - spec/codecs/netflow9_test_huawei_netstream_data.dat
 - spec/codecs/netflow9_test_huawei_netstream_tpl.dat
 - spec/codecs/netflow9_test_invalid01.dat
+- spec/codecs/netflow9_test_iptnetflow_reduced_size_encoding_tpldata260.dat
 - spec/codecs/netflow9_test_juniper_srx_tplopt.dat
 - spec/codecs/netflow9_test_macaddr_data.dat
 - spec/codecs/netflow9_test_macaddr_tpl.dat
@@ -191,6 +194,8 @@ test_files:
 - spec/codecs/benchmarks/netflow_bench_cisco_asr.py
 - spec/codecs/ipfix.dat
 - spec/codecs/ipfix_test_barracuda_data256.dat
+- spec/codecs/ipfix_test_barracuda_extended_uniflow_data256.dat
+- spec/codecs/ipfix_test_barracuda_extended_uniflow_tpl256.dat
 - spec/codecs/ipfix_test_barracuda_tpl.dat
 - spec/codecs/ipfix_test_mikrotik_data258.dat
 - spec/codecs/ipfix_test_mikrotik_data259.dat
@@ -248,6 +253,7 @@ test_files:
 - spec/codecs/netflow9_test_huawei_netstream_data.dat
 - spec/codecs/netflow9_test_huawei_netstream_tpl.dat
 - spec/codecs/netflow9_test_invalid01.dat
+- spec/codecs/netflow9_test_iptnetflow_reduced_size_encoding_tpldata260.dat
 - spec/codecs/netflow9_test_juniper_srx_tplopt.dat
 - spec/codecs/netflow9_test_macaddr_data.dat
 - spec/codecs/netflow9_test_macaddr_tpl.dat