logstash-codec-netflow 4.0.1 → 4.0.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 02d79c8d4e3e6e62b887d17e9161f30bdce7b397
-  data.tar.gz: ea75c8b08e31ab585ef4c47f06aefff54e23c75f
+  metadata.gz: c084b2195cdbabfc99579731def8db53f698c55f
+  data.tar.gz: 29cc163ea5580dfd7b1ee509284d0bbb7dd50892
 SHA512:
-  metadata.gz: dc7647f8b95859bccc245a83d991f7d757b5a34a86e6c8b2c4d407d61994fafd322da4628a2a6108d283e226cfc9b1f5ff38626952177af255dc34717b52de8d
-  data.tar.gz: 9a2627caa77a373b0a5223df7e6c2bd1a1f19340a515f4488f5129f409484ed39360a219561b8a1afec36e13bc0f87a47013190ab92995caf5dd30d7f2d76761
+  metadata.gz: 3d667327d46d640196d68ed90664f1af9eb257e0a0768f9f7bb548483a7de3fa29561db84cd6e09a5e5be26e1dd11c7d9a42d18df82348c40ba883e17283ce64
+  data.tar.gz: 87d7ff7dc82dae754633f58849c4a08beae62722cdb24b2eba6fe5d3fc94c19f3ddb1e6ffbcd046b6044560be74e7784aa434b20331ab8a26c7f9c2765b9657d

data/CHANGELOG.md

@@ -1,3 +1,7 @@
+## 4.0.2
+
+  - Fixed incorrect parsing of zero-filled Netflow 9 packets from Palo Alto
+
 ## 4.0.1
 
   - Fixed IPFIX options template parsing for Juniper MX240 JunOS 15.1 

data/CONTRIBUTORS

@@ -23,6 +23,7 @@ Contributors:
 * Magnus Kessler (kesslerm)
 * Marian Craciunescu (marian-craciunescu)
 * Matt Dainty (bodgit)
+* Max Caines (maxcaines)
 * Paul Warren (pwarren)
 * Pedro de Oliveira
 * Philipp Kahr

data/lib/logstash/codecs/netflow.rb

@@ -183,6 +183,11 @@ class LogStash::Codecs::Netflow < LogStash::Codecs::Base
   def decode_netflow9(flowset, record, metadata = nil)
     events = []
 
+    # Check for block of trailing padding
+    if record.flowset_length == 0
+      return events
+    end 
+
     case record.flowset_id
     when 0..1
       # Template flowset

data/lib/logstash/codecs/netflow/util.rb

@@ -379,7 +379,7 @@ end
 
 class NetflowTemplateFlowset < BinData::Record
   endian :big
-  array  :templates, :read_until => lambda { array.num_bytes == flowset_length - 4 } do
+  array  :templates, :read_until => lambda { flowset_length == 0 || array.num_bytes == flowset_length - 4 } do
     uint16 :template_id
     uint16 :field_count
     array  :record_fields, :initial_length => :field_count do
@@ -387,6 +387,7 @@ class NetflowTemplateFlowset < BinData::Record
       uint16 :field_length
     end
   end
+  rest  :rest, :onlyif => lambda { flowset_length == 0 }
 end
 
 class NetflowOptionFlowset < BinData::Record
@@ -417,7 +418,7 @@ class Netflow9PDU < BinData::Record
   uint32 :source_id
   array  :records, :read_until => :eof do
     uint16 :flowset_id, :assert => lambda { [0, 1, *(256..65535)].include?(flowset_id) }
-    uint16 :flowset_length, :assert => lambda { flowset_length > 4 }
+    uint16 :flowset_length, :assert => lambda { flowset_length == 0 || flowset_length > 4 }
     choice :flowset_data, :selection => :flowset_id do
       netflow_template_flowset 0
       netflow_option_flowset   1

data/logstash-codec-netflow.gemspec

@@ -1,7 +1,7 @@
 Gem::Specification.new do |s|
 
   s.name            = 'logstash-codec-netflow'
-  s.version         = '4.0.1'
+  s.version         = '4.0.2'
   s.licenses        = ['Apache License (2.0)']
   s.summary         = "Reads Netflow v5, Netflow v9 and IPFIX data"
   s.description     = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program"

data/spec/codecs/netflow_spec.rb

@@ -1101,6 +1101,60 @@ describe LogStash::Codecs::Netflow do
 
   end
 
+  context "Netflow 9 Palo Alto 1 flowset in large zero filled packet" do
+    let(:data) do
+      packets = []
+      packets << IO.read(File.join(File.dirname(__FILE__), "netflow9_test_paloalto_81_tpl256-263.dat"), :mode => "rb")
+      packets << IO.read(File.join(File.dirname(__FILE__), "netflow9_test_paloalto_81_data257_1flowset_in_large_zerofilled_packet.dat"), :mode => "rb")
+    end
+
+    let(:json_events) do
+      events = []
+      events << <<-END
+      {
+        "netflow": {
+          "output_snmp":500010002,
+          "icmp_type":0,
+          "in_pkts":3,
+          "src_tos":0,
+          "ipv4_dst_addr":"134.220.1.156",
+          "first_switched":"2018-06-06T13:20:03.000Z",
+          "flowset_id":257,
+          "l4_src_port":88,
+          "fw_event":2,
+          "version":9,
+          "flow_seq_num":970830115,
+          "ipv4_src_addr":"134.220.2.6",
+          "in_bytes":363,
+          "protocol":6,
+          "tcp_flags":94,
+          "input_snmp":500010024,
+          "last_switched":"2018-06-06T13:20:03.000Z",
+          "user_id":"unknown",
+          "conn_id":1428388,
+          "privateEnterpriseNumber":25461,
+          "l4_dst_port":50234,
+          "app_id":"kerberos",
+          "direction":0
+        },
+        "@timestamp":"2018-06-06T13:20:17.000Z",
+        "@version":"1"
+      }
+      END
+      events.map{|event| event.gsub(/\s+/, "")}
+    end
+
+    it "should decode raw data" do
+      expect(decode.size).to eq(1)
+      expect(decode[0].get("[netflow][app_id]")).to eq("kerberos")
+      expect(decode[0].get("[netflow][ipv4_src_addr]")).to eq("134.220.2.6")
+    end
+
+    it "should serialize to json" do
+      expect(JSON.parse(decode[0].to_json)).to eq(JSON.parse(json_events[0]))
+    end
+
+  end
 
   context "Netflow 9 Fortigate FortiOS 54x appid" do
     let(:data) do

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: logstash-codec-netflow
 version: !ruby/object:Gem::Version
-  version: 4.0.1
+  version: 4.0.2
 platform: ruby
 authors:
 - Elastic
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2018-06-02 00:00:00.000000000 Z
+date: 2018-06-24 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
@@ -154,6 +154,8 @@ files:
 - spec/codecs/netflow9_test_nprobe_data.dat
 - spec/codecs/netflow9_test_nprobe_dpi.dat
 - spec/codecs/netflow9_test_nprobe_tpl.dat
+- spec/codecs/netflow9_test_paloalto_81_data257_1flowset_in_large_zerofilled_packet.dat
+- spec/codecs/netflow9_test_paloalto_81_tpl256-263.dat
 - spec/codecs/netflow9_test_paloalto_panos_data.dat
 - spec/codecs/netflow9_test_paloalto_panos_tpl.dat
 - spec/codecs/netflow9_test_softflowd_tpl_data.dat
@@ -274,6 +276,8 @@ test_files:
 - spec/codecs/netflow9_test_nprobe_data.dat
 - spec/codecs/netflow9_test_nprobe_dpi.dat
 - spec/codecs/netflow9_test_nprobe_tpl.dat
+- spec/codecs/netflow9_test_paloalto_81_data257_1flowset_in_large_zerofilled_packet.dat
+- spec/codecs/netflow9_test_paloalto_81_tpl256-263.dat
 - spec/codecs/netflow9_test_paloalto_panos_data.dat
 - spec/codecs/netflow9_test_paloalto_panos_tpl.dat
 - spec/codecs/netflow9_test_softflowd_tpl_data.dat