logstash-codec-netflow 3.14.1 → 4.0.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 11eeb7718eb1146533c6b6f1745dfbbd3d267f24
-  data.tar.gz: 0618d031da52613490941fce6df6c92021791fa8
+  metadata.gz: 4d93e056a9df39efb7dff2feb887a661a4231313
+  data.tar.gz: f2e47894941ea4090de24a78028e4b6ec4efc3a6
 SHA512:
-  metadata.gz: 590fa8a90f09134a83af9a7907393c0f0bf1fa942c850db1a6daa749506e26347dfdd202a4e4a4d8cf078d0322e3510567b0eb00f020fe51f99a150a188df375
-  data.tar.gz: 5ad744cbb78da117c1b8936a01d1e0fc645feb11da895aa96214564655e4231b192ba52359817e6e26000c6f4190a9dff7676c14343bae5d5c32ca7b225d2acf
+  metadata.gz: a5cdc4f265bb0e91fcd0db9ce19e3b678bc04cb640a094742ba45b38b881526a563ba51342a982d377103eb55a9ee89cc455d03481cf4e409a6065166d2a772c
+  data.tar.gz: 64e22086639e4595a4b0a3fd988471d5c095b4370046df7d732d03531336bc47a60129c3a57fa267099660951e36f1b99603c04e1b02c431882086b0372b540d

data/CHANGELOG.md

@@ -1,3 +1,7 @@
+## 4.0.0
+
+  - Added support for RFC6759 decoding of application_id. **This is a breaking change to the way application_id is decoded. The format changes from e.g. 0:40567 to 0..12356..40567**
+
 ## 3.14.1
 
   - Fixes exception when receiving Netflow 9 from H3C devices

data/CONTRIBUTORS

@@ -36,6 +36,7 @@ Contributors:
 * Will Rigby (wrigby)
 * Yehonatan Devorkin (Devorkin)
 * Rojuinex
+* Sjaak01
 * debadair
 * HenryTheSir
 * hkshirish

data/lib/logstash/codecs/netflow.rb

@@ -472,6 +472,29 @@ class LogStash::Codecs::Netflow < LogStash::Codecs::Base
     field
   end # def string_field
 
+  def get_rfc6759_application_id_class(field,length)
+    case length
+    when 2
+      field[0] = :Application_Id16
+    when 3
+      field[0] = :Application_Id24
+    when 4
+      field[0] = :Application_Id32
+    when 5
+      field[0] = :Application_Id40
+    when 7
+      field[0] = :Application_Id56
+    when 8
+      field[0] = :Application_Id64
+    when 9
+      field[0] = :Application_Id72
+    else
+      @logger.warn("Unsupported application_id length encountered, skipping", :field => field, :length => length)
+      nil
+    end      
+    field[0]
+  end
+
   def netflow_field_for(type, length, template_id)
     if @netflow_fields.include?(type)
       field = @netflow_fields[type].clone
@@ -509,23 +532,7 @@ class LogStash::Codecs::Netflow < LogStash::Codecs::Base
           end
           field[0] = uint_field(length, field[0])
         when :application_id
-          case length
-          when 2
-            field[0] = :Application_Id16
-          when 3
-            field[0] = :Application_Id24
-          when 4
-            field[0] = :Application_Id32
-          when 5
-            field[0] = :Application_Id40
-          when 8
-            field[0] = :Application_Id64
-          when 9
-            field[0] = :Application_Id72
-          else
-            @logger.warn("Unsupported application_id length encountered, skipping", :field => field, :length => length)
-            nil
-          end      
+          field[0] = get_rfc6759_application_id_class(field,length)
         when :skip
           field += [nil, {:length => length.to_i}]
         when :string
@@ -573,6 +580,8 @@ class LogStash::Codecs::Netflow < LogStash::Codecs::Base
         field[0] = uint_field(length, 4)
       when :uint16
         field[0] = uint_field(length, 2)
+      when :application_id
+        field[0] = get_rfc6759_application_id_class(field,length)
       end
 
       @logger.debug("Definition complete", :field => field)

data/lib/logstash/codecs/netflow/util.rb

@@ -139,10 +139,11 @@ class Application_Id16 < BinData::Primitive
   end
 
   def get
-    self.classification_id.to_s + ":" + self.selector_id.to_s
+    self.classification_id.to_s + ".." + self.selector_id.to_s
   end
 end
 
+
 class Application_Id24 < BinData::Primitive
   endian :big
   uint8  :classification_id
@@ -156,10 +157,11 @@ class Application_Id24 < BinData::Primitive
   end
 
   def get
-    self.classification_id.to_s + ":" + self.selector_id.to_s
+    self.classification_id.to_s + ".." + self.selector_id.to_s
   end
 end
 
+
 class Application_Id32 < BinData::Primitive
   endian :big
   uint8  :classification_id
@@ -173,10 +175,11 @@ class Application_Id32 < BinData::Primitive
   end
 
   def get
-    self.classification_id.to_s + ":" + self.selector_id.to_s
+    self.classification_id.to_s + ".." + self.selector_id.to_s
   end
 end
 
+
 class Application_Id40 < BinData::Primitive
   endian :big
   uint8  :classification_id
@@ -190,41 +193,130 @@ class Application_Id40 < BinData::Primitive
   end
 
   def get
-    self.classification_id.to_s + ":" + self.selector_id.to_s
+    self.classification_id.to_s + ".." + self.selector_id.to_s
+  end
+end
+
+
+class Appid56PanaL7Pen < BinData::Record
+  # RFC6759 chapter 4.1: PANA-L7-PEN
+  # This implements the "application ids MAY be encoded in a smaller number of bytes"
+  # Used in Application_Id56 choice statement
+  endian :big
+  uint32 :pen_id
+  uint16 :selector_id
+end
+
+
+class Application_Id56 < BinData::Primitive
+  endian :big
+  uint8  :classification_id
+  choice :selector_id, :selection => :classification_id do
+    # for classification engine id 20 we switch to Appid64PanaL7Pen to decode
+    appid56_pana_l7_pen 20
+    uint48              :default
+  end
+
+  def set(val)
+    unless val.nil?
+      self.classification_id=val.to_i<<48
+      if self.classification_id == 20
+        # classification engine id 20 (PANA_L7_PEN) contains a 4-byte PEN:
+        self.pen_id            = val.to_i-((val.to_i>>48)<<48)>>16
+        self.selector_id       = val.to_i-((val.to_i>>16)<<16)
+      else
+        self.selector_id       = val.to_i-((val.to_i>>48)<<48)
+      end
+    end
+  end
+
+  def get
+    if self.classification_id == 20
+      self.classification_id.to_s + ".." + self.selector_id[:pen_id].to_s + ".." + self.selector_id[:selector_id].to_s
+    else
+      self.classification_id.to_s + ".." + self.selector_id.to_s
+    end
   end
 end
 
+
+class Appid64PanaL7Pen < BinData::Record
+  # RFC6759 chapter 4.1: PANA-L7-PEN
+  # This implements the 3 bytes default selector id length
+  # Used in Application_Id64 choice statement
+  endian :big
+  uint32 :pen_id
+  uint24 :selector_id
+end
+
 class Application_Id64 < BinData::Primitive
   endian :big
   uint8  :classification_id
-  uint56 :selector_id
+  choice :selector_id, :selection => :classification_id do
+    # for classification engine id 20 we switch to Appid64PanaL7Pen to decode
+    appid64_pana_l7_pen 20
+    uint56              :default
+  end
 
   def set(val)
     unless val.nil?
       self.classification_id=val.to_i<<56
-      self.selector_id = val.to_i-((val.to_i>>56)<<56)
+      if self.classification_id == 20
+        # classification engine id 20 (PANA_L7_PEN) contains a 4-byte PEN:
+        self.pen_id            = val.to_i-((val.to_i>>56)<<56)>>24
+        self.selector_id       = val.to_i-((val.to_i>>24)<<24)
+      else
+        self.selector_id       = val.to_i-((val.to_i>>56)<<56)
+      end
     end
   end
 
   def get
-    self.classification_id.to_s + ":" + self.selector_id.to_s
+    if self.classification_id == 20
+      self.classification_id.to_s + ".." + self.selector_id[:pen_id].to_s + ".." + self.selector_id[:selector_id].to_s
+    else
+      self.classification_id.to_s + ".." + self.selector_id.to_s
+    end
   end
 end
 
+class Appid72PanaL7Pen < BinData::Record
+  # RFC6759 chapter 4.1: PANA-L7-PEN
+  # This implements the "application ids MAY be encoded with a larger length"
+  # Used in Application_Id72 choice statement
+  endian :big
+  uint32 :pen_id
+  uint32 :selector_id
+end
+
 class Application_Id72 < BinData::Primitive
   endian :big
   uint8  :classification_id
-  uint64 :selector_id
+  choice :selector_id, :selection => :classification_id do
+    # for classification engine id 20 we switch to Appid72PanaL7Pen to decode
+    appid72_pana_l7_pen 20
+    uint64              :default
+  end
 
   def set(val)
     unless val.nil?
-      self.classification_id=val.to_i<<64
-      self.selector_id = val.to_i-((val.to_i>>64)<<64)
+      self.classification_id   = val.to_i<<64
+      if self.classification_id == 20 
+        # classification engine id 20 (PANA_L7_PEN) contains a 4-byte PEN:
+        self.pen_id            = val.to_i-((val.to_i>>64)<<64)>>32
+        self.selector_id       = val.to_i-((val.to_i>>32)<<32)
+      else
+        self.selector_id       = val.to_i-((val.to_i>>64)<<64)
+      end
     end
   end
 
   def get
-    self.classification_id.to_s + ":" + self.selector_id.to_s
+    if self.classification_id == 20
+      self.classification_id.to_s + ".." + self.selector_id[:pen_id].to_s + ".." + self.selector_id[:selector_id].to_s
+    else
+      self.classification_id.to_s + ".." + self.selector_id.to_s
+    end
   end
 end
 

data/logstash-codec-netflow.gemspec

@@ -1,7 +1,7 @@
 Gem::Specification.new do |s|
 
   s.name            = 'logstash-codec-netflow'
-  s.version         = '3.14.1'
+  s.version         = '4.0.0'
   s.licenses        = ['Apache License (2.0)']
   s.summary         = "Reads Netflow v5, Netflow v9 and IPFIX data"
   s.description     = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program"

data/spec/codecs/netflow_spec.rb

@@ -1024,8 +1024,6 @@ describe LogStash::Codecs::Netflow do
 
   end
 
-
-
   context "Netflow 9 IE150 IE151" do
     let(:data) do
       packets = []
@@ -1103,6 +1101,66 @@ describe LogStash::Codecs::Netflow do
 
   end
 
+
+  context "Netflow 9 Fortigate FortiOS 54x appid" do
+    let(:data) do
+      packets = []
+      packets << IO.read(File.join(File.dirname(__FILE__), "netflow9_test_fortigate_fortios_542_appid_tpl258-269.dat"), :mode => "rb")
+      packets << IO.read(File.join(File.dirname(__FILE__), "netflow9_test_fortigate_fortios_542_appid_data258_262.dat"), :mode => "rb")
+    end
+
+    let(:json_events) do
+      events = []
+      events << <<-END
+      {
+        "netflow": {
+          "output_snmp": 2,
+          "forwarding_status": {
+            "reason": 0,
+            "status": 1
+          },
+          "xlate_src_port": 45380,
+          "in_pkts": 6,
+          "ipv4_dst_addr": "182.50.136.239",
+          "first_switched": "2018-05-11T00:54:10.999Z",
+          "flowset_id": 262,
+          "l4_src_port": 45380,
+          "xlate_dst_port": 0,
+          "version": 9,
+          "application_id": "20..12356..36660",
+          "flow_seq_num": 350,
+          "ipv4_src_addr": "192.168.100.151",
+          "in_bytes": 748,
+          "protocol": 6,
+          "flow_end_reason": 3,
+          "last_switched": "2018-05-11T00:54:10.999Z",
+          "input_snmp": 8,
+          "out_pkts": 6,
+          "out_bytes": 748,
+          "xlate_src_addr_ipv4": "10.0.0.250",
+          "xlate_dst_addr_ipv4": "0.0.0.0",
+          "l4_dst_port": 80
+        },
+        "@timestamp": "2018-05-11T00:54:11.000Z",
+        "@version": "1"
+      }
+      END
+      events.map{|event| event.gsub(/\s+/, "")}
+    end
+
+    it "should decode raw data" do
+      expect(decode.size).to eq(17)
+      expect(decode[1].get("[netflow][application_id]")).to eq("20..12356..40568")
+      expect(decode[2].get("[netflow][application_id]")).to eq("20..12356..40568")
+      expect(decode[16].get("[netflow][application_id]")).to eq("20..12356..0")
+    end
+
+    it "should serialize to json" do
+      expect(JSON.parse(decode[0].to_json)).to eq(JSON.parse(json_events[0]))
+    end
+
+  end
+
   context "IPFIX Nokia BRAS" do
     let(:data) do
       packets = []
@@ -1352,7 +1410,7 @@ describe LogStash::Codecs::Netflow do
             "l4_src_port": 0,
             "nprobe_proto_name": "\u0000\u00c1\u0000\u0000\u0001\u00ac\u0010\u0000d\u00e4O\u00ef\u00ff\u00ff\u00fa\u0007",
             "version": 9,
-            "application_id": "0:82",
+            "application_id": "0..82",
             "flow_seq_num": 2,
             "ipv4_src_addr": "0.0.0.0",
             "protocol": 0,
@@ -1372,7 +1430,7 @@ describe LogStash::Codecs::Netflow do
     it "should decode raw data" do
       expect(decode.size).to eq(1)
       expect(decode[0].get("[netflow][nprobe_proto]")).to eq(82)
-      expect(decode[0].get("[netflow][application_id]")).to eq("0:82")
+      expect(decode[0].get("[netflow][application_id]")).to eq("0..82")
       expect(decode[0].get("[netflow][in_bytes]")).to eq(82)
     end
 
@@ -2291,7 +2349,7 @@ describe LogStash::Codecs::Netflow do
             "application_description": "ARGUS",
             "flowset_id": 260,
             "version": 9,
-            "application_id": "1:13"
+            "application_id": "1..13"
           },
           "@timestamp": "2017-02-14T11:09:59.000Z",
           "@version": "1"
@@ -2302,7 +2360,7 @@ describe LogStash::Codecs::Netflow do
 
     it "should decode raw data" do
       expect(decode.size).to eq(15)
-      expect(decode[14].get("[netflow][application_id]")).to eq("1:13")
+      expect(decode[14].get("[netflow][application_id]")).to eq("1..13")
       expect(decode[14].get("[netflow][application_description]")).to eq("ARGUS")
     end
 
@@ -2345,7 +2403,7 @@ describe LogStash::Codecs::Netflow do
             "udp_dst_port": 161,
             "src_mask": 0,
             "version": 9,
-            "application_id": "5:38",
+            "application_id": "5..38",
             "flow_seq_num": 1509134,
             "ipv4_src_addr": "10.10.172.60",
             "in_src_mac": "00:18:19:9e:6c:01",
@@ -2362,7 +2420,7 @@ describe LogStash::Codecs::Netflow do
 
     it "should decode raw data" do
       expect(decode.size).to eq(5)
-      expect(decode[4].get("[netflow][application_id]")).to eq("5:38")
+      expect(decode[4].get("[netflow][application_id]")).to eq("5..38")
     end
 
     it "should serialize to json" do
@@ -2388,7 +2446,7 @@ describe LogStash::Codecs::Netflow do
             "staMacAddress": "34:02:86:75:c0:51",
             "flowset_id": 261,
             "version": 9,
-            "application_id": "13:431",
+            "application_id": "13..431",
             "flow_seq_num": 78,
             "in_bytes": 80973880,
             "postIpDiffServCodePoint": 0,
@@ -2405,7 +2463,7 @@ describe LogStash::Codecs::Netflow do
 
     it "should decode raw data" do
       expect(decode.size).to eq(19)
-      expect(decode[18].get("[netflow][application_id]")).to eq("13:431")
+      expect(decode[18].get("[netflow][application_id]")).to eq("13..431")
     end
 
     it "should serialize to json" do

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: logstash-codec-netflow
 version: !ruby/object:Gem::Version
-  version: 3.14.1
+  version: 4.0.0
 platform: ruby
 authors:
 - Elastic
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2018-05-23 00:00:00.000000000 Z
+date: 2018-05-31 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
@@ -138,6 +138,8 @@ files:
 - spec/codecs/netflow9_test_fortigate_fortios_521_data256.dat
 - spec/codecs/netflow9_test_fortigate_fortios_521_data257.dat
 - spec/codecs/netflow9_test_fortigate_fortios_521_tpl.dat
+- spec/codecs/netflow9_test_fortigate_fortios_542_appid_data258_262.dat
+- spec/codecs/netflow9_test_fortigate_fortios_542_appid_tpl258-269.dat
 - spec/codecs/netflow9_test_h3c_data3281.dat
 - spec/codecs/netflow9_test_h3c_tpl3281.dat
 - spec/codecs/netflow9_test_huawei_netstream_data.dat
@@ -254,6 +256,8 @@ test_files:
 - spec/codecs/netflow9_test_fortigate_fortios_521_data256.dat
 - spec/codecs/netflow9_test_fortigate_fortios_521_data257.dat
 - spec/codecs/netflow9_test_fortigate_fortios_521_tpl.dat
+- spec/codecs/netflow9_test_fortigate_fortios_542_appid_data258_262.dat
+- spec/codecs/netflow9_test_fortigate_fortios_542_appid_tpl258-269.dat
 - spec/codecs/netflow9_test_h3c_data3281.dat
 - spec/codecs/netflow9_test_h3c_tpl3281.dat
 - spec/codecs/netflow9_test_huawei_netstream_data.dat