logstash-codec-netflow 3.5.2 → 3.6.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: dea55de2fab511c14f07d52974addff5c6d9418c
-  data.tar.gz: 4c6a9df871fc761a276a736049e21713d6bb93b9
+  metadata.gz: c7f0d6772017820e36296cf42defff63a59297ae
+  data.tar.gz: 029c09b6f7a1bf8d55a35fcbf3463be618584c04
 SHA512:
-  metadata.gz: 680bfc1c2de24e3f2e567b103f7c3457dc1cf415dd870760e8c729727d1911735e880a97fdd077c33f813b74591f3d09e1bf9b20b05e9f64ef4f1b0a1fa51859
-  data.tar.gz: 9164953d8fa32cd72ddfd669f372acb0348c2574810481b7fee1076f51069aecf0489f19d1ea2a7c833a9f279dc00776bb9b9c0bb48ab9c21a4bea566ae4757f
+  metadata.gz: f24188a8b785b89544fa711a531988761138ca42bd73f2900765bad3157adc927e76754b20f44f8aef853ebe7ea6ce8c286366fa42e9e0023a0d63d5740ecd3e
+  data.tar.gz: e3e67cd8db7fa5bd111024bfc33a2e3cb7b004bd009610d11580737753540ada6de5c1d9ef5286fadf94d6c1856829897810a9c60b51f6a30264f3b491a615e8

data/CHANGELOG.md

@@ -1,4 +1,10 @@
+## 3.6.0
+
+  - Added support for nprobe L7 DPI
+  - Added support for Fortigate FortiOS 5.4.x (application_id)
+
 ## 3.5.2
+
   - Fix some documentation issues
 
 ## 3.5.1

data/CONTRIBUTORS

@@ -11,6 +11,7 @@ Contributors:
 * Evgeniy Sudyr (ejectck)
 * G.J. Moed (gjmoed)
 * Gmoz Shih
+* Jason Keller (jasonkeller)
 * Jeremy Foran (jeremyforan)
 * Jordan Sissel (jordansissel)
 * Jorrit Folmer (jorritfolmer)

data/Gemfile

@@ -9,3 +9,4 @@ if Dir.exist?(logstash_path) && use_logstash_source
   gem 'logstash-core', :path => "#{logstash_path}/logstash-core"
   gem 'logstash-core-plugin-api', :path => "#{logstash_path}/logstash-core-plugin-api"
 end
+

data/docs/index.asciidoc

@@ -39,6 +39,7 @@ The following Netflow/IPFIX exporters are known to work with the most recent ver
 |Cisco ASA                |    | y  |       |  
 |Cisco ASR                |    | y  |       |  
 |Cisco IOS 12.x           |    | y  |       |  
+|Cisco ISR w/ HSL         |    | n  |       | https://github.com/logstash-plugins/logstash-codec-netflow/issues/93
 |Cisco WLC                |    | y  |       |
 |Citrix Netscaler         |    |    |   y   | Still some unknown fields, labeled netscalerUnknown<id>
 |fprobe                   |  y |    |       |
@@ -46,7 +47,7 @@ The following Netflow/IPFIX exporters are known to work with the most recent ver
 |ipt_NETFLOW              |  y | y  |   y   |
 |Juniper MX80             |  y |    |       | SW > 12.3R8
 |Mikrotik                 |  y |    |   y   | http://wiki.mikrotik.com/wiki/Manual:IP/Traffic_Flow
-|nProbe                   |  y | y  |   y   |  
+|nProbe                   |  y | y  |   y   | L7 DPI fields now also supported
 |OpenBSD pflow            |  y | n  |   y   | http://man.openbsd.org/OpenBSD-current/man4/pflow.4
 |Softflowd                |  y | y  |   y   | IPFIX supported in https://github.com/djmdjm/softflowd
 |Streamcore Streamgroomer |    | y  |       |
@@ -68,25 +69,23 @@ input {
 }
 --------------------------
 
-For high-performance production environments the configuration below will decode up to 6000 flows/sec on an 8 CPU instance. If your total flowrate exceeds 6000 flows/sec, you should use multiple Logstash instances.
-
+For high-performance production environments the configuration below will decode up to 15000 flows/sec on a dedicated 16 CPU instance. If your total flowrate exceeds 15000 flows/sec, you should use multiple Logstash instances.
 
 [source, ruby]
 --------------------------
 input {
   udp {
     port                 => 2055
-    receive_buffer_bytes => 16777216
     codec                => netflow
-    workers              => 6
+    receive_buffer_bytes => 16777216
+    workers              => 16
   }
 --------------------------
 
-Make sure to increase the Linux kernel receive buffer limit:
+To mitigate dropped packets, make sure to increase the Linux kernel receive buffer limit:
 
  # sysctl -w net.core.rmem_max=$((1024*1024*16))
 
-
 [id="plugins-{type}s-{plugin}-options"]
 ==== Netflow Codec Configuration Options
 

data/lib/logstash/codecs/netflow.rb

@@ -244,10 +244,11 @@ class LogStash::Codecs::Netflow < LogStash::Codecs::Base
 
       length = record.flowset_length - 4
 
-      # Template shouldn't be longer than the record and there should
-      # be at most 3 padding bytes
-      if template.num_bytes > length or ! (length % template.num_bytes).between?(0, 3)
-        @logger.warn("Template length doesn't fit cleanly into flowset", :template_id => record.flowset_id, :template_length => template.num_bytes, :record_length => length)
+      # Template shouldn't be longer than the record 
+      # As fas as padding is concerned, the RFC defines a SHOULD for 4-word alignment
+      # so we won't complain about that.
+      if template.num_bytes > length
+        @logger.warn("Template length exceeds flowset length, skipping", :template_id => record.flowset_id, :template_length => template.num_bytes, :record_length => length)
         return events
       end
 
@@ -460,9 +461,12 @@ class LogStash::Codecs::Netflow < LogStash::Codecs::Base
 
         # Small bit of fixup for:
         # - skip or string field types where the length is dynamic
-	# - for uint(8|16|24|32} where we use the length as specified by the
+	# - uint(8|16|24|32} where we use the length as specified by the
 	#   template instead of the YAML (e.g. ipv6_flow_label is 3 bytes in
 	#   the YAML and Cisco doc, but Cisco ASR9k sends 4 bytes)
+	# - application_id where we use the length as specified by the 
+	#   template and map it to custom types for handling.
+	#   
 	case field[0]
         when :uint8
           field[0] = uint_field(length, field[0])
@@ -472,6 +476,24 @@ class LogStash::Codecs::Netflow < LogStash::Codecs::Base
           field[0] = uint_field(length, field[0])
         when :uint32
           field[0] = uint_field(length, field[0])
+        when :application_id
+          case length
+          when 2
+            field[0] = :Application_Id16
+          when 3
+            field[0] = :Application_Id24
+          when 4
+            field[0] = :Application_Id32
+          when 5
+            field[0] = :Application_Id40
+          when 8
+            field[0] = :Application_Id64
+          when 9
+            field[0] = :Application_Id72
+          else
+            @logger.warn("Unsupported application_id length encountered, skipping", :field => field, :length => length)
+            nil
+          end      
         when :skip
           field += [nil, {:length => length.to_i}]
         when :string

data/lib/logstash/codecs/netflow/netflow.yaml

@@ -340,6 +340,9 @@
 367:
 - :mac_addr
 - :wtpMacAddress
+372:
+- :string
+- :applicationCategoryName
 8192:
 - :uint32
 - :streamcore_wan_rtt
@@ -451,3 +454,15 @@
 40005:
 - :uint8
 - :fw_event
+56701:
+- :string
+- :app_id
+56702:
+- :string
+- :user_id
+57590:
+- :uint16
+- :nprobe_proto
+57591:
+- :string
+- :nprobe_proto_name

data/lib/logstash/codecs/netflow/util.rb

@@ -108,7 +108,7 @@ class Forwarding_Status < BinData::Record
   bit6   :reason
 end
 
-class Application_Id < BinData::Primitive
+class Application_Id16 < BinData::Primitive
   endian :big
   uint8  :classification_id
   uint24 :selector_id
@@ -121,7 +121,81 @@ class Application_Id < BinData::Primitive
   def get
     self.classification_id.to_s + ":" + self.selector_id.to_s
   end
+end
+
+class Application_Id24 < BinData::Primitive
+  endian :big
+  uint8  :classification_id
+  uint16 :selector_id
+
+  def set(val)
+    self.classification_id=val.to_i<<16
+    self.selector_id = val.to_i-((val.to_i>>16)<<16)
+  end
+
+  def get
+    self.classification_id.to_s + ":" + self.selector_id.to_s
+  end
+end
+
+class Application_Id32 < BinData::Primitive
+  endian :big
+  uint8  :classification_id
+  uint24 :selector_id
+
+  def set(val)
+    self.classification_id=val.to_i<<24
+    self.selector_id = val.to_i-((val.to_i>>24)<<24)
+  end
+
+  def get
+    self.classification_id.to_s + ":" + self.selector_id.to_s
+  end
+end
+
+class Application_Id40 < BinData::Primitive
+  endian :big
+  uint8  :classification_id
+  uint32 :selector_id
 
+  def set(val)
+    self.classification_id=val.to_i<<32
+    self.selector_id = val.to_i-((val.to_i>>32)<<32)
+  end
+
+  def get
+    self.classification_id.to_s + ":" + self.selector_id.to_s
+  end
+end
+
+class Application_Id64 < BinData::Primitive
+  endian :big
+  uint8  :classification_id
+  uint56 :selector_id
+
+  def set(val)
+    self.classification_id=val.to_i<<56
+    self.selector_id = val.to_i-((val.to_i>>56)<<56)
+  end
+
+  def get
+    self.classification_id.to_s + ":" + self.selector_id.to_s
+  end
+end
+
+class Application_Id72 < BinData::Primitive
+  endian :big
+  uint8  :classification_id
+  uint64 :selector_id
+
+  def set(val)
+    self.classification_id=val.to_i<<64
+    self.selector_id = val.to_i-((val.to_i>>64)<<64)
+  end
+
+  def get
+    self.classification_id.to_s + ":" + self.selector_id.to_s
+  end
 end
 
 class OctetArray < BinData::Primitive

data/logstash-codec-netflow.gemspec

@@ -1,7 +1,7 @@
 Gem::Specification.new do |s|
 
   s.name            = 'logstash-codec-netflow'
-  s.version         = '3.5.2'
+  s.version         = '3.6.0'
   s.licenses        = ['Apache License (2.0)']
   s.summary         = "The netflow codec is for decoding Netflow v5/v9/v10 (IPFIX) flows."
   s.description     = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program"

data/spec/codecs/netflow_spec.rb

@@ -966,6 +966,55 @@ describe LogStash::Codecs::Netflow do
 
   end
 
+  context "Netflow 9 nprobe DPI L7" do
+    let(:data) do
+      packets = []
+      packets << IO.read(File.join(File.dirname(__FILE__), "netflow9_test_nprobe_dpi.dat"), :mode => "rb")
+    end
+
+    let(:json_events) do
+      events = []
+      events << <<-END
+        {
+          "netflow": {
+            "nprobe_proto": 82,
+            "in_pkts": 1,
+            "ipv4_dst_addr": "0.0.0.0",
+            "first_switched": "1970-01-01T00:08:33.000Z",
+            "flowset_id": 256,
+            "l4_src_port": 0,
+            "nprobe_proto_name": "\u0000\u00c1\u0000\u0000\u0001\u00ac\u0010\u0000d\u00e4O\u00ef\u00ff\u00ff\u00fa\u0007",
+            "version": 9,
+            "application_id": "0:82",
+            "flow_seq_num": 2,
+            "ipv4_src_addr": "0.0.0.0",
+            "protocol": 0,
+            "in_bytes": 82,
+            "application_name": "\u0000\u0000\u0000\u0000\u0000\"\u0000\u0000\u0000\u0000\u0004",
+            "last_switched": "1970-01-01T00:08:36.000Z",
+            "l4_dst_port": 0
+          },
+          "@timestamp": "1970-01-01T00:08:22.000Z",
+          "@version": "1",
+          "host": "172.16.32.201"
+        }
+        END
+      events.map{|event| event.gsub(/\s+/, "")}
+    end
+
+    it "should decode raw data" do
+      expect(decode.size).to eq(1)
+      expect(decode[0].get("[netflow][nprobe_proto]")).to eq(82)
+      expect(decode[0].get("[netflow][application_id]")).to eq("0:82")
+      expect(decode[0].get("[netflow][in_bytes]")).to eq(82)
+    end
+
+    it "should serialize to json" do
+      # We skip this due to unprintable characters in the proto_name and application_name
+      # expect(JSON.parse(decode[0].to_json)).to eq(JSON.parse(json_events[0]))
+    end
+  end
+
   context "Netflow 9 Fortigate FortiOS 5.2.1" do
     let(:data) do
       packets = []

metadata

@@ -1,22 +1,22 @@
 --- !ruby/object:Gem::Specification
 name: logstash-codec-netflow
 version: !ruby/object:Gem::Version
-  version: 3.5.2
+  version: 3.6.0
 platform: ruby
 authors:
 - Elastic
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2017-08-15 00:00:00.000000000 Z
+date: 2017-09-28 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - '>='
       - !ruby/object:Gem::Version
         version: '1.60'
-    - - "<="
+    - - <=
       - !ruby/object:Gem::Version
         version: '2.99'
   name: logstash-core-plugin-api
@@ -24,16 +24,16 @@ dependencies:
   type: :runtime
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - '>='
       - !ruby/object:Gem::Version
         version: '1.60'
-    - - "<="
+    - - <=
       - !ruby/object:Gem::Version
         version: '2.99'
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - '>='
       - !ruby/object:Gem::Version
         version: 1.5.0
   name: bindata
@@ -41,13 +41,13 @@ dependencies:
   type: :runtime
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - '>='
       - !ruby/object:Gem::Version
         version: 1.5.0
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - '>='
       - !ruby/object:Gem::Version
         version: 1.0.0
   name: logstash-devutils
@@ -55,7 +55,7 @@ dependencies:
   type: :development
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - '>='
       - !ruby/object:Gem::Version
         version: 1.0.0
 description: This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program
@@ -124,6 +124,7 @@ files:
 - spec/codecs/netflow9_test_macaddr_data.dat
 - spec/codecs/netflow9_test_macaddr_tpl.dat
 - spec/codecs/netflow9_test_nprobe_data.dat
+- spec/codecs/netflow9_test_nprobe_dpi.dat
 - spec/codecs/netflow9_test_nprobe_tpl.dat
 - spec/codecs/netflow9_test_softflowd_tpl_data.dat
 - spec/codecs/netflow9_test_streamcore_tpl_data256.dat
@@ -145,12 +146,12 @@ require_paths:
 - lib
 required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
-  - - ">="
+  - - '>='
     - !ruby/object:Gem::Version
       version: '0'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
-  - - ">="
+  - - '>='
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
@@ -205,6 +206,7 @@ test_files:
 - spec/codecs/netflow9_test_macaddr_data.dat
 - spec/codecs/netflow9_test_macaddr_tpl.dat
 - spec/codecs/netflow9_test_nprobe_data.dat
+- spec/codecs/netflow9_test_nprobe_dpi.dat
 - spec/codecs/netflow9_test_nprobe_tpl.dat
 - spec/codecs/netflow9_test_softflowd_tpl_data.dat
 - spec/codecs/netflow9_test_streamcore_tpl_data256.dat