logstash-codec-netflow 2.0.5 → 2.1.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 0ae020958c2421c6319d6f7b0ad7d9889329d988
-  data.tar.gz: 68ed6e1be5662bf00cc4d59aae6be8315485cb33
+  metadata.gz: 918358ce2cdaac6c82befb4288a60dbf12bd582d
+  data.tar.gz: 801135d166dc217611ca6a079eee87f27df9a0d5
 SHA512:
-  metadata.gz: d93c9c1608d2d4c838d9ce4f6625461c537e8009f75730c54e4fd2ccd77a867f758562b39e9aca1e17472bb00c70d8d40b7e11b22a3f745d41271dfe99e2ace6
-  data.tar.gz: f388c2d88d5e9a21da0c4966d2c7c2780e10665b60d83ddff636e384c0b8b422e7651493fa7933c39ac88455bf3aacfdb08cfabdd72d89002d14a7019b4029aa
+  metadata.gz: 4eb4a32af83f1485e23aca07ad96795988980e4da059c328e3870fb346482d7d0709248ff3d9c78e559f85f7b26b40ca036cbf80a7e6d406b83dab844a725b33
+  data.tar.gz: 6e9b9e41db2f5df5483eb66b0657a58ea4ff2ce594d64bb22d04337004bd3e68cb96057a86273ce73d77136d9d71cac3d6e53d0841e7ef74f0d5dbd4c290c367

data/CHANGELOG.md

@@ -1,11 +1,26 @@
+## 2.1.0
+
+  - Added IPFIX support
+  - Fixed exception if Netflow data contains MAC addresses (issue #26, issue #34)
+  - Fixed exceptions when receiving invalid Netflow v5 and v9 data (issue #17, issue 18)
+  - Fixed decoding Netflow templates from multiple (non-identical) exporters
+  - Add support for Cisco ASA fields
+  - Add support for Netflow 9 options template with scope fields 
+
 # 2.0.5
+
   - Depend on logstash-core-plugin-api instead of logstash-core, removing the need to mass update plugins on major releases of logstash
+
 # 2.0.4
+
   - New dependency requirements for logstash-core for the 5.0 release
+
 ## 2.0.3
+
  - Fixed JSON compare flaw in specs
 
 ## 2.0.0
+
  - Plugins were updated to follow the new shutdown semantic, this mainly allows Logstash to instruct input plugins to terminate gracefully,
    instead of using Thread.raise on the plugins' threads. Ref: https://github.com/elastic/logstash/pull/3895
  - Dependency on logstash-core update to 2.0

data/CONTRIBUTORS

@@ -3,12 +3,24 @@ reports, or in general have helped logstash along its way.
 
 Contributors:
 * Aaron Mildenstein (untergeek)
+* Adam Kaminski (thimslugga)
 * Colin Surprenant (colinsurprenant)
 * Jordan Sissel (jordansissel)
+* Jorrit Folmer (jorritfolmer)
 * Matt Dainty (bodgit)
+* Paul Warren (pwarren)
 * Pier-Hugues Pellerin (ph)
+* Pulkit Agrawal (propulkit)
 * Richard Pijnenburg (electrical)
+* Salvador Ferrer (salva-ferrer)
+* Will Rigby (wrigby)
+* Rojuinex
 * debadair
+* hkshirish
+* jstopinsek
+
+Maintainer:
+* Jorrit Folmer (jorritfolmer)
 
 Note: If you've sent us patches, bug reports, or otherwise contributed to
 Logstash, and you aren't on the list above and want to be, please let us know

data/Gemfile

@@ -1,2 +1,2 @@
 source 'https://rubygems.org'
-gemspec
+gemspec

data/LICENSE

@@ -1,4 +1,4 @@
-Copyright (c) 2012–2015 Elasticsearch <http://www.elastic.co>
+Copyright (c) 2012–2016 Elasticsearch <http://www.elastic.co>
 
 Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.

data/README.md

@@ -1,7 +1,6 @@
 # Logstash Plugin
 
-[![Build
-Status](http://build-eu-00.elastic.co/view/LS%20Plugins/view/LS%20Codecs/job/logstash-plugin-codec-netflow-unit/badge/icon)](http://build-eu-00.elastic.co/view/LS%20Plugins/view/LS%20Codecs/job/logstash-plugin-codec-netflow-unit/)
+[![Travis Build Status](https://travis-ci.org/logstash-plugins/logstash-codec-netflow.svg)](https://travis-ci.org/logstash-plugins/logstash-codec-netflow)
 
 This is a plugin for [Logstash](https://github.com/elastic/logstash).
 
@@ -56,7 +55,12 @@ gem "logstash-filter-awesome", :path => "/your/local/logstash-filter-awesome"
 ```
 - Install plugin
 ```sh
+# Logstash 2.3 and higher
+bin/logstash-plugin install --no-verify
+
+# Prior to Logstash 2.3
 bin/plugin install --no-verify
+
 ```
 - Run Logstash with your plugin
 ```sh
@@ -74,7 +78,12 @@ gem build logstash-filter-awesome.gemspec
 ```
 - Install the plugin from the Logstash home
 ```sh
-bin/plugin install /your/local/plugin/logstash-filter-awesome.gem
+# Logstash 2.3 and higher
+bin/logstash-plugin install --no-verify
+
+# Prior to Logstash 2.3
+bin/plugin install --no-verify
+
 ```
 - Start Logstash and proceed to test the plugin
 

data/lib/logstash/codecs/netflow.rb

@@ -3,7 +3,59 @@ require "logstash/filters/base"
 require "logstash/namespace"
 require "logstash/timestamp"
 
-# The "netflow" codec is for decoding Netflow v5/v9 flows.
+# The "netflow" codec is used for decoding Netflow v5/v9/v10 (IPFIX) flows.
+#
+# ==== Supported Netflow/IPFIX exporters
+#
+# The following Netflow/IPFIX exporters are known to work with the most recent version of the netflow codec:
+#
+# [cols="6,^2,^2,^2,12",options="header"]
+# |===========================================================================================
+# |Netflow exporter | v5 | v9 | IPFIX | Remarks
+# |Softflowd        |  y | y  |   y   | IPFIX supported in https://github.com/djmdjm/softflowd
+# |nProbe           |  y | y  |   y   |  
+# |ipt_NETFLOW      |  y | y  |   y   |
+# |Cisco ASA        |    | y  |       |  
+# |Cisco IOS 12.x   |    | y  |       |  
+# |fprobe           |  y |    |       |
+# |===========================================================================================
+#
+# ==== Usage
+#
+# Example Logstash configuration:
+#
+# [source]
+# -----------------------------
+# input {
+#   udp {
+#     host => localhost
+#     port => 2055
+#     codec => netflow {
+#       versions => [5, 9]
+#     }
+#     type => netflow
+#   }
+#   udp {
+#     host => localhost
+#         port => 4739
+#         codec => netflow {
+#       versions => [10]
+#       target => ipfix
+#     }
+#     type => ipfix
+#   }
+#   tcp {
+#     host => localhost
+#     port => 4739
+#     codec => netflow {
+#       versions => [10]
+#           target => ipfix
+#     }
+#     type => ipfix
+#   }
+# }
+# -----------------------------
+
 class LogStash::Codecs::Netflow < LogStash::Codecs::Base
   config_name "netflow"
 
@@ -14,7 +66,7 @@ class LogStash::Codecs::Netflow < LogStash::Codecs::Base
   config :target, :validate => :string, :default => "netflow"
 
   # Specify which Netflow versions you will accept.
-  config :versions, :validate => :array, :default => [5, 9]
+  config :versions, :validate => :array, :default => [5, 9, 10]
 
   # Override YAML file containing Netflow field definitions
   #
@@ -31,10 +83,36 @@ class LogStash::Codecs::Netflow < LogStash::Codecs::Base
   #    - :skip
   #
   # See <https://github.com/logstash-plugins/logstash-codec-netflow/blob/master/lib/logstash/codecs/netflow/netflow.yaml> for the base set.
-  config :definitions, :validate => :path
+  config :netflow_definitions, :validate => :path
+
+  # Override YAML file containing IPFIX field definitions
+  #
+  # Very similar to the Netflow version except there is a top level Private
+  # Enterprise Number (PEN) key added:
+  #
+  #    ---
+  #    pen:
+  #      id:
+  #      - :uintN or :ip4_addr or :ip6_addr or :mac_addr or :string
+  #      - :name
+  #      id:
+  #      - :skip
+  #
+  # There is an implicit PEN 0 for the standard fields.
+  #
+  # See <https://github.com/logstash-plugins/logstash-codec-netflow/blob/master/lib/logstash/codecs/netflow/ipfix.yaml> for the base set.
+  config :ipfix_definitions, :validate => :path
 
   NETFLOW5_FIELDS = ['version', 'flow_seq_num', 'engine_type', 'engine_id', 'sampling_algorithm', 'sampling_interval', 'flow_records']
   NETFLOW9_FIELDS = ['version', 'flow_seq_num']
+  NETFLOW9_SCOPES = {
+    1 => :scope_system,
+    2 => :scope_interface,
+    3 => :scope_line_card,
+    4 => :scope_netflow_cache,
+    5 => :scope_template,
+  }
+  IPFIX_FIELDS = ['version']
   SWITCHED = /_switched$/
   FLOWSET_ID = "flowset_id"
 
@@ -45,29 +123,19 @@ class LogStash::Codecs::Netflow < LogStash::Codecs::Base
 
   def register
     require "logstash/codecs/netflow/util"
-    @templates = Vash.new()
+    @netflow_templates = Vash.new()
+    @ipfix_templates = Vash.new()
 
     # Path to default Netflow v9 field definitions
     filename = ::File.expand_path('netflow/netflow.yaml', ::File.dirname(__FILE__))
+    @netflow_fields = load_definitions(filename, @netflow_definitions)
 
-    begin
-      @fields = YAML.load_file(filename)
-    rescue Exception => e
-      raise "#{self.class.name}: Bad syntax in definitions file #{filename}"
-    end
-
-    # Allow the user to augment/override/rename the supported Netflow fields
-    if @definitions
-      raise "#{self.class.name}: definitions file #{@definitions} does not exists" unless File.exists?(@definitions)
-      begin
-        @fields.merge!(YAML.load_file(@definitions))
-      rescue Exception => e
-        raise "#{self.class.name}: Bad syntax in definitions file #{@definitions}"
-      end
-    end
+    # Path to default IPFIX field definitions
+    filename = ::File.expand_path('netflow/ipfix.yaml', ::File.dirname(__FILE__))
+    @ipfix_fields = load_definitions(filename, @ipfix_definitions)
   end # def register
 
-  def decode(payload, &block)
+  def decode(payload, metadata = nil, &block)
     header = Header.read(payload)
 
     unless @versions.include?(header.version)
@@ -83,11 +151,22 @@ class LogStash::Codecs::Netflow < LogStash::Codecs::Base
     elsif header.version == 9
       flowset = Netflow9PDU.read(payload)
       flowset.records.each do |record|
-        decode_netflow9(flowset, record).each{|event| yield(event)}
+        if metadata != nil
+          decode_netflow9(flowset, record, metadata).each{|event| yield(event)}
+        else
+          decode_netflow9(flowset, record).each{|event| yield(event)}
+        end
+      end
+    elsif header.version == 10
+      flowset = IpfixPDU.read(payload)
+      flowset.records.each do |record|
+        decode_ipfix(flowset, record).each { |event| yield(event) }
       end
     else
       @logger.warn("Unsupported Netflow version v#{header.version}")
     end
+  rescue BinData::ValidityError, IOError => e
+    @logger.warn("Invalid netflow packet received (#{e})")
   end
 
   private
@@ -125,9 +204,11 @@ class LogStash::Codecs::Netflow < LogStash::Codecs::Base
     end
 
     LogStash::Event.new(event)
+  rescue BinData::ValidityError, IOError => e
+    @logger.warn("Invalid netflow packet received (#{e})")
   end
 
-  def decode_netflow9(flowset, record)
+  def decode_netflow9(flowset, record, metadata = nil)
     events = []
 
     case record.flowset_id
@@ -143,10 +224,14 @@ class LogStash::Codecs::Netflow < LogStash::Codecs::Base
           end
           # We get this far, we have a list of fields
           #key = "#{flowset.source_id}|#{event["source"]}|#{template.template_id}"
-          key = "#{flowset.source_id}|#{template.template_id}"
-          @templates[key, @cache_ttl] = BinData::Struct.new(:endian => :big, :fields => fields)
+          if metadata != nil
+            key = "#{flowset.source_id}|#{template.template_id}|#{metadata["host"]}|#{metadata["port"]}"
+          else
+            key = "#{flowset.source_id}|#{template.template_id}"
+          end
+          @netflow_templates[key, @cache_ttl] = BinData::Struct.new(:endian => :big, :fields => fields)
           # Purge any expired templates
-          @templates.cleanup!
+          @netflow_templates.cleanup!
         end
       end
     when 1
@@ -154,6 +239,9 @@ class LogStash::Codecs::Netflow < LogStash::Codecs::Base
       record.flowset_data.templates.each do |template|
         catch (:field) do
           fields = []
+          template.scope_fields.each do |field|
+            fields << [uint_field(0, field.field_length), NETFLOW9_SCOPES[field.field_type]]
+          end
           template.option_fields.each do |field|
             entry = netflow_field_for(field.field_type, field.field_length)
             throw :field unless entry
@@ -161,17 +249,25 @@ class LogStash::Codecs::Netflow < LogStash::Codecs::Base
           end
           # We get this far, we have a list of fields
           #key = "#{flowset.source_id}|#{event["source"]}|#{template.template_id}"
-          key = "#{flowset.source_id}|#{template.template_id}"
-          @templates[key, @cache_ttl] = BinData::Struct.new(:endian => :big, :fields => fields)
+          if metadata != nil
+            key = "#{flowset.source_id}|#{template.template_id}|#{metadata["host"]}|#{metadata["port"]}"
+          else
+            key = "#{flowset.source_id}|#{template.template_id}"
+          end
+          @netflow_templates[key, @cache_ttl] = BinData::Struct.new(:endian => :big, :fields => fields)
           # Purge any expired templates
-          @templates.cleanup!
+          @netflow_templates.cleanup!
         end
       end
     when 256..65535
       # Data flowset
       #key = "#{flowset.source_id}|#{event["source"]}|#{record.flowset_id}"
-      key = "#{flowset.source_id}|#{record.flowset_id}"
-      template = @templates[key]
+      if metadata != nil
+        key = "#{flowset.source_id}|#{record.flowset_id}|#{metadata["host"]}|#{metadata["port"]}"
+      else
+        key = "#{flowset.source_id}|#{record.flowset_id}"
+      end
+      template = @netflow_templates[key]
 
       unless template
         #@logger.warn("No matching template for flow id #{record.flowset_id} from #{event["source"]}")
@@ -224,6 +320,158 @@ class LogStash::Codecs::Netflow < LogStash::Codecs::Base
     end
 
     events
+  rescue BinData::ValidityError, IOError => e
+    @logger.warn("Invalid netflow packet received (#{e})")
+  end
+
+  def decode_ipfix(flowset, record)
+    events = []
+
+    case record.flowset_id
+    when 2
+      # Template flowset
+      record.flowset_data.templates.each do |template|
+        catch (:field) do
+          fields = []
+          template.fields.each do |field|
+            field_type = field.field_type
+            field_length = field.field_length
+            enterprise_id = field.enterprise ? field.enterprise_id : 0
+
+            if field.field_length == 0xffff
+              # FIXME
+              @logger.warn("Cowardly refusing to deal with variable length encoded field", :type => field_type, :enterprise => enterprise_id)
+              throw :field
+            end
+
+            if enterprise_id == 0
+              case field_type
+              when 291, 292, 293
+                # FIXME
+                @logger.warn("Cowardly refusing to deal with complex data types", :type => field_type, :enterprise => enterprise_id)
+                throw :field
+              end
+            end
+
+            entry = ipfix_field_for(field_type, enterprise_id, field.field_length)
+            throw :field unless entry
+            fields += entry
+          end
+          # FIXME Source IP address required in key
+          key = "#{flowset.observation_domain_id}|#{template.template_id}"
+          @ipfix_templates[key, @cache_ttl] = BinData::Struct.new(:endian => :big, :fields => fields)
+          # Purge any expired templates
+          @ipfix_templates.cleanup!
+        end
+      end
+    when 3
+      # Options template flowset
+      record.flowset_data.templates.each do |template|
+        catch (:field) do
+          fields = []
+          (template.scope_fields.to_ary + template.option_fields.to_ary).each do |field|
+            field_type = field.field_type
+            field_length = field.field_length
+            enterprise_id = field.enterprise ? field.enterprise_id : 0
+
+            if field.field_length == 0xffff
+              # FIXME
+              @logger.warn("Cowardly refusing to deal with variable length encoded field", :type => field_type, :enterprise => enterprise_id)
+              throw :field
+            end
+
+            if enterprise_id == 0
+              case field_type
+              when 291, 292, 293
+                # FIXME
+                @logger.warn("Cowardly refusing to deal with complex data types", :type => field_type, :enterprise => enterprise_id)
+                throw :field
+              end
+            end
+
+            entry = ipfix_field_for(field_type, enterprise_id, field.field_length)
+            throw :field unless entry
+            fields += entry
+          end
+          # FIXME Source IP address required in key
+          key = "#{flowset.observation_domain_id}|#{template.template_id}"
+          @ipfix_templates[key, @cache_ttl] = BinData::Struct.new(:endian => :big, :fields => fields)
+          # Purge any expired templates
+          @ipfix_templates.cleanup!
+        end
+      end
+    when 256..65535
+      # Data flowset
+      key = "#{flowset.observation_domain_id}|#{record.flowset_id}"
+      template = @ipfix_templates[key]
+
+      unless template
+        @logger.warn("No matching template for flow id #{record.flowset_id}")
+        next
+      end
+
+      array = BinData::Array.new(:type => template, :read_until => :eof)
+      records = array.read(record.flowset_data)
+
+      records.each do |r|
+        event = {
+          LogStash::Event::TIMESTAMP => LogStash::Timestamp.at(flowset.unix_sec),
+          @target => {}
+        }
+
+        IPFIX_FIELDS.each do |f|
+          event[@target][f] = flowset[f].snapshot
+        end
+
+        r.each_pair do |k, v|
+          case k.to_s
+          when /^flow(?:Start|End)Seconds$/
+            event[@target][k.to_s] = LogStash::Timestamp.at(v.snapshot).to_iso8601
+          when /^flow(?:Start|End)(Milli|Micro|Nano)seconds$/
+            divisor =
+              case $1
+              when 'Milli'
+                1_000
+              when 'Micro'
+                1_000_000
+              when 'Nano'
+                1_000_000_000
+              end
+            event[@target][k.to_s] = LogStash::Timestamp.at(v.snapshot.to_f / divisor).to_iso8601
+          else
+            event[@target][k.to_s] = v.snapshot
+          end
+        end
+
+        events << LogStash::Event.new(event)
+      end
+    else
+      @logger.warn("Unsupported flowset id #{record.flowset_id}")
+    end
+
+    events
+  rescue BinData::ValidityError => e
+    @logger.warn("Invalid IPFIX packet received (#{e})")
+  end
+
+  def load_definitions(defaults, extra)
+    begin
+      fields = YAML.load_file(defaults)
+    rescue Exception => e
+      raise "#{self.class.name}: Bad syntax in definitions file #{defaults}"
+    end
+
+    # Allow the user to augment/override/rename the default fields
+    if extra
+      raise "#{self.class.name}: definitions file #{extra} does not exist" unless File.exists?(extra)
+      begin
+        fields.merge!(YAML.load_file(extra))
+      rescue Exception => e
+        raise "#{self.class.name}: Bad syntax in definitions file #{extra}"
+      end
+    end
+
+    fields
   end
 
   def uint_field(length, default)
@@ -232,8 +480,8 @@ class LogStash::Codecs::Netflow < LogStash::Codecs::Base
   end # def uint_field
 
   def netflow_field_for(type, length)
-    if @fields.include?(type)
-      field = @fields[type]
+    if @netflow_fields.include?(type)
+      field = @netflow_fields[type].clone
       if field.is_a?(Array)
 
         field[0] = uint_field(length, field[0]) if field[0].is_a?(Integer)
@@ -259,4 +507,38 @@ class LogStash::Codecs::Netflow < LogStash::Codecs::Base
       nil
     end
   end # def netflow_field_for
+
+  def ipfix_field_for(type, enterprise, length)
+    if @ipfix_fields.include?(enterprise)
+      if @ipfix_fields[enterprise].include?(type)
+        field = @ipfix_fields[enterprise][type].clone
+      else
+        @logger.warn("Unsupported enterprise field", :type => type, :enterprise => enterprise, :length => length)
+      end
+    else
+      @logger.warn("Unsupported enterprise", :enterprise => enterprise)
+    end
+
+    return nil unless field
+
+    if field.is_a?(Array)
+      case field[0]
+      when :skip
+        field += [nil, {:length => length}]
+      when :string
+        field += [{:length => length, :trim_padding => true}]
+      when :uint64
+        field[0] = uint_field(length, 8)
+      when :uint32
+        field[0] = uint_field(length, 4)
+      when :uint16
+        field[0] = uint_field(length, 2)
+      end
+
+      @logger.debug("Definition complete", :field => field)
+      [field]
+    else
+      @logger.warn("Definition should be an array", :field => field)
+    end
+  end
 end # class LogStash::Filters::Netflow