logstash-codec-netflow 3.0.0 → 3.1.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 95e7e7d16a47ec1ae4535979d9792b6051e6d36f
-  data.tar.gz: b101a6c29ebc0fa9d65ade94fb14160cd7a1df33
+  metadata.gz: 39f14894de7025536b3188dc478d47f95f63fee0
+  data.tar.gz: 10b03be93979be247036c02dc77e8dcf04e9bfed
 SHA512:
-  metadata.gz: 776e788d36ccdb7c30844b7f4b7925eb7c4fc5a5c40371b3330376bb00b659a4338a312160634813dfe4d6f14d98fbf4f9d83c9da3533ec954c8460b749a7e04
-  data.tar.gz: 135bdba032f43b7855c4bf63fcf3468b40eb8faaefbb70ef9ada0ff63e2561cc230dcc87671d7f92afac0d702b1b406cccf46192ac4f12a15a351c46bb897f80
+  metadata.gz: 4e834252bcf93d1483a7d65bb2327420401a541531861a4bc3daeebda3e082ff5d359041af2d39b31f224e9a93a2aca8ec291f30a939b66d4cf8c71c80d798f2
+  data.tar.gz: 3f3bd7d64eee91851be697ace496edce09598d02e3799f4f1fb723224aa1c6a6fcb07dde053798bc35541a2d215276a6c1e2375045614a7cb4a467b2fc004d03

data/CHANGELOG.md

@@ -1,5 +1,14 @@
+## 3.1.0
+  - Added IPFIX support
+## 3.0.1
+  - Republish all the gems under jruby.
 ## 3.0.0
   - Update the plugin to the version 2.0 of the plugin api, this change is required for Logstash 5.0 compatibility. See https://github.com/elastic/logstash/issues/5141
+  - Fixed exception if Netflow data contains MAC addresses (issue #26, issue #34)
+  - Fixed exceptions when receiving invalid Netflow v5 and v9 data (issue #17, issue 18)
+  - Fixed decoding Netflow templates from multiple (non-identical) exporters
+  - Add support for Cisco ASA fields
+  - Add support for Netflow 9 options template with scope fields 
 # 2.0.5
   - Depend on logstash-core-plugin-api instead of logstash-core, removing the need to mass update plugins on major releases of logstash
 # 2.0.4

data/CONTRIBUTORS

@@ -3,12 +3,24 @@ reports, or in general have helped logstash along its way.
 
 Contributors:
 * Aaron Mildenstein (untergeek)
+* Adam Kaminski (thimslugga)
 * Colin Surprenant (colinsurprenant)
 * Jordan Sissel (jordansissel)
+* Jorrit Folmer (jorritfolmer)
 * Matt Dainty (bodgit)
+* Paul Warren (pwarren)
 * Pier-Hugues Pellerin (ph)
+* Pulkit Agrawal (propulkit)
 * Richard Pijnenburg (electrical)
+* Salvador Ferrer (salva-ferrer)
+* Will Rigby (wrigby)
+* Rojuinex
 * debadair
+* hkshirish
+* jstopinsek
+
+Maintainer:
+* Jorrit Folmer (jorritfolmer)
 
 Note: If you've sent us patches, bug reports, or otherwise contributed to
 Logstash, and you aren't on the list above and want to be, please let us know

data/lib/logstash/codecs/netflow.rb

@@ -3,7 +3,59 @@ require "logstash/filters/base"
 require "logstash/namespace"
 require "logstash/timestamp"
 
-# The "netflow" codec is for decoding Netflow v5/v9 flows.
+# The "netflow" codec is used for decoding Netflow v5/v9/v10 (IPFIX) flows.
+#
+# ==== Supported Netflow/IPFIX exporters
+#
+# The following Netflow/IPFIX exporters are known to work with the most recent version of the netflow codec:
+#
+# [cols="6,^2,^2,^2,12",options="header"]
+# |===========================================================================================
+# |Netflow exporter | v5 | v9 | IPFIX | Remarks
+# |Softflowd        |  y | y  |   y   | IPFIX supported in https://github.com/djmdjm/softflowd
+# |nProbe           |  y | y  |   y   |  
+# |ipt_NETFLOW      |  y | y  |   y   |
+# |Cisco ASA        |    | y  |       |  
+# |Cisco IOS 12.x   |    | y  |       |  
+# |fprobe           |  y |    |       |
+# |===========================================================================================
+#
+# ==== Usage
+#
+# Example Logstash configuration:
+#
+# [source]
+# -----------------------------
+# input {
+#   udp {
+#     host => localhost
+#     port => 2055
+#     codec => netflow {
+#       versions => [5, 9]
+#     }
+#     type => netflow
+#   }
+#   udp {
+#     host => localhost
+#         port => 4739
+#         codec => netflow {
+#       versions => [10]
+#       target => ipfix
+#     }
+#     type => ipfix
+#   }
+#   tcp {
+#     host => localhost
+#     port => 4739
+#     codec => netflow {
+#       versions => [10]
+#           target => ipfix
+#     }
+#     type => ipfix
+#   }
+# }
+# -----------------------------
+
 class LogStash::Codecs::Netflow < LogStash::Codecs::Base
   config_name "netflow"
 
@@ -14,7 +66,7 @@ class LogStash::Codecs::Netflow < LogStash::Codecs::Base
   config :target, :validate => :string, :default => "netflow"
 
   # Specify which Netflow versions you will accept.
-  config :versions, :validate => :array, :default => [5, 9]
+  config :versions, :validate => :array, :default => [5, 9, 10]
 
   # Override YAML file containing Netflow field definitions
   #
@@ -31,7 +83,25 @@ class LogStash::Codecs::Netflow < LogStash::Codecs::Base
   #    - :skip
   #
   # See <https://github.com/logstash-plugins/logstash-codec-netflow/blob/master/lib/logstash/codecs/netflow/netflow.yaml> for the base set.
-  config :definitions, :validate => :path
+  config :netflow_definitions, :validate => :path
+
+  # Override YAML file containing IPFIX field definitions
+  #
+  # Very similar to the Netflow version except there is a top level Private
+  # Enterprise Number (PEN) key added:
+  #
+  #    ---
+  #    pen:
+  #      id:
+  #      - :uintN or :ip4_addr or :ip6_addr or :mac_addr or :string
+  #      - :name
+  #      id:
+  #      - :skip
+  #
+  # There is an implicit PEN 0 for the standard fields.
+  #
+  # See <https://github.com/logstash-plugins/logstash-codec-netflow/blob/master/lib/logstash/codecs/netflow/ipfix.yaml> for the base set.
+  config :ipfix_definitions, :validate => :path
 
   NETFLOW5_FIELDS = ['version', 'flow_seq_num', 'engine_type', 'engine_id', 'sampling_algorithm', 'sampling_interval', 'flow_records']
   NETFLOW9_FIELDS = ['version', 'flow_seq_num']
@@ -42,6 +112,7 @@ class LogStash::Codecs::Netflow < LogStash::Codecs::Base
     4 => :scope_netflow_cache,
     5 => :scope_template,
   }
+  IPFIX_FIELDS = ['version']
   SWITCHED = /_switched$/
   FLOWSET_ID = "flowset_id"
 
@@ -52,26 +123,16 @@ class LogStash::Codecs::Netflow < LogStash::Codecs::Base
 
   def register
     require "logstash/codecs/netflow/util"
-    @templates = Vash.new()
+    @netflow_templates = Vash.new()
+    @ipfix_templates = Vash.new()
 
     # Path to default Netflow v9 field definitions
     filename = ::File.expand_path('netflow/netflow.yaml', ::File.dirname(__FILE__))
+    @netflow_fields = load_definitions(filename, @netflow_definitions)
 
-    begin
-      @fields = YAML.load_file(filename)
-    rescue Exception => e
-      raise "#{self.class.name}: Bad syntax in definitions file #{filename}"
-    end
-
-    # Allow the user to augment/override/rename the supported Netflow fields
-    if @definitions
-      raise "#{self.class.name}: definitions file #{@definitions} does not exists" unless File.exists?(@definitions)
-      begin
-        @fields.merge!(YAML.load_file(@definitions))
-      rescue Exception => e
-        raise "#{self.class.name}: Bad syntax in definitions file #{@definitions}"
-      end
-    end
+    # Path to default IPFIX field definitions
+    filename = ::File.expand_path('netflow/ipfix.yaml', ::File.dirname(__FILE__))
+    @ipfix_fields = load_definitions(filename, @ipfix_definitions)
   end # def register
 
   def decode(payload, metadata = nil, &block)
@@ -96,6 +157,11 @@ class LogStash::Codecs::Netflow < LogStash::Codecs::Base
           decode_netflow9(flowset, record).each{|event| yield(event)}
         end
       end
+    elsif header.version == 10
+      flowset = IpfixPDU.read(payload)
+      flowset.records.each do |record|
+        decode_ipfix(flowset, record).each { |event| yield(event) }
+      end
     else
       @logger.warn("Unsupported Netflow version v#{header.version}")
     end
@@ -163,9 +229,9 @@ class LogStash::Codecs::Netflow < LogStash::Codecs::Base
           else
             key = "#{flowset.source_id}|#{template.template_id}"
           end
-          @templates[key, @cache_ttl] = BinData::Struct.new(:endian => :big, :fields => fields)
+          @netflow_templates[key, @cache_ttl] = BinData::Struct.new(:endian => :big, :fields => fields)
           # Purge any expired templates
-          @templates.cleanup!
+          @netflow_templates.cleanup!
         end
       end
     when 1
@@ -188,9 +254,9 @@ class LogStash::Codecs::Netflow < LogStash::Codecs::Base
           else
             key = "#{flowset.source_id}|#{template.template_id}"
           end
-          @templates[key, @cache_ttl] = BinData::Struct.new(:endian => :big, :fields => fields)
+          @netflow_templates[key, @cache_ttl] = BinData::Struct.new(:endian => :big, :fields => fields)
           # Purge any expired templates
-          @templates.cleanup!
+          @netflow_templates.cleanup!
         end
       end
     when 256..65535
@@ -201,7 +267,7 @@ class LogStash::Codecs::Netflow < LogStash::Codecs::Base
       else
         key = "#{flowset.source_id}|#{record.flowset_id}"
       end
-      template = @templates[key]
+      template = @netflow_templates[key]
 
       unless template
         #@logger.warn("No matching template for flow id #{record.flowset_id} from #{event["source"]}")
@@ -258,14 +324,164 @@ class LogStash::Codecs::Netflow < LogStash::Codecs::Base
     @logger.warn("Invalid netflow packet received (#{e})")
   end
 
+  def decode_ipfix(flowset, record)
+    events = []
+
+    case record.flowset_id
+    when 2
+      # Template flowset
+      record.flowset_data.templates.each do |template|
+        catch (:field) do
+          fields = []
+          template.fields.each do |field|
+            field_type = field.field_type
+            field_length = field.field_length
+            enterprise_id = field.enterprise ? field.enterprise_id : 0
+
+            if field.field_length == 0xffff
+              # FIXME
+              @logger.warn("Cowardly refusing to deal with variable length encoded field", :type => field_type, :enterprise => enterprise_id)
+              throw :field
+            end
+
+            if enterprise_id == 0
+              case field_type
+              when 291, 292, 293
+                # FIXME
+                @logger.warn("Cowardly refusing to deal with complex data types", :type => field_type, :enterprise => enterprise_id)
+                throw :field
+              end
+            end
+
+            entry = ipfix_field_for(field_type, enterprise_id, field.field_length)
+            throw :field unless entry
+            fields += entry
+          end
+          # FIXME Source IP address required in key
+          key = "#{flowset.observation_domain_id}|#{template.template_id}"
+          @ipfix_templates[key, @cache_ttl] = BinData::Struct.new(:endian => :big, :fields => fields)
+          # Purge any expired templates
+          @ipfix_templates.cleanup!
+        end
+      end
+    when 3
+      # Options template flowset
+      record.flowset_data.templates.each do |template|
+        catch (:field) do
+          fields = []
+          (template.scope_fields.to_ary + template.option_fields.to_ary).each do |field|
+            field_type = field.field_type
+            field_length = field.field_length
+            enterprise_id = field.enterprise ? field.enterprise_id : 0
+
+            if field.field_length == 0xffff
+              # FIXME
+              @logger.warn("Cowardly refusing to deal with variable length encoded field", :type => field_type, :enterprise => enterprise_id)
+              throw :field
+            end
+
+            if enterprise_id == 0
+              case field_type
+              when 291, 292, 293
+                # FIXME
+                @logger.warn("Cowardly refusing to deal with complex data types", :type => field_type, :enterprise => enterprise_id)
+                throw :field
+              end
+            end
+
+            entry = ipfix_field_for(field_type, enterprise_id, field.field_length)
+            throw :field unless entry
+            fields += entry
+          end
+          # FIXME Source IP address required in key
+          key = "#{flowset.observation_domain_id}|#{template.template_id}"
+          @ipfix_templates[key, @cache_ttl] = BinData::Struct.new(:endian => :big, :fields => fields)
+          # Purge any expired templates
+          @ipfix_templates.cleanup!
+        end
+      end
+    when 256..65535
+      # Data flowset
+      key = "#{flowset.observation_domain_id}|#{record.flowset_id}"
+      template = @ipfix_templates[key]
+
+      unless template
+        @logger.warn("No matching template for flow id #{record.flowset_id}")
+        next
+      end
+
+      array = BinData::Array.new(:type => template, :read_until => :eof)
+      records = array.read(record.flowset_data)
+
+      records.each do |r|
+        event = {
+          LogStash::Event::TIMESTAMP => LogStash::Timestamp.at(flowset.unix_sec),
+          @target => {}
+        }
+
+        IPFIX_FIELDS.each do |f|
+          event[@target][f] = flowset[f].snapshot
+        end
+
+        r.each_pair do |k, v|
+          case k.to_s
+          when /^flow(?:Start|End)Seconds$/
+            event[@target][k.to_s] = LogStash::Timestamp.at(v.snapshot).to_iso8601
+          when /^flow(?:Start|End)(Milli|Micro|Nano)seconds$/
+            divisor =
+              case $1
+              when 'Milli'
+                1_000
+              when 'Micro'
+                1_000_000
+              when 'Nano'
+                1_000_000_000
+              end
+            event[@target][k.to_s] = LogStash::Timestamp.at(v.snapshot.to_f / divisor).to_iso8601
+          else
+            event[@target][k.to_s] = v.snapshot
+          end
+        end
+
+        events << LogStash::Event.new(event)
+      end
+    else
+      @logger.warn("Unsupported flowset id #{record.flowset_id}")
+    end
+
+    events
+  rescue BinData::ValidityError => e
+    @logger.warn("Invalid IPFIX packet received (#{e})")
+  end
+
+  def load_definitions(defaults, extra)
+    begin
+      fields = YAML.load_file(defaults)
+    rescue Exception => e
+      raise "#{self.class.name}: Bad syntax in definitions file #{defaults}"
+    end
+
+    # Allow the user to augment/override/rename the default fields
+    if extra
+      raise "#{self.class.name}: definitions file #{extra} does not exist" unless File.exists?(extra)
+      begin
+        fields.merge!(YAML.load_file(extra))
+      rescue Exception => e
+        raise "#{self.class.name}: Bad syntax in definitions file #{extra}"
+      end
+    end
+
+    fields
+  end
+
   def uint_field(length, default)
     # If length is 4, return :uint32, etc. and use default if length is 0
     ("uint" + (((length > 0) ? length : default) * 8).to_s).to_sym
   end # def uint_field
 
   def netflow_field_for(type, length)
-    if @fields.include?(type)
-      field = @fields[type].clone
+    if @netflow_fields.include?(type)
+      field = @netflow_fields[type].clone
       if field.is_a?(Array)
 
         field[0] = uint_field(length, field[0]) if field[0].is_a?(Integer)
@@ -291,4 +507,38 @@ class LogStash::Codecs::Netflow < LogStash::Codecs::Base
       nil
     end
   end # def netflow_field_for
+
+  def ipfix_field_for(type, enterprise, length)
+    if @ipfix_fields.include?(enterprise)
+      if @ipfix_fields[enterprise].include?(type)
+        field = @ipfix_fields[enterprise][type].clone
+      else
+        @logger.warn("Unsupported enterprise field", :type => type, :enterprise => enterprise, :length => length)
+      end
+    else
+      @logger.warn("Unsupported enterprise", :enterprise => enterprise)
+    end
+
+    return nil unless field
+
+    if field.is_a?(Array)
+      case field[0]
+      when :skip
+        field += [nil, {:length => length}]
+      when :string
+        field += [{:length => length, :trim_padding => true}]
+      when :uint64
+        field[0] = uint_field(length, 8)
+      when :uint32
+        field[0] = uint_field(length, 4)
+      when :uint16
+        field[0] = uint_field(length, 2)
+      end
+
+      @logger.debug("Definition complete", :field => field)
+      [field]
+    else
+      @logger.warn("Definition should be an array", :field => field)
+    end
+  end
 end # class LogStash::Filters::Netflow

data/lib/logstash/codecs/netflow/iana2yaml.rb

@@ -0,0 +1,77 @@
+#!/usr/bin/env ruby
+
+require 'open-uri'
+require 'csv'
+require 'yaml'
+
+# Convert IANA types to those used by BinData or created by ourselves
+def iana2bindata(type)
+  case type
+  when /^unsigned(\d+)$/
+    return 'uint' + $1
+  when /^signed(\d+)$/
+    return 'int' + $1
+  when 'float32'
+    return 'float'
+  when 'float64'
+    return 'double'
+  when 'ipv4Address'
+    return 'ip4_addr'
+  when 'ipv6Address'
+    return 'ip6_addr'
+  when 'macAddress'
+    return 'mac_addr'
+  when 'octetArray', 'string'
+    return 'string'
+  when 'dateTimeSeconds'
+    return 'uint32'
+  when 'dateTimeMilliseconds', 'dateTimeMicroseconds', 'dateTimeNanoseconds'
+    return 'uint64'
+  when 'boolean'
+    return 'uint8'
+  when 'basicList', 'subTemplateList', 'subTemplateMultiList'
+    return 'skip'
+  else
+    raise "Unknown type #{type}"
+  end
+end
+
+def iana2hash(url)
+  fields = { 0 => {} }
+
+  # Read in IANA-registered Information Elements (PEN 0)
+  CSV.new(open(url), :headers => :first_row, :converters => :numeric).each do |line|
+    # If it's not a Fixnum it's something like 'x-y' used to mark reserved blocks
+    next if line['ElementID'].class != Fixnum
+
+    # Blacklisted ID's
+    next if [0].include?(line['ElementID'])
+
+    # Skip any elements with no name
+    next unless line['Name'] and line['Data Type']
+
+    fields[0][line['ElementID']] = [iana2bindata(line['Data Type']).to_sym]
+    if fields[0][line['ElementID']][0] != :skip
+      fields[0][line['ElementID']] << line['Name'].to_sym
+    end
+  end
+
+  # Overrides
+  fields[0][210][0] = :skip # 210 is PaddingOctets so skip them properly
+  fields[0][210].delete_at(1)
+
+  # Generate the reverse PEN (PEN 29305)
+  reversed = fields[0].reject { |k|
+    # Excluded according to RFC 5103
+    [40,41,42,130,131,137,145,148,149,163,164,165,166,167,168,173,210,211,212,213,214,215,216,217,239].include?(k)
+  }.map { |k,v|
+    [k, v.size > 1 ? [v[0], ('reverse' + v[1].to_s.slice(0,1).capitalize + v[1].to_s.slice(1..-1)).to_sym] : [v[0]]]
+  }
+  fields[29305] = Hash[reversed]
+
+  return fields
+end
+
+ipfix_fields = iana2hash('http://www.iana.org/assignments/ipfix/ipfix-information-elements.csv')
+
+puts YAML.dump(ipfix_fields)