logstash-codec-netflow 2.1.1 → 3.0.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: fb68c1b451df00dd3cc4fb9a597b91cfa4892f23
-  data.tar.gz: b1ad1a99a601cc04f7694132154bc49d5b897912
+  metadata.gz: 95e7e7d16a47ec1ae4535979d9792b6051e6d36f
+  data.tar.gz: b101a6c29ebc0fa9d65ade94fb14160cd7a1df33
 SHA512:
-  metadata.gz: d591fd1d73dff5e68763d251182f79a8b73c1d32d2b7b16b0630d965b150d810b519be201ed079f29a1858cb59b239054938ce65f4b5e9faab39f12dea2e23e5
-  data.tar.gz: 996c051eed7e3fc779ac08dc3b076192360aa6365e2bc0bdd8175fee3141bb8d60f1ab7e2de417e6371c05720a03fa04e65e67a4257ec67b0504a80304754e7c
+  metadata.gz: 776e788d36ccdb7c30844b7f4b7925eb7c4fc5a5c40371b3330376bb00b659a4338a312160634813dfe4d6f14d98fbf4f9d83c9da3533ec954c8460b749a7e04
+  data.tar.gz: 135bdba032f43b7855c4bf63fcf3468b40eb8faaefbb70ef9ada0ff63e2561cc230dcc87671d7f92afac0d702b1b406cccf46192ac4f12a15a351c46bb897f80

data/CHANGELOG.md

@@ -1,30 +1,13 @@
-## 2.1.1
-
-  - Small update due to breaking change in BinData gem (issue #41)
-
-## 2.1.0
-
-  - Added IPFIX support
-  - Fixed exception if Netflow data contains MAC addresses (issue #26, issue #34)
-  - Fixed exceptions when receiving invalid Netflow v5 and v9 data (issue #17, issue 18)
-  - Fixed decoding Netflow templates from multiple (non-identical) exporters
-  - Add support for Cisco ASA fields
-  - Add support for Netflow 9 options template with scope fields 
-
+## 3.0.0
+  - Update the plugin to the version 2.0 of the plugin api, this change is required for Logstash 5.0 compatibility. See https://github.com/elastic/logstash/issues/5141
 # 2.0.5
-
   - Depend on logstash-core-plugin-api instead of logstash-core, removing the need to mass update plugins on major releases of logstash
-
 # 2.0.4
-
   - New dependency requirements for logstash-core for the 5.0 release
-
 ## 2.0.3
-
  - Fixed JSON compare flaw in specs
 
 ## 2.0.0
-
  - Plugins were updated to follow the new shutdown semantic, this mainly allows Logstash to instruct input plugins to terminate gracefully,
    instead of using Thread.raise on the plugins' threads. Ref: https://github.com/elastic/logstash/pull/3895
  - Dependency on logstash-core update to 2.0

data/CONTRIBUTORS

@@ -3,24 +3,12 @@ reports, or in general have helped logstash along its way.
 
 Contributors:
 * Aaron Mildenstein (untergeek)
-* Adam Kaminski (thimslugga)
 * Colin Surprenant (colinsurprenant)
 * Jordan Sissel (jordansissel)
-* Jorrit Folmer (jorritfolmer)
 * Matt Dainty (bodgit)
-* Paul Warren (pwarren)
 * Pier-Hugues Pellerin (ph)
-* Pulkit Agrawal (propulkit)
 * Richard Pijnenburg (electrical)
-* Salvador Ferrer (salva-ferrer)
-* Will Rigby (wrigby)
-* Rojuinex
 * debadair
-* hkshirish
-* jstopinsek
-
-Maintainer:
-* Jorrit Folmer (jorritfolmer)
 
 Note: If you've sent us patches, bug reports, or otherwise contributed to
 Logstash, and you aren't on the list above and want to be, please let us know

data/Gemfile

@@ -1,2 +1,4 @@
 source 'https://rubygems.org'
+
+# Specify your gem's dependencies in logstash-mass_effect.gemspec
 gemspec

data/lib/logstash/codecs/netflow.rb

@@ -3,59 +3,7 @@ require "logstash/filters/base"
 require "logstash/namespace"
 require "logstash/timestamp"
 
-# The "netflow" codec is used for decoding Netflow v5/v9/v10 (IPFIX) flows.
-#
-# ==== Supported Netflow/IPFIX exporters
-#
-# The following Netflow/IPFIX exporters are known to work with the most recent version of the netflow codec:
-#
-# [cols="6,^2,^2,^2,12",options="header"]
-# |===========================================================================================
-# |Netflow exporter | v5 | v9 | IPFIX | Remarks
-# |Softflowd        |  y | y  |   y   | IPFIX supported in https://github.com/djmdjm/softflowd
-# |nProbe           |  y | y  |   y   |  
-# |ipt_NETFLOW      |  y | y  |   y   |
-# |Cisco ASA        |    | y  |       |  
-# |Cisco IOS 12.x   |    | y  |       |  
-# |fprobe           |  y |    |       |
-# |===========================================================================================
-#
-# ==== Usage
-#
-# Example Logstash configuration:
-#
-# [source]
-# -----------------------------
-# input {
-#   udp {
-#     host => localhost
-#     port => 2055
-#     codec => netflow {
-#       versions => [5, 9]
-#     }
-#     type => netflow
-#   }
-#   udp {
-#     host => localhost
-#         port => 4739
-#         codec => netflow {
-#       versions => [10]
-#       target => ipfix
-#     }
-#     type => ipfix
-#   }
-#   tcp {
-#     host => localhost
-#     port => 4739
-#     codec => netflow {
-#       versions => [10]
-#           target => ipfix
-#     }
-#     type => ipfix
-#   }
-# }
-# -----------------------------
-
+# The "netflow" codec is for decoding Netflow v5/v9 flows.
 class LogStash::Codecs::Netflow < LogStash::Codecs::Base
   config_name "netflow"
 
@@ -66,7 +14,7 @@ class LogStash::Codecs::Netflow < LogStash::Codecs::Base
   config :target, :validate => :string, :default => "netflow"
 
   # Specify which Netflow versions you will accept.
-  config :versions, :validate => :array, :default => [5, 9, 10]
+  config :versions, :validate => :array, :default => [5, 9]
 
   # Override YAML file containing Netflow field definitions
   #
@@ -83,25 +31,7 @@ class LogStash::Codecs::Netflow < LogStash::Codecs::Base
   #    - :skip
   #
   # See <https://github.com/logstash-plugins/logstash-codec-netflow/blob/master/lib/logstash/codecs/netflow/netflow.yaml> for the base set.
-  config :netflow_definitions, :validate => :path
-
-  # Override YAML file containing IPFIX field definitions
-  #
-  # Very similar to the Netflow version except there is a top level Private
-  # Enterprise Number (PEN) key added:
-  #
-  #    ---
-  #    pen:
-  #      id:
-  #      - :uintN or :ip4_addr or :ip6_addr or :mac_addr or :string
-  #      - :name
-  #      id:
-  #      - :skip
-  #
-  # There is an implicit PEN 0 for the standard fields.
-  #
-  # See <https://github.com/logstash-plugins/logstash-codec-netflow/blob/master/lib/logstash/codecs/netflow/ipfix.yaml> for the base set.
-  config :ipfix_definitions, :validate => :path
+  config :definitions, :validate => :path
 
   NETFLOW5_FIELDS = ['version', 'flow_seq_num', 'engine_type', 'engine_id', 'sampling_algorithm', 'sampling_interval', 'flow_records']
   NETFLOW9_FIELDS = ['version', 'flow_seq_num']
@@ -112,7 +42,6 @@ class LogStash::Codecs::Netflow < LogStash::Codecs::Base
     4 => :scope_netflow_cache,
     5 => :scope_template,
   }
-  IPFIX_FIELDS = ['version']
   SWITCHED = /_switched$/
   FLOWSET_ID = "flowset_id"
 
@@ -123,16 +52,26 @@ class LogStash::Codecs::Netflow < LogStash::Codecs::Base
 
   def register
     require "logstash/codecs/netflow/util"
-    @netflow_templates = Vash.new()
-    @ipfix_templates = Vash.new()
+    @templates = Vash.new()
 
     # Path to default Netflow v9 field definitions
     filename = ::File.expand_path('netflow/netflow.yaml', ::File.dirname(__FILE__))
-    @netflow_fields = load_definitions(filename, @netflow_definitions)
 
-    # Path to default IPFIX field definitions
-    filename = ::File.expand_path('netflow/ipfix.yaml', ::File.dirname(__FILE__))
-    @ipfix_fields = load_definitions(filename, @ipfix_definitions)
+    begin
+      @fields = YAML.load_file(filename)
+    rescue Exception => e
+      raise "#{self.class.name}: Bad syntax in definitions file #{filename}"
+    end
+
+    # Allow the user to augment/override/rename the supported Netflow fields
+    if @definitions
+      raise "#{self.class.name}: definitions file #{@definitions} does not exists" unless File.exists?(@definitions)
+      begin
+        @fields.merge!(YAML.load_file(@definitions))
+      rescue Exception => e
+        raise "#{self.class.name}: Bad syntax in definitions file #{@definitions}"
+      end
+    end
   end # def register
 
   def decode(payload, metadata = nil, &block)
@@ -157,11 +96,6 @@ class LogStash::Codecs::Netflow < LogStash::Codecs::Base
           decode_netflow9(flowset, record).each{|event| yield(event)}
         end
       end
-    elsif header.version == 10
-      flowset = IpfixPDU.read(payload)
-      flowset.records.each do |record|
-        decode_ipfix(flowset, record).each { |event| yield(event) }
-      end
     else
       @logger.warn("Unsupported Netflow version v#{header.version}")
     end
@@ -217,7 +151,7 @@ class LogStash::Codecs::Netflow < LogStash::Codecs::Base
       record.flowset_data.templates.each do |template|
         catch (:field) do
           fields = []
-          template.record_fields.each do |field|
+          template.fields.each do |field|
             entry = netflow_field_for(field.field_type, field.field_length)
             throw :field unless entry
             fields += entry
@@ -229,9 +163,9 @@ class LogStash::Codecs::Netflow < LogStash::Codecs::Base
           else
             key = "#{flowset.source_id}|#{template.template_id}"
           end
-          @netflow_templates[key, @cache_ttl] = BinData::Struct.new(:endian => :big, :fields => fields)
+          @templates[key, @cache_ttl] = BinData::Struct.new(:endian => :big, :fields => fields)
           # Purge any expired templates
-          @netflow_templates.cleanup!
+          @templates.cleanup!
         end
       end
     when 1
@@ -254,9 +188,9 @@ class LogStash::Codecs::Netflow < LogStash::Codecs::Base
           else
             key = "#{flowset.source_id}|#{template.template_id}"
           end
-          @netflow_templates[key, @cache_ttl] = BinData::Struct.new(:endian => :big, :fields => fields)
+          @templates[key, @cache_ttl] = BinData::Struct.new(:endian => :big, :fields => fields)
           # Purge any expired templates
-          @netflow_templates.cleanup!
+          @templates.cleanup!
         end
       end
     when 256..65535
@@ -267,7 +201,7 @@ class LogStash::Codecs::Netflow < LogStash::Codecs::Base
       else
         key = "#{flowset.source_id}|#{record.flowset_id}"
       end
-      template = @netflow_templates[key]
+      template = @templates[key]
 
       unless template
         #@logger.warn("No matching template for flow id #{record.flowset_id} from #{event["source"]}")
@@ -324,164 +258,14 @@ class LogStash::Codecs::Netflow < LogStash::Codecs::Base
     @logger.warn("Invalid netflow packet received (#{e})")
   end
 
-  def decode_ipfix(flowset, record)
-    events = []
-
-    case record.flowset_id
-    when 2
-      # Template flowset
-      record.flowset_data.templates.each do |template|
-        catch (:field) do
-          fields = []
-          template.record_fields.each do |field|
-            field_type = field.field_type
-            field_length = field.field_length
-            enterprise_id = field.enterprise ? field.enterprise_id : 0
-
-            if field.field_length == 0xffff
-              # FIXME
-              @logger.warn("Cowardly refusing to deal with variable length encoded field", :type => field_type, :enterprise => enterprise_id)
-              throw :field
-            end
-
-            if enterprise_id == 0
-              case field_type
-              when 291, 292, 293
-                # FIXME
-                @logger.warn("Cowardly refusing to deal with complex data types", :type => field_type, :enterprise => enterprise_id)
-                throw :field
-              end
-            end
-
-            entry = ipfix_field_for(field_type, enterprise_id, field.field_length)
-            throw :field unless entry
-            fields += entry
-          end
-          # FIXME Source IP address required in key
-          key = "#{flowset.observation_domain_id}|#{template.template_id}"
-          @ipfix_templates[key, @cache_ttl] = BinData::Struct.new(:endian => :big, :fields => fields)
-          # Purge any expired templates
-          @ipfix_templates.cleanup!
-        end
-      end
-    when 3
-      # Options template flowset
-      record.flowset_data.templates.each do |template|
-        catch (:field) do
-          fields = []
-          (template.scope_fields.to_ary + template.option_fields.to_ary).each do |field|
-            field_type = field.field_type
-            field_length = field.field_length
-            enterprise_id = field.enterprise ? field.enterprise_id : 0
-
-            if field.field_length == 0xffff
-              # FIXME
-              @logger.warn("Cowardly refusing to deal with variable length encoded field", :type => field_type, :enterprise => enterprise_id)
-              throw :field
-            end
-
-            if enterprise_id == 0
-              case field_type
-              when 291, 292, 293
-                # FIXME
-                @logger.warn("Cowardly refusing to deal with complex data types", :type => field_type, :enterprise => enterprise_id)
-                throw :field
-              end
-            end
-
-            entry = ipfix_field_for(field_type, enterprise_id, field.field_length)
-            throw :field unless entry
-            fields += entry
-          end
-          # FIXME Source IP address required in key
-          key = "#{flowset.observation_domain_id}|#{template.template_id}"
-          @ipfix_templates[key, @cache_ttl] = BinData::Struct.new(:endian => :big, :fields => fields)
-          # Purge any expired templates
-          @ipfix_templates.cleanup!
-        end
-      end
-    when 256..65535
-      # Data flowset
-      key = "#{flowset.observation_domain_id}|#{record.flowset_id}"
-      template = @ipfix_templates[key]
-
-      unless template
-        @logger.warn("No matching template for flow id #{record.flowset_id}")
-        next
-      end
-
-      array = BinData::Array.new(:type => template, :read_until => :eof)
-      records = array.read(record.flowset_data)
-
-      records.each do |r|
-        event = {
-          LogStash::Event::TIMESTAMP => LogStash::Timestamp.at(flowset.unix_sec),
-          @target => {}
-        }
-
-        IPFIX_FIELDS.each do |f|
-          event[@target][f] = flowset[f].snapshot
-        end
-
-        r.each_pair do |k, v|
-          case k.to_s
-          when /^flow(?:Start|End)Seconds$/
-            event[@target][k.to_s] = LogStash::Timestamp.at(v.snapshot).to_iso8601
-          when /^flow(?:Start|End)(Milli|Micro|Nano)seconds$/
-            divisor =
-              case $1
-              when 'Milli'
-                1_000
-              when 'Micro'
-                1_000_000
-              when 'Nano'
-                1_000_000_000
-              end
-            event[@target][k.to_s] = LogStash::Timestamp.at(v.snapshot.to_f / divisor).to_iso8601
-          else
-            event[@target][k.to_s] = v.snapshot
-          end
-        end
-
-        events << LogStash::Event.new(event)
-      end
-    else
-      @logger.warn("Unsupported flowset id #{record.flowset_id}")
-    end
-
-    events
-  rescue BinData::ValidityError => e
-    @logger.warn("Invalid IPFIX packet received (#{e})")
-  end
-
-  def load_definitions(defaults, extra)
-    begin
-      fields = YAML.load_file(defaults)
-    rescue Exception => e
-      raise "#{self.class.name}: Bad syntax in definitions file #{defaults}"
-    end
-
-    # Allow the user to augment/override/rename the default fields
-    if extra
-      raise "#{self.class.name}: definitions file #{extra} does not exist" unless File.exists?(extra)
-      begin
-        fields.merge!(YAML.load_file(extra))
-      rescue Exception => e
-        raise "#{self.class.name}: Bad syntax in definitions file #{extra}"
-      end
-    end
-
-    fields
-  end
-
   def uint_field(length, default)
     # If length is 4, return :uint32, etc. and use default if length is 0
     ("uint" + (((length > 0) ? length : default) * 8).to_s).to_sym
   end # def uint_field
 
   def netflow_field_for(type, length)
-    if @netflow_fields.include?(type)
-      field = @netflow_fields[type].clone
+    if @fields.include?(type)
+      field = @fields[type].clone
       if field.is_a?(Array)
 
         field[0] = uint_field(length, field[0]) if field[0].is_a?(Integer)
@@ -507,38 +291,4 @@ class LogStash::Codecs::Netflow < LogStash::Codecs::Base
       nil
     end
   end # def netflow_field_for
-
-  def ipfix_field_for(type, enterprise, length)
-    if @ipfix_fields.include?(enterprise)
-      if @ipfix_fields[enterprise].include?(type)
-        field = @ipfix_fields[enterprise][type].clone
-      else
-        @logger.warn("Unsupported enterprise field", :type => type, :enterprise => enterprise, :length => length)
-      end
-    else
-      @logger.warn("Unsupported enterprise", :enterprise => enterprise)
-    end
-
-    return nil unless field
-
-    if field.is_a?(Array)
-      case field[0]
-      when :skip
-        field += [nil, {:length => length}]
-      when :string
-        field += [{:length => length, :trim_padding => true}]
-      when :uint64
-        field[0] = uint_field(length, 8)
-      when :uint32
-        field[0] = uint_field(length, 4)
-      when :uint16
-        field[0] = uint_field(length, 2)
-      end
-
-      @logger.debug("Definition complete", :field => field)
-      [field]
-    else
-      @logger.warn("Definition should be an array", :field => field)
-    end
-  end
 end # class LogStash::Filters::Netflow