logstash-codec-netflow 3.5.0 → 3.5.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 32f71436a65fa46010df46996c298d6ee3fd69c3
-  data.tar.gz: 3546a333cad653c94bc66a9eadc7a0b84169b2cf
+  metadata.gz: 936d05f955f0c16ab55b3a6998302ca2f283c2d2
+  data.tar.gz: f6f16b2055779fe96113d8f61d281fb27e4673cf
 SHA512:
-  metadata.gz: 829dadfc0855a0185db386d859a81d151639b51ffd663ceb51bc6126ab870b0bf8a8f49fbd600f2dda17bb290a17ab469e98ae28e7039807a41b0dac4cc3c2e4
-  data.tar.gz: ef4c100952e730e612fb20c08dd624ad498e20f4de7b8e04602d93c2b162e844767d903c4554811db9e237d31512415bc19496a242cbeb1733834718d26239fc
+  metadata.gz: e1e1bcb5abd65d89dc491122bfb99bda3b4fd20a9e17cc58753779e14b4b255c43c9d3a50ede39a5e6ed51a41999131bc1c844b2ee277863318a8724d9e995be
+  data.tar.gz: 1358d6678af2221b0087953f0e327de3f7916dbbf2513872000a2ef261701695c69a55a17f505ffbeb74a5c4e30419cb6cfe2b539a8b1dc09adf567643877f97

data/CHANGELOG.md

@@ -1,3 +1,10 @@
+## 3.5.1
+
+  - Added test for Fortigate FortiOS 5.2 (Netflow v9)
+  - Added permission check to templates cache (Issue #80)
+  - Clarified confusing warning about missing templates
+  - Added test for Barracuda firewall (IPFIX)
+
 ## 3.5.0
 
   - Added support for Cisco WLC (Netflow v9)

data/CONTRIBUTORS

@@ -9,11 +9,14 @@ Contributors:
 * Diyaldine Maoulida
 * Evgeniy Sudyr (ejectck)
 * G.J. Moed (gjmoed)
+* Gmoz Shih
+* Jeremy Foran (jeremyforan)
 * Jordan Sissel (jordansissel)
 * Jorrit Folmer (jorritfolmer)
 * Keenan Tims (ktims)
 * Matt Dainty (bodgit)
 * Paul Warren (pwarren)
+* Philipp Kahr
 * Pier-Hugues Pellerin (ph)
 * Pulkit Agrawal (propulkit)
 * Raju Nair (rajutech76)

data/docs/index.asciidoc

@@ -12,7 +12,7 @@ START - GENERATED VARIABLES, DO NOT EDIT!
 END - GENERATED VARIABLES, DO NOT EDIT!
 ///////////////////////////////////////////
 
-[id="plugins-{type}-{plugin}"]
+[id="plugins-{type}s-{plugin}"]
 
 === Netflow codec plugin
 
@@ -24,56 +24,46 @@ The "netflow" codec is used for decoding Netflow v5/v9/v10 (IPFIX) flows.
 
 ==== Supported Netflow/IPFIX exporters
 
+This codec supports:
+
+* Netflow v5
+* Netflow v9
+* IPFIX
+
 The following Netflow/IPFIX exporters are known to work with the most recent version of the netflow codec:
 
 [cols="6,^2,^2,^2,12",options="header"]
 |===========================================================================================
-|Netflow exporter      | v5 | v9 | IPFIX | Remarks
-|Softflowd             |  y | y  |   y   | IPFIX supported in https://github.com/djmdjm/softflowd
-|nProbe                |  y | y  |   y   |  
-|ipt_NETFLOW           |  y | y  |   y   |
-|Cisco ASA             |    | y  |       |  
-|Cisco IOS 12.x        |    | y  |       |  
-|fprobe                |  y |    |       |
-|Juniper MX80          |  y |    |       | SW > 12.3R8
-|OpenBSD pflow         |  y | n  |   y   | http://man.openbsd.org/OpenBSD-current/man4/pflow.4
-|Mikrotik 6.35.4       |  y |    |   n   | http://wiki.mikrotik.com/wiki/Manual:IP/Traffic_Flow
-|Ubiquiti Edgerouter X |    | y  |       | With MPLS labels
-|Citrix Netscaler      |    |    |   y   | Still some unknown fields, labeled netscalerUnknown<id>
+|Netflow exporter         | v5 | v9 | IPFIX | Remarks
+|Barracuda Firewall       |    |    |   y   |
+|Cisco ASA                |    | y  |       |  
+|Cisco ASR                |    | y  |       |  
+|Cisco IOS 12.x           |    | y  |       |  
+|Cisco WLC                |    | y  |       |
+|Citrix Netscaler         |    |    |   y   | Still some unknown fields, labeled netscalerUnknown<id>
+|fprobe                   |  y |    |       |
+|Fortigate FortiOS 5.2    |    | y  |       |
+|ipt_NETFLOW              |  y | y  |   y   |
+|Juniper MX80             |  y |    |       | SW > 12.3R8
+|Mikrotik 6.35.4          |  y |    |   n   | http://wiki.mikrotik.com/wiki/Manual:IP/Traffic_Flow
+|nProbe                   |  y | y  |   y   |  
+|OpenBSD pflow            |  y | n  |   y   | http://man.openbsd.org/OpenBSD-current/man4/pflow.4
+|Softflowd                |  y | y  |   y   | IPFIX supported in https://github.com/djmdjm/softflowd
+|Streamcore Streamgroomer |    | y  |       |
+|Ubiquiti Edgerouter X    |    | y  |       | With MPLS labels
+|VMware VDS               |    |    |   y   | Still some unknown fields
 |===========================================================================================
 
 ==== Usage
 
-Example Logstash configuration:
+Example Logstash configuration that will listen on 2055/udp for Netflow v5,v9 and IPFIX:
 
 [source, ruby]
 --------------------------
 input {
   udp {
-    host => localhost
     port => 2055
-    codec => netflow {
-      versions => [5, 9]
-    }
-    type => netflow
-  }
-  udp {
-    host => localhost
-    port => 4739
-    codec => netflow {
-      versions => [10]
-      target => ipfix
-   }
-   type => ipfix
-  }
-  tcp {
-    host => localhost
-    port => 4739
-    codec => netflow {
-      versions => [10]
-      target => ipfix
-    }
-    type => ipfix
+    codec => netflow
   }
 }
 --------------------------
@@ -101,10 +91,15 @@ input {
   * Value type is <<path,path>>
   * There is no default value for this setting.
 
-Where to save the template cache
-This helps speed up processing when restarting logstash
-(So you don't have to await the arrival of templates)
-cache will save as path/netflow_templates.cache and/or path/ipfix_templates.cache
+Enables the template cache and saves it in the specified directory. This
+minimizes data loss after Logstash restarts because the codec doesn't have to
+wait for the arrival of templates, but instead reload already received
+templates received during previous runs.
+
+Template caches are saved as:
+
+* <<path,path>>/netflow_templates.cache for Netflow v9 templates.
+* <<path,path>>/ipfix_templates.cache for IPFIX templates.
 
 [id="plugins-{type}s-{plugin}-cache_ttl"]
 ===== `cache_ttl` 

data/lib/logstash/codecs/netflow.rb

@@ -5,63 +5,7 @@ require "logstash/timestamp"
 #require "logstash/json"
 require "json"
 
-# The "netflow" codec is used for decoding Netflow v5/v9/v10 (IPFIX) flows.
-#
-# ==== Supported Netflow/IPFIX exporters
-#
-# The following Netflow/IPFIX exporters are known to work with the most recent version of the netflow codec:
-#
-# [cols="6,^2,^2,^2,12",options="header"]
-# |===========================================================================================
-# |Netflow exporter      | v5 | v9 | IPFIX | Remarks
-# |Softflowd             |  y | y  |   y   | IPFIX supported in https://github.com/djmdjm/softflowd
-# |nProbe                |  y | y  |   y   |  
-# |ipt_NETFLOW           |  y | y  |   y   |
-# |Cisco ASA             |    | y  |       |  
-# |Cisco IOS 12.x        |    | y  |       |  
-# |fprobe                |  y |    |       |
-# |Juniper MX80          |  y |    |       | SW > 12.3R8
-# |OpenBSD pflow         |  y | n  |   y   | http://man.openbsd.org/OpenBSD-current/man4/pflow.4
-# |Mikrotik 6.35.4       |  y |    |   n   | http://wiki.mikrotik.com/wiki/Manual:IP/Traffic_Flow
-# |Ubiquiti Edgerouter X |    | y  |       | With MPLS labels
-# |Citrix Netscaler      |    |    |   y   | Still some unknown fields, labeled netscalerUnknown<id>
-# |===========================================================================================
-#
-# ==== Usage
-#
-# Example Logstash configuration:
-#
-# [source, ruby]
-# --------------------------
-# input {
-#   udp {
-#     host => localhost
-#     port => 2055
-#     codec => netflow {
-#       versions => [5, 9]
-#     }
-#     type => netflow
-#   }
-#   udp {
-#     host => localhost
-#     port => 4739
-#     codec => netflow {
-#       versions => [10]
-#       target => ipfix
-#    }
-#    type => ipfix
-#   }
-#   tcp {
-#     host => localhost
-#     port => 4739
-#     codec => netflow {
-#       versions => [10]
-#       target => ipfix
-#     }
-#     type => ipfix
-#   }
-# }
-# --------------------------
+# Documentation moved to docs/
 
 class LogStash::Codecs::Netflow < LogStash::Codecs::Base
   config_name "netflow"
@@ -87,42 +31,9 @@ class LogStash::Codecs::Netflow < LogStash::Codecs::Base
   config :versions, :validate => :array, :default => [5, 9, 10]
 
   # Override YAML file containing Netflow field definitions
-  #
-  # Each Netflow field is defined like so:
-  #
-  # [source,yaml]
-  # --------------------------
-  # id:
-  # - default length in bytes
-  # - :name
-  # id:
-  # - :uintN or :ip4_addr or :ip6_addr or :mac_addr or :string
-  # - :name
-  # id:
-  # - :skip
-  # --------------------------
-  #
-  # See <https://github.com/logstash-plugins/logstash-codec-netflow/blob/master/lib/logstash/codecs/netflow/netflow.yaml> for the base set.
   config :netflow_definitions, :validate => :path
 
   # Override YAML file containing IPFIX field definitions
-  #
-  # Very similar to the Netflow version except there is a top level Private
-  # Enterprise Number (PEN) key added:
-  #
-  # [source,yaml]
-  # --------------------------
-  # pen:
-  # id:
-  # - :uintN or :ip4_addr or :ip6_addr or :mac_addr or :string
-  # - :name
-  # id:
-  # - :skip
-  # --------------------------
-  #
-  # There is an implicit PEN 0 for the standard fields.
-  #
-  # See <https://github.com/logstash-plugins/logstash-codec-netflow/blob/master/lib/logstash/codecs/netflow/ipfix.yaml> for the base set.
   config :ipfix_definitions, :validate => :path
 
   NETFLOW5_FIELDS = ['version', 'flow_seq_num', 'engine_type', 'engine_id', 'sampling_algorithm', 'sampling_interval', 'flow_records']
@@ -158,19 +69,25 @@ class LogStash::Codecs::Netflow < LogStash::Codecs::Base
 
     if @cache_save_path
       if @versions.include?(9)
-        if File.exists?("#{@cache_save_path}/netflow_templates.cache")
+        cache_save_file_netflow = "#{@cache_save_path}/netflow_templates.cache"
+        if File.exists?(cache_save_file_netflow)
+          raise "#{self.class.name}: Template cache file #{cache_save_file_netflow} not writable" unless File.writable?(cache_save_file_netflow)
           @netflow_templates_cache = load_templates_cache("#{@cache_save_path}/netflow_templates.cache")
           @netflow_templates_cache.each{ |key, fields| @netflow_templates[key, @cache_ttl] = BinData::Struct.new(:endian => :big, :fields => fields) }
         else
+          raise "#{self.class.name}: Template cache directory #{cache_save_path} not writable" unless File.writable?(cache_save_path)
           @netflow_templates_cache = {}
         end
       end
 
       if @versions.include?(10)
-        if File.exists?("#{@cache_save_path}/ipfix_templates.cache")
+        cache_save_file_ipfix = "#{@cache_save_path}/ipfix_templates.cache"
+        if File.exists?(cache_save_file_ipfix)
+          raise "#{self.class.name}: Template cache file #{cache_save_file_ipfix} not writable" unless File.writable?(cache_save_file_ipfix)
           @ipfix_templates_cache = load_templates_cache("#{@cache_save_path}/ipfix_templates.cache")
           @ipfix_templates_cache.each{ |key, fields| @ipfix_templates[key, @cache_ttl] = BinData::Struct.new(:endian => :big, :fields => fields) }
         else
+          raise "#{self.class.name}: Template cache directory #{cache_save_path} not writable" unless File.writable?(cache_save_path)
           @ipfix_templates_cache = {}
         end
       end
@@ -321,8 +238,7 @@ class LogStash::Codecs::Netflow < LogStash::Codecs::Base
       template = @netflow_templates[key]
 
       unless template
-        #@logger.warn("No matching template for flow id #{record.flowset_id} from #{event["source"]}")
-        @logger.warn("No matching template for flow id #{record.flowset_id}")
+        @logger.warn("Can't (yet) decode flowset id #{record.flowset_id} from source id #{flowset.source_id}, because no template to decode it with has been received. This message will usually go away after 1 minute.")
         return events
       end
 
@@ -414,7 +330,7 @@ class LogStash::Codecs::Netflow < LogStash::Codecs::Base
       template = @ipfix_templates[key]
 
       unless template
-        @logger.warn("No matching template for flow id #{record.flowset_id}")
+        @logger.warn("Can't (yet) decode flowset id #{record.flowset_id} from observation domain id #{flowset.observation_domain_id}, because no template to decode it with has been received. This message will usually go away after 1 minute.")
         return events
       end
 
@@ -491,6 +407,7 @@ class LogStash::Codecs::Netflow < LogStash::Codecs::Base
   def load_templates_cache(file_path)
     templates_cache = {}
     begin
+      @logger.debug? and @logger.debug("Loading templates from template cache #{file_path}")
       templates_cache = JSON.parse(File.read(file_path))
     rescue Exception => e
       raise "#{self.class.name}: templates cache file corrupt (#{file_path})"
@@ -501,6 +418,7 @@ class LogStash::Codecs::Netflow < LogStash::Codecs::Base
 
   def save_templates_cache(templates_cache, file_path)
     begin
+      @logger.debug? and @logger.debug("Writing templates to template cache #{file_path}")
       File.open(file_path, 'w') {|file| file.write templates_cache.to_json }
     rescue Exception => e
       raise "#{self.class.name}: saving templates cache file failed (#{file_path}) with error #{e}"

data/logstash-codec-netflow.gemspec

@@ -1,7 +1,7 @@
 Gem::Specification.new do |s|
 
   s.name            = 'logstash-codec-netflow'
-  s.version         = '3.5.0'
+  s.version         = '3.5.1'
   s.licenses        = ['Apache License (2.0)']
   s.summary         = "The netflow codec is for decoding Netflow v5/v9/v10 (IPFIX) flows."
   s.description     = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program"

data/spec/codecs/netflow_spec.rb

@@ -966,6 +966,76 @@ describe LogStash::Codecs::Netflow do
 
   end
 
+  context "Netflow 9 Fortigate FortiOS 5.2.1" do
+    let(:data) do
+      packets = []
+      packets << IO.read(File.join(File.dirname(__FILE__), "netflow9_test_fortigate_fortios_521_tpl.dat"), :mode => "rb")
+      packets << IO.read(File.join(File.dirname(__FILE__), "netflow9_test_fortigate_fortios_521_data256.dat"), :mode => "rb")
+      packets << IO.read(File.join(File.dirname(__FILE__), "netflow9_test_fortigate_fortios_521_data257.dat"), :mode => "rb")
+    end
+
+    let(:json_events) do
+      events = []
+      events << <<-END
+        {
+          "netflow": {
+            "flow_seq_num": 13641,
+            "scope_system": 1,
+            "total_bytes_exp": 6871319015,
+            "total_flows_exp": 107864,
+            "flow_active_timeout": 1800,
+            "flow_inactive_timeout": 15,
+            "flowset_id": 256,
+            "total_pkts_exp": 11920854,
+            "version": 9,
+            "sampling_algorithm": 1,
+            "sampling_interval": 1
+          },
+          "@timestamp": "2017-07-18T05:42:14.000Z",
+          "@version": "1"
+        }
+        END
+
+      events << <<-END
+        {
+          "netflow": {
+            "output_snmp": 3,
+            "in_pkts": 3,
+            "ipv4_dst_addr": "31.13.87.36",
+            "first_switched": "2017-07-25T04:44:29.999Z",
+            "flowset_id": 257,
+            "l4_src_port": 61910,
+            "version": 9,
+            "flow_seq_num": 13635,
+            "ipv4_src_addr": "192.168.99.7",
+            "in_bytes": 152,
+            "protocol": 6,
+            "last_switched": "2017-07-25T04:44:38.999Z",
+            "input_snmp": 9,
+            "out_pkts": 0,
+            "out_bytes": 0,
+            "l4_dst_port": 443
+          },
+          "@timestamp": "2017-07-18T05:41:59.000Z",
+          "@version": "1"
+        }
+        END
+      events.map{|event| event.gsub(/\s+/, "")}
+    end
+
+    it "should decode raw data" do
+      expect(decode.size).to eq(2)
+      expect(decode[0].get("[netflow][total_bytes_exp]")).to eq(6871319015)
+      expect(decode[1].get("[netflow][ipv4_src_addr]")).to eq("192.168.99.7")
+    end
+
+    it "should serialize to json" do
+      expect(JSON.parse(decode[0].to_json)).to eq(JSON.parse(json_events[0]))
+      expect(JSON.parse(decode[1].to_json)).to eq(JSON.parse(json_events[1]))
+    end
+
+  end
+
   context "Netflow 9 Streamcore" do
     let(:data) do
       packets = []
@@ -1741,6 +1811,54 @@ describe LogStash::Codecs::Netflow do
     end
   end
 
+  context "IPFIX Barracuda firewall" do
+    let(:data) do
+      packets = []
+      packets << IO.read(File.join(File.dirname(__FILE__), "ipfix_test_barracuda_tpl.dat"), :mode => "rb")
+      packets << IO.read(File.join(File.dirname(__FILE__), "ipfix_test_barracuda_data256.dat"), :mode => "rb")
+    end
+
+    let(:json_events) do
+      events = []
+      events << <<-END
+        {
+          "netflow": {
+            "destinationIPv4Address": "10.99.168.140",
+            "octetTotalCount": 113,
+            "destinationTransportPort": 50294,
+            "flowStartSysUpTime": 2395374954,
+            "sourceIPv4Address": "10.98.243.20",
+            "flowEndSysUpTime": 2395395322,
+            "flowDurationMilliseconds": 20368,
+            "ingressInterface": 41874,
+            "version": 10,
+            "packetDeltaCount": 1,
+            "firewallEvent": 2,
+            "protocolIdentifier": 17,
+            "sourceMacAddress": "00:00:00:00:00:00",
+            "egressInterface": 48660,
+            "octetDeltaCount": 113,
+            "sourceTransportPort": 53,
+            "packetTotalCount": 1
+          },
+          "@timestamp": "2017-06-29T13:58:28.000Z",
+          "@version": "1"
+        }
+        END
+      events.map{|event| event.gsub(/\s+/, "")}
+    end
+
+    it "should decode raw data" do
+      expect(decode.size).to eq(8)
+      expect(decode[7].get("[netflow][firewallEvent]")).to eq(2)
+    end
+
+    it "should serialize to json" do
+      expect(JSON.parse(decode[7].to_json)).to eq(JSON.parse(json_events[0]))
+    end
+  end
+
+
 
 
 end
@@ -1779,7 +1897,7 @@ describe LogStash::Codecs::Netflow, 'missing templates, no template caching conf
     end
 
     it "should report missing templates" do
-      expect(logger).to receive(:warn).with(/No matching template for flow id/)
+      expect(logger).to receive(:warn).with(/Can't \(yet\) decode flowset id/)
       decode[0]
     end
   end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: logstash-codec-netflow
 version: !ruby/object:Gem::Version
-  version: 3.5.0
+  version: 3.5.1
 platform: ruby
 authors:
 - Elastic
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2017-06-23 00:00:00.000000000 Z
+date: 2017-07-18 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: logstash-core-plugin-api
@@ -116,10 +116,15 @@ files:
 - spec/codecs/netflow9_test_cisco_nbar_tpl262.dat
 - spec/codecs/netflow9_test_cisco_wlc_tpl.dat
 - spec/codecs/netflow9_test_cisco_wlc_data261.dat
+- spec/codecs/ipfix_test_barracuda_tpl.dat
+- spec/codecs/ipfix_test_barracuda_data256.dat
+- spec/codecs/netflow9_test_fortigate_fortios_521_data257.dat
 - spec/codecs/netflow_spec.rb
+- spec/codecs/netflow9_test_fortigate_fortios_521_tpl.dat
+- spec/codecs/netflow9_test_fortigate_fortios_521_data256.dat
 - logstash-codec-netflow.gemspec
-- CHANGELOG.md
 - README.md
+- CHANGELOG.md
 - CONTRIBUTORS
 - Gemfile
 - LICENSE
@@ -197,4 +202,9 @@ test_files:
 - spec/codecs/netflow9_test_cisco_nbar_tpl262.dat
 - spec/codecs/netflow9_test_cisco_wlc_tpl.dat
 - spec/codecs/netflow9_test_cisco_wlc_data261.dat
+- spec/codecs/ipfix_test_barracuda_tpl.dat
+- spec/codecs/ipfix_test_barracuda_data256.dat
+- spec/codecs/netflow9_test_fortigate_fortios_521_data257.dat
 - spec/codecs/netflow_spec.rb
+- spec/codecs/netflow9_test_fortigate_fortios_521_tpl.dat
+- spec/codecs/netflow9_test_fortigate_fortios_521_data256.dat