logstash-codec-netflow 3.2.2 → 3.3.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 05a98ccdb2cc8a75bdda0d5186e17385e4fbb3f2
-  data.tar.gz: e78c49964d3d50a6904895ec9329da7607dcfc81
+  metadata.gz: c13bc81edc56109e3a9c795d38d565d68b40ae44
+  data.tar.gz: 30620977b21b47ce7b77c2d54427b34e060ed5c9
 SHA512:
-  metadata.gz: 84be91763bb7159eefb3f924176955deca396b95fbe09a189c1554d892bac06f34591b2147d91302964b29e69441753139e0f859bbc7741b99ec51ba38f0e642
-  data.tar.gz: a2530632510c75b0aa30a153f1f3f199a9c178777e376d2a9dcaf602140de9d866e94775b230e9955c6cbb0921e97aa6da9187d909830a85906b1f82b76b6531
+  metadata.gz: 88656d03bee9bcae65cfe6b0a25830cd5f2d0831c369d38e7e8f24b1dea976bf391d8197d3c355670ac012244a78c958ed6472d9c31514c00b7decf0fcad7559
+  data.tar.gz: c269177cae5252f6bb7fc7d475b36c9c956fa71b5a21468c1ad5d0a537b80f7c89ed7a6ea244191f0af77fa85e65ec801a9d4c1b03aaa32f6bbaac3ee49d3cb3

data/CHANGELOG.md

@@ -1,3 +1,21 @@
+## 3.3.0
+
+  - Added support for Cisco ASR 9000 (Netflow v9)
+
+## 3.2.5
+
+  - Added support for Streamcore StreamGroomer (Netflow v9)
+  - Fixed docs so they can generate
+
+## 3.2.4
+
+  - Fixed 0-length template field length (Netflow 9)
+
+## 3.2.3
+
+  - Fixed 0-length scope field length (Netflow 9, Juniper SRX)
+  - Fixed JRuby 9K compatibility
+
 ## 3.2.2
 
   - Added support for VMware VDS IPFIX although field definitions are unknown

data/CONTRIBUTORS

@@ -10,10 +10,12 @@ Contributors:
 * G.J. Moed (gjmoed)
 * Jordan Sissel (jordansissel)
 * Jorrit Folmer (jorritfolmer)
+* Keenan Tims (ktims)
 * Matt Dainty (bodgit)
 * Paul Warren (pwarren)
 * Pier-Hugues Pellerin (ph)
 * Pulkit Agrawal (propulkit)
+* Raju Nair (rajutech76)
 * Richard Pijnenburg (electrical)
 * Salvador Ferrer (salva-ferrer)
 * Will Rigby (wrigby)

data/lib/logstash/codecs/netflow.rb

@@ -30,8 +30,8 @@ require "logstash/json"
 #
 # Example Logstash configuration:
 #
-# [source]
-# -----------------------------
+# [source, ruby]
+# --------------------------
 # input {
 #   udp {
 #     host => localhost
@@ -43,24 +43,24 @@ require "logstash/json"
 #   }
 #   udp {
 #     host => localhost
-#         port => 4739
-#         codec => netflow {
+#     port => 4739
+#     codec => netflow {
 #       versions => [10]
 #       target => ipfix
-#     }
-#     type => ipfix
+#    }
+#    type => ipfix
 #   }
 #   tcp {
 #     host => localhost
 #     port => 4739
 #     codec => netflow {
 #       versions => [10]
-#           target => ipfix
+#       target => ipfix
 #     }
 #     type => ipfix
 #   }
 # }
-# -----------------------------
+# --------------------------
 
 class LogStash::Codecs::Netflow < LogStash::Codecs::Base
   config_name "netflow"
@@ -89,15 +89,17 @@ class LogStash::Codecs::Netflow < LogStash::Codecs::Base
   #
   # Each Netflow field is defined like so:
   #
-  #    ---
-  #    id:
-  #    - default length in bytes
-  #    - :name
-  #    id:
-  #    - :uintN or :ip4_addr or :ip6_addr or :mac_addr or :string
-  #    - :name
-  #    id:
-  #    - :skip
+  # [source,yaml]
+  # --------------------------
+  # id:
+  # - default length in bytes
+  # - :name
+  # id:
+  # - :uintN or :ip4_addr or :ip6_addr or :mac_addr or :string
+  # - :name
+  # id:
+  # - :skip
+  # --------------------------
   #
   # See <https://github.com/logstash-plugins/logstash-codec-netflow/blob/master/lib/logstash/codecs/netflow/netflow.yaml> for the base set.
   config :netflow_definitions, :validate => :path
@@ -107,13 +109,15 @@ class LogStash::Codecs::Netflow < LogStash::Codecs::Base
   # Very similar to the Netflow version except there is a top level Private
   # Enterprise Number (PEN) key added:
   #
-  #    ---
-  #    pen:
-  #      id:
-  #      - :uintN or :ip4_addr or :ip6_addr or :mac_addr or :string
-  #      - :name
-  #      id:
-  #      - :skip
+  # [source,yaml]
+  # --------------------------
+  # pen:
+  # id:
+  # - :uintN or :ip4_addr or :ip6_addr or :mac_addr or :string
+  # - :name
+  # id:
+  # - :skip
+  # --------------------------
   #
   # There is an implicit PEN 0 for the standard fields.
   #
@@ -186,6 +190,7 @@ class LogStash::Codecs::Netflow < LogStash::Codecs::Base
         yield(decode_netflow5(flowset, record))
       end
     elsif header.version == 9
+#     BinData::trace_reading do
       flowset = Netflow9PDU.read(payload)
       flowset.records.each do |record|
         if metadata != nil
@@ -193,7 +198,8 @@ class LogStash::Codecs::Netflow < LogStash::Codecs::Base
         else
           decode_netflow9(flowset, record).each{|event| yield(event)}
         end
-      end
+#      end
+     end
     elsif header.version == 10
       flowset = IpfixPDU.read(payload)
       flowset.records.each do |record|
@@ -254,21 +260,28 @@ class LogStash::Codecs::Netflow < LogStash::Codecs::Base
       record.flowset_data.templates.each do |template|
         catch (:field) do
           fields = []
+          template_length = 0
           # Template flowset (0) or Options template flowset (1) ?
           if record.flowset_id == 0
             template.record_fields.each do |field|
-              entry = netflow_field_for(field.field_type, field.field_length)
-              throw :field unless entry
-              fields += entry
+              if field.field_length > 0
+                entry = netflow_field_for(field.field_type, field.field_length, template.template_id)
+                throw :field unless entry
+                fields += entry
+                template_length += field.field_length
+              end
             end
           else
             template.scope_fields.each do |field|
-              fields << [uint_field(0, field.field_length), NETFLOW9_SCOPES[field.field_type]]
+              if field.field_length > 0
+                fields << [uint_field(0, field.field_length), NETFLOW9_SCOPES[field.field_type]]
+              end
             end
             template.option_fields.each do |field|
-              entry = netflow_field_for(field.field_type, field.field_length)
+              entry = netflow_field_for(field.field_type, field.field_length, template.template_id)
               throw :field unless entry
               fields += entry
+              template_length += field.field_length
             end
           end
           # We get this far, we have a list of fields
@@ -279,6 +292,11 @@ class LogStash::Codecs::Netflow < LogStash::Codecs::Base
             key = "#{flowset.source_id}|#{template.template_id}"
           end
           @netflow_templates[key, @cache_ttl] = BinData::Struct.new(:endian => :big, :fields => fields)
+          @logger.debug("Received template #{template.template_id} with fields #{fields.inspect}")
+          @logger.debug("Received template #{template.template_id} of size #{template_length} bytes. Representing in #{@netflow_templates[key].num_bytes} BinData bytes")
+          if template_length != @netflow_templates[key].num_bytes
+            @logger.warn("Received template #{template.template_id} of size (#{template_length} bytes) doesn't match BinData representation we built (#{@netflow_templates[key].num_bytes} bytes)")
+          end
           # Purge any expired templates
           @netflow_templates.cleanup!
           if @cache_save_path
@@ -300,7 +318,7 @@ class LogStash::Codecs::Netflow < LogStash::Codecs::Base
       unless template
         #@logger.warn("No matching template for flow id #{record.flowset_id} from #{event["source"]}")
         @logger.warn("No matching template for flow id #{record.flowset_id}")
-        next
+        return events
       end
 
       length = record.flowset_length - 4
@@ -309,7 +327,7 @@ class LogStash::Codecs::Netflow < LogStash::Codecs::Base
       # be at most 3 padding bytes
       if template.num_bytes > length or ! (length % template.num_bytes).between?(0, 3)
         @logger.warn("Template length doesn't fit cleanly into flowset", :template_id => record.flowset_id, :template_length => template.num_bytes, :record_length => length)
-        next
+        return events
       end
 
       array = BinData::Array.new(:type => template, :initial_length => length / template.num_bytes)
@@ -389,7 +407,7 @@ class LogStash::Codecs::Netflow < LogStash::Codecs::Base
 
       unless template
         @logger.warn("No matching template for flow id #{record.flowset_id}")
-        next
+        return events
       end
 
       array = BinData::Array.new(:type => template, :read_until => :eof)
@@ -507,23 +525,34 @@ class LogStash::Codecs::Netflow < LogStash::Codecs::Base
     field
   end # def string_field
 
-  def netflow_field_for(type, length)
+  def netflow_field_for(type, length, template_id)
     if @netflow_fields.include?(type)
       field = @netflow_fields[type].clone
       if field.is_a?(Array)
 
         field[0] = uint_field(length, field[0]) if field[0].is_a?(Integer)
 
-        # Small bit of fixup for skip or string field types where the length
-        # is dynamic
-        case field[0]
+        # Small bit of fixup for:
+        # - skip or string field types where the length is dynamic
+	# - for uint(8|16|24|32} where we use the length as specified by the
+	#   template instead of the YAML (e.g. ipv6_flow_label is 3 bytes in
+	#   the YAML and Cisco doc, but Cisco ASR9k sends 4 bytes)
+	case field[0]
+        when :uint8
+          field[0] = uint_field(length, field[0])
+        when :uint16
+          field[0] = uint_field(length, field[0])
+        when :uint24
+          field[0] = uint_field(length, field[0])
+        when :uint32
+          field[0] = uint_field(length, field[0])
         when :skip
           field += [nil, {:length => length.to_i}]
         when :string
           field += [{:length => length.to_i, :trim_padding => true}]
         end
 
-        @logger.debug? and @logger.debug("Definition complete", :field => field)
+        @logger.debug? and @logger.debug("Field definition complete for template #{template_id}", :field => field)
 
         [field]
       else
@@ -531,7 +560,7 @@ class LogStash::Codecs::Netflow < LogStash::Codecs::Base
         nil
       end
     else
-      @logger.warn("Unsupported field", :type => type, :length => length)
+      @logger.warn("Unsupported field in template #{template_id}", :type => type, :length => length)
       nil
     end
   end # def netflow_field_for

data/lib/logstash/codecs/netflow/netflow.yaml

@@ -189,8 +189,6 @@
 64:
 - :uint32
 - :ipv6_option_headers
-64:
-- :skip
 65:
 - :skip
 66:
@@ -234,6 +232,9 @@
 148:
 - :uint32
 - :conn_id
+152:
+- 8
+- :flow_start_msec
 176:
 - :uint8
 - :icmp_type
@@ -246,6 +247,18 @@
 179:
 - :uint8
 - :icmp_code_ipv6
+180:
+- :uint16
+- :udp_src_port
+181:
+- :uint16
+- :udp_dst_port
+182:
+- :uint16
+- :tcp_src_port
+183:
+- :uint16
+- :tcp_dst_port
 201:
 - mpls_label_stack_octets
 - mpls_label_stack_octets
@@ -261,36 +274,132 @@
 228:
 - :uint16
 - :xlate_dst_port
+231:
+- :uint32 
+- :fwd_flow_delta_bytes
+232:
+- :uint32
+- :rev_flow_delta_bytes
 233:
 - :uint8
 - :fw_event
+234:
+- :uint32
+- :ingressVRFID
+235:
+- :uint32
+- :egressVRFID
+236:
+- :string
+- :VRFname
 281:
 - :ip6_addr
 - :xlate_src_addr_ipv6
 282:
 - :ip6_addr
 - :xlate_dst_addr_ipv6
-33002:
-- :uint16
-- :fw_ext_event
 323:
 - 8
 - :event_time_msec
-152:
-- 8
-- :flow_start_msec
-231:
-- :uint32 
-- :fwd_flow_delta_bytes
-232:
+361:
+- :uint16
+- :postNATPortBlockStart
+362:
+- :uint16
+- :postNATPortBlockEnd
+8192:
 - :uint32
-- :rev_flow_delta_bytes
+- :streamcore_wan_rtt
+8193:
+- :uint32
+- :streamcore_net_app_resp_time
+8194:
+- :uint32
+- :streamcore_total_app_resp_time
+8195:
+- :uint16
+- :streamcore_tcp_retrans_rate
+8196:
+- :uint8
+- :streamcore_call_direction
+8256:
+- :string
+- :streamcore_hostname
+8257:
+- :string
+- :streamcore_url
+8258:
+- :string
+- :streamcore_ssl_cn
+8259:
+- :string
+- :streamcore_ssl_org
+8320:
+- :uint16
+- :streamcore_mos_lq
+8321:
+- :uint16
+- :streamcore_net_delay
+8322:
+- :uint16
+- :streamcore_net_loss
+8323:
+- :uint16
+- :streamcore_net_jitter
+8324:
+- :uint16
+- :streamcore_net_discard
+8325:
+- :uint8
+- :streamcore_rtp_clockrate_in
+8326:
+- :uint8
+- :streamcore_rtp_clockrate_out
+8327:
+- :uint8
+- :streamcore_codec_in
+8328:
+- :uint8
+- :streamcore_codec_out
+8384:
+- :uint32
+- :streamcore_id_rule_1
+8385:
+- :uint32
+- :streamcore_id_rule_2
+8386:
+- :uint32
+- :streamcore_id_rule_3
+8387:
+- :uint32
+- :streamcore_id_rule_4
+8388:
+- :uint32
+- :streamcore_id_rule_5
+8389:
+- :uint32
+- :streamcore_id_rule_6
+8390:
+- :uint32
+- :streamcore_id_rule_7
+8391:
+- :uint32
+- :streamcore_id_rule_8
+8392:
+- :uint32
+- :streamcore_id_rule_9
+8393:
+- :uint32
+- :streamcore_id_rule_10
 33000:
 - :acl_id_asa
 - :ingress_acl_id
 33001:
 - :acl_id_asa
 - egress_acl_id
+33002:
+- :uint16
+- :fw_ext_event
 40000:
 - :string
 - :username
@@ -309,18 +418,3 @@
 40005:
 - :uint8
 - :fw_event
-95:
-- :uint32
-- :application_id
-180:
-- :uint16
-- :udp_src_port
-181:
-- :uint16
-- :udp_dst_port
-182:
-- :uint16
-- :tcp_src_port
-183:
-- :uint16
-- :tcp_dst_port

data/lib/logstash/codecs/netflow/util.rb

@@ -179,15 +179,15 @@ class NetflowOptionFlowset < BinData::Record
   endian :big
   array  :templates, :read_until => lambda { flowset_length - 4 - array.num_bytes <= 2 } do
     uint16 :template_id
-    uint16 :scope_length
-    uint16 :option_length
+    uint16 :scope_length, :assert => lambda { scope_length > 0 }
+    uint16 :option_length, :assert => lambda { option_length > 0 }
     array  :scope_fields, :initial_length => lambda { scope_length / 4 } do
       uint16 :field_type
       uint16 :field_length
     end
     array  :option_fields, :initial_length => lambda { option_length / 4 } do
       uint16 :field_type
-      uint16 :field_length
+      uint16 :field_length, :assert => lambda { field_length > 0 }
     end
   end
   skip   :length => lambda { templates.length.odd? ? 2 : 0 }

data/logstash-codec-netflow.gemspec

@@ -1,7 +1,7 @@
 Gem::Specification.new do |s|
 
   s.name            = 'logstash-codec-netflow'
-  s.version         = '3.2.2'
+  s.version         = '3.3.0'
   s.licenses        = ['Apache License (2.0)']
   s.summary         = "The netflow codec is for decoding Netflow v5/v9/v10 (IPFIX) flows."
   s.description     = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program"

data/spec/codecs/netflow_spec.rb

@@ -966,6 +966,186 @@ describe LogStash::Codecs::Netflow do
 
   end
 
+  context "Netflow 9 Streamcore" do
+    let(:data) do
+      packets = []
+      packets << IO.read(File.join(File.dirname(__FILE__), "netflow9_test_streamcore_tpl_data256.dat"), :mode => "rb")
+      packets << IO.read(File.join(File.dirname(__FILE__), "netflow9_test_streamcore_tpl_data260.dat"), :mode => "rb")
+    end
+
+    let(:json_events) do
+      events = []
+      events << <<-END
+        {
+          "netflow": {
+            "in_pkts": 3,
+            "first_switched": "2017-01-11T11:47:23.999Z",
+            "flowset_id": 256,
+            "l4_src_port": 8080,
+            "streamcore_id_rule_1": 1171,
+            "streamcore_id_rule_2": 1179,
+            "in_bytes": 128,
+            "protocol": 6,
+            "streamcore_id_rule_5": 0,
+            "tcp_flags": 19,
+            "streamcore_id_rule_3": 1192,
+            "streamcore_id_rule_4": 1435,
+            "streamcore_net_app_resp_time": 0,
+            "l4_dst_port": 50073,
+            "output_snmp": 1148,
+            "streamcore_call_direction": 1,
+            "src_tos": 40,
+            "ipv4_dst_addr": "10.231.128.150",
+            "version": 9,
+            "streamcore_tcp_retrans_rate": 0,
+            "flow_seq_num": 2143054578,
+            "ipv4_src_addr": "100.78.40.201",
+            "input_snmp": 1152,
+            "last_switched": "2017-01-11T11:47:29.999Z",
+            "streamcore_wan_rtt": 0,
+            "streamcore_total_app_resp_time": 0
+          },
+          "@timestamp": "2017-01-11T11:48:15.000Z",
+          "@version": "1"
+        }
+      END
+
+      events << <<-END
+        {
+          "netflow": {
+            "in_pkts": 4,
+            "first_switched": "2017-01-11T11:47:23.999Z",
+            "flowset_id": 256,
+            "l4_src_port": 50073,
+            "streamcore_id_rule_1": 1171,
+            "streamcore_id_rule_2": 1179,
+            "in_bytes": 172,
+            "protocol": 6,
+            "streamcore_id_rule_5": 0,
+            "tcp_flags": 19,
+            "streamcore_id_rule_3": 1192,
+            "streamcore_id_rule_4": 1435,
+            "streamcore_net_app_resp_time": 0,
+            "l4_dst_port": 8080,
+            "output_snmp": 1152,
+            "streamcore_call_direction": 0,
+            "src_tos": 40,
+            "ipv4_dst_addr": "100.78.40.201",
+            "version": 9,
+            "streamcore_tcp_retrans_rate": 0,
+            "flow_seq_num": 2143054578,
+            "ipv4_src_addr": "10.231.128.150",
+            "input_snmp": 1148,
+            "last_switched": "2017-01-11T11:47:29.999Z",
+            "streamcore_wan_rtt": 0,
+            "streamcore_total_app_resp_time": 0
+          },
+          "@timestamp": "2017-01-11T11:48:15.000Z",
+          "@version": "1"
+        }
+      END
+
+      events << <<-END
+        {
+          "netflow": {
+            "streamcore_id_rule_10": 0,
+            "in_pkts": 10,
+            "first_switched": "2017-01-11T11:22:44.999Z",
+            "flowset_id": 260,
+            "l4_src_port": 8080,
+            "reamcore_id_rule_1": 1171,
+            "streamcore_id_rule_2": 1179,
+            "in_bytes": 3943,
+            "protocol": 6,
+            "streamcore_id_rule_5": 0,
+            "tcp_flags": 26,
+            "streamcore_id_rule_6": 0,
+            "streamcore_id_rule_3": 1192,
+            "streamcore_id_rule_4": 1435,
+            "streamcore_id_rule_9": 0,
+            "streamcore_id_rule_7": 0,
+            "streamcore_id_rule_8": 0,
+            "streamcore_net_app_resp_time": 17,
+            "l4_dst_port": 53483,
+            "output_snmp": 1148,
+            "streamcore_hostname": "live.lemde.fr",
+            "streamcore_call_direction": 1,
+            "src_tos": 40,
+            "ipv4_dst_addr": "10.27.8.20",
+            "version": 9,
+            "streamcore_tcp_retrans_rate": 0,
+            "flow_seq_num": 2142545188,
+            "ipv4_src_addr": "100.78.40.201",
+            "input_snmp": 1152,
+            "last_switched": "2017-01-11T11:23:35.999Z",
+            "streamcore_url": "\/mux.json",
+            "streamcore_wan_rtt": 0,
+            "streamcore_total_app_resp_time": 19
+          },
+          "@timestamp": "2017-01-11T11:23:51.000Z",
+          "@version": "1"
+        }
+      END
+
+      events << <<-END
+        {
+          "netflow": {
+            "streamcore_id_rule_10": 0,
+            "in_pkts": 11,
+            "first_switched": "2017-01-11T11:22:44.999Z",
+            "flowset_id": 260,
+            "l4_src_port": 53483,
+            "streamcore_id_rule_1": 1171,
+            "streamcore_id_rule_2": 1179,
+            "in_bytes": 3052,
+            "protocol": 6,
+            "streamcore_id_rule_5": 0,
+            "tcp_flags": 26,
+            "streamcore_id_rule_6": 0,
+            "streamcore_id_rule_3": 1192,
+            "streamcore_id_rule_4": 1435,
+            "streamcore_id_rule_9": 0,
+            "streamcore_id_rule_7": 0,
+            "streamcore_id_rule_8": 0,
+            "streamcore_net_app_resp_time": 17,
+            "l4_dst_port": 8080,
+            "output_snmp": 1152,
+            "streamcore_hostname": "live.lemde.fr",
+            "streamcore_call_direction": 0,
+            "src_tos": 40,
+            "ipv4_dst_addr": "100.78.40.201",
+            "version": 9,
+            "streamcore_tcp_retrans_rate": 0,
+            "flow_seq_num": 2142545188,
+            "ipv4_src_addr": "10.27.8.20",
+            "input_snmp": 1148,
+            "last_switched": "2017-01-11T11:23:35.999Z",
+            "streamcore_url": "\/mux.json",
+            "streamcore_wan_rtt": 0,
+            "streamcore_total_app_resp_time": 19
+          },
+          "@timestamp": "2017-01-11T11:23:51.000Z",
+          "@version": "1"
+        }
+      END
+
+      events.map{|event| event.gsub(/\s+/, "")}
+    end
+
+    it "should decode raw data" do
+      expect(decode.size).to eq(4)
+      expect(decode[0].get("[netflow][streamcore_id_rule_1]")).to eq(1171)
+      expect(decode[3].get("[netflow][streamcore_hostname]")).to eq("live.lemde.fr")
+    end
+
+    it "should serialize to json" do
+      expect(JSON.parse(decode[0].to_json)).to eq(JSON.parse(json_events[0]))
+      expect(JSON.parse(decode[3].to_json)).to eq(JSON.parse(json_events[3]))
+    end
+
+  end
+
+
   context "IPFIX Netscaler with variable length fields" do
     let(:data) do
       # this ipfix raw data was produced by a Netscaler appliance and captured with wireshark
@@ -1236,14 +1416,192 @@ describe LogStash::Codecs::Netflow do
 
     it "should serialize to json" do
       expect(JSON.parse(decode[0].to_json)).to eq(JSON.parse(json_events[0]))
-      expect(JSON.parse(decode[1].to_json)).to eq(JSON.parse(json_events[1]))
-      expect(JSON.parse(decode[2].to_json)).to eq(JSON.parse(json_events[2]))
-      expect(JSON.parse(decode[3].to_json)).to eq(JSON.parse(json_events[3]))
-      expect(JSON.parse(decode[4].to_json)).to eq(JSON.parse(json_events[4]))
     end
 
   end
 
+  context "Juniper SRX options template with 0 scope field length" do
+    let(:data) do
+      packets = []
+      packets << IO.read(File.join(File.dirname(__FILE__), "netflow9_test_juniper_srx_tplopt.dat"), :mode => "rb")
+    end
+
+    let(:json_events) do
+      events = []
+      events << <<-END
+        {
+          "netflow": {
+            "flow_seq_num": 338,
+            "flowset_id": 256,
+            "version":9,
+            "sampling_algorithm":2,
+            "sampling_interval":1
+          },
+          "@timestamp":"2016-11-29T00:21:56.000Z",
+          "@version":"1"
+        }
+        END
+      events.map{|event| event.gsub(/\s+/, "")}
+    end
+
+    it "should decode raw data" do
+      expect(decode.size).to eq(1)
+      expect(decode[0].get("[netflow][sampling_algorithm]")).to eq(2)
+    end
+
+    it "should serialize to json" do
+      expect(JSON.parse(decode[0].to_json)).to eq(JSON.parse(json_events[0]))
+    end
+
+  end
+
+  context "Netflow 9 template with 0 length fields" do
+    let(:data) do
+      packets = []
+      packets << IO.read(File.join(File.dirname(__FILE__), "netflow9_test_0length_fields_tpl_data.dat"), :mode => "rb")
+    end
+
+    let(:json_events) do
+      events = []
+      events << <<-END
+        {
+          "netflow":{  
+             "output_snmp":3,
+             "dst_mask":32,
+             "in_pkts":0,
+             "ipv4_dst_addr":"239.255.255.250",
+             "first_switched":"2016-12-23T01:34:52.999Z",
+             "flowset_id":256,
+             "l4_src_port":0,
+             "src_mask":32,
+             "version":9,
+             "flow_seq_num":100728833,
+             "ipv4_src_addr":"192.168.1.33",
+             "in_bytes":0,
+             "protocol":2,
+             "input_snmp":2,
+             "last_switched":"2016-12-23T01:34:52.999Z",
+             "tcp_flags":0,
+             "engine_id":1,
+             "out_pkts":1,
+             "out_bytes":32,
+             "l4_dst_port":0,
+             "direction":1
+          },
+          "@timestamp":"2016-12-23T01:35:31.000Z",
+          "@version":"1"
+        }
+      END
+      events.map{|event| event.gsub(/\s+/, "")}
+    end
+
+    it "should decode raw data" do
+      expect(decode.size).to eq(10)
+      expect(decode[9].get("[netflow][ipv4_src_addr]")).to eq("192.168.1.33")
+    end
+
+    it "should serialize to json" do
+      expect(JSON.parse(decode[9].to_json)).to eq(JSON.parse(json_events[0]))
+    end
+
+  end
+
+  context "Netflow 9 Cisco ASR 9000 series options template 256" do
+    let(:data) do
+      packets = []
+      packets << IO.read(File.join(File.dirname(__FILE__), "netflow9_test_cisco_asr9k_opttpl256.dat"), :mode => "rb")
+      packets << IO.read(File.join(File.dirname(__FILE__), "netflow9_test_cisco_asr9k_data256.dat"), :mode => "rb")
+    end
+
+    let(:json_events) do
+      events = []
+      events << <<-END
+        {
+          "netflow": {
+            "flow_seq_num": 24496783,
+            "scope_system": 3250896451,
+            "input_snmp": 104,
+            "if_desc": "TenGigE0_6_0_2",
+            "flowset_id": 256,
+            "version": 9
+          },
+          "@timestamp": "2016-12-06T10:09:48.000Z",
+          "@version": "1"
+        }
+        END
+      events.map{|event| event.gsub(/\s+/, "")}
+    end
+
+    it "should decode raw data" do
+      expect(decode.size).to eq(19)
+      expect(decode[18].get("[netflow][if_desc]")).to eq("TenGigE0_6_0_2")
+    end
+
+    it "should serialize to json" do
+      expect(JSON.parse(decode[18].to_json)).to eq(JSON.parse(json_events[0]))
+    end
+
+  end
+
+  context "Netflow 9 Cisco ASR 9000 series template 260" do
+    let(:data) do
+      packets = []
+      packets << IO.read(File.join(File.dirname(__FILE__), "netflow9_test_cisco_asr9k_tpl260.dat"), :mode => "rb")
+      packets << IO.read(File.join(File.dirname(__FILE__), "netflow9_test_cisco_asr9k_data260.dat"), :mode => "rb")
+    end
+
+    let(:json_events) do
+      events = []
+      events << <<-END
+        {
+          "netflow": {
+            "dst_as": 64498,
+            "forwarding_status": {
+              "reason": 0,
+              "status": 1
+            },
+            "in_pkts": 2,
+            "first_switched": "2016-12-06T10:08:53.999Z",
+            "flowset_id": 260,
+            "l4_src_port": 443,
+            "in_bytes": 112,
+            "protocol": 6,
+            "tcp_flags": 18,
+            "ingressVRFID": 1610612736,
+            "l4_dst_port": 52364,
+            "src_as": 15169,
+            "direction": 1,
+            "output_snmp": 158,
+            "dst_mask": 24,
+            "ipv4_dst_addr": "10.0.15.38",
+            "src_tos": 0,
+            "src_mask": 24,
+            "version": 9,
+            "flow_seq_num": 24495777,
+            "ipv4_src_addr": "10.0.29.46",
+            "egressVRFID": 1610612736,
+            "input_snmp": 75,
+            "last_switched": "2016-12-06T10:08:54.999Z",
+            "flow_sampler_id": 1,
+            "bgp_ipv4_next_hop": "10.0.14.27"
+          },
+          "@timestamp": "2016-12-06T10:09:24.000Z",
+          "@version": "1"
+        }
+        END
+      events.map{|event| event.gsub(/\s+/, "")}
+    end
+
+    it "should decode raw data" do
+      expect(decode.size).to eq(21)
+      expect(decode[20].get("[netflow][egressVRFID]")).to eq(1610612736)
+    end
+
+    it "should serialize to json" do
+      expect(JSON.parse(decode[20].to_json)).to eq(JSON.parse(json_events[0]))
+    end
+  end
+
 end
 
 describe LogStash::Codecs::Netflow, 'missing templates, no template caching configured' do

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: logstash-codec-netflow
 version: !ruby/object:Gem::Version
-  version: 3.2.2
+  version: 3.3.0
 platform: ruby
 authors:
 - Elastic
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2016-12-25 00:00:00.000000000 Z
+date: 2017-02-12 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
@@ -90,17 +90,28 @@ files:
 - spec/codecs/netflow5_test_invalid02.dat
 - spec/codecs/netflow5_test_juniper_mx80.dat
 - spec/codecs/netflow5_test_microtik.dat
+- spec/codecs/netflow9_test_0length_fields_tpl_data.dat
 - spec/codecs/netflow9_test_cisco_asa_1_data.dat
 - spec/codecs/netflow9_test_cisco_asa_1_tpl.dat
 - spec/codecs/netflow9_test_cisco_asa_2_data.dat
 - spec/codecs/netflow9_test_cisco_asa_2_tpl_26x.dat
 - spec/codecs/netflow9_test_cisco_asa_2_tpl_27x.dat
+- spec/codecs/netflow9_test_cisco_asr9k_data256.dat
+- spec/codecs/netflow9_test_cisco_asr9k_data260.dat
+- spec/codecs/netflow9_test_cisco_asr9k_opttpl256.dat
+- spec/codecs/netflow9_test_cisco_asr9k_opttpl257.dat
+- spec/codecs/netflow9_test_cisco_asr9k_opttpl334.dat
+- spec/codecs/netflow9_test_cisco_asr9k_tpl260.dat
+- spec/codecs/netflow9_test_cisco_asr9k_tpl266.dat
 - spec/codecs/netflow9_test_invalid01.dat
+- spec/codecs/netflow9_test_juniper_srx_tplopt.dat
 - spec/codecs/netflow9_test_macaddr_data.dat
 - spec/codecs/netflow9_test_macaddr_tpl.dat
 - spec/codecs/netflow9_test_nprobe_data.dat
 - spec/codecs/netflow9_test_nprobe_tpl.dat
 - spec/codecs/netflow9_test_softflowd_tpl_data.dat
+- spec/codecs/netflow9_test_streamcore_tpl_data256.dat
+- spec/codecs/netflow9_test_streamcore_tpl_data260.dat
 - spec/codecs/netflow9_test_ubnt_edgerouter_data1024.dat
 - spec/codecs/netflow9_test_ubnt_edgerouter_data1025.dat
 - spec/codecs/netflow9_test_ubnt_edgerouter_tpl.dat
@@ -147,17 +158,28 @@ test_files:
 - spec/codecs/netflow5_test_invalid02.dat
 - spec/codecs/netflow5_test_juniper_mx80.dat
 - spec/codecs/netflow5_test_microtik.dat
+- spec/codecs/netflow9_test_0length_fields_tpl_data.dat
 - spec/codecs/netflow9_test_cisco_asa_1_data.dat
 - spec/codecs/netflow9_test_cisco_asa_1_tpl.dat
 - spec/codecs/netflow9_test_cisco_asa_2_data.dat
 - spec/codecs/netflow9_test_cisco_asa_2_tpl_26x.dat
 - spec/codecs/netflow9_test_cisco_asa_2_tpl_27x.dat
+- spec/codecs/netflow9_test_cisco_asr9k_data256.dat
+- spec/codecs/netflow9_test_cisco_asr9k_data260.dat
+- spec/codecs/netflow9_test_cisco_asr9k_opttpl256.dat
+- spec/codecs/netflow9_test_cisco_asr9k_opttpl257.dat
+- spec/codecs/netflow9_test_cisco_asr9k_opttpl334.dat
+- spec/codecs/netflow9_test_cisco_asr9k_tpl260.dat
+- spec/codecs/netflow9_test_cisco_asr9k_tpl266.dat
 - spec/codecs/netflow9_test_invalid01.dat
+- spec/codecs/netflow9_test_juniper_srx_tplopt.dat
 - spec/codecs/netflow9_test_macaddr_data.dat
 - spec/codecs/netflow9_test_macaddr_tpl.dat
 - spec/codecs/netflow9_test_nprobe_data.dat
 - spec/codecs/netflow9_test_nprobe_tpl.dat
 - spec/codecs/netflow9_test_softflowd_tpl_data.dat
+- spec/codecs/netflow9_test_streamcore_tpl_data256.dat
+- spec/codecs/netflow9_test_streamcore_tpl_data260.dat
 - spec/codecs/netflow9_test_ubnt_edgerouter_data1024.dat
 - spec/codecs/netflow9_test_ubnt_edgerouter_data1025.dat
 - spec/codecs/netflow9_test_ubnt_edgerouter_tpl.dat