fluent-plugin-netflow 0.2.0 → 0.2.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: d5a011a605cc39a2aff47556d09b98973bd84eb9
-  data.tar.gz: 779a28e68e6cd2bcd480d74ddcaa69605111839b
+  metadata.gz: 3cd8781b6ecc14e1dd982e4046b795a34412e768
+  data.tar.gz: be9f57fbb0fd1323fba31f718146632e6d2d5fdc
 SHA512:
-  metadata.gz: 0c9396c4d0b6f9b8f1d6fda6ca0a53bdb4c0772fb12780e5a52930e840c886f73b4b6221cf6225773053038cf532319aab0c3439797b244239a20c3816462ae1
-  data.tar.gz: 6e6e44c7685bff5996bd068b068308934245f32928ab0e334623dfadd0359e7dc14658d96a7c85dded55dd58a7c9c155f0f3e033626cf3923295870f4dcc3977
+  metadata.gz: aca8f4e3c4f146ad05e0b0f166e1a7bd65e77352e4ce0927101e5d2d6b3f0d6ad443a39ba976c8c47b917366bf363ff2d1a17ed0bc0b068e4be81879b9d9f27e
+  data.tar.gz: 14d3a412b02acccd57ef7723ee1b2b32e87f3b3c34d5f4e14f6b85bb5fc6f23f78d575c809a24a56e410879af6b6a22f139a62279a400340f3a827ebcc257f2b

data/README.md

@@ -1,8 +1,12 @@
 # Netflow plugin for Fluentd
 
-Accept Netflow logs.
+[![Build Status](https://travis-ci.org/repeatedly/fluent-plugin-netflow.svg)](https://travis-ci.org/repeatedly/fluent-plugin-netflow)
+
+
+## Overview
+
+[Fluentd](http://fluentd.org/) input plugin that acts as Netflow v5/v9 collector.
 
-Netflow parser is based on [Logstash's netflow codes](https://github.com/elasticsearch/logstash/blob/master/lib/logstash/codecs/netflow.rb).
 
 ## Installation
 
@@ -10,6 +14,7 @@ Use RubyGems:
 
     fluent-gem install fluent-plugin-netflow
 
+
 ## Configuration
 
     <source>
@@ -17,16 +22,99 @@ Use RubyGems:
       tag netflow.event
 
       # optional parameters
-      bind 127.0.0.1
-      port 5140
-
-      # optional parser parameters
+      bind 192.168.0.1
+      port 2055
       cache_ttl 6000
       versions [5, 9]
     </source>
 
+**bind**
+
+IP address on which the plugin will accept Netflow.  
+(Default: '0.0.0.0')
+
+**port**
+
+UDP port number on which tpe plugin will accept Netflow.  
+(Default: 5140)
+
+**cache_ttl**
+
+Template cache TTL for Netflow v9 in seconds. Templates not refreshed from the Netflow v9 exporter within the TTL are expired at the plugin.  
+(Default: 4000)
+
+**versions**
+
+Netflow versions which are acceptable.  
+(Default:[5, 9])
+
+**switched_times_from_uptime**
+
+When set to true, the plugin stores system uptime for ```first_switched``` and ```last_switched``` instead of ISO8601-formatted absolute time.  
+(Defaults: false)
+
+
+## Performance Evaluation
+
+Benchmark for v5 protocol on Macbook Air (Early 2014, 1.7 GHz Intel Core i7):
+* 0 packets dropped in 32,000 records/second (for 3,000,000 packets)
+* 45,000 records/second in maximum (for flooding netflow packets)
+
+Tested with the packet generator below:
+
+* https://github.com/mshindo/NetFlow-Generator
+* `./flowgen -n3000000 -i50 -w1 -p5140 localhost`
+
+And configuration:
+
+    <source>
+      @type  netflow
+      tag netflow.event
+      bind 0.0.0.0
+      port 5140
+      switched_times_from_uptime yes
+    </source>
+    <match netflow.event>
+      @type flowcounter
+      unit minute
+      count_keys count # missing column for counting events only
+      tag flowcount
+    </match>
+    <match flowcount>
+      @type stdout
+    </match>
+
+
+## Tips
+
+### Use netflow parser in other plugins
+
+```ruby
+require 'fluent/plugin/parser_netflow'
+
+parser = TextParser::NetflowParser.new
+parser.configure(conf)
+
+# Netflow v5
+parser.call(payload) do |time, record|
+  # do something
+end
+
+# Netflow v9
+parser.call(payload, source_ip_address) do |time, record|
+  # do something
+end
+```
+
+**NOTE:**
+If the plugin receives Netflow v9 from multiple sources, provide ```source_ip_address``` argument to parse correctly.
+
+### More speed ?
+
+:bullettrain_side: Try ```switched_times_from_uptime true``` option !
+
+
 ## TODO
 
-- Support TCP protocol? TCP is needed?
-- Use Fluentd feature instead of own handlers
-- Need another maintainer who uses Netflow in production!
+* Netflow v9 protocol parser optimization
+* Use Fluentd feature instead of own handlers

data/VERSION

@@ -1 +1 @@
-0.2.0
+0.2.1

data/lib/fluent/plugin/in_netflow.rb

@@ -69,7 +69,7 @@ module Fluent
     def receive_data(host, data)
       log.on_debug { log.debug "received logs", :host => host, :data => data }
 
-      @parser.call(data) { |time, record|
+      @parser.call(data, host) { |time, record|
         unless time && record
           log.warn "pattern not match: #{data.inspect}"
           return

data/lib/fluent/plugin/netflow_fields.yaml

@@ -0,0 +1,258 @@
+---
+option:
+  1:
+  - 4
+  - :in_bytes
+  2:
+  - 4
+  - :in_pkts
+  3:
+  - 4
+  - :flows
+  4:
+  - :uint8
+  - :protocol
+  5:
+  - :uint8
+  - :src_tos
+  6:
+  - :uint8
+  - :tcp_flags
+  7:
+  - :uint16
+  - :l4_src_port
+  8:
+  - :ip4_addr
+  - :ipv4_src_addr
+  9:
+  - :uint8
+  - :src_mask
+  10:
+  - 2
+  - :input_snmp
+  11:
+  - :uint16
+  - :l4_dst_port
+  12:
+  - :ip4_addr
+  - :ipv4_dst_addr
+  13:
+  - :uint8
+  - :dst_mask
+  14:
+  - 2
+  - :output_snmp
+  15:
+  - :ip4_addr
+  - :ipv4_next_hop
+  16:
+  - 2
+  - :src_as
+  17:
+  - 2
+  - :dst_as
+  18:
+  - :ip4_addr
+  - :bgp_ipv4_next_hop
+  19:
+  - 4
+  - :mul_dst_pkts
+  20:
+  - 4
+  - :mul_dst_bytes
+  21:
+  - :uint32
+  - :last_switched
+  22:
+  - :uint32
+  - :first_switched
+  23:
+  - 4
+  - :out_bytes
+  24:
+  - 4
+  - :out_pkts
+  25:
+  - :uint16
+  - :min_pkt_length
+  26:
+  - :uint16
+  - :max_pkt_length
+  27:
+  - :ip6_addr
+  - :ipv6_src_addr
+  28:
+  - :ip6_addr
+  - :ipv6_dst_addr
+  29:
+  - :uint8
+  - :ipv6_src_mask
+  30:
+  - :uint8
+  - :ipv6_dst_mask
+  31:
+  - 3
+  - :ipv6_flow_label
+  32:
+  - :uint16
+  - :icmp_type
+  33:
+  - :uint8
+  - :mul_igmp_type
+  34:
+  - :uint32
+  - :sampling_interval
+  35:
+  - :uint8
+  - :sampling_algorithm
+  36:
+  - :uint16
+  - :flow_active_timeout
+  37:
+  - :uint16
+  - :flow_inactive_timeout
+  38:
+  - :uint8
+  - :engine_type
+  39:
+  - :uint8
+  - :engine_id
+  40:
+  - 4
+  - :total_bytes_exp
+  41:
+  - 4
+  - :total_pkts_exp
+  42:
+  - 4
+  - :total_flows_exp
+  43:
+  - :skip
+  44:
+  - :ip4_addr
+  - :ipv4_src_prefix
+  45:
+  - :ip4_addr
+  - :ipv4_dst_prefix
+  46:
+  - :uint8
+  - :mpls_top_label_type
+  47:
+  - :uint32
+  - :mpls_top_label_ip_addr
+  48:
+  - 1
+  - :flow_sampler_id
+  49:
+  - :uint8
+  - :flow_sampler_mode
+  50:
+  - :uint32
+  - :flow_sampler_random_interval
+  51:
+  - :skip
+  52:
+  - :uint8
+  - :min_ttl
+  53:
+  - :uint8
+  - :max_ttl
+  54:
+  - :uint16
+  - :ipv4_ident
+  55:
+  - :uint8
+  - :dst_tos
+  56:
+  - :mac_addr
+  - :in_src_mac
+  57:
+  - :mac_addr
+  - :out_dst_mac
+  58:
+  - :uint16
+  - :src_vlan
+  59:
+  - :uint16
+  - :dst_vlan
+  60:
+  - :uint8
+  - :ip_protocol_version
+  61:
+  - :uint8
+  - :direction
+  62:
+  - :ip6_addr
+  - :ipv6_next_hop
+  63:
+  - :ip6_addr
+  - :bgp_ipv6_next_hop
+  64:
+  - :uint32
+  - :ipv6_option_headers
+  65:
+  - :skip
+  66:
+  - :skip
+  67:
+  - :skip
+  68:
+  - :skip
+  69:
+  - :skip
+  70:
+  - :mpls_label
+  - :mpls_label_1
+  71:
+  - :mpls_label
+  - :mpls_label_2
+  72:
+  - :mpls_label
+  - :mpls_label_3
+  80:
+  - :mac_addr
+  - :in_dst_mac
+  81:
+  - :mac_addr
+  - :out_src_mac
+  82:
+  - :string
+  - :if_name
+  83:
+  - :string
+  - :if_desc
+  84:
+  - :string
+  - :sampler_name
+  89:
+  - :uint8
+  - :forwarding_status
+  91:
+  - :uint8
+  - :mpls_prefix_len
+  234:
+  - :uint32
+  - :ingress_vrf_id
+  235:
+  - :uint32
+  - :egress_vrf_id
+  236:
+  - :string
+  - :vrf_name
+
+scope:
+  1:
+  - :ip4_addr
+  - :system
+  2:
+  - :skip
+  - :interface
+  3:
+  - :skip
+  - :line_card
+  4:
+  - :skip
+  - :netflow_cache
+  5:
+  - :skip
+  - :template

data/lib/fluent/plugin/parser_netflow.rb

@@ -26,43 +26,36 @@ module Fluent
         super
 
         @templates = Vash.new()
+        @samplers_v9 = Vash.new()
         # Path to default Netflow v9 field definitions
-        filename = File.expand_path('../netflow_option_fields.yaml', __FILE__)
+        filename = File.expand_path('../netflow_fields.yaml', __FILE__)
 
         begin
           @fields = YAML.load_file(filename)
         rescue => e
-          raise "Bad syntax in definitions file #{filename}", error_class: e.class, error: e.message
+          raise ConfigError, "Bad syntax in definitions file #{filename}, error_class = #{e.class.name}, error = #{e.message}"
         end
 
         # Allow the user to augment/override/rename the supported Netflow fields
         if @definitions
-          raise "definitions file #{@definitions} does not exists" unless File.exist?(@definitions)
+          raise ConfigError, "definitions file #{@definitions} does not exists" unless File.exist?(@definitions)
           begin
-            @fields.merge!(YAML.load_file(@definitions))
+            @fields['option'].merge!(YAML.load_file(@definitions))
           rescue => e
-            raise "Bad syntax in definitions file #{@definitions}", error_class: e.class, error: e.message
+            raise ConfigError, "Bad syntax in definitions file #{@definitions}, error_class = #{e.class.name}, error = #{e.message}"
           end
         end
-        # Path to default Netflow v9 scope field definitions
-        filename = File.expand_path('../netflow_scope_fields.yaml', __FILE__)
-
-        begin
-          @scope_fields = YAML.load_file(filename)
-        rescue => e
-          raise "Bad syntax in scope definitions file #{filename}", error_class: e.class, error: e.message
-        end
       end
 
-      def call(payload, &block)
+      def call(payload, host=nil, &block)
         version,_ = payload[0,2].unpack('n')
         case version
         when 5
           forV5(payload, block)
         when 9
           # TODO: implement forV9
-          flowset = Netflow9PDU.read(payload)
-          handle_v9(flowset, block)
+          pdu = Netflow9PDU.read(payload)
+          handle_v9(host, pdu, block)
         else
           $log.warn "Unsupported Netflow version v#{version}: #{version.class}"
         end
@@ -190,34 +183,33 @@ module Fluent
         end
       end
 
-      def handle_v9(flowset, block)
-        flowset.records.each do |record|
-          case record.flowset_id
+      def handle_v9(host, pdu, block)
+        pdu.records.each do |flowset|
+          case flowset.flowset_id
           when 0
-            handle_v9_flowset_template(flowset, record)
+            handle_v9_flowset_template(host, pdu, flowset)
           when 1
-            handle_v9_flowset_options_template(flowset, record)
+            handle_v9_flowset_options_template(host, pdu, flowset)
           when 256..65535
-            handle_v9_flowset_data(flowset, record, block)
+            handle_v9_flowset_data(host, pdu, flowset, block)
           else
-            $log.warn "Unsupported flowset id #{record.flowset_id}"
+            $log.warn "Unsupported flowset id #{flowset.flowset_id}"
           end
         end
       end
 
-      def handle_v9_flowset_template(flowset, record)
-        record.flowset_data.templates.each do |template|
+      def handle_v9_flowset_template(host, pdu, flowset)
+        flowset.flowset_data.templates.each do |template|
           catch (:field) do
             fields = []
             template.fields.each do |field|
-              entry = netflow_field_for(field.field_type, field.field_length, @fields)
-              if !entry
-                throw :field
-              end
+              entry = netflow_field_for(field.field_type, field.field_length)
+              throw :field unless entry
+
               fields += entry
             end
             # We get this far, we have a list of fields
-            key = "#{flowset.source_id}|#{template.template_id}"
+            key = "#{host}|#{pdu.source_id}|#{template.template_id}"
             @templates[key, @cache_ttl] = BinData::Struct.new(endian: :big, fields: fields)
             # Purge any expired templates
             @templates.cleanup!
@@ -225,26 +217,24 @@ module Fluent
         end
       end
 
-      def handle_v9_flowset_options_template(flowset, record)
-        record.flowset_data.templates.each do |template|
+      NETFLOW_V9_FIELD_CATEGORIES = ['scope', 'option']
+
+      def handle_v9_flowset_options_template(host, pdu, flowset)
+        flowset.flowset_data.templates.each do |template|
           catch (:field) do
             fields = []
-            template.scope_fields.each do |field|
-              entry = netflow_field_for(field.field_type, field.field_length, @scope_fields)
-              if ! entry
-                throw :field
-              end
-              fields += entry
-            end
-            template.option_fields.each do |field|
-              entry = netflow_field_for(field.field_type, field.field_length, @fields)
-              if ! entry
-                throw :field
+
+            NETFLOW_V9_FIELD_CATEGORIES.each do |category|
+              template["#{category}_fields"].each do |field|
+                entry = netflow_field_for(field.field_type, field.field_length, category)
+                throw :field unless entry
+
+                fields += entry
               end
-              fields += entry
             end
+
             # We get this far, we have a list of fields
-            key = "#{flowset.source_id}|#{template.template_id}"
+            key = "#{host}|#{pdu.source_id}|#{template.template_id}"
             @templates[key, @cache_ttl] = BinData::Struct.new(endian: :big, fields: fields)
             # Purge any expired templates
             @templates.cleanup!
@@ -254,48 +244,55 @@ module Fluent
 
       FIELDS_FOR_COPY_V9 = ['version', 'flow_seq_num']
 
-      def handle_v9_flowset_data(flowset, record, block)
-        key = "#{flowset.source_id}|#{record.flowset_id}"
-        template = @templates[key]
+      def handle_v9_flowset_data(host, pdu, flowset, block)
+        template_key = "#{host}|#{pdu.source_id}|#{flowset.flowset_id}"
+        template = @templates[template_key]
         if ! template
-          $log.warn("No matching template for flow id #{record.flowset_id}")
+          $log.warn("No matching template for flow id #{flowset.flowset_id}")
           return
         end
 
-        length = record.flowset_length - 4
+        length = flowset.flowset_length - 4
 
-        # Template shouldn't be longer than the record and there should
+        # Template shouldn't be longer than the flowset and there should
         # be at most 3 padding bytes
         if template.num_bytes > length or ! (length % template.num_bytes).between?(0, 3)
           $log.warn "Template length doesn't fit cleanly into flowset",
-                    template_id: record.flowset_id, template_length: template.num_bytes, record_length: length
+                    template_id: flowset.flowset_id, template_length: template.num_bytes, flowset_length: length
           return
         end
 
         array = BinData::Array.new(type: template, initial_length: length / template.num_bytes)
 
-        records = array.read(record.flowset_data)
-        records.each do |r|
-          time = flowset.unix_sec
+        fields = array.read(flowset.flowset_data)
+        fields.each do |r|
+          if is_sampler?(r)
+            sampler_key = "#{host}|#{pdu.source_id}|#{r.flow_sampler_id}"
+            register_sampler_v9 sampler_key, r
+            next
+          end
+
+          time = pdu.unix_sec  # TODO: Fluent::EventTime (see: forV5)
           event = {}
 
           # Fewer fields in the v9 header
           FIELDS_FOR_COPY_V9.each do |f|
-            event[f] = flowset[f]
+            event[f] = pdu[f]
+          end
+
+          event['flowset_id'] = flowset.flowset_id
+
+          r.each_pair {|k,v| event[k.to_s] = v }
+          unless @switched_times_from_uptime
+            event['first_switched'] = format_for_switched(msec_from_boot_to_time(event['first_switched'], pdu.uptime, time, 0))
+            event['last_switched']  = format_for_switched(msec_from_boot_to_time(event['last_switched'] , pdu.uptime, time, 0))
           end
 
-          event['flowset_id'] = record.flowset_id
-
-          r.each_pair do |k,v|
-            case k.to_s
-            when /_switched$/
-              millis = flowset.uptime - v
-              seconds = flowset.unix_sec - (millis / 1000)
-              # v9 did away with the nanosecs field
-              micros = 1000000 - (millis % 1000)
-              event[k.to_s] = Time.at(seconds, micros).utc.strftime("%Y-%m-%dT%H:%M:%S.%3NZ")
-            else
-              event[k.to_s] = v
+          if sampler_id = r['flow_sampler_id']
+            sampler_key = "#{host}|#{pdu.source_id}|#{sampler_id}"
+            if sampler = @samplers_v9[sampler_key]
+              event['sampling_algorithm'] ||= sampler['flow_sampler_mode']
+              event['sampling_interval'] ||= sampler['flow_sampler_random_interval']
             end
           end
 
@@ -308,9 +305,9 @@ module Fluent
         ("uint" + (((length > 0) ? length : default) * 8).to_s).to_sym
       end
 
-      def netflow_field_for(type, length, field_definitions)
-        if field_definitions.include?(type)
-          field = field_definitions[type]
+      def netflow_field_for(type, length, category='option')
+        if @fields[category].include?(type)
+          field = @fields[category][type]
           if field.is_a?(Array)
 
             if field[0].is_a?(Integer)
@@ -336,6 +333,16 @@ module Fluent
           nil
         end
       end
+
+      # covers Netflow v9 and v10 (a.k.a IPFIX)
+      def is_sampler?(record)
+        record['flow_sampler_id'] && record['flow_sampler_mode'] && record['flow_sampler_random_interval']
+      end
+
+      def register_sampler_v9(key, sampler)
+        @samplers_v9[key, @cache_ttl] = sampler
+        @samplers_v9.cleanup!
+      end
     end
   end
 end

data/test/test_parser_netflow.rb

@@ -20,7 +20,7 @@ class NetflowParserTest < Test::Unit::TestCase
   test 'parse v5 binary data, dumped by netflow-generator' do
     # generated by https://github.com/mshindo/NetFlow-Generator
     parser = create_parser
-    raw_data = File.open(File.expand_path('../netflow.v5.dump', __FILE__)){|f| f.read }
+    raw_data = File.open(File.expand_path('../dump/netflow.v5.dump', __FILE__)){|f| f.read }
     bytes_for_1record = 72
     assert_equal bytes_for_1record, raw_data.size
     parsed = []

data/test/test_parser_netflow9.rb

@@ -0,0 +1,130 @@
+require 'helper'
+
+class Netflow9ParserTest < Test::Unit::TestCase
+  def setup
+    Fluent::Test.setup
+  end
+
+  def create_parser(conf={})
+    parser = Fluent::TextParser::NetflowParser.new
+    parser.configure(Fluent::Config::Element.new('ROOT', '', conf, []))
+    parser
+  end
+
+  def raw_template
+    @raw_template ||= File.read(File.expand_path('../dump/netflow.v9.template.dump', __FILE__))
+  end
+
+  def raw_data
+    @raw_data ||= File.read(File.expand_path('../dump/netflow.v9.dump', __FILE__))
+  end
+
+  def raw_sampler_template
+    @raw_sampler_template ||= File.read(File.expand_path('../dump/netflow.v9.sampler_template.dump', __FILE__))
+  end
+
+  def raw_sampler_data
+    @raw_sampler_data ||= File.read(File.expand_path('../dump/netflow.v9.sampler.dump', __FILE__))
+  end
+
+  DEFAULT_HOST = '127.0.0.1'
+
+  test 'parse netflow v9 binary data before loading corresponding template' do
+    parser = create_parser
+
+    assert_equal 92, raw_data.size
+    parser.call(raw_data, DEFAULT_HOST) do |time, record|
+      assert false, 'nothing emitted'
+    end
+  end
+
+  test 'parse netflow v9 binary data' do
+    parser = create_parser
+
+    parsed = []
+    parser.call raw_template, DEFAULT_HOST
+    parser.call(raw_data, DEFAULT_HOST) do |time, record|
+      parsed << [time, record]
+    end
+
+    assert_equal 1, parsed.size
+    assert_equal Time.parse('2016-02-12T04:02:25Z').to_i, parsed.first[0]
+    expected_record = {
+      # header
+      'version'      => 9,
+      'flow_seq_num' => 4645895,
+      'flowset_id'   => 260,
+
+      # flowset
+      'in_pkts'           => 1,
+      'in_bytes'          => 60,
+      'ipv4_src_addr'     => '192.168.0.1',
+      'ipv4_dst_addr'     => '192.168.0.2',
+      'input_snmp'        => 54,
+      'output_snmp'       => 29,
+      'last_switched'     => '2016-02-12T04:02:09.053Z',
+      'first_switched'    => '2016-02-12T04:02:09.053Z',
+      'l4_src_port'       => 80,
+      'l4_dst_port'       => 32822,
+      'src_as'            => 0,
+      'dst_as'            => 65000,
+      'bgp_ipv4_next_hop' => '192.168.0.3',
+      'src_mask'          => 24,
+      'dst_mask'          => 24,
+      'protocol'          => 6,
+      'tcp_flags'         => 0x12,
+      'src_tos'           => 0x0,
+      'direction'         => 0,
+      'forwarding_status' => 0b01000000,
+      'flow_sampler_id'   => 1,
+      'ingress_vrf_id'    => 1610612736,
+      'egress_vrf_id'     => 1610612736
+    }
+    assert_equal expected_record, parsed.first[1]
+  end
+
+  test 'parse netflow v9 binary data after sampler data is cached' do
+    parser = create_parser
+
+    parsed = []
+    [raw_sampler_template, raw_sampler_data, raw_template].each {|raw| parser.call(raw, DEFAULT_HOST){} }
+    parser.call(raw_data, DEFAULT_HOST) do |time, record|
+      parsed << [time, record]
+    end
+
+    assert_equal 2,    parsed.first[1]['sampling_algorithm']
+    assert_equal 5000, parsed.first[1]['sampling_interval']
+  end
+
+  test 'parse netflow v9 binary data with host-based template cache' do
+    parser = create_parser
+    another_host = DEFAULT_HOST.next
+
+    parsed = []
+    parser.call raw_template, DEFAULT_HOST
+    parser.call(raw_data, another_host) do |time, record|
+      assert false, 'nothing emitted'
+    end
+    parser.call raw_template, another_host
+    parser.call(raw_data, another_host) do |time, record|
+      parsed << [time, record]
+    end
+
+    assert_equal 1, parsed.size
+  end
+
+  test 'parse netflow v9 binary data with host-based sampler cache' do
+    parser = create_parser
+    another_host = DEFAULT_HOST.next
+
+    parsed = []
+    [raw_sampler_template, raw_sampler_data, raw_template].each {|raw| parser.call(raw, DEFAULT_HOST){} }
+    parser.call(raw_template, another_host){}
+    parser.call(raw_data, another_host) do |time, record|
+      parsed << [time, record]
+    end
+
+    assert_equal nil, parsed.first[1]['sampling_algorithm']
+    assert_equal nil, parsed.first[1]['sampling_interval']
+  end
+end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-netflow
 version: !ruby/object:Gem::Version
-  version: 0.2.0
+  version: 0.2.1
 platform: ruby
 authors:
 - Masahiro Nakagawa
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2016-03-01 00:00:00.000000000 Z
+date: 2016-03-24 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fluentd
@@ -87,15 +87,19 @@ files:
 - example/fluentd.conf
 - fluent-plugin-netflow.gemspec
 - lib/fluent/plugin/in_netflow.rb
-- lib/fluent/plugin/netflow_option_fields.yaml
+- lib/fluent/plugin/netflow_fields.yaml
 - lib/fluent/plugin/netflow_records.rb
-- lib/fluent/plugin/netflow_scope_fields.yaml
 - lib/fluent/plugin/parser_netflow.rb
 - lib/fluent/plugin/vash.rb
+- test/dump/netflow.v5.dump
+- test/dump/netflow.v9.dump
+- test/dump/netflow.v9.sampler.dump
+- test/dump/netflow.v9.sampler_template.dump
+- test/dump/netflow.v9.template.dump
 - test/helper.rb
-- test/netflow.v5.dump
 - test/test_in_netflow.rb
 - test/test_parser_netflow.rb
+- test/test_parser_netflow9.rb
 homepage: https://github.com/repeatedly/fluent-plugin-netflow
 licenses:
 - Apache License (2.0)
@@ -121,7 +125,12 @@ signing_key:
 specification_version: 4
 summary: Netflow plugin for Fluentd
 test_files:
+- test/dump/netflow.v5.dump
+- test/dump/netflow.v9.dump
+- test/dump/netflow.v9.sampler.dump
+- test/dump/netflow.v9.sampler_template.dump
+- test/dump/netflow.v9.template.dump
 - test/helper.rb
-- test/netflow.v5.dump
 - test/test_in_netflow.rb
 - test/test_parser_netflow.rb
+- test/test_parser_netflow9.rb

data/lib/fluent/plugin/netflow_option_fields.yaml

@@ -1,237 +0,0 @@
----
-1:
-- 4
-- :in_bytes
-2:
-- 4
-- :in_pkts
-3:
-- 4
-- :flows
-4:
-- :uint8
-- :protocol
-5:
-- :uint8
-- :src_tos
-6:
-- :uint8
-- :tcp_flags
-7:
-- :uint16
-- :l4_src_port
-8:
-- :ip4_addr
-- :ipv4_src_addr
-9:
-- :uint8
-- :src_mask
-10:
-- 2
-- :input_snmp
-11:
-- :uint16
-- :l4_dst_port
-12:
-- :ip4_addr
-- :ipv4_dst_addr
-13:
-- :uint8
-- :dst_mask
-14:
-- 2
-- :output_snmp
-15:
-- :ip4_addr
-- :ipv4_next_hop
-16:
-- 2
-- :src_as
-17:
-- 2
-- :dst_as
-18:
-- :ip4_addr
-- :bgp_ipv4_next_hop
-19:
-- 4
-- :mul_dst_pkts
-20:
-- 4
-- :mul_dst_bytes
-21:
-- :uint32
-- :last_switched
-22:
-- :uint32
-- :first_switched
-23:
-- 4
-- :out_bytes
-24:
-- 4
-- :out_pkts
-25:
-- :uint16
-- :min_pkt_length
-26:
-- :uint16
-- :max_pkt_length
-27:
-- :ip6_addr
-- :ipv6_src_addr
-28:
-- :ip6_addr
-- :ipv6_dst_addr
-29:
-- :uint8
-- :ipv6_src_mask
-30:
-- :uint8
-- :ipv6_dst_mask
-31:
-- 3
-- :ipv6_flow_label
-32:
-- :uint16
-- :icmp_type
-33:
-- :uint8
-- :mul_igmp_type
-34:
-- :uint32
-- :sampling_interval
-35:
-- :uint8
-- :sampling_algorithm
-36:
-- :uint16
-- :flow_active_timeout
-37:
-- :uint16
-- :flow_inactive_timeout
-38:
-- :uint8
-- :engine_type
-39:
-- :uint8
-- :engine_id
-40:
-- 4
-- :total_bytes_exp
-41:
-- 4
-- :total_pkts_exp
-42:
-- 4
-- :total_flows_exp
-43:
-- :skip
-44:
-- :ip4_addr
-- :ipv4_src_prefix
-45:
-- :ip4_addr
-- :ipv4_dst_prefix
-46:
-- :uint8
-- :mpls_top_label_type
-47:
-- :uint32
-- :mpls_top_label_ip_addr
-48:
-- 1
-- :flow_sampler_id
-49:
-- :uint8
-- :flow_sampler_mode
-50:
-- :uint32
-- :flow_sampler_random_interval
-51:
-- :skip
-52:
-- :uint8
-- :min_ttl
-53:
-- :uint8
-- :max_ttl
-54:
-- :uint16
-- :ipv4_ident
-55:
-- :uint8
-- :dst_tos
-56:
-- :mac_addr
-- :in_src_mac
-57:
-- :mac_addr
-- :out_dst_mac
-58:
-- :uint16
-- :src_vlan
-59:
-- :uint16
-- :dst_vlan
-60:
-- :uint8
-- :ip_protocol_version
-61:
-- :uint8
-- :direction
-62:
-- :ip6_addr
-- :ipv6_next_hop
-63:
-- :ip6_addr
-- :bgp_ipv6_next_hop
-64:
-- :uint32
-- :ipv6_option_headers
-65:
-- :skip
-66:
-- :skip
-67:
-- :skip
-68:
-- :skip
-69:
-- :skip
-70:
-- :mpls_label
-- :mpls_label_1
-71:
-- :mpls_label
-- :mpls_label_2
-72:
-- :mpls_label
-- :mpls_label_3
-80:
-- :mac_addr
-- :in_dst_mac
-81:
-- :mac_addr
-- :out_src_mac
-82:
-- :string
-- :if_name
-83:
-- :string
-- :if_desc
-84:
-- :string
-- :sampler_name
-89:
-- :uint8
-- :forwarding_status
-91:
-- :uint8
-- :mpls_prefix_len
-234:
-- :uint32
-- :ingress_vrf_id
-235:
-- :uint32
-- :egress_vrf_id

data/lib/fluent/plugin/netflow_scope_fields.yaml

@@ -1,16 +0,0 @@
----
-1:
-- :ip4_addr
-- :system
-2:
-- :skip
-- :interface
-3:
-- :skip
-- :line_card
-4:
-- :skip
-- :netflow_cache
-5:
-- :skip
-- :template