fluent-plugin-netflow 0.2.1 → 0.2.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 3cd8781b6ecc14e1dd982e4046b795a34412e768
-  data.tar.gz: be9f57fbb0fd1323fba31f718146632e6d2d5fdc
+  metadata.gz: 5d48e590e8c0d4358dbec1e7fb8613cd828a2eff
+  data.tar.gz: 9245b86b319963bf4ea64573028003ac195b7ebe
 SHA512:
-  metadata.gz: aca8f4e3c4f146ad05e0b0f166e1a7bd65e77352e4ce0927101e5d2d6b3f0d6ad443a39ba976c8c47b917366bf363ff2d1a17ed0bc0b068e4be81879b9d9f27e
-  data.tar.gz: 14d3a412b02acccd57ef7723ee1b2b32e87f3b3c34d5f4e14f6b85bb5fc6f23f78d575c809a24a56e410879af6b6a22f139a62279a400340f3a827ebcc257f2b
+  metadata.gz: e18722b820dfc5d0e28846ee42722407d3cd1d276758e9930f1305ca5fb06e23bf2257b715dd6e17111e33c9319957d6f86e5843fb9140747afb66b0b1da3cb9
+  data.tar.gz: 9b6a67536069dd3f9f17f76fecd384f91b1d24ba403ff09838ea28480069246075b0a3f41ebb1a8404da7f20c27a95907b11cb0fa71333179dc539758acdec07

data/README.md

@@ -109,6 +109,51 @@ end
 **NOTE:**
 If the plugin receives Netflow v9 from multiple sources, provide ```source_ip_address``` argument to parse correctly.
 
+### Field definition for Netflow v9
+
+Both option and scope fields for Netflow v9 are defined in [YAML](https://www.ietf.org/rfc/rfc3954.txt) where two parameters are described for each field value like:
+
+```yaml
+option:
+  ...
+  4:           # field value
+  - :uint8     # field length
+  - :protocol  # field type
+```
+
+See [RFC3954 document](https://www.ietf.org/rfc/rfc3954.txt) for more details.
+
+When int value specified for field length, the template parser in this plugin will prefer a field length in received template flowset over YAML. The int value in YAML will be used as a default value only when the length in received flowset is invalid.
+
+```yaml
+option:
+  1:
+  - 4          # means :unit32, which is just a default
+  - :in_bytes
+```
+
+When ```:skip``` is described for a field, the template parser will learn the length from received template flowset and skip the field when data flowsets are processed.
+
+```yaml
+option:
+  ...
+  43:
+  - :skip
+```
+
+**NOTE:**
+The definitions don't exactly reflect RFC3954 in order to cover some illegal implementations which export Netflow v9 in bad field length.
+
+```yaml
+   31:
+   - 3  # Some system exports in 4 bytes despite of RFC
+   - :ipv6_flow_label
+   ...
+   48:
+   - 1  # Some system exports in 2 bytes despite of RFC
+   - :flow_sampler_id
+```
+
 ### More speed ?
 
 :bullettrain_side: Try ```switched_times_from_uptime true``` option !

data/VERSION

@@ -1 +1 @@
-0.2.1
+0.2.2

data/lib/fluent/plugin/netflow_fields.yaml

@@ -209,6 +209,27 @@ option:
   72:
   - :mpls_label
   - :mpls_label_3
+  73:
+  - :mpls_label
+  - :mpls_label_4
+  74:
+  - :mpls_label
+  - :mpls_label_5
+  75:
+  - :mpls_label
+  - :mpls_label_6
+  76:
+  - :mpls_label
+  - :mpls_label_7
+  77:
+  - :mpls_label
+  - :mpls_label_8
+  78:
+  - :mpls_label
+  - :mpls_label_9
+  79:
+  - :mpls_label
+  - :mpls_label_10
   80:
   - :mac_addr
   - :in_dst_mac
@@ -246,13 +267,9 @@ scope:
   - :system
   2:
   - :skip
-  - :interface
   3:
   - :skip
-  - :line_card
   4:
   - :skip
-  - :netflow_cache
   5:
   - :skip
-  - :template

data/lib/fluent/plugin/parser_netflow.rb

@@ -38,7 +38,7 @@ module Fluent
 
         # Allow the user to augment/override/rename the supported Netflow fields
         if @definitions
-          raise ConfigError, "definitions file #{@definitions} does not exists" unless File.exist?(@definitions)
+          raise ConfigError, "definitions file #{@definitions} doesn't exist" unless File.exist?(@definitions)
           begin
             @fields['option'].merge!(YAML.load_file(@definitions))
           rescue => e
@@ -193,7 +193,7 @@ module Fluent
           when 256..65535
             handle_v9_flowset_data(host, pdu, flowset, block)
           else
-            $log.warn "Unsupported flowset id #{flowset.flowset_id}"
+            $log.warn 'Unsupported flowset', flowset_id: flowset.flowset_id
           end
         end
       end
@@ -248,7 +248,8 @@ module Fluent
         template_key = "#{host}|#{pdu.source_id}|#{flowset.flowset_id}"
         template = @templates[template_key]
         if ! template
-          $log.warn("No matching template for flow id #{flowset.flowset_id}")
+          $log.warn 'No matching template for',
+                    host: host, source_id: pdu.source_id, flowset_id: flowset.flowset_id
           return
         end
 
@@ -306,31 +307,26 @@ module Fluent
       end
 
       def netflow_field_for(type, length, category='option')
-        if @fields[category].include?(type)
-          field = @fields[category][type]
-          if field.is_a?(Array)
-
-            if field[0].is_a?(Integer)
-              field[0] = uint_field(length, field[0])
-            end
+        unless field = @fields[category][type]
+          $log.warn "Skip unsupported field", type: type, length: length
+          return [:skip, nil, {length: length}]
+        end
 
-            # Small bit of fixup for skip or string field types where the length
-            # is dynamic
-            case field[0]
-            when :skip
-              field += [nil, {length: length}]
-            when :string
-              field += [{length: length, trim_padding: true}]
-            end
+        unless field.is_a?(Array)
+          $log.warn "Skip non-Array definition", field: field
+          return [:skip, nil, {length: length}]
+        end
 
-            [field]
-          else
-            $log.warn "Definition should be an array", field: field
-            nil
-          end
+        # Small bit of fixup for numeric value, :skip or :string field length, which are dynamic
+        case field[0]
+        when Integer
+          [[uint_field(length, field[0]), field[1]]]
+        when :skip
+          [field + [nil, {length: length}]]
+        when :string
+          [field + [{length: length, trim_padding: true}]]
         else
-          $log.warn "Unsupported field", type: type, length: length
-          nil
+          [field]
         end
       end
 

data/test/test_parser_netflow9.rb

@@ -15,10 +15,18 @@ class Netflow9ParserTest < Test::Unit::TestCase
     @raw_template ||= File.read(File.expand_path('../dump/netflow.v9.template.dump', __FILE__))
   end
 
+  def raw_mpls_template
+    @raw_mpls_template ||= File.read(File.expand_path('../dump/netflow.v9.mpls-template.dump', __FILE__))
+  end
+
   def raw_data
     @raw_data ||= File.read(File.expand_path('../dump/netflow.v9.dump', __FILE__))
   end
 
+  def raw_mpls_data
+    @raw_mpls_data ||= File.read(File.expand_path('../dump/netflow.v9.mpls-data.dump', __FILE__))
+  end
+
   def raw_sampler_template
     @raw_sampler_template ||= File.read(File.expand_path('../dump/netflow.v9.sampler_template.dump', __FILE__))
   end
@@ -27,6 +35,10 @@ class Netflow9ParserTest < Test::Unit::TestCase
     @raw_sampler_data ||= File.read(File.expand_path('../dump/netflow.v9.sampler.dump', __FILE__))
   end
 
+  def raw_2byte_as_template
+    @raw_2byte_as_template ||= File.read(File.expand_path('../dump/netflow.v9.template.as2.dump', __FILE__))
+  end
+
   DEFAULT_HOST = '127.0.0.1'
 
   test 'parse netflow v9 binary data before loading corresponding template' do
@@ -127,4 +139,32 @@ class Netflow9ParserTest < Test::Unit::TestCase
     assert_equal nil, parsed.first[1]['sampling_algorithm']
     assert_equal nil, parsed.first[1]['sampling_interval']
   end
+
+  test 'parse netflow v9 binary data with templates whose AS field length varies' do
+    parser = create_parser
+
+    parsed = []
+    [raw_2byte_as_template, raw_template].each {|raw| parser.call(raw, DEFAULT_HOST){} }
+    parser.call(raw_data, DEFAULT_HOST) do |time, record|
+      parsed << [time, record]
+    end
+
+    assert_equal 1, parsed.size
+    assert_equal     0, parsed.first[1]['src_as']
+    assert_equal 65000, parsed.first[1]['dst_as']
+  end
+
+  test 'parse netflow v9 binary data contains mpls information' do
+    parser = create_parser
+
+    parsed = []
+    [raw_sampler_template, raw_sampler_data, raw_mpls_template].each {|raw| parser.call(raw, DEFAULT_HOST){} }
+    parser.call(raw_mpls_data, DEFAULT_HOST) do |time, record|
+      parsed << [time, record]
+    end
+
+    assert_equal 24002, parsed.first[1]['mpls_label_1']
+    assert_equal '192.168.32.100', parsed.first[1]['ipv4_src_addr']
+    assert_equal '172.16.32.2', parsed.first[1]['ipv4_dst_addr']
+  end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-netflow
 version: !ruby/object:Gem::Version
-  version: 0.2.1
+  version: 0.2.2
 platform: ruby
 authors:
 - Masahiro Nakagawa
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2016-03-24 00:00:00.000000000 Z
+date: 2016-04-07 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fluentd
@@ -93,8 +93,11 @@ files:
 - lib/fluent/plugin/vash.rb
 - test/dump/netflow.v5.dump
 - test/dump/netflow.v9.dump
+- test/dump/netflow.v9.mpls-data.dump
+- test/dump/netflow.v9.mpls-template.dump
 - test/dump/netflow.v9.sampler.dump
 - test/dump/netflow.v9.sampler_template.dump
+- test/dump/netflow.v9.template.as2.dump
 - test/dump/netflow.v9.template.dump
 - test/helper.rb
 - test/test_in_netflow.rb
@@ -127,8 +130,11 @@ summary: Netflow plugin for Fluentd
 test_files:
 - test/dump/netflow.v5.dump
 - test/dump/netflow.v9.dump
+- test/dump/netflow.v9.mpls-data.dump
+- test/dump/netflow.v9.mpls-template.dump
 - test/dump/netflow.v9.sampler.dump
 - test/dump/netflow.v9.sampler_template.dump
+- test/dump/netflow.v9.template.as2.dump
 - test/dump/netflow.v9.template.dump
 - test/helper.rb
 - test/test_in_netflow.rb