fluent-plugin-netflow-enchanced 1.0.0.rc1

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 4c76eebf8b18910e2ac39ab145f110dd01237496
+  data.tar.gz: e4b6d605e9fe93e9d26024e3bca00abd940e5341
+SHA512:
+  metadata.gz: e3160293b97bece1d8dba37b608e044c90782907b17ee2c055ac4309ee72b44e2d98fb10f9e23f0b2318cf850bd6718b8ad6f62f1a2f1fd7063d9c38d6f26c5b
+  data.tar.gz: 5b1dabb47633997098275b699ae9c642a8956fbb0869688844f0db8790f5007a8ca625045c19f0cc65c262c2c2b6c562a95ce086744b06ae2dfbdd8c98b45556

data/.gitignore

@@ -0,0 +1,24 @@
+*.gem
+*.rbc
+.bundle
+.config
+.yardoc
+Gemfile.lock
+InstalledFiles
+_yardoc
+coverage
+doc/
+lib/bundler/man
+pkg
+rdoc
+spec/reports
+test/tmp
+test/version_tmp
+tmp
+# For TextMate, emacs, vim
+*.tmproj
+tmtags
+*~
+\#*
+.\#*
+*.swp

data/.travis.yml

@@ -0,0 +1,18 @@
+language: ruby
+
+rvm:
+  - 2.1
+  - 2.2
+  - 2.3.1
+  - 2.4.0
+  - ruby-head
+  - rbx
+
+matrix:
+  allow_failures:
+    - rvm: ruby-head
+    - rvm: rbx
+
+before_install: gem update bundler
+
+script: bundle exec rake test

data/Gemfile

@@ -0,0 +1,3 @@
+source "https://rubygems.org"
+
+gemspec

data/README.md

@@ -0,0 +1,180 @@
+# Netflow plugin for Fluentd
+
+[![Build Status](https://travis-ci.org/repeatedly/fluent-plugin-netflow.svg)](https://travis-ci.org/repeatedly/fluent-plugin-netflow)
+
+
+## Overview
+
+[Fluentd](http://fluentd.org/) input plugin that acts as Netflow v5/v9 collector.
+
+
+## Installation
+
+Use RubyGems:
+
+    fluent-gem install fluent-plugin-netflow
+
+
+## Configuration
+
+    <source>
+      type netflow
+      tag netflow.event
+
+      # optional parameters
+      bind 192.168.0.1
+      port 2055
+      cache_ttl 6000
+      versions [5, 9]
+      definitions /path/to/custom_fields.yaml
+    </source>
+
+**bind**
+
+IP address on which the plugin will accept Netflow.  
+(Default: '0.0.0.0')
+
+**port**
+
+UDP port number on which tpe plugin will accept Netflow.  
+(Default: 5140)
+
+**cache_ttl**
+
+Template cache TTL for Netflow v9 in seconds. Templates not refreshed from the Netflow v9 exporter within the TTL are expired at the plugin.  
+(Default: 4000)
+
+**versions**
+
+Netflow versions which are acceptable.  
+(Default:[5, 9])
+
+**switched_times_from_uptime**
+
+When set to true, the plugin stores system uptime for ```first_switched``` and ```last_switched``` instead of ISO8601-formatted absolute time.  
+(Defaults: false)
+
+**definitions**
+
+YAML file containing Netflow field definitions to overfide pre-defined templates. Example is like below
+
+    ---
+    4:          # field value
+    - :uint8    # field length
+    - :protocol # field type
+
+
+## Performance Evaluation
+
+Benchmark for v5 protocol on Macbook Air (Early 2014, 1.7 GHz Intel Core i7):
+* 0 packets dropped in 32,000 records/second (for 3,000,000 packets)
+* 45,000 records/second in maximum (for flooding netflow packets)
+
+Tested with the packet generator below:
+
+* https://github.com/mshindo/NetFlow-Generator
+* `./flowgen -n3000000 -i50 -w1 -p5140 localhost`
+
+And configuration:
+
+    <source>
+      @type  netflow
+      tag netflow.event
+      bind 0.0.0.0
+      port 5140
+      switched_times_from_uptime yes
+    </source>
+    <match netflow.event>
+      @type flowcounter
+      unit minute
+      count_keys count # missing column for counting events only
+      tag flowcount
+    </match>
+    <match flowcount>
+      @type stdout
+    </match>
+
+
+## Tips
+
+### Use netflow parser in other plugins
+
+```ruby
+require 'fluent/plugin/parser_netflow'
+
+parser = Fluent::Plugin::NetflowParser.new
+parser.configure(conf)
+
+# Netflow v5
+parser.call(payload) do |time, record|
+  # do something
+end
+
+# Netflow v9
+parser.call(payload, source_ip_address) do |time, record|
+  # do something
+end
+```
+
+**NOTE:**
+If the plugin receives Netflow v9 from multiple sources, provide ```source_ip_address``` argument to parse correctly.
+
+### Field definition for Netflow v9
+
+Both option and scope fields for Netflow v9 are defined in [YAML](https://www.ietf.org/rfc/rfc3954.txt) where two parameters are described for each field value like:
+
+```yaml
+option:
+  ...
+  4:           # field value
+  - :uint8     # field length
+  - :protocol  # field type
+```
+
+See [RFC3954 document](https://www.ietf.org/rfc/rfc3954.txt) for more details.
+
+When int value specified for field length, the template parser in this plugin will prefer a field length in received template flowset over YAML. The int value in YAML will be used as a default value only when the length in received flowset is invalid.
+
+```yaml
+option:
+  1:
+  - 4          # means :unit32, which is just a default
+  - :in_bytes
+```
+
+When ```:skip``` is described for a field, the template parser will learn the length from received template flowset and skip the field when data flowsets are processed.
+
+```yaml
+option:
+  ...
+  43:
+  - :skip
+```
+
+**NOTE:**
+The definitions don't exactly reflect RFC3954 in order to cover some illegal implementations which export Netflow v9 in bad field length.
+
+```yaml
+   31:
+   - 3  # Some system exports in 4 bytes despite of RFC
+   - :ipv6_flow_label
+   ...
+   48:
+   - 1  # Some system exports in 2 bytes despite of RFC
+   - :flow_sampler_id
+```
+
+### PaloAlto Netflow
+
+PaloAlto Netflow has different field definitionas:
+See this definitions for PaloAlto Netflow: https://github.com/repeatedly/fluent-plugin-netflow/issues/27#issuecomment-269197495
+
+### More speed ?
+
+:bullettrain_side: Try ```switched_times_from_uptime true``` option !
+
+
+## TODO
+
+* Netflow v9 protocol parser optimization
+* Use Fluentd feature instead of own handlers

data/Rakefile

@@ -0,0 +1,14 @@
+
+require 'bundler'
+Bundler::GemHelper.install_tasks
+
+require 'rake/testtask'
+
+Rake::TestTask.new(:test) do |test|
+  test.libs << 'lib' << 'test'
+  test.test_files = FileList['test/**/test_*.rb']
+  test.verbose = true
+end
+
+task :default => [:build]
+

data/VERSION

@@ -0,0 +1 @@
+1.0.0.rc1

data/example/fluentd.conf

@@ -0,0 +1,9 @@
+<source>
+  @type netflow
+  bind 127.0.0.1
+  tag  example.netflow
+</source>
+
+<match example.netflow>
+  @type stdout
+</match>

data/fluent-plugin-netflow.gemspec

@@ -0,0 +1,24 @@
+# encoding: utf-8
+$:.push File.expand_path('../lib', __FILE__)
+
+Gem::Specification.new do |gem|
+  gem.name        = "fluent-plugin-netflow-enchanced"
+  gem.description = "Netflow plugin for Fluentd"
+  gem.homepage    = "https://github.com/repeatedly/fluent-plugin-netflow"
+  gem.summary     = gem.description
+  gem.version     = File.read("VERSION").strip
+  gem.authors     = ["Masahiro Nakagawa"]
+  gem.email       = "repeatedly@gmail.com"
+  gem.has_rdoc    = false
+  #gem.platform    = Gem::Platform::RUBY
+  gem.license     = 'Apache License (2.0)'
+  gem.files       = `git ls-files`.split("\n")
+  gem.test_files  = `git ls-files -- {test,spec,features}/*`.split("\n")
+  gem.executables = `git ls-files -- bin/*`.split("\n").map{ |f| File.basename(f) }
+  gem.require_paths = ['lib']
+
+  gem.add_dependency "fluentd", [">= 0.14.10", "< 2"]
+  gem.add_dependency "bindata", "~> 2.1"
+  gem.add_development_dependency "rake", ">= 0.9.2"
+  gem.add_development_dependency "test-unit", "~> 3.0"
+end

data/lib/fluent/plugin/in_netflow.rb

@@ -0,0 +1,80 @@
+#
+# Fluent
+#
+# Copyright (C) 2014 Masahiro Nakagawa
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+
+require 'fluent/plugin/input'
+require 'fluent/plugin/parser_netflow'
+
+module Fluent::Plugin
+  class NetflowInput < Input
+    Fluent::Plugin.register_input('netflow', self)
+
+    helpers :server
+
+    config_param :port, :integer, default: 5140
+    config_param :bind, :string, default: '0.0.0.0'
+    config_param :tag, :string
+    config_param :protocol_type, default: :udp do |val|
+      case val.downcase
+      when 'udp'
+        :udp
+      else
+        raise Fluent::ConfigError, "netflow input protocol type should be 'udp'"
+      end
+    end
+    config_param :max_bytes, :integer, default: 2048
+
+    def configure(conf)
+      super
+
+      @parser = Fluent::Plugin::NetflowParser.new
+      @parser.configure(conf)
+    end
+
+    def start
+      super
+      server_create(:in_netflow_server, @port, bind: @bind, proto: @protocol_type, max_bytes: @max_bytes) do |data, sock|
+        receive_data(sock.remote_host, data)
+      end
+    end
+
+    def shutdown
+      super
+    end
+
+    protected
+
+    def receive_data(host, data)
+      log.on_debug { log.debug "received logs", :host => host, :data => data }
+
+      @parser.call(data, host) { |time, record|
+        unless time && record
+          log.warn "pattern not match: #{data.inspect}"
+          return
+        end
+
+        record['host'] = host
+        record['process_time'] = Time.new.strftime("%Y-%m-%dT%H:%M:%S.%LZ")
+
+        router.emit(@tag, time, record)
+      }
+    rescue => e
+      log.warn "unexpected error on parsing", data: data.dump, error_class: e.class, error: e.message
+      log.warn_backtrace
+    end
+  end
+end

data/lib/fluent/plugin/netflow_fields.yaml

@@ -0,0 +1,302 @@
+---
+option:
+  1:
+  - 4
+  - :in_bytes
+  2:
+  - 4
+  - :in_pkts
+  3:
+  - 4
+  - :flows
+  4:
+  - :uint8
+  - :protocol
+  5:
+  - :uint8
+  - :src_tos
+  6:
+  - :uint8
+  - :tcp_flags
+  7:
+  - :uint16
+  - :l4_src_port
+  8:
+  - :ip4_addr
+  - :ipv4_src_addr
+  9:
+  - :uint8
+  - :src_mask
+  10:
+  - 2
+  - :input_snmp
+  11:
+  - :uint16
+  - :l4_dst_port
+  12:
+  - :ip4_addr
+  - :ipv4_dst_addr
+  13:
+  - :uint8
+  - :dst_mask
+  14:
+  - 2
+  - :output_snmp
+  15:
+  - :ip4_addr
+  - :ipv4_next_hop
+  16:
+  - 2
+  - :src_as
+  17:
+  - 2
+  - :dst_as
+  18:
+  - :ip4_addr
+  - :bgp_ipv4_next_hop
+  19:
+  - 4
+  - :mul_dst_pkts
+  20:
+  - 4
+  - :mul_dst_bytes
+  21:
+  - :uint32
+  - :last_switched
+  22:
+  - :uint32
+  - :first_switched
+  23:
+  - 4
+  - :out_bytes
+  24:
+  - 4
+  - :out_pkts
+  25:
+  - :uint16
+  - :min_pkt_length
+  26:
+  - :uint16
+  - :max_pkt_length
+  27:
+  - :ip6_addr
+  - :ipv6_src_addr
+  28:
+  - :ip6_addr
+  - :ipv6_dst_addr
+  29:
+  - :uint8
+  - :ipv6_src_mask
+  30:
+  - :uint8
+  - :ipv6_dst_mask
+  31:
+  - 3
+  - :ipv6_flow_label
+  32:
+  - :uint16
+  - :icmp_type
+  33:
+  - :uint8
+  - :mul_igmp_type
+  34:
+  - :uint32
+  - :sampling_interval
+  35:
+  - :uint8
+  - :sampling_algorithm
+  36:
+  - :uint16
+  - :flow_active_timeout
+  37:
+  - :uint16
+  - :flow_inactive_timeout
+  38:
+  - :uint8
+  - :engine_type
+  39:
+  - :uint8
+  - :engine_id
+  40:
+  - 4
+  - :total_bytes_exp
+  41:
+  - 4
+  - :total_pkts_exp
+  42:
+  - 4
+  - :total_flows_exp
+  43:
+  - :skip
+  44:
+  - :ip4_addr
+  - :ipv4_src_prefix
+  45:
+  - :ip4_addr
+  - :ipv4_dst_prefix
+  46:
+  - :uint8
+  - :mpls_top_label_type
+  47:
+  - :uint32
+  - :mpls_top_label_ip_addr
+  48:
+  - 1
+  - :flow_sampler_id
+  49:
+  - :uint8
+  - :flow_sampler_mode
+  50:
+  - :uint32
+  - :flow_sampler_random_interval
+  51:
+  - :skip
+  52:
+  - :uint8
+  - :min_ttl
+  53:
+  - :uint8
+  - :max_ttl
+  54:
+  - :uint16
+  - :ipv4_ident
+  55:
+  - :uint8
+  - :dst_tos
+  56:
+  - :mac_addr
+  - :in_src_mac
+  57:
+  - :mac_addr
+  - :out_dst_mac
+  58:
+  - :uint16
+  - :src_vlan
+  59:
+  - :uint16
+  - :dst_vlan
+  60:
+  - :uint8
+  - :ip_protocol_version
+  61:
+  - :uint8
+  - :direction
+  62:
+  - :ip6_addr
+  - :ipv6_next_hop
+  63:
+  - :ip6_addr
+  - :bgp_ipv6_next_hop
+  64:
+  - :uint32
+  - :ipv6_option_headers
+  65:
+  - :skip
+  66:
+  - :skip
+  67:
+  - :skip
+  68:
+  - :skip
+  69:
+  - :skip
+  70:
+  - :mpls_label
+  - :mpls_label_1
+  71:
+  - :mpls_label
+  - :mpls_label_2
+  72:
+  - :mpls_label
+  - :mpls_label_3
+  73:
+  - :mpls_label
+  - :mpls_label_4
+  74:
+  - :mpls_label
+  - :mpls_label_5
+  75:
+  - :mpls_label
+  - :mpls_label_6
+  76:
+  - :mpls_label
+  - :mpls_label_7
+  77:
+  - :mpls_label
+  - :mpls_label_8
+  78:
+  - :mpls_label
+  - :mpls_label_9
+  79:
+  - :mpls_label
+  - :mpls_label_10
+  80:
+  - :mac_addr
+  - :in_dst_mac
+  81:
+  - :mac_addr
+  - :out_src_mac
+  82:
+  - :string
+  - :if_name
+  83:
+  - :string
+  - :if_desc
+  84:
+  - :string
+  - :sampler_name
+  89:
+  - :uint8
+  - :forwarding_status
+  91:
+  - :uint8
+  - :mpls_prefix_len
+  95:
+  - 4
+  - :app_id
+  150:
+  - :uint32
+  - :flowStartSeconds
+  151:
+  - :uint32
+  - :flowEndSeconds
+  152:
+  - :uint64
+  - :flowStartMilliseconds
+  153:
+  - :uint64
+  - :flowEndMilliseconds
+  154:
+  - :uint64
+  - :flowStartMicroseconds
+  155:
+  - :uint64
+  - :flowEndMicroseconds
+  156:
+  - :uint64
+  - :flowStartNanoseconds
+  157:
+  - :uint64
+  - :flowEndNanoseconds
+  234:
+  - :uint32
+  - :ingress_vrf_id
+  235:
+  - :uint32
+  - :egress_vrf_id
+  236:
+  - :string
+  - :vrf_name
+
+scope:
+  1:
+  - :ip4_addr
+  - :system
+  2:
+  - :skip
+  3:
+  - :skip
+  4:
+  - :skip
+  5:
+  - :skip