fluent-plugin-netflow 0.2.8 → 1.0.0.rc1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: c9584ad91d1208c8ad6bcbc2823dbff74d0227cf
-  data.tar.gz: 934185e7c819deefb8ba1e371e7242ffbfe8c935
+  metadata.gz: 4d4c63738c6b09eef497f26c714a2834fdce3e78
+  data.tar.gz: d6883ea9b5f1f7ab29cec7557ca305c5260e74b5
 SHA512:
-  metadata.gz: 0d398b04b33ae4ccbdb4642b77f1c52daccc78f59e003e0ca995732fd93c0f20d4d5e76e66282b984b4ab9045a1ae00923378bfc4dfc1770af3746ba571ad0fd
-  data.tar.gz: 05b97f663f248fc1ab79360ff241773019f4154be0348128701e57ebb088a43c5925c6f8167c76734978a6baedef7b45976223f94c36e186d5cf4aefba693de0
+  metadata.gz: 9c0577832be1c6aab86590a93df190bdb02a532ad8a3ab0f99925ae18f27b4a927c9cc76fb97dcd0aa4fa900e2e58a26726ec3e08b057c08ba73f19157a2b282
+  data.tar.gz: '0712933f971b7aa748e8b97dd1ce0ee0526ddadc110d8db0ccd6b536f78ba01ffbbd17a6a3dd481ec3d057628c401d76b9c9ea8f6e094f1a0907038be43e1d68'

data/.travis.yml

@@ -4,6 +4,7 @@ rvm:
   - 2.1
   - 2.2
   - 2.3.1
+  - 2.4.0
   - ruby-head
   - rbx
 

data/README.md

@@ -26,6 +26,7 @@ Use RubyGems:
       port 2055
       cache_ttl 6000
       versions [5, 9]
+      definitions /path/to/custom_fields.yaml
     </source>
 
 **bind**
@@ -53,6 +54,15 @@ Netflow versions which are acceptable.
 When set to true, the plugin stores system uptime for ```first_switched``` and ```last_switched``` instead of ISO8601-formatted absolute time.  
 (Defaults: false)
 
+**definitions**
+
+YAML file containing Netflow field definitions to overfide pre-defined templates. Example is like below
+
+    ---
+    4:          # field value
+    - :uint8    # field length
+    - :protocol # field type
+
 
 ## Performance Evaluation
 
@@ -92,7 +102,7 @@ And configuration:
 ```ruby
 require 'fluent/plugin/parser_netflow'
 
-parser = TextParser::NetflowParser.new
+parser = Fluent::Plugin::NetflowParser.new
 parser.configure(conf)
 
 # Netflow v5
@@ -154,6 +164,11 @@ The definitions don't exactly reflect RFC3954 in order to cover some illegal imp
    - :flow_sampler_id
 ```
 
+### PaloAlto Netflow
+
+PaloAlto Netflow has different field definitionas:
+See this definitions for PaloAlto Netflow: https://github.com/repeatedly/fluent-plugin-netflow/issues/27#issuecomment-269197495
+
 ### More speed ?
 
 :bullettrain_side: Try ```switched_times_from_uptime true``` option !

data/VERSION

@@ -1 +1 @@
-0.2.8
+1.0.0.rc1

data/fluent-plugin-netflow.gemspec

@@ -17,7 +17,7 @@ Gem::Specification.new do |gem|
   gem.executables = `git ls-files -- bin/*`.split("\n").map{ |f| File.basename(f) }
   gem.require_paths = ['lib']
 
-  gem.add_dependency "fluentd", [">= 0.10.17", "< 2"]
+  gem.add_dependency "fluentd", [">= 0.14.10", "< 2"]
   gem.add_dependency "bindata", "~> 2.1"
   gem.add_development_dependency "rake", ">= 0.9.2"
   gem.add_development_dependency "test-unit", "~> 3.0"

data/lib/fluent/plugin/in_netflow.rb

@@ -15,14 +15,15 @@
 #    See the License for the specific language governing permissions and
 #    limitations under the License.
 #
-require 'cool.io'
-require 'fluent/input'
-require 'fluent/plugin/socket_util'
+
+require 'fluent/plugin/input'
 require 'fluent/plugin/parser_netflow'
 
-module Fluent
+module Fluent::Plugin
   class NetflowInput < Input
-    Plugin.register_input('netflow', self)
+    Fluent::Plugin.register_input('netflow', self)
+
+    helpers :server
 
     config_param :port, :integer, default: 5140
     config_param :bind, :string, default: '0.0.0.0'
@@ -32,41 +33,29 @@ module Fluent
       when 'udp'
         :udp
       else
-        raise ConfigError, "netflow input protocol type should be 'udp'"
+        raise Fluent::ConfigError, "netflow input protocol type should be 'udp'"
       end
     end
+    config_param :max_bytes, :integer, default: 2048
 
     def configure(conf)
       super
 
-      @parser = TextParser::NetflowParser.new
+      @parser = Fluent::Plugin::NetflowParser.new
       @parser.configure(conf)
     end
 
     def start
       super
-      @loop = Coolio::Loop.new
-      @handler = listen(method(:receive_data))
-      @loop.attach(@handler)
-
-      @thread = Thread.new(&method(:run))
+      server_create(:in_netflow_server, @port, bind: @bind, proto: @protocol_type, max_bytes: @max_bytes) do |data, sock|
+        receive_data(sock.remote_host, data)
+      end
     end
 
     def shutdown
-      @loop.watchers.each { |w| w.detach }
-      @loop.stop
-      @handler.close
-      @thread.join
       super
     end
 
-    def run
-      @loop.run
-    rescue => e
-      log.error "unexpected error", error_class: e.class, error: e.message
-      log.error_backtrace
-    end
-
     protected
 
     def receive_data(host, data)
@@ -85,34 +74,5 @@ module Fluent
       log.warn "unexpected error on parsing", data: data.dump, error_class: e.class, error: e.message
       log.warn_backtrace
     end
-
-    private
-
-    def listen(callback)
-      log.info "listening netflow socket on #{@bind}:#{@port} with #{@protocol_type}"
-      if @protocol_type == :udp
-        @usock = SocketUtil.create_udp_socket(@bind)
-        @usock.bind(@bind, @port)
-        UdpHandler.new(@usock, callback)
-      else
-        Coolio::TCPServer.new(@bind, @port, TcpHandler, log, callback)
-      end
-    end
-
-    class UdpHandler < Coolio::IO
-      def initialize(io, callback)
-        super(io)
-        @io = io
-        @callback = callback
-      end
-
-      def on_readable
-        msg, addr = @io.recvfrom_nonblock(4096)
-        @callback.call(addr[3], msg)
-      rescue => e
-        log.error "unexpected error on reading from socket", error_class: e.class, error: e.message
-        log.error_backtrace
-      end
-    end
   end
 end

data/lib/fluent/plugin/netflow_records.rb

@@ -1,7 +1,7 @@
 require "bindata"
 
 module Fluent
-  class TextParser
+  module Plugin
     class NetflowParser < Parser
       class IP4Addr < BinData::Primitive
         endian :big

data/lib/fluent/plugin/parser_netflow.rb

@@ -1,16 +1,16 @@
 require "ipaddr"
 require 'yaml'
 
-require 'fluent/parser'
+require 'fluent/plugin/parser'
 
 require_relative 'netflow_records'
 require_relative 'vash'
 
 module Fluent
-  class TextParser
+  module Plugin
     # port from logstash's netflow parser
     class NetflowParser < Parser
-      Plugin.register_parser('netflow', self)
+      Fluent::Plugin.register_parser('netflow', self)
 
       config_param :switched_times_from_uptime, :bool, default: false
       config_param :cache_ttl, :integer, default: 4000
@@ -33,16 +33,16 @@ module Fluent
         begin
           @template_fields = YAML.load_file(filename)
         rescue => e
-          raise ConfigError, "Bad syntax in definitions file #{filename}, error_class = #{e.class.name}, error = #{e.message}"
+          raise Fluent::ConfigError, "Bad syntax in definitions file #{filename}, error_class = #{e.class.name}, error = #{e.message}"
         end
 
         # Allow the user to augment/override/rename the supported Netflow fields
         if @definitions
-          raise ConfigError, "definitions file #{@definitions} doesn't exist" unless File.exist?(@definitions)
+          raise Fluent::ConfigError, "definitions file #{@definitions} doesn't exist" unless File.exist?(@definitions)
           begin
             @template_fields['option'].merge!(YAML.load_file(@definitions))
           rescue => e
-            raise ConfigError, "Bad syntax in definitions file #{@definitions}, error_class = #{e.class.name}, error = #{e.message}"
+            raise Fluent::ConfigError, "Bad syntax in definitions file #{@definitions}, error_class = #{e.class.name}, error = #{e.message}"
           end
         end
       end
@@ -145,7 +145,7 @@ module Fluent
         sampling_algorithm = (sampling & 0b1100000000000000) >> 14
         sampling_interval = sampling & 0b0011111111111111
 
-        time = unix_sec.to_i
+        time = Time.at(unix_sec, unix_nsec / 1000).to_i # TODO: Fluent::EventTime
 
         records_bytes = payload.bytesize - NETFLOW_V5_HEADER_BYTES
 
@@ -289,7 +289,7 @@ module Fluent
             next
           end
 
-          time = pdu.unix_sec.to_i
+          time = pdu.unix_sec  # TODO: Fluent::EventTime (see: forV5)
           event = {}
 
           # Fewer fields in the v9 header

data/lib/fluent/plugin/vash.rb

@@ -1,5 +1,5 @@
 module Fluent
-  class TextParser
+  module Plugin
     class NetflowParser < Parser
       # https://gist.github.com/joshaven/184837
       class Vash < Hash

data/test/test_in_netflow.rb

@@ -1,4 +1,5 @@
 require 'helper'
+require 'fluent/test/driver/input'
 
 class NetflowInputTest < Test::Unit::TestCase
   def setup
@@ -13,7 +14,7 @@ class NetflowInputTest < Test::Unit::TestCase
   ]
 
   def create_driver(conf=CONFIG)
-    Fluent::Test::InputTestDriver.new(Fluent::NetflowInput).configure(conf)
+    Fluent::Test::Driver::Input.new(Fluent::Plugin::NetflowInput).configure(conf)
   end
 
   def test_configure
@@ -22,6 +23,7 @@ class NetflowInputTest < Test::Unit::TestCase
     assert_equal '127.0.0.1', d.instance.bind
     assert_equal 'test.netflow', d.instance.tag
     assert_equal :udp, d.instance.protocol_type
+    assert_equal 2048, d.instance.max_bytes
 
     assert_raise Fluent::ConfigError do
       d = create_driver CONFIG + %[

data/test/test_parser_netflow.rb

@@ -1,4 +1,5 @@
 require 'helper'
+require 'fluent/test/driver/parser'
 
 class NetflowParserTest < Test::Unit::TestCase
   def setup
@@ -6,7 +7,7 @@ class NetflowParserTest < Test::Unit::TestCase
   end
 
   def create_parser(conf={})
-    parser = Fluent::TextParser::NetflowParser.new
+    parser = Fluent::Plugin::NetflowParser.new
     parser.configure(Fluent::Config::Element.new('ROOT', '', conf, []))
     parser
   end
@@ -218,7 +219,6 @@ class NetflowParserTest < Test::Unit::TestCase
     end
 
     assert_equal 1, parsed.size
-    assert_instance_of Integer, parsed.first[0]
     assert_equal time1, parsed.first[0]
 
     event = parsed.first[1]
@@ -311,25 +311,25 @@ class NetflowParserTest < Test::Unit::TestCase
 
   require 'fluent/plugin/netflow_records'
   def ipv4addr(v)
-    addr = Fluent::TextParser::NetflowParser::IP4Addr.new
+    addr = Fluent::Plugin::NetflowParser::IP4Addr.new
     addr.set(v)
     addr
   end
 
   def ipv6addr(v)
-    addr = Fluent::TextParser::NetflowParser::IP6Addr.new
+    addr = Fluent::Plugin::NetflowParser::IP6Addr.new
     addr.set(v)
     addr
   end
 
   def macaddr(v)
-    addr = Fluent::TextParser::NetflowParser::MacAddr.new
+    addr = Fluent::Plugin::NetflowParser::MacAddr.new
     addr.set(v)
     addr
   end
 
   def mplslabel(v)
-    label = Fluent::TextParser::NetflowParser::MplsLabel.new
+    label = Fluent::Plugin::NetflowParser::MplsLabel.new
     label.set(v)
     label
   end
@@ -366,7 +366,7 @@ class NetflowParserTest < Test::Unit::TestCase
       end
       r
     }
-    Fluent::TextParser::NetflowParser::Netflow5PDU.new(hash)
+    Fluent::Plugin::NetflowParser::Netflow5PDU.new(hash)
   end
 
   def v9_template(hash)

data/test/test_parser_netflow9.rb

@@ -6,7 +6,7 @@ class Netflow9ParserTest < Test::Unit::TestCase
   end
 
   def create_parser(conf={})
-    parser = Fluent::TextParser::NetflowParser.new
+    parser = Fluent::Plugin::NetflowParser.new
     parser.configure(Fluent::Config::Element.new('ROOT', '', conf, []))
     parser
   end
@@ -68,7 +68,6 @@ class Netflow9ParserTest < Test::Unit::TestCase
     end
 
     assert_equal 1, parsed.size
-    assert_instance_of Integer, parsed.first[0]
     assert_equal Time.parse('2016-02-12T04:02:25Z').to_i, parsed.first[0]
     expected_record = {
       # header

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-netflow
 version: !ruby/object:Gem::Version
-  version: 0.2.8
+  version: 1.0.0.rc1
 platform: ruby
 authors:
 - Masahiro Nakagawa
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2017-05-16 00:00:00.000000000 Z
+date: 2017-02-09 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fluentd
@@ -16,7 +16,7 @@ dependencies:
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 0.10.17
+        version: 0.14.10
     - - "<"
       - !ruby/object:Gem::Version
         version: '2'
@@ -26,7 +26,7 @@ dependencies:
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 0.10.17
+        version: 0.14.10
     - - "<"
       - !ruby/object:Gem::Version
         version: '2'
@@ -120,12 +120,12 @@ required_ruby_version: !ruby/object:Gem::Requirement
       version: '0'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
-  - - ">="
+  - - ">"
     - !ruby/object:Gem::Version
-      version: '0'
+      version: 1.3.1
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.6.11
+rubygems_version: 2.5.2
 signing_key: 
 specification_version: 4
 summary: Netflow plugin for Fluentd