fluent-plugin-netflowipfix 1.0.2 → 1.1.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 4d367541e96f10fe8e2a3746029b5400606addc2
-  data.tar.gz: 2c9d9cf3e5e73b608997468f6841e593acab2bb1
+  metadata.gz: '069c7af312866b4319eaa05bbf3a85b21dbe7284'
+  data.tar.gz: 347842300a58938ced6d12a552df6df416a74765
 SHA512:
-  metadata.gz: 2ab9ec4274282f2ffd07adba1004dc9269a628e6a1256e2e9fbc2b8e66d829b532ed6174712df58c5fc264a18bf8399159ce57ff7034c39c47ad5488367f404f
-  data.tar.gz: c4566318627dcd6a0181175912ea91f588a0a439587c10655c83cb44db09ed7d26acab539545df34f8d04c4daedd4ff49d829684af49c29eb35fc93f86cbccd9
+  metadata.gz: 5319acf004e86e279528485faa5ad760acaeb4b2a970aed0551a399ff34ef5b9a9692d3efd4c88072b65a76692ad96f46b19d1e84a8ede30cdce611d261cd16d
+  data.tar.gz: df25aa1db4ba2990a1428d99f9072c1c8f7ddd21fa9838d314d9acb218bfcd04808c9d129cc3be140508b8cf63f21550f1bffb41c6d49626f84ad5c711f4c622

data/fluent-plugin-netflowipfix.gemspec

@@ -3,7 +3,7 @@ $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
 
 Gem::Specification.new do |spec|
   spec.name    = "fluent-plugin-netflowipfix"
-  spec.version = "1.0.2"
+  spec.version = "1.1.0"
   spec.authors = ["Yves Desharnaus"]
   spec.email   = ["yvesbd@gmail.com"]
 

data/lib/fluent/plugin/in_netflowipfix.rb

@@ -1,5 +1,5 @@
 #
-# Copyright 2018 Yves Desharnais
+# Copyright 2018-2019 Yves Desharnais
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -13,149 +13,291 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+
+require "socket"
 require "fluent/plugin/input"
-require 'cool.io'
-require 'fluent/plugin/socket_util'
-# require_relative 'parser_netflowipfix'
+
 require_relative 'parser_netflow_v5'
 require_relative 'parser_netflow_v9'
 require_relative 'netflowipfix_records'
 require_relative 'vash'
 
+
 module Fluent
-	module Plugin
-		class NetflowipfixInput < Fluent::Plugin::Input
-			Fluent::Plugin.register_input("netflowipfix", self)
-
-      config_param :cache_ttl, :integer, default: 4000
-      config_param :definitions, :string, default: nil
-
-			config_param :debug, :bool, default: false
-			config_param :port, :integer, default: 5140
-			config_param :bind, :string, default: '0.0.0.0'
-			config_param :tag, :string
-			config_param :protocol_type, default: :udp do |val|
-				case val.downcase
-				when 'udp'
-					:udp
-				else
-					raise ConfigError, "netflow input protocol type should be 'udp'"
-				end
-			end # config_param :protocol_type
-
-			def configure(conf)
-				super
-				@nbpackets = 0
-				@parser_v5 = ParserNetflowv5.new
-				@parser_v9 = ParserNetflowv9.new
-				@parser_v9.configure(@cache_ttl, @definitions)
-				@parser_v10 = ParserIPfixv10.new
-				@parser_v10.configure(@cache_ttl, @definitions)
-			end # def configure
-
-			def start
-				@loop = Coolio::Loop.new
-				@handler = listen(method(:receive_data))
-				@loop.attach(@handler)
-				@thread = Thread.new(&method(:run))
-			end # def start
-
-			def shutdown
-				@loop.watchers.each { |w| w.detach }
-				@loop.stop
-				@handler.close
-				@thread.join
-			end # def shutdown
-
-			def run
-				@loop.run
-				rescue => e
-					log.error "unexpected error", error_class: e.class, error: e.message
-					log.error_backtrace
-			end # def run
-
-			protected
-
-			def receive_data(host, data)
-				# if (@debug) 
-#				log.on_debug { log.debug "received logs", :host => host, :data => data }
-				call(data, host) { |time, record|
-				unless time && record
-					log.warn "pattern not match: #{data.inspect}"
-					return
+  module Plugin
+    class NetflowipfixInput < Fluent::Plugin::Input
+      Fluent::Plugin.register_input("netflowipfix", self)
+      include DetachMultiProcessMixin
+
+class PortConnection
+	def initialize(bind, port, tag, cache_ttl, definitions, queuesleep)
+		@bind = bind
+		@port = port
+		@tag = tag
+		@cache_ttl = cache_ttl
+		@definitions = definitions
+		@eventQueue = Queue.new
+		@udpQueue = Queue.new
+		@queuesleep = queuesleep
+	end
+	
+	def bind
+		@bind
+	end
+	def port
+		@port
+	end
+	def tag
+		@tag
+	end
+	
+	def start
+		@thread_udp = UdpListenerThread.new(@bind, @port, @udpQueue, @tag)
+		@thread_parser = ParserThread.new(@udpQueue, @queuesleep, @eventQueue, @cache_ttl, @definitions)
+		@thread_udp.start
+		@thread_parser.start
+	end # def start
+	def stop
+			@thread_udp.close
+			@thread_udp.join
+			@thread_parser.close
+			@thread_parser.join
+	end # def stop
+	
+	def event_pop
+		@eventQueue.pop
+	end
+	
+	def event_queue_length
+		@eventQueue.length
+	end
+
+
+#	def udpqueue_pop
+#		@udpQueue.pop
+#	end
+	
+#	def udpqueue_length
+#		@udpQueue.length
+#	end
+end #class PortConnection
+
+		config_param :tag, :string
+		config_param :port, :integer, default: nil
+		config_param :bind, :string, :default => '0.0.0.0'
+		config_param :queuesleep, :integer, default: 10
+
+		def configure(conf)
+			super
+			$log.debug "NetflowipfixInput::configure: #{@bind}:#{@port}"
+			@@connections ||=  {}
+			if @@connections.nil?
+			end
+			@@connections[@port] = PortConnection.new(@bind, @port, @tag, @cache_ttl, @definitions, @queuesleep)
+			log.debug "NetflowipfixInput::configure NB=#{@@connections.length}"	
+			@total = 0
+		end
+		
+		def start
+			super
+			
+			$log.debug "NetflowipfixInput::start NB=#{@@connections.length}"	
+			if @@connections.nil?
+			else
+				@@connections.each do | port, conn |
+					$log.debug "start listening UDP on #{conn.bind}:#{conn.port}"
+					conn.start				
 				end
+			end
+			
+			
+#			@eventQueue = Queue.new
+#			@udpQueue = Queue.new
+#			@thread_udp = UdpListenerThread.new(@bind, @port, @udpQueue)
+#			@thread_parser = ParserThread.new(@udpQueue, @queuesleep, @eventQueue, @cache_ttl, @definitions)
+#			@thread_udp.start
+#			@thread_parser.start
+			waitForEvents
+		end
 
-#			if (@debug) log.info "ready to emit ", time:time, tag:@tag
-
-				record['host'] = host
-				router.emit(@tag, EventTime.new(time), record)
-				} # call
-				rescue => e
-				log.warn "unexpected error on parsing", data: data.dump, error_class: e.class, error: e.message
-				log.warn_backtrace
-			end # def receive_data
-
-			private
-
-			def listen(callback)
-				log.info "listening netflow socket on #{@bind}:#{@port} with #{@protocol_type}"
-				if @protocol_type == :udp
-					@usock = SocketUtil.create_udp_socket(@bind)
-					@usock.bind(@bind, @port)
-					UdpHandler.new(@usock, callback)
-				else
-					Coolio::TCPServer.new(@bind, @port, TcpHandler, log, callback)
+		def shutdown
+			super
+			$log.debug "NetflowipfixInput::shutdown NB=#{@@connections.length}"	
+			if @@connections.nil?
+			else
+#				$log.debug "listening UDP on #{@bind}:#{@port}"
+#				@connections[@port].stop
+				@@connections.each do | port, conn |
+					$log.debug "shutdown listening UDP on #{conn.bind}:#{conn.port}"
+					conn.stop				
 				end
-			end # def listen
-    
-			def call(payload, host=nil, &block)
+				@@connections = nil
+			end
+
+#			@thread_udp.close
+#			@thread_udp.join
+#			@thread_parser.close
+#			@thread_parser.join
+		end		
+		
+
+		def waitForEvents
+#		puts "Main::run begin #{@eventQueue.length}"
+			loop do
+					@@connections.each do | port, conn |
+						if (conn.event_queue_length > 0) 
+							$log.debug "waitForEvents: #{conn.bind}:#{conn.port}"
+							ar = conn.event_pop			
+							time = ar[0]
+							record = ar[1]
+							router.emit(conn.tag, EventTime.new(time.to_i), record)
+						end
+					end
+					sleep(@queuesleep)
+
+#				if @eventQueue.length > 0
+				
+				
+#					ar = @eventQueue.pop
+#					time = ar[0]
+#					record = ar[1]
+#					router.emit(@tag, EventTime.new(time.to_i), record)
+
+
+#		puts "Main::pop before #{@eventQueue.length} #{@tag} #{time}" # #{record.to_s}"
+#		puts "Main::pop after  #{@eventQueue.length}"
+#				else
+#				end
+			end
+#		puts "Main::run end #{@eventQueue.length}"
+
+		end
+
+		private
+
+
+class UdpListenerThread
+
+	def initialize(bind, port, udpQueue, tag)
+		@port = port
+		@udpQueue = udpQueue
+		@udp_socket = UDPSocket.new
+		@udp_socket.bind(bind, port)
+		@total = 0
+		@tag = tag
+	end
+
+	def start
+		@thread = Thread.new(&method(:run))
+		puts "UdpListenerThread::start"
+	end 
+	
+	def close
+			@udp_socket.close
+	end
+	
+	def join
+			@thread.join
+	end
+	
+	
+	def run
+			loop do
+				msg, sender =  @udp_socket.recvfrom(4096)
+				@total = @total + msg.length
+#		puts "UdpListenerThread::recvfrom #{msg.length} bytes for #{@total} total on UDP/#{@port}"
+#				log.debug "Received #{msg.length} bytes for #{@total} total"
+				record = {}
+				record["message"] = msg
+				record["length"] = msg.length
+				record["total"] = @total
+				record["sender"] = sender
+				record["port"] = @port
+#				time = EventTime.new()
+				time = Time.now.getutc
+#				router.emit(@tag, EventTime.new(), record)
+				@udpQueue << [time, record]
+			end
+	end
+end # class UdpListenerThread
+		
+class ParserThread
+	def initialize(udpQueue, queuesleep, eventQueue, cache_ttl, definitions)
+		@udpQueue = udpQueue
+		@queuesleep = queuesleep
+		@eventQueue = eventQueue
+
+		@parser_v5 = NetflowipfixInput::ParserNetflowv5.new
+		@parser_v9 = NetflowipfixInput::ParserNetflowv9.new
+		@parser_v10 = NetflowipfixInput::ParserIPfixv10.new
+
+		@parser_v9.configure(cache_ttl, definitions)
+		@parser_v10.configure(cache_ttl, definitions)
+	end
+	def start
+		@thread = Thread.new(&method(:run))
+		puts "ParserThread::start"
+	end 
+	
+	def close
+	end
+	
+	def join
+			@thread.join
+	end
+	
+	def run
+#		puts "ParserThread::run start #{@udpQueue.length}"
+
+		loop do
+			if @udpQueue.length == 0
+#		puts "ParserThread::run sleep #{@queuesleep}"
+				sleep(@queuesleep)
+
+			else
+				block = method(:emit)
+#block = nil
+				ar = @udpQueue.pop
+				time = ar[0]
+				msg = ar[1]
+				payload = msg["message"]
+				host = msg["sender"]
+				
 				version,_ = payload[0,2].unpack('n')
-				@nbpackets += 1
-				#	nb = @nbpackets
-				if (@debug) 
-					log.debug "Packet #{@nbpackets} with version #{version}"
-				end
+#		puts "ParserThread::pop #{@udpQueue.length} v#{version}"
+
+
 				case version
 					when 5          
-						packet = Netflow5Packet.read(payload)
+						packet = NetflowipfixInput::Netflow5Packet.read(payload)
 						@parser_v5.handle_v5(host, packet, block)
 					when 9
-						packet = Netflow9Packet.read(payload)
+						packet = NetflowipfixInput::Netflow9Packet.read(payload)
 						@parser_v9.handle_v9(host, packet, block)
 					when 10
-						packet = Netflow10Packet.read(payload)
+						packet = NetflowipfixInput::Netflow10Packet.read(payload)
 						@parser_v10.handle_v10(host, packet, block)
 					else
-						$log.warn "Unsupported Netflow version v#{version}: #{version.class}"
+#						$log.warn "Unsupported Netflow version v#{version}: #{version.class}"
 				end # case
-			end # def call
-
-		end # class NetflowipfixInput
 
-		class UdpHandler < Coolio::IO
-			def initialize(io, callback)
-				super(io)
-				@io = io
-				@callback = callback
-			end # def initialize
+#				parent_call(time, record)
+#				@eventQueue << [time, record]
+			end
+		end # loop do
+	end # def run
+	def emit(time, event, host = nil)
+		if !host.nil?
+			event["host"] = host
+		end
+		@eventQueue << [time, event]
+# puts "ParserThread::emit #{@eventQueue.length}"
+	end # def emit
 
-			def on_readable
-				msg, addr = @io.recvfrom_nonblock(4096)
-				@callback.call(addr[3], msg)
-				rescue => e
-					log.error "unexpected error on reading from socket", error_class: e.class, error: e.message
-				log.error_backtrace
-			end # def on_readable
-		end # class UdpHandler
+end # class ParserThread
 
 
-	end # module Plugin
+    end # class DnsCacheOuput
+  end # module Plugin
 end # module Fluent
 
-
-
-    
-
-
-=begin
-=end

data/lib/fluent/plugin/parser_netflow_v5.rb

@@ -71,7 +71,6 @@ module Fluent
 
 
 
-
       def handle_v5(host, packet, block)
         packet.records.each do |flowset|
           # handle_flowset_data(host, packet, flowset, block, null, null)
@@ -111,7 +110,7 @@ module Fluent
           end # unless
 
           time = Time.at(packet.unix_sec, packet.unix_nsec / 1000).to_i # TODO: Fluent::EventTime
-          block.call(time, record)
+          block.call(time, record, host)
         end # do flowset
       end # def handle_v5
 

data/lib/fluent/plugin/parser_netflow_v9.rb

@@ -219,7 +219,13 @@ module Fluent
 							#  end
 							#end
 
-							block.call(time, event)
+#							block.call(time, event)
+#							if (defined?(block)).nil?
+#								log.error "*** handle_flowset_data block is blank"
+#							else
+#							end
+							  block.call(time, event, host)
+
 						end # fields = array.read
 					end # def handle_flowset_data
 

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-netflowipfix
 version: !ruby/object:Gem::Version
-  version: 1.0.2
+  version: 1.1.0
 platform: ruby
 authors:
 - Yves Desharnaus
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2018-12-08 00:00:00.000000000 Z
+date: 2019-01-03 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rake