fluent-plugin-netflowipfix 1.0.0

data/lib/fluent/plugin/netflow_fields.yaml

@@ -0,0 +1,346 @@
+---
+option:
+  1:
+  - 4
+  - :in_bytes
+  2:
+  - 4
+  - :in_pkts
+  3:
+  - 4
+  - :flows
+  4:
+  - :uint8
+  - :protocol
+  5:
+  - :uint8
+  - :src_tos
+  6:
+  - :uint8
+  - :tcp_flags
+  7:
+  - :uint16
+  - :l4_src_port
+  8:
+  - :ip4_addr
+  - :ipv4_src_addr
+  9:
+  - :uint8
+  - :src_mask
+  10:
+  - 2
+  - :input_snmp
+  11:
+  - :uint16
+  - :l4_dst_port
+  12:
+  - :ip4_addr
+  - :ipv4_dst_addr
+  13:
+  - :uint8
+  - :dst_mask
+  14:
+  - 2
+  - :output_snmp
+  15:
+  - :ip4_addr
+  - :ipv4_next_hop
+  16:
+  - 2
+  - :src_as
+  17:
+  - 2
+  - :dst_as
+  18:
+  - :ip4_addr
+  - :bgp_ipv4_next_hop
+  19:
+  - 4
+  - :mul_dst_pkts
+  20:
+  - 4
+  - :mul_dst_bytes
+  21:
+  - :uint32
+  - :last_switched
+  22:
+  - :uint32
+  - :first_switched
+  23:
+  - 4
+  - :out_bytes
+  24:
+  - 4
+  - :out_pkts
+  25:
+  - :uint16
+  - :min_pkt_length
+  26:
+  - :uint16
+  - :max_pkt_length
+  27:
+  - :ip6_addr
+  - :ipv6_src_addr
+  28:
+  - :ip6_addr
+  - :ipv6_dst_addr
+  29:
+  - :uint8
+  - :ipv6_src_mask
+  30:
+  - :uint8
+  - :ipv6_dst_mask
+  31:
+  - 3
+  - :ipv6_flow_label
+  32:
+  - :uint16
+  - :icmp_type
+  33:
+  - :uint8
+  - :mul_igmp_type
+  34:
+  - :uint32
+  - :sampling_interval
+  35:
+  - :uint8
+  - :sampling_algorithm
+  36:
+  - :uint16
+  - :flow_active_timeout
+  37:
+  - :uint16
+  - :flow_inactive_timeout
+  38:
+  - :uint8
+  - :engine_type
+  39:
+  - :uint8
+  - :engine_id
+  40:
+  - 4
+  - :total_bytes_exp
+  41:
+  - 4
+  - :total_pkts_exp
+  42:
+  - 4
+  - :total_flows_exp
+  43:
+  - 4
+  - :skip43
+  44:
+  - :ip4_addr
+  - :ipv4_src_prefix
+  45:
+  - :ip4_addr
+  - :ipv4_dst_prefix
+  46:
+  - :uint8
+  - :mpls_top_label_type
+  47:
+  - :uint32
+  - :mpls_top_label_ip_addr
+  48:
+  - 1
+  - :flow_sampler_id
+  49:
+  - :uint8
+  - :flow_sampler_mode
+  50:
+  - :uint32
+  - :flow_sampler_random_interval
+  51:
+  - 1
+  - :skip51
+  52:
+  - :uint8
+  - :min_ttl
+  53:
+  - :uint8
+  - :max_ttl
+  54:
+  - :uint16
+  - :ipv4_ident
+  55:
+  - :uint8
+  - :dst_tos
+  56:
+  - :mac_addr
+  - :in_src_mac
+  57:
+  - :mac_addr
+  - :out_dst_mac
+  58:
+  - :uint16
+  - :src_vlan
+  59:
+  - :uint16
+  - :dst_vlan
+  60:
+  - :uint8
+  - :ip_protocol_version
+  61:
+  - :uint8
+  - :direction
+  62:
+  - :ip6_addr
+  - :ipv6_next_hop
+  63:
+  - :ip6_addr
+  - :bgp_ipv6_next_hop
+  64:
+  - :uint32
+  - :ipv6_option_headers
+  65:
+  - 1
+  - :skip65
+  66:
+  - 1
+  - :skip66
+  67:
+  - 1
+  - :skip67
+  68:
+  - 1
+  - :skip68
+  69:
+  - 1
+  - :skip69
+  70:
+  - :mpls_label
+  - :mpls_label_1
+  71:
+  - :mpls_label
+  - :mpls_label_2
+  72:
+  - :mpls_label
+  - :mpls_label_3
+  73:
+  - :mpls_label
+  - :mpls_label_4
+  74:
+  - :mpls_label
+  - :mpls_label_5
+  75:
+  - :mpls_label
+  - :mpls_label_6
+  76:
+  - :mpls_label
+  - :mpls_label_7
+  77:
+  - :mpls_label
+  - :mpls_label_8
+  78:
+  - :mpls_label
+  - :mpls_label_9
+  79:
+  - :mpls_label
+  - :mpls_label_10
+  80:
+  - :mac_addr
+  - :in_dst_mac
+  81:
+  - :mac_addr
+  - :out_src_mac
+  82:
+  - :string
+  - :if_name
+  83:
+  - :string
+  - :if_desc
+  84:
+  - :string
+  - :sampler_name
+  89:
+  - :uint8
+  - :forwarding_status
+  91:
+  - :uint8
+  - :mpls_prefix_len
+  234:
+  - :uint32
+  - :ingress_vrf_id
+  235:
+  - :uint32
+  - :egress_vrf_id
+  236:
+  - :string
+  - :vrf_name
+  148:
+  - 4
+  - :NF_F_CONN_ID
+  176:
+  - 1
+  - :NF_F_ICMP_TYPE
+  177:
+  - 1
+  - :NF_F_ICMP_CODE
+  152:
+  - 8
+  - :NF_F_FLOW_CREATE_TIME_MSEC
+  233:
+  - 1
+  - :NF_F_FW_EVENT
+  323:
+  - 8
+  - :NF_F_EVENT_TIME_MSEC
+  33000:
+  - 12
+  - :NF_F_INGRESS_ACL_ID
+  33001:
+  - 12
+  - :NF_F_EGRESS_ACL_ID
+  33002:
+  - 2
+  - :NF_F_FW_EXT_EVENT
+  40000:
+  - 20
+  - :NF_F_USERNAME
+  40001:
+  - 4
+  - skip40001
+  40002:
+  - 4
+  - skip40002
+  40003:
+  - 2
+  - skip40003
+  40004:
+  - 2
+  - skip40004
+  40005:
+  - 1
+  - skip40005
+  85:
+  - 4
+  - skip85
+  178:
+  - 1
+  - skip178
+  179:
+  - 1
+  - skip179
+  231:
+  - 4
+  - skip231
+#  232:
+#  - 4
+#  - skip232
+
+scope:
+  1:
+  - :ip4_addr
+  - :system
+  2:
+  - 1
+  - :skip2
+  3:
+  - 1
+  - :skip3
+  4:
+  - 1
+  - :skip4
+  5:
+  - 1
+  - :skip5

data/lib/fluent/plugin/netflowipfix_records.rb

@@ -0,0 +1,243 @@
+require "bindata"
+
+module Fluent
+  module Plugin
+		class NetflowipfixInput < Fluent::Plugin::Input
+
+
+			class IP4Addr < BinData::Primitive
+				endian :big
+				uint32 :storage
+
+				def set(val)
+					ip = IPAddr.new(val)
+					if ! ip.ipv4?
+					  raise ArgumentError, "invalid IPv4 address '#{val}'"
+					end
+					self.storage = ip.to_i
+				end # set
+
+				def get
+					IPAddr.new_ntoh([self.storage].pack('N')).to_s
+				end # get
+			end # class
+
+
+			class IP6Addr < BinData::Primitive
+				endian  :big
+				uint128 :storage
+
+				def set(val)
+				  ip = IPAddr.new(val)
+				  if ! ip.ipv6?
+					raise ArgumentError, "invalid IPv6 address `#{val}'"
+				  end
+				  self.storage = ip.to_i
+				end
+
+				def get
+				  IPAddr.new_ntoh((0..7).map { |i|
+					  (self.storage >> (112 - 16 * i)) & 0xffff
+					}.pack('n8')).to_s
+				end
+			end
+
+  
+			class MacAddr < BinData::Primitive
+				endian :big
+				array :bytes, type: :uint8, initial_length: 6
+
+				def set(val)
+				  ints = val.split(/:/).collect { |int| int.to_i(16) }
+				  self.bytes = ints
+				end
+
+				def get
+				  self.bytes.collect { |byte| byte.value.to_s(16).rjust(2,'0') }.join(":")
+				end
+			end
+
+			class MplsLabel < BinData::Primitive
+				endian :big
+				bit20 :label
+				bit3  :exp
+				bit1  :bottom
+				def set(val)
+				  self.label = val >> 4
+				  self.exp = (val & 0b1111) >> 1
+				  self.bottom = val & 0b1
+				end
+				def get
+					self.label
+				end
+			end
+
+			class Header < BinData::Record
+				endian :big
+				uint16 :version
+			end
+
+			class Netflow5Packet < BinData::Record
+				endian :big
+				uint16 :version
+				uint16 :flow_records
+				uint32 :uptime
+				uint32 :unix_sec
+				uint32 :unix_nsec
+				uint32 :flow_seq_num
+				uint8  :engine_type
+				uint8  :engine_id
+				bit2   :sampling_algorithm
+				bit14  :sampling_interval
+				array  :records, initial_length: :flow_records do
+				  ip4_addr :ipv4_src_addr
+				  ip4_addr :ipv4_dst_addr
+				  ip4_addr :ipv4_next_hop
+				  uint16   :input_snmp
+				  uint16   :output_snmp
+				  uint32   :in_pkts
+				  uint32   :in_bytes
+				  uint32   :first_switched
+				  uint32   :last_switched
+				  uint16   :l4_src_port
+				  uint16   :l4_dst_port
+				  skip     length: 1
+				  uint8    :tcp_flags # Split up the TCP flags maybe?
+				  uint8    :protocol
+				  uint8    :src_tos
+				  uint16   :src_as
+				  uint16   :dst_as
+				  uint8    :src_mask
+				  uint8    :dst_mask
+				  skip     length: 2
+				end
+			end
+
+			# Template format for v9 and v10 - shared field must use same name to simplify code
+			class Template9 < BinData::Record
+				endian :big
+				array  :templates, read_until: lambda { array.num_bytes == flowset_length - 4 } do
+				  uint16 :template_id
+				  uint16 :field_count
+				  array  :template_fields, initial_length: :field_count do
+					uint16 :field_type
+					uint16 :field_length
+				  end # array fields
+				end # array templates
+			end #class
+
+			class Template10 < BinData::Record
+				endian :big
+				array  :templates, read_until: lambda { array.num_bytes == flowset_length - 4 } do
+				  uint16 :template_id
+				  uint16 :field_count
+				  array  :template_fields, initial_length: :field_count do
+					uint16 :field_type
+					uint16 :field_length
+						# TODO: if upperbit (enterprise_bit) is set, then we have an enterprise # of 4 bytes (uint32)
+					  uint32 :enterpriseNumber, :onlyif => lambda { field_type >= 0x8000 }
+				  end # array fields
+				end # array templates
+			end #class
+
+			class Option9 < BinData::Record
+				endian :big
+				array  :templates, read_until: lambda { flowset_length - 4 - array.num_bytes <= 2 } do
+				  uint16 :template_id
+				  uint16 :scope_length
+				  uint16 :option_length
+				  array  :scope_fields, initial_length: lambda { scope_length / 4 } do
+					uint16 :field_type
+					uint16 :field_length
+				  end # array scope_fields
+				  array  :option_fields, initial_length: lambda { option_length / 4 } do
+					uint16 :field_type
+					uint16 :field_length
+				  end # array option_fields
+				end # array templates
+				skip   length: lambda { templates.length.odd? ? 2 : 0 }
+			end #class
+
+			class Option10 < BinData::Record
+				endian :big
+				array  :templates, read_until: lambda { flowset_length - 4 - array.num_bytes <= 2 } do
+				  uint16 :template_id
+				  uint16 :field_count
+				  uint16 :scope_field_count
+				  array  :scope_fields, initial_length: :scope_field_count do
+					uint16 :field_type
+					uint16 :field_length
+						  # TODO: if upperbit (enterprise_bit) is set, then we have an enterprise # of 4 bytes (uint32)
+					  uint32 :enterpriseNumber, :onlyif => lambda { field_type >= 0x8000 }
+				  end # array scope_fields
+				  array  :option_fields, initial_length: lambda { field_count - scope_field_count } do
+					uint16 :field_type
+					uint16 :field_length
+						  # TODO: if upperbit (enterprise_bit) is set, then we have an enterprise # of 4 bytes (uint32)
+					  uint32 :enterpriseNumber, :onlyif => lambda { field_type >= 0x8000 }
+				  end # array option_fields
+				end # array templates
+			end #class
+
+			class Netflow9Packet < BinData::Record
+				endian :big
+				uint16 :version
+				uint16 :flow_records
+				uint32 :uptime
+				uint32 :unix_sec
+				uint32 :flow_seq_num
+				uint32 :source_id
+				array  :records, read_until: :eof do
+				  uint16 :flowset_id
+				  uint16 :flowset_length
+				  choice :flowset_data, selection: :flowset_id do
+					template9 0
+					option9   1
+					string           :default, read_length: lambda { flowset_length - 4 }
+				  end # choice
+				end # array records
+			end #class
+
+			class Netflow10Packet < BinData::Record
+				endian :big
+				uint16 :version
+				uint16 :ipfix_length #flow_records
+				uint32 :unix_sec #export_time #uptime
+				# uint32 :
+				uint32 :flow_seq_num # seq_num
+				uint32 :source_id # observation_domain_id
+				array  :records, read_until: :eof do
+					# set header
+				  uint16 :flowset_id # 2 = template, 3 = options, >= 256 = data sets
+				  uint16 :flowset_length # in octets
+					  # record
+				  choice :flowset_data, selection: :flowset_id do
+					template10 2
+					option10   3
+					string           :default, read_length: lambda { flowset_length - 4 }
+				  end # choice
+				end # array 
+			end # class
+
+
+			class OctetArray1 < BinData::Array
+				endian :big
+				uint8 :storage
+			end
+
+			class OctetArray2 < BinData::Primitive
+				array :bytes, type: :uint8, initial_length: 2
+
+				def set(val)
+				  ints = val.split(/:/).collect { |int| int.to_i(16) }
+				  self.bytes = ints
+				end
+
+				def get
+				  self.bytes.collect { |byte| byte.value.to_s(16).rjust(2,'0') }.join(":")
+				end
+			end
+
+		end # class NetflowipfixInput
+	end # module Plugin
+end # module Fluent