fluent-plugin-sflow 0.1.0

data/lib/sflow/lib/sflow.rb

@@ -0,0 +1,10 @@
+require 'bindata'
+require 'eventmachine'
+require 'yaml'
+
+dir = File.expand_path(File.join(File.dirname(__FILE__),  'sflow'))
+['config','models/ipv4header', 'models/tcpheader', 'models/udpheader', 'models/protocol', 'models/binary_models','parsers/parsers','storage/storage', 'collector','snmp/iface_names'].each do |req|
+  require File.join(dir, req)
+end
+
+Process.daemon(true) if $daemonize == true

data/lib/sflow/lib/sflow/collector.rb

@@ -0,0 +1,69 @@
+class SflowCollector
+  module Collector
+  Thread.abort_on_exception=true
+  require 'socket'
+    def post_init
+      puts "Server listening."
+    end
+
+    def receive_data(data)
+      operation = proc do
+        begin
+          if data != nil
+            sflow = SflowParser.parse_packet(data)
+          end
+        rescue Exception => e
+          puts Time.now
+          puts sflow.inspect
+          puts e.message
+          puts e.backtrace
+        end
+      end
+
+      callback = proc do |sflow|
+        begin
+          if sflow != nil
+            SflowStorage.send_udpjson(sflow)
+          end
+        rescue Exception => e
+          puts Time.now
+          puts sflow.inspect if sflow != nil
+          puts e.message
+          puts e.backtrace
+        end
+      end
+
+      EM.defer(operation,callback)
+
+    end
+  end
+
+  def self.start_collector(bind_ip = '0.0.0.0', bind_port = 6343)
+    begin
+      config = SflowConfig.new
+      if config.logstash_host and config.logstash_port 
+        puts "Connecting to Logstash: #{config.logstash_host}:#{config.logstash_port}"
+        $logstash = UDPSocket.new
+        $logstash.connect(config.logstash_host, config.logstash_port)
+      else
+        puts "no host:port given"
+        exit 1
+      end
+      $switch_hash = config.switch_hash
+      if config.switch_hash != nil
+        $switchportnames = SNMPwalk.new(config.switch_hash.each_key)
+      end
+      EventMachine::run do
+        EventMachine::open_datagram_socket(bind_ip, bind_port, Collector)
+      end
+    rescue Exception => e
+      puts Time.now
+      puts e.message
+      puts e.backtrace
+      raise "unable to start sflow collector"
+    end
+  end
+
+end
+
+

data/lib/sflow/lib/sflow/config.rb

@@ -0,0 +1,15 @@
+class SflowConfig
+  attr_reader :switch_hash
+  attr_reader :logstash_host
+  attr_reader :logstash_port
+  attr_reader :daemonize
+
+  def initialize
+    config = YAML.load_file("etc/config.yaml")
+    @switch_hash = config['switch']
+    @logstash_host = config['logstash_host']
+    @logstash_port = config['logstash_port']
+    @daemonize = config['daemonize']
+  end
+end
+

data/lib/sflow/lib/sflow/models/binary_models.rb

@@ -0,0 +1,176 @@
+class Header < BinData::Record
+  endian :big
+  uint32 :version
+  uint32 :address_type
+  uint32 :agent_address
+  uint32 :sub_agent_id
+  uint32 :seq_number
+  uint32 :sys_uptime
+  uint32 :num_samples
+  array :flow_samples, :initial_length => :num_samples do
+    uint16 :enterprise_std
+    uint16 :sflow_sample_type
+    uint32 :sample_length
+    string :sample_data, :length => :sample_length
+  end
+end
+
+class Sflow5sampleheader1 < BinData::Record
+  endian :big
+  uint32 :seq_number
+  uint32 :source_id_type
+  uint32 :sampling_rate
+  uint32 :sample_pool
+  uint32 :dropped_packets
+  uint32 :i_iface_value
+  uint32 :o_iface_value
+  uint32 :num_records
+  array :records, :initial_length => :num_records do
+    uint16 :enterprise
+    uint16 :format
+    uint32 :flow_length
+    string :record_data, :length => :flow_length
+  end
+
+end
+
+class Sflow5sampleheader3 < BinData::Record
+  endian :big
+  uint32 :seq_number
+  uint32 :source_id_type
+  uint32 :source_id_index
+  uint32 :sampling_rate
+  uint32 :sample_pool
+  uint32 :dropped_packets
+  uint32 :i_iface_format
+  uint32 :i_iface_value
+  uint32 :o_iface_format
+  uint32 :o_iface_value
+  uint32 :num_records
+  array :records, :initial_length => :num_records do
+    uint16 :enterprise
+    uint16 :format
+    uint32 :flow_length
+    string :record_data, :length => :flow_length
+  end
+
+end
+
+
+class Sflow5counterheader4 < BinData::Record
+  endian :big
+  uint32 :seq_number
+  uint32 :source_id_type
+  uint32 :source_id_index
+  uint32 :num_records
+  array :records, :initial_length => :num_records do
+    uint16 :enterprise
+    uint16 :format
+    uint32 :record_length
+    string :record_data, :length => :record_length
+  end
+end
+
+class Sflow5counterheader2 < BinData::Record
+  endian :big
+  uint32 :seq_number
+  uint32 :source_id_type
+  uint32 :num_records
+  array :records, :initial_length => :num_records do
+    uint16 :enterprise
+    uint16 :format
+    uint32 :record_length
+    string :record_data, :length => :record_length
+  end
+end
+
+
+class Sflow5rawpacket < BinData::Record 
+  endian :big
+  uint32 :header_protocol
+  uint32 :frame_length
+  uint32 :payload
+  uint32 :xy
+  array :rawpacket_data, :read_until => :eof do
+    string :data, :length => 1
+  end
+end
+
+class Sflow5extswitch < BinData::Record
+  endian :big
+  uint32 :src_vlan
+  uint32 :src_priority
+  uint32 :dst_vlan
+  uint32 :dst_priority
+end
+
+class Sflow5genericcounter < BinData::Record
+  endian :big
+  uint32 :int_index
+  uint32 :int_type
+  uint64 :int_speed
+  uint32 :int_direction
+  uint16 :int_admin_status
+  uint16 :int_oper_status
+  uint64 :input_octets
+  uint32 :input_packets
+  uint32 :input_packets_multi
+  uint32 :input_packets_broad
+  uint32 :input_packets_discard
+  uint32 :input_packets_error
+  uint32 :unknown_proto
+  uint64 :output_octets
+  uint32 :output_packets
+  uint32 :output_packets_multi
+  uint32 :output_packets_broad
+  uint32 :output_packets_discard
+  uint32 :output_packets_error
+  uint32 :prom_mode
+end
+
+class Sflow5ethcounter < BinData::Record
+  endian :big
+  uint32 :alignment_errors
+  uint32 :fcs_errors
+  uint32 :single_collision_frames
+  uint32 :multi_collision_frames
+  uint32 :sqe_test_errors
+  uint32 :deffered_transmission
+  uint32 :late_collision
+  uint32 :excessive_collision
+  uint32 :internal_mac_transmit_errors
+  uint32 :carrier_sense_errors
+  uint32 :frame_too_long
+  uint32 :internal_mac_receive_errors
+  uint32 :symbol_errors
+end
+
+class Sflow5rawpacketheaderEthernet < BinData::Record
+  endian :big
+  string :eth_src, :length => 6
+  string :eth_dst, :length => 6
+  uint16 :eth_type
+  array :ethernetdata, :read_until => :eof do
+    string :data, :length => 1
+  end
+end
+
+class Sflow5rawpacketdata < BinData::Record
+  endian :big
+  string :eth, :length => 14
+  string :vlan_tag, :length => 2
+  string :vlan_tag_p, :length => 2
+  string :vlana, :length => 2
+  string :vlanb, :length => 2
+  string :ip_packet, :length => 40
+end
+
+class Sflow5rawpacketdataVLAN < BinData::Record
+  endian :big
+  uint16 :prio
+  uint16 :type
+  array :vlandata, :read_until => :eof do
+    string :data, :length => 1
+  end
+end
+

data/lib/sflow/lib/sflow/models/ipv4header.rb

@@ -0,0 +1,69 @@
+# coding: utf-8
+require_relative 'protocol'
+
+# TODO: チェックサムを確認する
+class IPv4Header
+
+  attr_reader :version,:header_length,:packet_length,:identification,:frag_dont,:frag_more,:frag_offset,:ttl,:protocol,:checksum,:sndr_addr,:dest_addr,
+    :data_length
+
+  def initialize(packet,offset=0)
+    @packet = packet.force_encoding("ASCII-8BIT")
+    @offset = offset
+    header = packet.unpack("x#{offset}n10")
+    @version = header[0] >> 12
+    @header_length  = ((header[0] >> 8) & 0x0f)*4
+    @packet_length = header[1]
+    @identification = header[2]
+    @frag_dont = (header[3] >> 14) & 0x01 != 0
+    @frag_more  = (header[3] >> 13) & 0x01 != 0
+    @frag_offset = header[3] & 0x1fff
+    @ttl = header[4] >> 8
+    @protocol = header[4] & 0x00ff
+    @checksum = header[5]
+    @sndr_addr = ip_to_s(packet[12..15])
+    @dest_addr = ip_to_s(packet[16..19])
+    @data_length = @packet_length - @header_length
+
+    @virtual_header = packet[12..19] + [0,6,@data_length].pack("CCn")
+
+  end
+
+  def upper
+    upper_header = Protocol.to_class(@protocol)
+    offset = @offset+@header_length
+    upper_header.new(@packet,offset,@data_length,self)
+  end
+  
+  def data
+    start = @offset+@header_length
+    @packet[start..start+@data_length]
+  end
+
+  def get_virtual_header
+    @virtual_header
+  end
+
+  def ip_to_s(ip)
+    ip = ip.unpack("n2")
+    sprintf("%d.%d.%d.%d",ip[0]>>8,ip[0]&0x00ff,ip[1]>>8,ip[1]&0x00ff)
+  end
+
+  def to_s
+    "IPv4 Header\n" <<
+    "  Version            : #{@version}\n" <<
+    "  Header Length      : #{@header_length}\n" <<
+    "  Packet Length      : #{@packet_length}\n" <<
+    "  Identification     : #{@identification}\n" <<
+    "  Don't fragment     : #{@frag_dont}\n" <<
+    "  More fragments     : #{@frag_more}\n" <<
+    "  Fragment Offset    : #{@frag_offset}\n" <<
+    "  TTL                : #{@ttl}\n" <<
+    "  Protocol           : #{Protocol.to_s(@protocol)}\n" <<
+    "  Header Checksum    : #{@checksum}\n" <<
+    "  Sender Address     : #{@sndr_addr}\n" <<
+    "  Destination Address: #{@dest_addr}\n" <<
+    "  (Data Length)      : #{@data_length}"
+  end
+
+end

data/lib/sflow/lib/sflow/models/protocol.rb

@@ -0,0 +1,47 @@
+require_relative 'ipv4header'
+require_relative 'udpheader'
+require_relative 'tcpheader'
+
+class Protocol
+  ICMP = 0x01
+  IGMP = 0x02
+  TCP  = 0x06
+  UDP  = 0x11
+  IPv6 = 0x29
+
+  def self.to_class protocol
+    case protocol
+    when Protocol::ICMP
+      raise "ICMP is not supported"
+    when Protocol::IGMP
+      raise "IGMP is not supported"
+    when Protocol::TCP
+      TCPHeader
+    when Protocol::UDP
+      UDPHeader
+    when Protocol::IPv6
+      raise "IPv6 is not supported"
+    else
+      raise "Protocol:"+sprintf("0x%2X",protocol)+" is not supported"
+    end
+  end
+ 
+  def self.to_s protocol
+    case protocol
+    when Protocol::ICMP
+      "ICMP"
+    when Protocol::IGMP
+      "IGMP"
+    when Protocol::TCP
+      "TCP"
+    when Protocol::UDP
+      "UDP"
+    when Protocol::IPv6
+      "IPv6"
+    else
+      sprintf("0x%2X",protocol)
+    end
+  end
+
+end
+

data/lib/sflow/lib/sflow/models/tcpheader.rb

@@ -0,0 +1,82 @@
+class TCPHeader
+
+  attr_reader :sndr_port,:dest_port,:seq_num,:ack_num,:header_length,
+    :urg,:ack,:psh,:rst,:syn,:fin,:win_size,:checksum,:emgcy_ptr,
+    :packet_length,:data_length,:lower
+
+  def initialize(packet,offset=0,length=nil,lower=nil)
+    @packet = packet.force_encoding("ASCII-8BIT")
+    @offset = offset
+    @length = length || packet.bytesize-offset
+    header = packet.unpack("x#{offset}n2N2n4")
+    @sndr_port = header[0]
+    @dest_port = header[1]
+    @seq_num = header[2]
+    @ack_num = header[3]
+    @header_length = (header[4]>>12)*4
+    @urg = (header[4] & 0b100000) != 0
+    @ack = (header[4] & 0b010000) != 0
+    @psh = (header[4] & 0b001000) != 0
+    @rst = (header[4] & 0b000100) != 0
+    @syn = (header[4] & 0b000010) != 0
+    @fin = (header[4] & 0b000001) != 0
+    @win_size = header[5]
+    @checksum = header[6]
+    @emgcy_ptr = header[7]
+
+    @packet_length = @length
+    @data_length = @packet_length-@header_length
+
+    @lower = lower
+
+    # check checksum
+    calc_cs = false
+    if calc_cs
+    tmp =  @packet[@offset..@offset+@length]
+    if (tmp.length % 2) != 0
+      tmp += "\0"
+    end
+    data = @lower.get_virtual_header + tmp
+    sum = 0
+    list = data.unpack("n*")
+    list.each do |d|
+      sum += d
+    end
+    sum  = (sum & 0xffff) + (sum >> 16)
+    sum  = (sum & 0xffff) + (sum >> 16)
+    raise if sum != 65535
+    end
+
+
+
+
+  end
+
+  def data
+    if(@data_length>0)
+      @packet[@offset+@header_length..@offset+@length]
+    else
+      ""
+    end
+  end
+
+  def to_s
+    "TCP Header\n" <<
+    "  Sender Port     : #{@sndr_port}\n" <<
+    "  Destination Port: #{@dest_port}\n" <<
+    "  Sequence Number : #{@seq_num}\n" <<
+    "  ACK Number      : #{@ack_num}\n" <<
+    "  Header Length   : #{@header_length}\n" <<
+    "  URG             : #{@urg}\n" <<
+    "  ACK             : #{@ack}\n" <<
+    "  PSH             : #{@psh}\n" <<
+    "  RST             : #{@rst}\n" <<
+    "  SYN             : #{@syn}\n" <<
+    "  FIN             : #{@fin}\n" <<
+    "  Window Size     : #{@win_size}\n" <<
+    "  Checksum        : #{@checksum}\n" <<
+    "  Emergency Ptr   : #{@emgcy_ptr}\n" <<
+    "  (Packet Length) : #{@packet_length}\n" <<
+    "  (Data Length)   : #{@data_length}"
+  end
+end