logstash-codec-sflow 0.1.0 → 0.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 7734945c0489502693b7f17044c6f96bae64637b
-  data.tar.gz: 9b353b730c693c2397f5edb013c3b0811a5f133c
+  metadata.gz: a9773b581653d998b843923848706abfe5851dad
+  data.tar.gz: 92c48583c7b35ecdf55bd707e4196b671149cc91
 SHA512:
-  metadata.gz: 5767825ecfb36ad9a4738cf770fb0718f9a785264b7e87fabbf6e292afaa058ff49fa6761cc5e456ffbcec4f4875cc7cfd48a19597d74a8fb0ac0d3f133fada6
-  data.tar.gz: 90e4aa370af0b55f121aa7b922e577d6153b166724048fdb3eaee630366214360b8dc0d3088a01d56a6d1ed8e98fb2475fc0036f1acbb4bc27faeeb918cfd4b0
+  metadata.gz: 74eb65d450dbc9b1dc05dd784a1d0b7efca1eb8f82cc09d978049d672a98802dbdcd36ce652475baa31222c9cfadb5c814a7bed2277845584db6c1ecb8877331
+  data.tar.gz: 68dcb680539c692397aeaecaa4cc7f80fa5f381f7852450115e9b27c09609b19cf9acd3f822bc648d1c20defa63cd79944fea6ebd0bb20f8c630b8155eeaa943

data/README.md

@@ -65,7 +65,7 @@ bundle exec rspec
 
 - Edit Logstash `tools/Gemfile` and add the local plugin path, for example:
 ```ruby
-gem "logstash-filter-awesome", :path => "/your/local/logstash-filter-awesome"
+gem "logstash-codec-sflow", :path => "/your/local/logstash-filter-awesome"
 ```
 - Update Logstash dependencies
 ```sh

data/lib/logstash/codecs/sflow/flow_record.rb

@@ -2,6 +2,7 @@
 
 require 'bindata'
 require 'logstash/codecs/sflow/util'
+require 'logstash/codecs/sflow/packet_header'
 
 class RawPacketHeader < BinData::Record
   endian :big
@@ -9,14 +10,20 @@ class RawPacketHeader < BinData::Record
   uint32 :frame_length
   uint32 :stripped
   uint32 :header_size
-  skip :length => :header_size
+  choice :sample_header, :selection => :protocol do
+    ethernet_header 1, :size_header => lambda { header_size * 8 }
+    ip_header 11, :size_header => lambda { header_size * 8 }
+    skip :default, :length => :header_size
+  end
 end
 
 class EthernetFrameData < BinData::Record
   endian :big
   uint32 :packet_length
   mac_address :src_mac
+  skip :length => 2
   mac_address :dst_mac
+  skip :length => 2
   uint32 :type
 end
 

data/lib/logstash/codecs/sflow/packet_header.rb

@@ -0,0 +1,100 @@
+# encoding: utf-8
+
+require 'bindata'
+require 'logstash/codecs/sflow/util'
+
+
+class TcpHeader < BinData::Record
+  mandatory_parameter :size_header
+
+  endian :big
+  uint16 :src_port
+  uint16 :dst_port
+  uint32 :tcp_seq_number
+  uint32 :tcp_ack_number
+  bit4 :tcp_header_length # times 4
+  bit3 :tcp_reserved
+  bit1 :tcp_is_nonce
+  bit1 :tcp_is_cwr
+  bit1 :tcp_is_ecn_echo
+  bit1 :tcp_is_urgent
+  bit1 :tcp_is_ack
+  bit1 :tcp_is_push
+  bit1 :tcp_is_reset
+  bit1 :tcp_is_syn
+  bit1 :tcp_is_fin
+  uint16 :tcp_window_size
+  uint16 :tcp_checksum
+  uint16 :tcp_urgent_pointer
+  array :tcp_options, :initial_length => lambda { (((tcp_header_length * 4) - 20)/4).ceil }, :onlyif => :is_options? do
+    string :tcp_option, :length => 4, :pad_byte => "\0"
+  end
+  bit :nbits => lambda { size_header - (tcp_header_length * 4 * 8) }
+
+  def is_options?
+    tcp_header_length.to_i > 5
+  end
+end
+
+class UdpHeader < BinData::Record
+  endian :big
+  uint16 :src_port
+  uint16 :dst_port
+  uint16 :udp_length
+  uint16 :udp_checksum
+  skip :length => lambda { udp_length - 64 } #skip udp data
+end
+
+class IPV4Header < BinData::Record
+  mandatory_parameter :size_header
+
+  endian :big
+  bit4 :ip_header_length # times 4
+  bit6 :ip_dscp
+  bit2 :ip_ecn
+  uint16 :ip_total_length
+  uint16 :ip_identification
+  bit3 :ip_flags
+  bit13 :ip_fragment_offset
+  uint8 :ip_ttl
+  uint8 :ip_protocol
+  uint16 :ip_checksum
+  ip4_addr :src_ip
+  ip4_addr :dst_ip
+  array :ip_options, :initial_length => lambda { (((ip_header_length * 4) - 20)/4).ceil }, :onlyif => :is_options? do
+    string :ip_option, :length => 4, :pad_byte => "\0"
+  end
+  choice :layer4, :selection => :ip_protocol do
+    tcp_header 6, :size_header => lambda { size_header - (ip_header_length * 4 * 8) }
+    udp_header 17
+    bit :default, :nbits => lambda { size_header - (ip_header_length * 4 * 8) }
+  end
+
+  def is_options?
+    ip_header_length.to_i > 5
+  end
+end
+
+class IPHeader < BinData::Record
+  mandatory_parameter :size_header
+
+  endian :big
+  bit4 :ip_version
+  choice :header, :selection => :ip_version do
+    ipv4_header 4, :size_header => :size_header
+    bit :default, :nbits => lambda { size_header - 4 }
+  end
+end
+
+class EthernetHeader < BinData::Record
+  mandatory_parameter :size_header
+
+  endian :big
+  mac_address :eth_dst
+  mac_address :eth_src
+  uint16 :eth_type
+  choice :layer3, :selection => :eth_type do
+    ip_header 2048, :size_header => lambda { size_header - (14 * 8) }
+    bit :default, :nbits => lambda { size_header - (14 * 8) }
+  end
+end

data/lib/logstash/codecs/sflow/sample.rb

@@ -19,13 +19,13 @@ class FlowSample < BinData::Record
     bit20 :record_entreprise
     bit12 :record_format
     uint32 :record_length
-    choice :record_data, :selection => :record_format do
-      raw_packet_header 1
-      ethernet_frame_data 2
-      ip4_data 3
-      ip6_data 4
-      extended_switch_data 1001
-      extended_router_data 1002
+    choice :record_data, :selection => lambda { "#{record_entreprise}-#{record_format}" } do
+      raw_packet_header "0-1"
+      ethernet_frame_data "0-2"
+      ip4_data "0-3"
+      ip6_data "0-4"
+      extended_switch_data "0-1001"
+      extended_router_data "0-1002"
       skip :default, :length => :record_length
     end
   end

data/lib/logstash/codecs/sflow/util.rb

@@ -5,7 +5,6 @@ require 'ipaddr'
 
 class MacAddress < BinData::Primitive
   array :bytes, :type => :uint8, :initial_length => 6
-  bit16 :padbytes
 
   def set(val)
     ints = val.split(/:/).collect { |int| int.to_i(16) }

data/lib/logstash/codecs/sflow.rb

@@ -1,21 +1,30 @@
 # encoding: utf-8
-require "logstash/codecs/base"
-require "logstash/namespace"
+require 'logstash/codecs/base'
+require 'logstash/namespace'
 
 # The "sflow" codec is for decoding sflow v5 flows.
 class LogStash::Codecs::Sflow < LogStash::Codecs::Base
-  config_name "sflow"
+  config_name 'sflow'
 
   # Specify which sflow must not be send in the event
-  config :removed_field, :validate => :array, :default => ['record_length', 'record_count', 'record_entreprise',
-                                                           'record_format', 'sample_entreprise', 'sample_format',
-                                                           'sample_length', 'sample_count', 'sflow_version',
-                                                           'ip_version']
+  config :optional_removed_field, :validate => :array, :default => ['sflow_version', 'ip_version', 'header_size',
+                                                                    'ip_header_length', 'ip_dscp', 'ip_ecn',
+                                                                    'ip_total_length', 'ip_identification', 'ip_flags',
+                                                                    'ip_fragment_offset', 'ip_ttl', 'ip_checksum',
+                                                                    'ip_options', 'tcp_seq_number', 'tcp_ack_number',
+                                                                    'tcp_header_length', 'tcp_reserved', 'tcp_is_nonce',
+                                                                    'tcp_is_cwr', 'tcp_is_ecn_echo', 'tcp_is_urgent',
+                                                                    'tcp_is_ack', 'tcp_is_push', 'tcp_is_reset',
+                                                                    'tcp_is_syn', 'tcp_is_fin', 'tcp_window_size',
+                                                                    'tcp_checksum', 'tcp_urgent_pointer', 'tcp_options']
 
 
   def initialize(params = {})
     super(params)
     @threadsafe = false
+    @removed_field = ['record_length', 'record_count', 'record_entreprise', 'record_format', 'sample_entreprise',
+                      'sample_format', 'sample_length', 'sample_count', 'sample_header', 'layer3', 'layer4',
+                      'tcp_nbits', 'ip_nbits'] | @optional_removed_field
   end
 
   # def initialize
@@ -29,56 +38,132 @@ class LogStash::Codecs::Sflow < LogStash::Codecs::Base
 
   public
   def decode(payload)
+
     decoded = SFlow.read(payload)
 
     events = []
 
     decoded['samples'].each do |sample|
+      #Treat case with no flow decoded (Unknown flow)
       if sample['sample_data'].to_s.eql? ''
         @logger.warn("Unknown sample entreprise #{sample['sample_entreprise'].to_s} - format #{sample['sample_format'].to_s}")
         next
       end
-      sample['sample_data']['records'].each do |record|
-        # Ensure that some data exist for the record
-        if record['record_data'].to_s.eql? ''
-          @logger.warn("Unknown record entreprise #{record['record_entreprise'].to_s}, format #{record['record_format'].to_s}")
-          next
-        end
 
+      #treat sample flow
+      if sample['sample_entreprise'] == 0 && sample['sample_format'] == 1
         # Create the logstash event
         event = {
             LogStash::Event::TIMESTAMP => LogStash::Timestamp.now
         }
+        sample['sample_data']['records'].each do |record|
+          # Ensure that some data exist for the record
+          if record['record_data'].to_s.eql? ''
+            @logger.warn("Unknown record entreprise #{record['record_entreprise'].to_s}, format #{record['record_format'].to_s}")
+            next
+          end
 
-        decoded.each_pair do |k, v|
-          unless k.to_s.eql? 'samples' or @removed_field.include? k.to_s
-            event["#{k}"] = v
+          decoded.each_pair do |k, v|
+            unless k.to_s.eql? 'samples' or @removed_field.include? k.to_s
+              event["#{k}"] = v
+            end
           end
-        end
 
-        sample.each_pair do |k, v|
-          unless k.to_s.eql? 'sample_data' or @removed_field.include? k.to_s
-            event["#{k}"] = v
+          sample.each_pair do |k, v|
+            unless k.to_s.eql? 'sample_data' or @removed_field.include? k.to_s
+              event["#{k}"] = v
+            end
           end
-        end
 
-        sample['sample_data'].each_pair do |k, v|
-          unless k.to_s.eql? 'records' or @removed_field.include? k.to_s
-            event["#{k}"] = v
+          sample['sample_data'].each_pair do |k, v|
+            unless k.to_s.eql? 'records' or @removed_field.include? k.to_s
+              event["#{k}"] = v
+            end
           end
+
+          record.each_pair do |k, v|
+            unless k.to_s.eql? 'record_data' or @removed_field.include? k.to_s
+              event["#{k}"] = v
+            end
+          end
+
+          record['record_data'].each_pair do |k, v|
+            unless k.to_s.eql? 'record_data' or @removed_field.include? k.to_s
+              event["#{k}"] = v
+            end
+          end
+
+          unless record['record_data']['sample_header'].to_s.eql? ''
+            record['record_data']['sample_header'].each_pair do |k, v|
+              unless k.to_s.eql? 'record_data' or @removed_field.include? k.to_s
+                event["#{k}"] = v
+              end
+            end
+
+            if record['record_data']['sample_header'].has_key?("layer3")
+              record['record_data']['sample_header']['layer3']['header'].each_pair do |k, v|
+                unless k.to_s.eql? 'record_data' or @removed_field.include? k.to_s
+                  event["#{k}"] = v
+                end
+              end
+            end
+
+            unless record['record_data']['sample_header']['layer3']['header']['layer4'].to_s.eql? ''
+              record['record_data']['sample_header']['layer3']['header']['layer4'].each_pair do |k, v|
+                unless k.to_s.eql? 'record_data' or @removed_field.include? k.to_s
+                  event["#{k}"] = v
+                end
+              end
+            end
+          end
+
         end
+        events.push(event)
+
+        #treat counter flow
+      elsif sample['sample_entreprise'] == 0 && sample['sample_format'] == 2
+        sample['sample_data']['records'].each do |record|
+          # Ensure that some data exist for the record
+          if record['record_data'].to_s.eql? ''
+            @logger.warn("Unknown record entreprise #{record['record_entreprise'].to_s}, format #{record['record_format'].to_s}")
+            next
+          end
+
+          # Create the logstash event
+          event = {
+              LogStash::Event::TIMESTAMP => LogStash::Timestamp.now
+          }
+
+          decoded.each_pair do |k, v|
+            unless k.to_s.eql? 'samples' or @removed_field.include? k.to_s
+              event["#{k}"] = v
+            end
+          end
+
+          sample.each_pair do |k, v|
+            unless k.to_s.eql? 'sample_data' or @removed_field.include? k.to_s
+              event["#{k}"] = v
+            end
+          end
 
-        record.each_pair do |k, v|
-          unless k.to_s.eql? 'record_data' or @removed_field.include? k.to_s
+          sample['sample_data'].each_pair do |k, v|
+            unless k.to_s.eql? 'records' or @removed_field.include? k.to_s
+              event["#{k}"] = v
+            end
+          end
+
+          record.each_pair do |k, v|
+            unless k.to_s.eql? 'record_data' or @removed_field.include? k.to_s
+              event["#{k}"] = v
+            end
+          end
+
+          record['record_data'].each_pair do |k, v|
             event["#{k}"] = v
           end
-        end
 
-        record['record_data'].each_pair do |k, v|
-          event["#{k}"] = v
+          events.push(event)
         end
-
-        events.push(event)
       end
     end
 

data/logstash-codec-sflow.gemspec

@@ -1,7 +1,7 @@
 Gem::Specification.new do |s|
 
   s.name = 'logstash-codec-sflow'
-  s.version = '0.1.0'
+  s.version = '0.2.0'
   s.licenses = ['Apache License (2.0)']
   s.summary = "The sflow codec is for decoding SFlow v5 flows."
   s.description = "This gem is a logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/plugin install gemname. This gem is not a stand-alone program"

data/spec/codecs/sflow/packet_header_spec.rb

@@ -0,0 +1,94 @@
+# encoding: utf-8
+
+require "logstash/devutils/rspec/spec_helper"
+require "logstash/codecs/sflow/packet_header"
+
+describe UdpHeader do
+  it "should decode udp header" do
+    payload = IO.read(File.join(File.dirname(__FILE__), "udp.dat"), :mode => "rb")
+    decoded = UdpHeader.read(payload)
+
+    decoded["src_port"].to_s.should eq("20665")
+    decoded["dst_port"].to_s.should eq("514")
+    decoded["udp_length"].to_s.should eq("147")
+  end
+end
+
+
+describe TcpHeader do
+  it "should decode tcp header" do
+    payload = IO.read(File.join(File.dirname(__FILE__), "tcp.dat"), :mode => "rb")
+    decoded = TcpHeader.new(:size_header => payload.bytesize * 8).read(payload)
+
+    decoded["src_port"].to_s.should eq("5672")
+    decoded["dst_port"].to_s.should eq("59451")
+    decoded["tcp_seq_number"].to_s.should eq("2671357038")
+    decoded["tcp_ack_number"].to_s.should eq("2651945969")
+    (decoded["tcp_header_length"].to_i*4).to_s.should eq("32")
+    decoded["tcp_is_nonce"].to_s.should eq("0")
+    decoded["tcp_is_cwr"].to_s.should eq("0")
+    decoded["tcp_is_ecn_echo"].to_s.should eq("0")
+    decoded["tcp_is_urgent"].to_s.should eq("0")
+    decoded["tcp_is_ack"].to_s.should eq("1")
+    decoded["tcp_is_push"].to_s.should eq("1")
+    decoded["tcp_is_reset"].to_s.should eq("0")
+    decoded["tcp_is_syn"].to_s.should eq("0")
+    decoded["tcp_is_fin"].to_s.should eq("0")
+    decoded["tcp_window_size"].to_s.should eq("147")
+    decoded["tcp_checksum"].to_s.should eq("13042")
+    decoded["tcp_urgent_pointer"].to_s.should eq("0")
+  end
+end
+
+
+describe IPHeader do
+  it "should decode ipv4 over tcp header" do
+    payload = IO.read(File.join(File.dirname(__FILE__), "ipv4_over_tcp_header.dat"), :mode => "rb")
+    decoded = IPHeader.new(:size_header => payload.bytesize * 8).read(payload)
+
+    decoded["ip_version"].to_s.should eq("4")
+    decoded["header"]["ip_header_length"].to_s.should eq("5")
+    decoded["header"]["ip_dscp"].to_s.should eq("0")
+    decoded["header"]["ip_ecn"].to_s.should eq("0")
+    decoded["header"]["ip_total_length"].to_s.should eq("476")
+    decoded["header"]["ip_identification"].to_s.should eq("30529")
+    decoded["header"]["ip_flags"].to_s.should eq("2")
+    decoded["header"]["ip_fragment_offset"].to_s.should eq("0")
+    decoded["header"]["ip_ttl"].to_s.should eq("62")
+    decoded["header"]["ip_protocol"].to_s.should eq("6")
+    decoded["header"]["ip_checksum"].to_s.should eq("37559")
+    decoded["header"]["src_ip"].to_s.should eq("10.243.27.17")
+    decoded["header"]["dst_ip"].to_s.should eq("10.243.0.45")
+    decoded["header"]["layer4"]["src_port"].to_s.should eq("5672")
+    decoded["header"]["layer4"]["dst_port"].to_s.should eq("59451")
+    decoded["header"]["layer4"]["tcp_seq_number"].to_s.should eq("2671357038")
+    decoded["header"]["layer4"]["tcp_ack_number"].to_s.should eq("2651945969")
+    (decoded["header"]["layer4"]["tcp_header_length"].to_i*4).to_s.should eq("32")
+    decoded["header"]["layer4"]["tcp_is_nonce"].to_s.should eq("0")
+    decoded["header"]["layer4"]["tcp_is_cwr"].to_s.should eq("0")
+    decoded["header"]["layer4"]["tcp_is_ecn_echo"].to_s.should eq("0")
+    decoded["header"]["layer4"]["tcp_is_urgent"].to_s.should eq("0")
+    decoded["header"]["layer4"]["tcp_is_ack"].to_s.should eq("1")
+    decoded["header"]["layer4"]["tcp_is_push"].to_s.should eq("1")
+    decoded["header"]["layer4"]["tcp_is_reset"].to_s.should eq("0")
+    decoded["header"]["layer4"]["tcp_is_syn"].to_s.should eq("0")
+    decoded["header"]["layer4"]["tcp_is_fin"].to_s.should eq("0")
+    decoded["header"]["layer4"]["tcp_window_size"].to_s.should eq("147")
+    decoded["header"]["layer4"]["tcp_checksum"].to_s.should eq("13042")
+    decoded["header"]["layer4"]["tcp_urgent_pointer"].to_s.should eq("0")
+  end
+end
+
+
+describe EthernetHeader do
+  it "should decode ipv4 over udp header" do
+    payload = IO.read(File.join(File.dirname(__FILE__), "ethernet_ipv4_over_udp_header.dat"), :mode => "rb")
+    decoded = EthernetHeader.new(:size_header => payload.bytesize * 8).read(payload)
+
+    decoded["eth_dst"].to_s.should eq("00:23:e9:78:16:c6")
+    decoded["eth_src"].to_s.should eq("58:f3:9c:81:4b:81")
+    decoded["eth_type"].to_s.should eq("2048")
+    decoded["layer3"]["header"]["dst_ip"].to_s.should eq("10.243.27.9")
+    decoded["layer3"]["header"]["layer4"]["dst_port"].to_s.should eq("514")
+  end
+end

data/spec/codecs/sflow_spec.rb

@@ -2,19 +2,22 @@
 
 require "logstash/devutils/rspec/spec_helper"
 require "logstash/codecs/sflow"
+require "logstash/codecs/sflow/datagram"
 
-describe LogStash::Codecs::Sflow do
-  before :each do
-    @subject = LogStash::Codecs::Sflow.new
+describe SFlow do
+  it "should decode sflow counters" do
     payload = IO.read(File.join(File.dirname(__FILE__), "sflow_counters_sample.dat"), :mode => "rb")
-    @subject.decode(payload)
-    payload = IO.read(File.join(File.dirname(__FILE__), "sflow_flow_sample.dat"), :mode => "rb")
-    @subject.decode(payload)
+    decoded = SFlow.read(payload)
+  end
+
+  it "should decode sflow 1 counters" do
+    payload = IO.read(File.join(File.dirname(__FILE__), "sflow_1_counters_sample.dat"), :mode => "rb")
+    decoded = SFlow.read(payload)
   end
 
-  describe "#new" do
-    it "LogStash::Codecs::Sflow" do
-      @subject.should be_an_instance_of LogStash::Codecs::Sflow
-    end
+
+  it "should decode sflow sample" do
+    payload = IO.read(File.join(File.dirname(__FILE__), "sflow_flow_sample.dat"), :mode => "rb")
+    decoded = SFlow.read(payload)
   end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: logstash-codec-sflow
 version: !ruby/object:Gem::Version
-  version: 0.1.0
+  version: 0.2.0
 platform: ruby
 authors:
 - Nicolas Fraison
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2015-12-08 00:00:00.000000000 Z
+date: 2015-12-13 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
@@ -74,9 +74,16 @@ files:
 - lib/logstash/codecs/sflow/counter_record.rb
 - lib/logstash/codecs/sflow/datagram.rb
 - lib/logstash/codecs/sflow/flow_record.rb
+- lib/logstash/codecs/sflow/packet_header.rb
 - lib/logstash/codecs/sflow/sample.rb
 - lib/logstash/codecs/sflow/util.rb
 - logstash-codec-sflow.gemspec
+- spec/codecs/sflow/ethernet_ipv4_over_udp_header.dat
+- spec/codecs/sflow/ipv4_over_tcp_header.dat
+- spec/codecs/sflow/packet_header_spec.rb
+- spec/codecs/sflow/tcp.dat
+- spec/codecs/sflow/udp.dat
+- spec/codecs/sflow_1_counters_sample.dat
 - spec/codecs/sflow_counters_sample.dat
 - spec/codecs/sflow_flow_sample.dat
 - spec/codecs/sflow_spec.rb
@@ -107,6 +114,12 @@ signing_key:
 specification_version: 4
 summary: The sflow codec is for decoding SFlow v5 flows.
 test_files:
+- spec/codecs/sflow/ethernet_ipv4_over_udp_header.dat
+- spec/codecs/sflow/ipv4_over_tcp_header.dat
+- spec/codecs/sflow/packet_header_spec.rb
+- spec/codecs/sflow/tcp.dat
+- spec/codecs/sflow/udp.dat
+- spec/codecs/sflow_1_counters_sample.dat
 - spec/codecs/sflow_counters_sample.dat
 - spec/codecs/sflow_flow_sample.dat
 - spec/codecs/sflow_spec.rb