logstash-codec-sflow 1.1.1 → 1.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: ddc03024121f42f455dd1275cfe5cc6413f5b203
-  data.tar.gz: 84f0a827141d0fe80c01e1f8b373a649229be489
+  metadata.gz: c28971730da8f9f302c926868c06bdac25a0d9b0
+  data.tar.gz: 6d114bd259a7194a9d61c8759c4c4bc2b400efce
 SHA512:
-  metadata.gz: 49681626b90b77e12f4091c96c1bd7cedbee381c15be3a13cb56a20796c1df6b061de9e3c77799087535d0959eb1789b758670e13e0997f5dd711154c9d8760f
-  data.tar.gz: dcdbcc07970c0c8c15960676a3925e6bebd070a2f0b6c084e4011301343bf9de0b8ed071f72e288a9e90a99d5dd4de32946e365470030cf5de85611489304961
+  metadata.gz: a6ef3af6d5a7f001997fa986c739eb0d2f23fb1857724ac4dbed38a211e69ca7f3518d778aed2e19d0e162a7c51695f02952d636bccf37bb4c22533e6221f0b7
+  data.tar.gz: 36e2350656250450e8d3299062e299c1e959f30c13e08d1cde9ba96825be6fe919a7c9102a7d5f3286419abd7b70251a4faff639dfb1168e42aaab06976bde87

data/README.md

@@ -2,11 +2,11 @@
 ## Description
 Logstash codec plugin to decode sflow codec.
 
-This codec manage flow sample and counter flow.
+This codec manage flow sample, counter flow, expanded flow sample and expanded counter flow
 
-For the flow sample it is able to decode Ethernet, 802.1Q VLAN, IPv4, UDP and TCP header
+For the (expanded) flow sample it is able to decode Ethernet, 802.1Q VLAN, IPv4, UDP and TCP header
 
-For the counter flow it is able to decode some records of type:
+For the (expanded) counter flow it is able to decode some records of type:
 
 - Generic Interface
 - Ethernet Interface
@@ -21,6 +21,25 @@ If needed you can aks for some to be added.
 Please provide a pcap file containing the sflow events of the counter/protocol
 to add in order to be able to implement it.
 
+## Tune reported fields
+By default all those fields are removed from the emitted event:
+    
+    %w(sflow_version header_size ip_header_length ip_dscp ip_ecn ip_total_length ip_identification ip_flags 
+    ip_fragment_offset ip_ttl ip_checksum ip_options tcp_seq_number tcp_ack_number tcp_header_length tcp_reserved 
+    tcp_is_nonce tcp_is_cwr tcp_is_ecn_echo tcp_is_urgent tcp_is_ack tcp_is_push tcp_is_reset tcp_is_syn tcp_is_fin 
+    tcp_window_size tcp_checksum tcp_urgent_pointer tcp_options vlan_cfi sequence_number flow_sequence_number vlan_type 
+    udp_length udp_checksum)
+    
+You can tune the list of removed fields by setting this parameter to the sflow codec *optional_removed_field*
+
+## frame_length_times_sampling_rate output field on (expanded) flow sample
+
+This field is the length of the frame times the sampling rate. It permits to approximate the number of bits send/receive 
+on an interface/socket.
+
+You must first ensure to have well configured the sampling rate to have an accurate output metric (See: http://blog.sflow.com/2009/06/sampling-rates.html)
+
+
 ## Human Readable Protocol
 In order to translate protocols value to a human readable protocol, you can use the
 logstash-filter-translate plugin
@@ -55,7 +74,8 @@ filter {
       translate {
         field => ip_protocol
         dictionary => [ "6", "TCP",
-                        "17", "UDP"
+                        "17", "UDP",
+                        "50", "Encapsulating Security Payload"
                       ]
         fallback => "UNKNOWN"
         destination => ip_protocol

data/lib/logstash/codecs/sflow.rb

@@ -107,16 +107,18 @@ class LogStash::Codecs::Sflow < LogStash::Codecs::Base
     events = []
 
     decoded['samples'].each do |sample|
+      @logger.debug("sample: #{sample}")
       #Treat case with no flow decoded (Unknown flow)
       if sample['sample_data'].to_s.eql? ''
         @logger.warn("Unknown sample entreprise #{sample['sample_entreprise'].to_s} - format #{sample['sample_format'].to_s}")
         next
       end
 
-      #treat sample flow
-      if sample['sample_entreprise'] == 0 && sample['sample_format'] == 1
+      #treat sample flow and expanded sample flow
+      if sample['sample_entreprise'] == 0 && (sample['sample_format'] == 1 || sample['sample_format'] == 3)
         # Create the logstash event
         event = LogStash::Event.new({})
+
         common_sflow(event, decoded, sample)
 
         sample['sample_data']['records'].each do |record|
@@ -134,15 +136,19 @@ class LogStash::Codecs::Sflow < LogStash::Codecs::Base
           event["frame_length_times_sampling_rate"] = event['frame_length'].to_i * event['sampling_rate'].to_i
         end
 
-        event["sflow_type"] = 'sample'
+        if sample['sample_format'] == 1
+          event["sflow_type"] = 'flow_sample'
+        else
+          event["sflow_type"] = 'expanded_flow_sample'
+        end
 
         #Get interface dfescr if snmp_interface true
         snmp_call(event)
 
         events.push(event)
 
-      #treat counter flow
-      elsif sample['sample_entreprise'] == 0 && sample['sample_format'] == 2
+      #treat counter flow and expanded counter flow
+      elsif sample['sample_entreprise'] == 0 && (sample['sample_format'] == 2 || sample['sample_format'] == 4)
         sample['sample_data']['records'].each do |record|
           # Ensure that some data exist for the record
           if record['record_data'].to_s.eql? ''
@@ -156,7 +162,12 @@ class LogStash::Codecs::Sflow < LogStash::Codecs::Base
 
           assign_key_value(event, record)
 
-          event["sflow_type"] = 'counter'
+          if sample['sample_format'] == 2
+            event["sflow_type"] = 'counter_sample'
+          else
+            event["sflow_type"] = 'expanded_counter_sample'
+          end
+
 
           #Get interface dfescr if snmp_interface true
           snmp_call(event)

data/lib/logstash/codecs/sflow/datagram.rb

@@ -30,6 +30,8 @@ class SFlow < BinData::Record
     choice :sample_data, :selection => lambda { "#{sample_entreprise}-#{sample_format}" } do
       flow_sample '0-1'
       counter_sample '0-2'
+      expanded_flow_sample '0-3'
+      expanded_counter_sample '0-4'
       skip :default, :length => :sample_length
     end
   end

data/lib/logstash/codecs/sflow/flow_record.rb

@@ -6,6 +6,8 @@ require 'logstash/codecs/sflow/packet_header'
 
 # noinspection RubyResolve
 class RawPacketHeader < BinData::Record
+  mandatory_parameter :record_length
+
   endian :big
   uint32 :protocol
   uint32 :frame_length
@@ -16,6 +18,7 @@ class RawPacketHeader < BinData::Record
     ip_header 11, :size_header => lambda { header_size * 8 }
     skip :default, :length => :header_size
   end
+  bit :padded, :nbits => lambda { (record_length - (header_size + 16)) * 8 } #padded data
 end
 
 # noinspection RubyResolve

data/lib/logstash/codecs/sflow/packet_header.rb

@@ -55,7 +55,6 @@ class UdpHeader < BinData::Record
   uint16 :dst_port
   uint16 :udp_length
   uint16 :udp_checksum
-  #skip :length => lambda { udp_length - 64 } #skip udp data
   bit :data, :nbits => lambda { size_header - 64 } #skip udp data
 end
 

data/lib/logstash/codecs/sflow/sample.rb

@@ -21,7 +21,7 @@ class FlowSample < BinData::Record
     bit12 :record_format
     uint32 :record_length
     choice :record_data, :selection => lambda { "#{record_entreprise}-#{record_format}" } do
-      raw_packet_header '0-1'
+      raw_packet_header '0-1', :record_length => :record_length
       ethernet_frame_data '0-2'
       ip4_data '0-3'
       ip6_data '0-4'
@@ -56,3 +56,58 @@ class CounterSample < BinData::Record
     #processor_information :record_data
   end
 end
+
+# noinspection RubyResolve
+class ExpandedFlowSample < BinData::Record
+  endian :big
+  uint32 :flow_sequence_number
+  uint32 :source_id_type
+  uint32 :source_id_index
+  uint32 :sampling_rate
+  uint32 :sample_pool
+  uint32 :drops
+  uint32 :input_interface_format
+  uint32 :input_interface_value
+  uint32 :output_interface_format
+  uint32 :output_interface_value
+  uint32 :record_count
+  array :records, :initial_length => :record_count do
+    bit20 :record_entreprise
+    bit12 :record_format
+    uint32 :record_length
+    choice :record_data, :selection => lambda { "#{record_entreprise}-#{record_format}" } do
+      raw_packet_header '0-1', :record_length => :record_length
+          ethernet_frame_data '0-2'
+      ip4_data '0-3'
+      ip6_data '0-4'
+      extended_switch_data '0-1001'
+      extended_router_data '0-1002'
+      skip :default, :length => :record_length
+    end
+  end
+end
+
+# noinspection RubyResolve
+class ExpandedCounterSample < BinData::Record
+  endian :big
+  uint32 :sample_seq_number
+  uint32 :source_id_type
+  uint32 :source_id_index
+  uint32 :record_count
+  array :records, :initial_length => :record_count do
+    bit20 :record_entreprise
+    bit12 :record_format
+    uint32 :record_length
+    choice :record_data, :selection => lambda { "#{record_entreprise}-#{record_format}" } do
+      generic_interface '0-1'
+      ethernet_interfaces '0-2'
+      token_ring '0-3'
+      hundred_base_vg '0-4'
+      vlan '0-5'
+      processor_information '0-1001'
+      http_counters '0-2201'
+      skip :default, :length => :record_length
+    end
+    #processor_information :record_data
+  end
+end

data/logstash-codec-sflow.gemspec

@@ -1,7 +1,7 @@
 Gem::Specification.new do |s|
 
   s.name = 'logstash-codec-sflow'
-  s.version = '1.1.1'
+  s.version = '1.2.0'
   s.licenses = ['Apache License (2.0)']
   s.summary = 'The sflow codec is for decoding SFlow v5 flows.'
   s.description = 'This gem is a logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/plugin install gemname. This gem is not a stand-alone program'

data/spec/codecs/sflow_spec.rb

@@ -7,21 +7,21 @@ require "logstash/codecs/sflow/datagram"
 describe SFlow do
   it "should decode sflow counters" do
     payload = IO.read(File.join(File.dirname(__FILE__), "sflow_counters_sample.dat"), :mode => "rb")
-    decoded = SFlow.read(payload)
+    SFlow.read(payload)
   end
 
   it "should decode sflow 1 counters" do
     payload = IO.read(File.join(File.dirname(__FILE__), "sflow_1_counters_sample.dat"), :mode => "rb")
-    decoded = SFlow.read(payload)
+    SFlow.read(payload)
   end
 
   it "should decode sflow sample" do
     payload = IO.read(File.join(File.dirname(__FILE__), "sflow_flow_sample.dat"), :mode => "rb")
-    decoded = SFlow.read(payload)
+    SFlow.read(payload)
   end
 
   it "should decode sflow sample eth vlan" do
     payload = IO.read(File.join(File.dirname(__FILE__), "sflow_flow_sample_eth_vlan.dat"), :mode => "rb")
-    decoded = SFlow.read(payload)
+    SFlow.read(payload)
   end
 end

metadata

@@ -1,22 +1,22 @@
 --- !ruby/object:Gem::Specification
 name: logstash-codec-sflow
 version: !ruby/object:Gem::Version
-  version: 1.1.1
+  version: 1.2.0
 platform: ruby
 authors:
 - Nicolas Fraison
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2016-08-01 00:00:00.000000000 Z
+date: 2016-08-04 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: 1.4.0
-    - - <
+    - - "<"
       - !ruby/object:Gem::Version
         version: 3.0.0
   name: logstash-core
@@ -24,16 +24,16 @@ dependencies:
   type: :runtime
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: 1.4.0
-    - - <
+    - - "<"
       - !ruby/object:Gem::Version
         version: 3.0.0
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: 2.3.0
   name: bindata
@@ -41,13 +41,13 @@ dependencies:
   type: :runtime
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: 2.3.0
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: 1.1.0
   name: lru_redux
@@ -55,13 +55,13 @@ dependencies:
   type: :runtime
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: 1.1.0
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: 1.2.0
   name: snmp
@@ -69,13 +69,13 @@ dependencies:
   type: :runtime
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: 1.2.0
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
   name: logstash-devutils
@@ -83,7 +83,7 @@ dependencies:
   type: :development
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
 description: This gem is a logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/plugin install gemname. This gem is not a stand-alone program
@@ -130,17 +130,17 @@ require_paths:
 - lib
 required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
-  - - '>='
+  - - ">="
     - !ruby/object:Gem::Version
       version: '0'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
-  - - '>='
+  - - ">="
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
 rubyforge_project:
-rubygems_version: 2.4.5
+rubygems_version: 2.4.8
 signing_key:
 specification_version: 4
 summary: The sflow codec is for decoding SFlow v5 flows.