logstash-codec-netflow 3.8.1 → 3.8.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 5b85cd8c1849c1914c49727e17a1775257aee259
-  data.tar.gz: fb97641f2b79c06cc380c27eba5933f40dfc8c87
+  metadata.gz: eb35147ecbca3f014d8ba75ae96d5b5190cca121
+  data.tar.gz: 2faec212bb770aa4bcae045c2f30d4046b12cbdd
 SHA512:
-  metadata.gz: 2ee8918200546ef2e9d454eadf84bce73abab9fc24c268ce2c90808ddc1c4dba70d8503bad0213b774e71385df5a00a986c738e0abea1105afa9f842e5ed66ce
-  data.tar.gz: 578e1d35533e5053a4b43189bc2581a3e511ae0d152974fa4f3d8bb7aa80c35bc2f6dad837b433b905a7de8c7d644bfb9d81e6efd39b760713bcb82683aaf882
+  metadata.gz: 6fecbcfdb821d74677230e402afd21964ab4a7d6190a548439cf8b2118e1af584365ad5a951231ba45647cfdee33a01cb2a51e5f5344b69b1202f969332ece1d
+  data.tar.gz: 3df0991010965dc1630ff39b7fdefc96ca102b51241a12c55d5cb829cadc76ccaf840684272b769a7452a5abe4293684a4717894df4139a1155c7090ea2a05c3

data/CHANGELOG.md

@@ -1,3 +1,11 @@
+## 3.8.3
+
+  - Fixed a race condition that could cause some errors when running in a multithreaded input
+
+## 3.8.2
+
+  - Fixed exceptions due to NilClass in util.rb and netflow.rb
+
 ## 3.8.1
 
   - Prevent Netflow and IPFIX templates from being modified concurrently

data/CONTRIBUTORS

@@ -4,6 +4,7 @@ reports, or in general have helped logstash along its way.
 Contributors:
 * Aaron Mildenstein (untergeek)
 * Adam Kaminski (thimslugga)
+* Andrew Cholakian (andrewvc)
 * Bjørn Ruberg (bruberg)
 * Colin Surprenant (colinsurprenant)
 * Daniel Nägele (analogbyte)

data/lib/logstash/codecs/netflow.rb

@@ -50,6 +50,7 @@ class LogStash::Codecs::Netflow < LogStash::Codecs::Base
   FLOWSET_ID = "flowset_id"
 
   def initialize(params = {})
+    @file_cache_mutex = Mutex.new
     super(params)
     @threadsafe = true
     @decode_mutex_netflow = Mutex.new
@@ -240,9 +241,10 @@ class LogStash::Codecs::Netflow < LogStash::Codecs::Base
       else
         key = "#{flowset.source_id}|#{record.flowset_id}"
       end
-      template = @netflow_templates[key]
 
-      unless template
+      template = @decode_mutex_netflow.synchronize { @netflow_templates[key] }
+
+      if !template
         @logger.warn("Can't (yet) decode flowset id #{record.flowset_id} from source id #{flowset.source_id}, because no template to decode it with has been received. This message will usually go away after 1 minute.")
         return events
       end
@@ -252,9 +254,11 @@ class LogStash::Codecs::Netflow < LogStash::Codecs::Base
       # Template shouldn't be longer than the record 
       # As fas as padding is concerned, the RFC defines a SHOULD for 4-word alignment
       # so we won't complain about that.
-      if template.num_bytes > length
-        @logger.warn("Template length exceeds flowset length, skipping", :template_id => record.flowset_id, :template_length => template.num_bytes, :record_length => length)
-        return events
+      if template.num_bytes != nil
+        if template.num_bytes > length
+          @logger.warn("Template length exceeds flowset length, skipping", :template_id => record.flowset_id, :template_length => template.num_bytes, :record_length => length)
+          return events
+        end
       end
 
       array = BinData::Array.new(:type => template, :initial_length => length / template.num_bytes)
@@ -336,9 +340,9 @@ class LogStash::Codecs::Netflow < LogStash::Codecs::Base
     when 256..65535
       # Data flowset
       key = "#{flowset.observation_domain_id}|#{record.flowset_id}"
-      template = @ipfix_templates[key]
-
-      unless template
+      if @ipfix_templates[key] != nil
+        template = @ipfix_templates[key]
+      else
         @logger.warn("Can't (yet) decode flowset id #{record.flowset_id} from observation domain id #{flowset.observation_domain_id}, because no template to decode it with has been received. This message will usually go away after 1 minute.")
         return events
       end
@@ -417,9 +421,10 @@ class LogStash::Codecs::Netflow < LogStash::Codecs::Base
     templates_cache = {}
     begin
       @logger.debug? and @logger.debug("Loading templates from template cache #{file_path}")
-      templates_cache = JSON.parse(File.read(file_path))
+      file_data = @file_cache_mutex.synchronize { File.read(file_path)}
+      templates_cache = JSON.parse(file_data)
     rescue Exception => e
-      raise "#{self.class.name}: templates cache file corrupt (#{file_path})"
+      raise "#{self.class.name}: templates cache file could not be read @ (#{file_path}: #{e.class.name} #{e.message})"
     end
 
     templates_cache
@@ -428,7 +433,9 @@ class LogStash::Codecs::Netflow < LogStash::Codecs::Base
   def save_templates_cache(templates_cache, file_path)
     begin
       @logger.debug? and @logger.debug("Writing templates to template cache #{file_path}")
-      File.open(file_path, 'w') {|file| file.write templates_cache.to_json }
+      @file_cache_mutex.synchronize do
+        File.open(file_path, 'w') {|file| file.write templates_cache.to_json }
+      end
     rescue Exception => e
       raise "#{self.class.name}: saving templates cache file failed (#{file_path}) with error #{e}"
     end

data/lib/logstash/codecs/netflow/util.rb

@@ -7,15 +7,19 @@ class IP4Addr < BinData::Primitive
   uint32 :storage
 
   def set(val)
-    ip = IPAddr.new(val)
-    if ! ip.ipv4?
-      raise ArgumentError, "invalid IPv4 address '#{val}'"
+    unless val.nil?
+      ip = IPAddr.new(val)
+      if ! ip.ipv4?
+        raise ArgumentError, "invalid IPv4 address '#{val}'"
+      end
+      self.storage = ip.to_i
     end
-    self.storage = ip.to_i
   end
 
   def get
-    IPAddr.new_ntoh([self.storage].pack('N')).to_s
+    unless self.storage.nil?
+      IPAddr.new_ntoh([self.storage].pack('N')).to_s
+    end
   end
 end
 
@@ -24,17 +28,21 @@ class IP6Addr < BinData::Primitive
   uint128 :storage
 
   def set(val)
-    ip = IPAddr.new(val)
-    if ! ip.ipv6?
-      raise ArgumentError, "invalid IPv6 address `#{val}'"
+    unless val.nil?
+      ip = IPAddr.new(val)
+      if ! ip.ipv6?
+        raise ArgumentError, "invalid IPv6 address `#{val}'"
+      end
+      self.storage = ip.to_i
     end
-    self.storage = ip.to_i
   end
 
   def get
-    IPAddr.new_ntoh((0..7).map { |i|
-      (self.storage >> (112 - 16 * i)) & 0xffff
-    }.pack('n8')).to_s
+    unless self.storage.nil?
+      IPAddr.new_ntoh((0..7).map { |i|
+        (self.storage >> (112 - 16 * i)) & 0xffff
+      }.pack('n8')).to_s
+    end
   end
 end
 
@@ -42,12 +50,18 @@ class MacAddr < BinData::Primitive
   array :bytes, :type => :uint8, :initial_length => 6
 
   def set(val)
-    ints = val.split(/:/).collect { |int| int.to_i(16) }
-    self.bytes = ints
+    unless val.nil?
+      ints = val.split(/:/).collect { |int| int.to_i(16) }
+      self.bytes = ints
+    end
   end
 
   def get
-    self.bytes.collect { |byte| byte.value.to_s(16).rjust(2,'0') }.join(":")
+    self.bytes.collect { |byte| 
+      unless byte.nil?
+        byte.value.to_s(16).rjust(2,'0')
+      end
+    }.join(":")
   end
 end
 
@@ -85,11 +99,17 @@ class ACLIdASA < BinData::Primitive
   array :bytes, :type => :uint8, :initial_length => 12
 
   def set(val)
-    self.bytes = val.split("-").collect { |aclid| aclid.scan(/../).collect { |hex| hex.to_i(16)} }.flatten
+    unless val.nil?
+      self.bytes = val.split("-").collect { |aclid| aclid.scan(/../).collect { |hex| hex.to_i(16)} }.flatten
+    end
   end
 
   def get
-    hexstring = self.bytes.collect { |byte| byte.value.to_s(16).rjust(2,'0') }.join
+    hexstring = self.bytes.collect { |byte| 
+      unless byte.nil?
+        byte.value.to_s(16).rjust(2,'0') 
+      end
+    }.join
     hexstring.scan(/......../).collect { |aclid| aclid }.join("-")
   end
 end
@@ -114,8 +134,10 @@ class Application_Id16 < BinData::Primitive
   uint24 :selector_id
 
   def set(val)
-    self.classification_id=val.to_i<<24
-    self.selector_id = val.to_i-((val.to_i>>24)<<24)
+    unless val.nil?
+      self.classification_id=val.to_i<<24
+      self.selector_id = val.to_i-((val.to_i>>24)<<24)
+    end
   end
 
   def get
@@ -129,8 +151,10 @@ class Application_Id24 < BinData::Primitive
   uint16 :selector_id
 
   def set(val)
-    self.classification_id=val.to_i<<16
-    self.selector_id = val.to_i-((val.to_i>>16)<<16)
+    unless val.nil?
+      self.classification_id=val.to_i<<16
+      self.selector_id = val.to_i-((val.to_i>>16)<<16)
+    end
   end
 
   def get
@@ -144,8 +168,10 @@ class Application_Id32 < BinData::Primitive
   uint24 :selector_id
 
   def set(val)
-    self.classification_id=val.to_i<<24
-    self.selector_id = val.to_i-((val.to_i>>24)<<24)
+    unless val.nil?
+      self.classification_id=val.to_i<<24
+      self.selector_id = val.to_i-((val.to_i>>24)<<24)
+    end
   end
 
   def get
@@ -159,8 +185,10 @@ class Application_Id40 < BinData::Primitive
   uint32 :selector_id
 
   def set(val)
-    self.classification_id=val.to_i<<32
-    self.selector_id = val.to_i-((val.to_i>>32)<<32)
+    unless val.nil?
+      self.classification_id=val.to_i<<32
+      self.selector_id = val.to_i-((val.to_i>>32)<<32)
+    end
   end
 
   def get
@@ -174,8 +202,10 @@ class Application_Id64 < BinData::Primitive
   uint56 :selector_id
 
   def set(val)
-    self.classification_id=val.to_i<<56
-    self.selector_id = val.to_i-((val.to_i>>56)<<56)
+    unless val.nil?
+      self.classification_id=val.to_i<<56
+      self.selector_id = val.to_i-((val.to_i>>56)<<56)
+    end
   end
 
   def get
@@ -189,8 +219,10 @@ class Application_Id72 < BinData::Primitive
   uint64 :selector_id
 
   def set(val)
-    self.classification_id=val.to_i<<64
-    self.selector_id = val.to_i-((val.to_i>>64)<<64)
+    unless val.nil?
+      self.classification_id=val.to_i<<64
+      self.selector_id = val.to_i-((val.to_i>>64)<<64)
+    end
   end
 
   def get
@@ -204,7 +236,9 @@ class OctetArray < BinData::Primitive
   array :bytes, :type => :uint8, :initial_length => :initial_length
 
   def set(val)
-    self.bytes = val.scan(/../).collect { |hex| hex.to_i(16)}
+    unless val.nil?
+      self.bytes = val.scan(/../).collect { |hex| hex.to_i(16)}
+    end
   end
 
   def get

data/logstash-codec-netflow.gemspec

@@ -1,7 +1,7 @@
 Gem::Specification.new do |s|
 
   s.name            = 'logstash-codec-netflow'
-  s.version         = '3.8.1'
+  s.version         = '3.8.3'
   s.licenses        = ['Apache License (2.0)']
   s.summary         = "Reads Netflow v5 and Netflow v9 data"
   s.description     = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program"

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: logstash-codec-netflow
 version: !ruby/object:Gem::Version
-  version: 3.8.1
+  version: 3.8.3
 platform: ruby
 authors:
 - Elastic
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2017-11-19 00:00:00.000000000 Z
+date: 2017-11-23 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: logstash-core-plugin-api