logstash-codec-netflow 4.3.1 → 4.3.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 1977177c31346ff62addb2a2140b21adcf35d1becf6c995604e16f91e1998d94
-  data.tar.gz: f448694fb80229e0257dc48f2c3200d8d19d21055a9e17057488c81d8fce2a84
+  metadata.gz: 1077f3fabf491578e56f3bdf62833fcc57fa84e7a45339a86041d0d3cab20ea9
+  data.tar.gz: 483b793cab3301353f7631cba07eb82e6a560b62c9656cc0e718ef0286cc61ab
 SHA512:
-  metadata.gz: 87bb8f9f6cf7070f786225e7557552873c7f9be274a18cf97d14e213dbd898c1b6c91bc4008cc57221e1911206ae1f839680b09219a207c57c74ae7d9c996dc3
-  data.tar.gz: 4963f7cf398aaf8552805f14f3be1b6423316a9880340edd226caaa591ae84f76fcb2db714b2da686964551b0d6a97928c1fe3e5c238b9da2893f2d6b6370e1d
+  metadata.gz: 67390c9542344deb69f6d862de01458e06d11932ecad4afc6ad824bc4d22abf22188786d309738294d7b311e688acebe0735d6ab8a5007f52cf8852acc5745cd
+  data.tar.gz: b0ef1675c310fec9dd98ac6db3e8088c62feaa230a6a368d640b67c8ee25f2970edba1d52c49d8d3d619be38894a27f141f7c9c15cf554584210c7c3ffe2b44e

data/CHANGELOG.md

@@ -1,3 +1,10 @@
+## 4.3.3
+  - Fix NoMethodError when decode fails. [214](https://github.com/logstash-plugins/logstash-codec-netflow/pull/214)
+
+## 4.3.2
+  - Updates the milliseconds rounding for IPFIX start/end milliseconds fields. 
+  - Fix the test to run on Logstash 8 with microseconds precision. [#206](https://github.com/logstash-plugins/logstash-codec-netflow/pull/206)
+
 ## 4.3.1
   - Fixed unable to initialize the plugin with Logstash 8.10+ [#205](https://github.com/logstash-plugins/logstash-codec-netflow/pull/205)
 

data/lib/logstash/codecs/netflow.rb

@@ -88,16 +88,32 @@ class LogStash::Codecs::Netflow < LogStash::Codecs::Base
     if header.version == 5
       flowset = Netflow5PDU.read(payload)
       flowset.records.each do |record|
-        yield(decode_netflow5(flowset, record))
+        begin
+          yield(decode_netflow5(flowset, record))
+        rescue BinData::ValidityError, IOError => e
+          @logger.warn("Invalid Netflow v5 record (#{e})")
+          if @logger.debug?
+            @logger.debug("Netflow v5 decode error", :flow_seq_num => flowset.flow_seq_num)
+          end
+        end
       end
     elsif header.version == 9
 #     BinData::trace_reading do
       flowset = Netflow9PDU.read(payload)
       flowset.records.each do |record|
-        if metadata != nil
-          decode_netflow9(flowset, record, metadata).each{|event| yield(event)}
-        else
-          decode_netflow9(flowset, record).each{|event| yield(event)}
+        begin
+          if metadata != nil
+            decode_netflow9(flowset, record, metadata).each{|event| yield(event)}
+          else
+            decode_netflow9(flowset, record).each{|event| yield(event)}
+          end
+        rescue BinData::ValidityError, IOError => e
+          @logger.warn("Invalid Netflow v9 record (#{e})")
+          if @logger.debug?
+            host = metadata ? metadata["host"] : nil
+            port = metadata ? metadata["port"] : nil
+            @logger.debug("Netflow v9 decode error", :source_id => flowset.source_id, :flowset_id => record.flowset_id, :host => host, :port => port)
+          end
         end
 #      end
      end
@@ -105,7 +121,14 @@ class LogStash::Codecs::Netflow < LogStash::Codecs::Base
 #     BinData::trace_reading do
       flowset = IpfixPDU.read(payload)
       flowset.records.each do |record|
-        decode_ipfix(flowset, record).each { |event| yield(event) }
+        begin
+          decode_ipfix(flowset, record).each { |event| yield(event) }
+        rescue BinData::ValidityError, IOError => e
+          @logger.warn("Invalid IPFIX record (#{e})")
+          if @logger.debug?
+            @logger.debug("IPFIX decode error", :observation_domain_id => flowset.observation_domain_id, :flowset_id => record.flowset_id)
+          end
+        end
       end
 #     end
     else
@@ -151,8 +174,6 @@ class LogStash::Codecs::Netflow < LogStash::Codecs::Base
     end
 
     event_factory.new_event(event)
-  rescue BinData::ValidityError, IOError => e
-    @logger.warn("Invalid netflow packet received (#{e})")
   end
 
   def decode_netflow9(flowset, record, metadata = nil)
@@ -280,8 +301,6 @@ class LogStash::Codecs::Netflow < LogStash::Codecs::Base
     end
 
     events
-  rescue BinData::ValidityError, IOError => e
-    @logger.warn("Invalid netflow packet received (#{e})")
   end
 
   def decode_ipfix(flowset, record)
@@ -343,7 +362,13 @@ class LogStash::Codecs::Netflow < LogStash::Codecs::Base
           when /^flow(?:Start|End)(Milli|Micro|Nano)seconds$/
             case $1
             when 'Milli'
-              event[@target][k.to_s] = LogStash::Timestamp.at(v.snapshot.to_f / 1_000).to_iso8601
+              secs = v.snapshot.to_i / 1000
+              micros = (v.snapshot.to_i % 1000) * 1000
+              # Use the 2 args Timestamp.at to avoid the precision under milliseconds. Doing math division (like /1000 on a float)
+              # could introduce error in representation that makes 0.192 millis to be expressed like 0.192000001 nanoseconds,
+              # so here we cut to millis, but there is a rounding when representing to to_iso8601, so 191998 micros becomes
+              # 192 millis in LogStash 8 while in previous versions it appears truncated like 191.
+              event[@target][k.to_s] = LogStash::Timestamp.at(secs, micros).to_iso8601
             when 'Micro', 'Nano'
               # For now we'll stick to assuming ntp timestamps,
               # Netscaler implementation may be buggy though:
@@ -365,8 +390,6 @@ class LogStash::Codecs::Netflow < LogStash::Codecs::Base
     end
 
     events
-  rescue BinData::ValidityError => e
-    @logger.warn("Invalid IPFIX packet received (#{e})")
   end
 
   def load_definitions(defaults, extra)

data/logstash-codec-netflow.gemspec

@@ -1,7 +1,7 @@
 Gem::Specification.new do |s|
 
   s.name            = 'logstash-codec-netflow'
-  s.version         = '4.3.1'
+  s.version         = '4.3.3'
   s.licenses        = ['Apache License (2.0)']
   s.summary         = "Reads Netflow v5, Netflow v9 and IPFIX data"
   s.description     = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program"

data/spec/codecs/netflow_spec.rb

@@ -17,6 +17,11 @@ describe LogStash::Codecs::Netflow do
     end
   end
 
+  let(:is_LS_8) do
+    logstash_version = Gem::Version.create(LOGSTASH_CORE_VERSION)
+    Gem::Requirement.create('>= 8.0').satisfied_by?(logstash_version)
+  end
+
   ### NETFLOW v5
  
   context "Netflow 5 valid 01" do
@@ -28,11 +33,13 @@ describe LogStash::Codecs::Netflow do
       data << IO.read(File.join(File.dirname(__FILE__), "netflow5.dat"), :mode => "rb")
     end
 
+    let(:micros) { is_LS_8 ? "328" : "" }
+
     let(:json_events) do
       events = []
       events << <<-END
         {
-          "@timestamp": "2015-05-02T18:38:08.280Z",
+          "@timestamp": "2015-05-02T18:38:08.280#{micros}Z",
           "netflow": {
             "version": 5,
             "flow_seq_num": 0,
@@ -48,8 +55,8 @@ describe LogStash::Codecs::Netflow do
             "output_snmp": 0,
             "in_pkts": 5,
             "in_bytes": 230,
-            "first_switched": "2015-06-21T11:40:52.194Z",
-            "last_switched": "2015-05-02T18:38:08.476Z",
+            "first_switched": "2015-06-21T11:40:52.194#{micros}Z",
+            "last_switched": "2015-05-02T18:38:08.476#{micros}Z",
             "l4_src_port": 54435,
             "l4_dst_port": 22,
             "tcp_flags": 16,
@@ -66,7 +73,7 @@ describe LogStash::Codecs::Netflow do
 
       events << <<-END
         {
-          "@timestamp": "2015-05-02T18:38:08.280Z",
+          "@timestamp": "2015-05-02T18:38:08.280#{micros}Z",
           "netflow": {
             "version": 5,
             "flow_seq_num": 0,
@@ -82,8 +89,8 @@ describe LogStash::Codecs::Netflow do
             "output_snmp": 0,
             "in_pkts": 4,
             "in_bytes": 304,
-            "first_switched": "2015-06-21T11:40:52.194Z",
-            "last_switched": "2015-05-02T18:38:08.476Z",
+            "first_switched": "2015-06-21T11:40:52.194#{micros}Z",
+            "last_switched": "2015-05-02T18:38:08.476#{micros}Z",
             "l4_src_port": 22,
             "l4_dst_port": 54435,
             "tcp_flags": 24,
@@ -835,11 +842,13 @@ describe LogStash::Codecs::Netflow do
       packets << IO.read(File.join(File.dirname(__FILE__), "netflow5_test_microtik.dat"), :mode => "rb")
     end
 
+    let(:micros) { is_LS_8 ? "932" : "" }
+
     let(:json_events) do
       events = []
       events << <<-END
         {
-          "@timestamp": "2016-07-21T13:51:57.514Z",
+          "@timestamp": "2016-07-21T13:51:57.514#{micros}Z",
           "netflow": {
             "version": 5,
             "flow_seq_num": 8140050,
@@ -855,8 +864,8 @@ describe LogStash::Codecs::Netflow do
             "output_snmp": 46,
             "in_pkts": 13,
             "in_bytes": 11442,
-            "first_switched": "2016-07-21T13:51:42.254Z",
-            "last_switched": "2016-07-21T13:51:42.254Z",
+            "first_switched": "2016-07-21T13:51:42.254#{micros}Z",
+            "last_switched": "2016-07-21T13:51:42.254#{micros}Z",
             "l4_src_port": 80,
             "l4_dst_port": 51826,
             "tcp_flags": 82,
@@ -1330,7 +1339,7 @@ describe LogStash::Codecs::Netflow do
           "ixiaDstLongitude": 100.33540344238281,
           "ixiaHttpUserAgent": "",
           "ixiaDeviceName": "unknown",
-          "flowStartMilliseconds": "2018-10-25T12:24:19.881Z",
+          "flowStartMilliseconds": "2018-10-25T12:24:19.882Z",
           "destinationIPv4Address": "202.170.60.247",
           "ixiaDeviceId": 0,
           "ixiaL7AppName": "unknown",
@@ -2029,6 +2038,9 @@ describe LogStash::Codecs::Netflow do
       data << IO.read(File.join(File.dirname(__FILE__), "ipfix_test_netscaler_data.dat"), :mode => "rb")
     end
 
+    # in LS 8 the precision is up to nanos in LS 7 is up to millis
+    let(:nanos) { is_LS_8 ? "128468" : "" }
+
     let(:json_events) do
       events = []
       events << <<-END
@@ -2038,7 +2050,7 @@ describe LogStash::Codecs::Netflow do
             "netscalerHttpReqUserAgent": "Mozilla/5.0 (Commodore 64;  kobo.com) Gecko/20100101 Firefox/75.0",
             "destinationTransportPort": 443,
             "netscalerHttpReqCookie": "beer=123456789abcdefghijklmnopqrstuvw; AnotherCookie=1234567890abcdefghijklmnopqr; Shameless.Plug=Thankyou.Rakuten.Kobo.Inc.For.Allowing.me.time.to.work.on.this.and.contribute.back.to.the.community; Padding=aaaaaaaaaaaaaaaaaaaaaaaaaabbbbbbbbbbbbbbbbbbccccccccccccccddddddddddddddddddddddeeeeeeeeeeeeeeeeeeeeeffffffffffffffffffffffgggggggggggggggggggggggghhhhhhhhhhhhhhhhhiiiiiiiiiiiiiiiiiiiiiijjjjjjjjjjjjjjjjjjjjjjjjkkkkkkkkkkkkkkkkkklllllllllllllllmmmmmmmmmm; more=less; GJquote=There.is.no.spoon; GarrySays=Nice!!; LastPadding=aaaaaaaaaaaaabbbbbbbbbbbbbbbbbbbbcccccccccccccccccccdddddddddddeeeeeeee",
-            "flowEndMicroseconds": "2016-11-11T12:09:19.000Z",
+            "flowEndMicroseconds": "2016-11-11T12:09:19.000#{nanos}Z",
             "netscalerHttpReqUrl": "/aa/bb/ccccc/ddddddddddddddddddddddddd",
             "sourceIPv4Address": "192.168.0.1",
             "netscalerHttpReqMethod": "GET",
@@ -2057,7 +2069,7 @@ describe LogStash::Codecs::Netflow do
             "netscalerHttpReqVia": "1.1 akamai.net(ghost) (AkamaiGHost)",
             "netscalerConnectionId": 14460661,
             "tcpControlBits": 24,
-            "flowStartMicroseconds": "2016-11-11T12:09:19.000Z",
+            "flowStartMicroseconds": "2016-11-11T12:09:19.000#{nanos}Z",
             "ingressInterface": 8,
             "version": 10,
             "packetDeltaCount": 2,
@@ -2085,7 +2097,6 @@ describe LogStash::Codecs::Netflow do
       expect(decode[0].get("[netflow][version]")).to eq(10)
       expect(decode[0].get("[netflow][sourceIPv4Address]")).to eq('192.168.0.1')
       expect(decode[0].get("[netflow][destinationIPv4Address]")).to eq('10.0.0.1')
-      expect(decode[0].get("[netflow][flowEndMicroseconds]")).to eq('2016-11-11T12:09:19.000Z')
       expect(decode[0].get("[netflow][netscalerConnectionId]")).to eq(14460661)
       expect(decode[1].get("[netflow][version]")).to eq(10)
       expect(decode[1].get("[netflow][flowId]")).to eq(14460662)
@@ -2097,6 +2108,16 @@ describe LogStash::Codecs::Netflow do
       expect(decode[2].get("[netflow][netscalerHttpReqXForwardedFor]")).to eq('11.222.33.255')
     end
 
+    if Gem::Requirement.create('>= 8.0').satisfied_by?(Gem::Version.create(LOGSTASH_CORE_VERSION))
+      it "should decode raw data decoding flowEndMicroseconds with nano precision" do
+        expect(decode[0].get("[netflow][flowEndMicroseconds]")).to eq('2016-11-11T12:09:19.000127768Z')
+      end
+    else
+      it "should decode raw data decoding flowEndMicroseconds with millis precision" do
+        expect(decode[0].get("[netflow][flowEndMicroseconds]")).to eq('2016-11-11T12:09:19.000Z')
+      end
+    end
+
     it "should decode variable length fields" do
       expect(decode[2].get("[netflow][netscalerHttpReqUrl]")).to eq('/aa/bb/ccccc/ddddddddddddddddddddddddd')
       expect(decode[2].get("[netflow][netscalerHttpReqHost]")).to eq('www.kobo.com')
@@ -2962,7 +2983,7 @@ describe LogStash::Codecs::Netflow do
           "tcpSequenceNumber": 340533701,
           "silkAppLabel": 0,
           "sourceTransportPort": 63499,
-          "flowEndMilliseconds": "2016-12-25T12:58:34.346Z",
+          "flowEndMilliseconds": "2016-12-25T12:58:34.347Z",
           "flowAttributes": 0,
           "destinationIPv4Address": "172.16.32.215",
           "octetTotalCount": 172,
@@ -3065,6 +3086,11 @@ end
 
 # New subject with config, ordered testing since we need caching before data processing
 describe LogStash::Codecs::Netflow, 'configured with template caching', :order => :defined do
+  let(:is_LS_8) do
+    logstash_version = Gem::Version.create(LOGSTASH_CORE_VERSION)
+    Gem::Requirement.create('>= 8.0').satisfied_by?(logstash_version)
+  end
+
   context "IPFIX Netscaler with variable length fields" do
     subject do
       LogStash::Codecs::Netflow.new(cache_config).tap do |codec|
@@ -3171,10 +3197,13 @@ describe LogStash::Codecs::Netflow, 'configured with template caching', :order =
       expect(JSON.parse(File.read("#{tmp_dir}/ipfix_templates.cache"))).to eq(JSON.parse(cached_templates))
     end
 
+    # in LS 8 the precision is up to nanos in LS 7 is up to millis
+    let(:nanos) { is_LS_8 ? "127768" : "" }
+
     it "should decode raw data based on cached templates" do
       expect(decode.size).to eq(3)
       expect(decode[0].get("[netflow][version]")).to eq(10)
-      expect(decode[0].get("[netflow][flowEndMicroseconds]")).to eq('2016-11-11T12:09:19.000Z')
+      expect(decode[0].get("[netflow][flowEndMicroseconds]")).to eq("2016-11-11T12:09:19.000#{nanos}Z")
       expect(decode[0].get("[netflow][netscalerConnectionId]")).to eq(14460661)
       expect(decode[1].get("[netflow][version]")).to eq(10)
       expect(decode[1].get("[netflow][observationPointId]")).to eq(167954698)
@@ -3215,7 +3244,6 @@ describe LogStash::Codecs::Netflow, 'configured with include_flowset_id for ipfi
   it "should decode raw data" do
     expect(decode.size).to eq(3)
     expect(decode[0].get("[netflow][version]")).to eq(10)
-    expect(decode[0].get("[netflow][flowEndMicroseconds]")).to eq('2016-11-11T12:09:19.000Z')
     expect(decode[0].get("[netflow][netscalerConnectionId]")).to eq(14460661)
     expect(decode[1].get("[netflow][version]")).to eq(10)
     expect(decode[1].get("[netflow][observationPointId]")).to eq(167954698)
@@ -3224,6 +3252,16 @@ describe LogStash::Codecs::Netflow, 'configured with include_flowset_id for ipfi
     expect(decode[2].get("[netflow][netscalerAppUnitNameAppId]")).to eq(239927296)
   end
 
+  if Gem::Requirement.create('>= 8.0').satisfied_by?(Gem::Version.create(LOGSTASH_CORE_VERSION))
+    it "should decode raw data decoding flowEndMicroseconds with nano precision" do
+      expect(decode[0].get("[netflow][flowEndMicroseconds]")).to eq('2016-11-11T12:09:19.000127768Z')
+    end
+  else
+    it "should decode raw data decoding flowEndMicroseconds with millis precision" do
+      expect(decode[0].get("[netflow][flowEndMicroseconds]")).to eq('2016-11-11T12:09:19.000Z')
+    end
+  end
+
   it "should include flowset_id" do
     expect(decode[0].get("[netflow][flowset_id]")).to eq(258)
     expect(decode[1].get("[netflow][flowset_id]")).to eq(257)

metadata

@@ -1,66 +1,65 @@
 --- !ruby/object:Gem::Specification
 name: logstash-codec-netflow
 version: !ruby/object:Gem::Version
-  version: 4.3.1
+  version: 4.3.3
 platform: ruby
 authors:
 - Elastic
-autorequire:
 bindir: bin
 cert_chain: []
-date: 2023-12-14 00:00:00.000000000 Z
+date: 2026-01-21 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
+  name: logstash-core-plugin-api
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '2.0'
-  name: logstash-core-plugin-api
-  prerelease: false
   type: :runtime
+  prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '2.0'
 - !ruby/object:Gem::Dependency
+  name: logstash-mixin-event_support
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '1.0'
-  name: logstash-mixin-event_support
-  prerelease: false
   type: :runtime
+  prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '1.0'
 - !ruby/object:Gem::Dependency
+  name: bindata
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
         version: 1.5.0
-  name: bindata
-  prerelease: false
   type: :runtime
+  prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
         version: 1.5.0
 - !ruby/object:Gem::Dependency
+  name: logstash-devutils
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
         version: 1.0.0
-  name: logstash-devutils
-  prerelease: false
   type: :development
+  prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
@@ -200,7 +199,6 @@ licenses:
 metadata:
   logstash_plugin: 'true'
   logstash_group: codec
-post_install_message:
 rdoc_options: []
 require_paths:
 - lib
@@ -215,8 +213,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.2.33
-signing_key:
+rubygems_version: 3.6.3
 specification_version: 4
 summary: Reads Netflow v5, Netflow v9 and IPFIX data
 test_files: