logstash-codec-netflow 4.3.1 → 4.3.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 1977177c31346ff62addb2a2140b21adcf35d1becf6c995604e16f91e1998d94
-  data.tar.gz: f448694fb80229e0257dc48f2c3200d8d19d21055a9e17057488c81d8fce2a84
+  metadata.gz: 25aee1405937eb3d007e9c41fc86295eea44c81ca8f4559168fa892a3a37925c
+  data.tar.gz: 151a0535226ecff1c22013a847156f8f4ff4fe3e7f2abae74f2ec49719f0c844
 SHA512:
-  metadata.gz: 87bb8f9f6cf7070f786225e7557552873c7f9be274a18cf97d14e213dbd898c1b6c91bc4008cc57221e1911206ae1f839680b09219a207c57c74ae7d9c996dc3
-  data.tar.gz: 4963f7cf398aaf8552805f14f3be1b6423316a9880340edd226caaa591ae84f76fcb2db714b2da686964551b0d6a97928c1fe3e5c238b9da2893f2d6b6370e1d
+  metadata.gz: 61aaf20a488e709118f96620414c16c5440cf17f1a3bd2d8a51aea3bd435ade159e13f774573d2e8f016f3bafa0e2ec213b259c085d44138dd44b1fdc9aa70c7
+  data.tar.gz: b76dc8fd387086eef2c476ca79b116f6b541dd41a9947c4341941d9eba1d0874d937d5c558b29d2a56d03db790805be12b0aa0f4a0f8b1789a3646f0e98b5cc8

data/CHANGELOG.md

@@ -1,3 +1,7 @@
+## 4.3.2
+  - Updates the milliseconds rounding for IPFIX start/end milliseconds fields. 
+  - Fix the test to run on Logstash 8 with microseconds precision. [#206](https://github.com/logstash-plugins/logstash-codec-netflow/pull/206)
+
 ## 4.3.1
   - Fixed unable to initialize the plugin with Logstash 8.10+ [#205](https://github.com/logstash-plugins/logstash-codec-netflow/pull/205)
 

data/lib/logstash/codecs/netflow.rb

@@ -343,7 +343,13 @@ class LogStash::Codecs::Netflow < LogStash::Codecs::Base
           when /^flow(?:Start|End)(Milli|Micro|Nano)seconds$/
             case $1
             when 'Milli'
-              event[@target][k.to_s] = LogStash::Timestamp.at(v.snapshot.to_f / 1_000).to_iso8601
+              secs = v.snapshot.to_i / 1000
+              micros = (v.snapshot.to_i % 1000) * 1000
+              # Use the 2 args Timestamp.at to avoid the precision under milliseconds. Doing math division (like /1000 on a float)
+              # could introduce error in representation that makes 0.192 millis to be expressed like 0.192000001 nanoseconds,
+              # so here we cut to millis, but there is a rounding when representing to to_iso8601, so 191998 micros becomes
+              # 192 millis in LogStash 8 while in previous versions it appears truncated like 191.
+              event[@target][k.to_s] = LogStash::Timestamp.at(secs, micros).to_iso8601
             when 'Micro', 'Nano'
               # For now we'll stick to assuming ntp timestamps,
               # Netscaler implementation may be buggy though:

data/logstash-codec-netflow.gemspec

@@ -1,7 +1,7 @@
 Gem::Specification.new do |s|
 
   s.name            = 'logstash-codec-netflow'
-  s.version         = '4.3.1'
+  s.version         = '4.3.2'
   s.licenses        = ['Apache License (2.0)']
   s.summary         = "Reads Netflow v5, Netflow v9 and IPFIX data"
   s.description     = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program"

data/spec/codecs/netflow_spec.rb

@@ -17,6 +17,11 @@ describe LogStash::Codecs::Netflow do
     end
   end
 
+  let(:is_LS_8) do
+    logstash_version = Gem::Version.create(LOGSTASH_CORE_VERSION)
+    Gem::Requirement.create('>= 8.0').satisfied_by?(logstash_version)
+  end
+
   ### NETFLOW v5
  
   context "Netflow 5 valid 01" do
@@ -28,11 +33,13 @@ describe LogStash::Codecs::Netflow do
       data << IO.read(File.join(File.dirname(__FILE__), "netflow5.dat"), :mode => "rb")
     end
 
+    let(:micros) { is_LS_8 ? "328" : "" }
+
     let(:json_events) do
       events = []
       events << <<-END
         {
-          "@timestamp": "2015-05-02T18:38:08.280Z",
+          "@timestamp": "2015-05-02T18:38:08.280#{micros}Z",
           "netflow": {
             "version": 5,
             "flow_seq_num": 0,
@@ -48,8 +55,8 @@ describe LogStash::Codecs::Netflow do
             "output_snmp": 0,
             "in_pkts": 5,
             "in_bytes": 230,
-            "first_switched": "2015-06-21T11:40:52.194Z",
-            "last_switched": "2015-05-02T18:38:08.476Z",
+            "first_switched": "2015-06-21T11:40:52.194#{micros}Z",
+            "last_switched": "2015-05-02T18:38:08.476#{micros}Z",
             "l4_src_port": 54435,
             "l4_dst_port": 22,
             "tcp_flags": 16,
@@ -66,7 +73,7 @@ describe LogStash::Codecs::Netflow do
 
       events << <<-END
         {
-          "@timestamp": "2015-05-02T18:38:08.280Z",
+          "@timestamp": "2015-05-02T18:38:08.280#{micros}Z",
           "netflow": {
             "version": 5,
             "flow_seq_num": 0,
@@ -82,8 +89,8 @@ describe LogStash::Codecs::Netflow do
             "output_snmp": 0,
             "in_pkts": 4,
             "in_bytes": 304,
-            "first_switched": "2015-06-21T11:40:52.194Z",
-            "last_switched": "2015-05-02T18:38:08.476Z",
+            "first_switched": "2015-06-21T11:40:52.194#{micros}Z",
+            "last_switched": "2015-05-02T18:38:08.476#{micros}Z",
             "l4_src_port": 22,
             "l4_dst_port": 54435,
             "tcp_flags": 24,
@@ -835,11 +842,13 @@ describe LogStash::Codecs::Netflow do
       packets << IO.read(File.join(File.dirname(__FILE__), "netflow5_test_microtik.dat"), :mode => "rb")
     end
 
+    let(:micros) { is_LS_8 ? "932" : "" }
+
     let(:json_events) do
       events = []
       events << <<-END
         {
-          "@timestamp": "2016-07-21T13:51:57.514Z",
+          "@timestamp": "2016-07-21T13:51:57.514#{micros}Z",
           "netflow": {
             "version": 5,
             "flow_seq_num": 8140050,
@@ -855,8 +864,8 @@ describe LogStash::Codecs::Netflow do
             "output_snmp": 46,
             "in_pkts": 13,
             "in_bytes": 11442,
-            "first_switched": "2016-07-21T13:51:42.254Z",
-            "last_switched": "2016-07-21T13:51:42.254Z",
+            "first_switched": "2016-07-21T13:51:42.254#{micros}Z",
+            "last_switched": "2016-07-21T13:51:42.254#{micros}Z",
             "l4_src_port": 80,
             "l4_dst_port": 51826,
             "tcp_flags": 82,
@@ -1330,7 +1339,7 @@ describe LogStash::Codecs::Netflow do
           "ixiaDstLongitude": 100.33540344238281,
           "ixiaHttpUserAgent": "",
           "ixiaDeviceName": "unknown",
-          "flowStartMilliseconds": "2018-10-25T12:24:19.881Z",
+          "flowStartMilliseconds": "2018-10-25T12:24:19.882Z",
           "destinationIPv4Address": "202.170.60.247",
           "ixiaDeviceId": 0,
           "ixiaL7AppName": "unknown",
@@ -2029,6 +2038,9 @@ describe LogStash::Codecs::Netflow do
       data << IO.read(File.join(File.dirname(__FILE__), "ipfix_test_netscaler_data.dat"), :mode => "rb")
     end
 
+    # in LS 8 the precision is up to nanos in LS 7 is up to millis
+    let(:nanos) { is_LS_8 ? "128468" : "" }
+
     let(:json_events) do
       events = []
       events << <<-END
@@ -2038,7 +2050,7 @@ describe LogStash::Codecs::Netflow do
             "netscalerHttpReqUserAgent": "Mozilla/5.0 (Commodore 64;  kobo.com) Gecko/20100101 Firefox/75.0",
             "destinationTransportPort": 443,
             "netscalerHttpReqCookie": "beer=123456789abcdefghijklmnopqrstuvw; AnotherCookie=1234567890abcdefghijklmnopqr; Shameless.Plug=Thankyou.Rakuten.Kobo.Inc.For.Allowing.me.time.to.work.on.this.and.contribute.back.to.the.community; Padding=aaaaaaaaaaaaaaaaaaaaaaaaaabbbbbbbbbbbbbbbbbbccccccccccccccddddddddddddddddddddddeeeeeeeeeeeeeeeeeeeeeffffffffffffffffffffffgggggggggggggggggggggggghhhhhhhhhhhhhhhhhiiiiiiiiiiiiiiiiiiiiiijjjjjjjjjjjjjjjjjjjjjjjjkkkkkkkkkkkkkkkkkklllllllllllllllmmmmmmmmmm; more=less; GJquote=There.is.no.spoon; GarrySays=Nice!!; LastPadding=aaaaaaaaaaaaabbbbbbbbbbbbbbbbbbbbcccccccccccccccccccdddddddddddeeeeeeee",
-            "flowEndMicroseconds": "2016-11-11T12:09:19.000Z",
+            "flowEndMicroseconds": "2016-11-11T12:09:19.000#{nanos}Z",
             "netscalerHttpReqUrl": "/aa/bb/ccccc/ddddddddddddddddddddddddd",
             "sourceIPv4Address": "192.168.0.1",
             "netscalerHttpReqMethod": "GET",
@@ -2057,7 +2069,7 @@ describe LogStash::Codecs::Netflow do
             "netscalerHttpReqVia": "1.1 akamai.net(ghost) (AkamaiGHost)",
             "netscalerConnectionId": 14460661,
             "tcpControlBits": 24,
-            "flowStartMicroseconds": "2016-11-11T12:09:19.000Z",
+            "flowStartMicroseconds": "2016-11-11T12:09:19.000#{nanos}Z",
             "ingressInterface": 8,
             "version": 10,
             "packetDeltaCount": 2,
@@ -2085,7 +2097,6 @@ describe LogStash::Codecs::Netflow do
       expect(decode[0].get("[netflow][version]")).to eq(10)
       expect(decode[0].get("[netflow][sourceIPv4Address]")).to eq('192.168.0.1')
       expect(decode[0].get("[netflow][destinationIPv4Address]")).to eq('10.0.0.1')
-      expect(decode[0].get("[netflow][flowEndMicroseconds]")).to eq('2016-11-11T12:09:19.000Z')
       expect(decode[0].get("[netflow][netscalerConnectionId]")).to eq(14460661)
       expect(decode[1].get("[netflow][version]")).to eq(10)
       expect(decode[1].get("[netflow][flowId]")).to eq(14460662)
@@ -2097,6 +2108,16 @@ describe LogStash::Codecs::Netflow do
       expect(decode[2].get("[netflow][netscalerHttpReqXForwardedFor]")).to eq('11.222.33.255')
     end
 
+    if Gem::Requirement.create('>= 8.0').satisfied_by?(Gem::Version.create(LOGSTASH_CORE_VERSION))
+      it "should decode raw data decoding flowEndMicroseconds with nano precision" do
+        expect(decode[0].get("[netflow][flowEndMicroseconds]")).to eq('2016-11-11T12:09:19.000127768Z')
+      end
+    else
+      it "should decode raw data decoding flowEndMicroseconds with millis precision" do
+        expect(decode[0].get("[netflow][flowEndMicroseconds]")).to eq('2016-11-11T12:09:19.000Z')
+      end
+    end
+
     it "should decode variable length fields" do
       expect(decode[2].get("[netflow][netscalerHttpReqUrl]")).to eq('/aa/bb/ccccc/ddddddddddddddddddddddddd')
       expect(decode[2].get("[netflow][netscalerHttpReqHost]")).to eq('www.kobo.com')
@@ -2962,7 +2983,7 @@ describe LogStash::Codecs::Netflow do
           "tcpSequenceNumber": 340533701,
           "silkAppLabel": 0,
           "sourceTransportPort": 63499,
-          "flowEndMilliseconds": "2016-12-25T12:58:34.346Z",
+          "flowEndMilliseconds": "2016-12-25T12:58:34.347Z",
           "flowAttributes": 0,
           "destinationIPv4Address": "172.16.32.215",
           "octetTotalCount": 172,
@@ -3065,6 +3086,11 @@ end
 
 # New subject with config, ordered testing since we need caching before data processing
 describe LogStash::Codecs::Netflow, 'configured with template caching', :order => :defined do
+  let(:is_LS_8) do
+    logstash_version = Gem::Version.create(LOGSTASH_CORE_VERSION)
+    Gem::Requirement.create('>= 8.0').satisfied_by?(logstash_version)
+  end
+
   context "IPFIX Netscaler with variable length fields" do
     subject do
       LogStash::Codecs::Netflow.new(cache_config).tap do |codec|
@@ -3171,10 +3197,13 @@ describe LogStash::Codecs::Netflow, 'configured with template caching', :order =
       expect(JSON.parse(File.read("#{tmp_dir}/ipfix_templates.cache"))).to eq(JSON.parse(cached_templates))
     end
 
+    # in LS 8 the precision is up to nanos in LS 7 is up to millis
+    let(:nanos) { is_LS_8 ? "127768" : "" }
+
     it "should decode raw data based on cached templates" do
       expect(decode.size).to eq(3)
       expect(decode[0].get("[netflow][version]")).to eq(10)
-      expect(decode[0].get("[netflow][flowEndMicroseconds]")).to eq('2016-11-11T12:09:19.000Z')
+      expect(decode[0].get("[netflow][flowEndMicroseconds]")).to eq("2016-11-11T12:09:19.000#{nanos}Z")
       expect(decode[0].get("[netflow][netscalerConnectionId]")).to eq(14460661)
       expect(decode[1].get("[netflow][version]")).to eq(10)
       expect(decode[1].get("[netflow][observationPointId]")).to eq(167954698)
@@ -3215,7 +3244,6 @@ describe LogStash::Codecs::Netflow, 'configured with include_flowset_id for ipfi
   it "should decode raw data" do
     expect(decode.size).to eq(3)
     expect(decode[0].get("[netflow][version]")).to eq(10)
-    expect(decode[0].get("[netflow][flowEndMicroseconds]")).to eq('2016-11-11T12:09:19.000Z')
     expect(decode[0].get("[netflow][netscalerConnectionId]")).to eq(14460661)
     expect(decode[1].get("[netflow][version]")).to eq(10)
     expect(decode[1].get("[netflow][observationPointId]")).to eq(167954698)
@@ -3224,6 +3252,16 @@ describe LogStash::Codecs::Netflow, 'configured with include_flowset_id for ipfi
     expect(decode[2].get("[netflow][netscalerAppUnitNameAppId]")).to eq(239927296)
   end
 
+  if Gem::Requirement.create('>= 8.0').satisfied_by?(Gem::Version.create(LOGSTASH_CORE_VERSION))
+    it "should decode raw data decoding flowEndMicroseconds with nano precision" do
+      expect(decode[0].get("[netflow][flowEndMicroseconds]")).to eq('2016-11-11T12:09:19.000127768Z')
+    end
+  else
+    it "should decode raw data decoding flowEndMicroseconds with millis precision" do
+      expect(decode[0].get("[netflow][flowEndMicroseconds]")).to eq('2016-11-11T12:09:19.000Z')
+    end
+  end
+
   it "should include flowset_id" do
     expect(decode[0].get("[netflow][flowset_id]")).to eq(258)
     expect(decode[1].get("[netflow][flowset_id]")).to eq(257)

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: logstash-codec-netflow
 version: !ruby/object:Gem::Version
-  version: 4.3.1
+  version: 4.3.2
 platform: ruby
 authors:
 - Elastic
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2023-12-14 00:00:00.000000000 Z
+date: 2023-12-22 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement