logstash-codec-netflow 3.7.1 → 3.8.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
-SHA256:
-  metadata.gz: 565c6ab7ab13d7903e04ee30abcc47d29af927f1689358f805036df90302ae87
-  data.tar.gz: ea82070a59a0281f891a07d98a5bebdf71478284935c869ef3253266f8eea93b
+SHA1:
+  metadata.gz: 0d51a96934f6215fa848ef5826f1da03ee4bf555
+  data.tar.gz: 01285e9dad3666666178a98dd50af9275fedb43b
 SHA512:
-  metadata.gz: c9524679b1ca9860e3be04c68563a531328aa1119363d245927788bc905da15b683f2dfae48ef8e01d6559a66e19320841fddb72bf07f48616d600d320b08a46
-  data.tar.gz: 9f6b5cff6f7662b392d7f38d672cb3fae272c2d2e28a656a2e7d7d2911f3f6f9a5a5a88aee4dd41a379605782673d1635ae7d0051a88c3f477e7a28b3027ed01
+  metadata.gz: a9e15da9958d4435051ba14da84768bd3db6b96ce0c8d7c80de3848ac2bfbcc5205f8d0cfd0415668447f3481fdedae3b5cea15bd9747124ac4aecbd648cc692
+  data.tar.gz: 1064815fd03ec5d7439f2fd3df9a9f50421742b8af83d002e62a3553005eb8cb1fa00ad1a43a8b19754fc2dd019bd963c35bb200a7b37ef6dbd8eea8a579c43f

data/CHANGELOG.md

@@ -1,10 +1,12 @@
-## 3.7.1
-  - Update gemspec summary
+## 3.8.0
+
+  - Added initial YAF support with applabel and silk (but without DPI plugins because of complex data types)
 
 ## 3.7.1
 
+  - Update gemspec summary
   - Added support for CISCO1941/K9 software 15.1 
-
+  - Added undocumented Netscaler fields
 
 ## 3.7.0
 

data/lib/logstash/codecs/netflow/ipfix.yaml

@@ -2100,6 +2100,301 @@
   541:
   - :uint32
   - :netscalerUnknown541
+6871:
+  14:
+  - :uint8
+  - :initialTCPFlags
+  15:
+  - :uint8
+  - :unionTCPFlags
+  18:
+  - :string
+  - :payload
+  21:
+  - :uint32
+  - :reverseFlowDeltaMilliseconds
+  33:
+  - :uint16
+  - :silkAppLabel
+  35:
+  - :uint8
+  - :payloadEntropy
+  36:
+  - :string
+  - :osName
+  37:
+  - :string
+  - :osVersion
+  38:
+  - :string
+  - :firstPacketBanner
+  39:
+  - :string
+  - :secondPacketBanner
+  40:
+  - :uint16
+  - :flowAttributes
+  100:
+  - :uint32
+  - :expiredFragmentCount
+  101:
+  - :uint32
+  - :assembledFragmentCount
+  102:
+  - :uint32
+  - :meanFlowRate
+  103:
+  - :uint32
+  - :meanPacketRate
+  104:
+  - :uint32
+  - :flowTableFlushEventCount
+  105:
+  - :uint32
+  - :flowTablePeakCount
+  107:
+  - :string
+  - :osFingerPrint
+  126:
+  - :string
+  - :tftpFilename
+  127:
+  - :string
+  - :tftpMode
+  174:
+  - :uint8
+  - :dnsQueryResponse
+  175:
+  - :uint16
+  - :dnsQRType
+  176:
+  - :uint8
+  - :dnsAuthoritative
+  177:
+  - :uint8
+  - :dnsNXDomain
+  178:
+  - :uint8
+  - :dnsRRSection
+  179:
+  - :string
+  - :dnsQName
+  180:
+  - :string
+  - :dnsCName
+  181:
+  - :uint16
+  - :dnsMXPreference
+  182:
+  - :string
+  - :dnsMXExchange
+  183:
+  - :string
+  - :dnsNSDName
+  184:
+  - :string
+  - :dnsPTRDName
+  185:
+  - :string
+  - :sslCipher
+  186:
+  - :uint8
+  - :sslClientVersion
+  187:
+  - :uint32
+  - :sslServerCipher
+  188:
+  - :uint8
+  - :sslCompressionMethod
+  189:
+  - :uint8
+  - :sslCertVersion
+  190:
+  - :string
+  - :sslCertSignature
+  199:
+  - :uint32
+  - :dnsTTL
+  208:
+  - :string
+  - :dnsTXTData
+  209:
+  - :uint32
+  - :dnsSOASerial
+  210:
+  - :uint32
+  - :dnsSOARefresh
+  211:
+  - :uint32
+  - :dnsSOARetry
+  212:
+  - :uint32
+  - :dnsSOAExpire
+  213:
+  - :uint32
+  - :dnsSOAMinimum
+  214:
+  - :string
+  - :dnsSOAMName
+  215:
+  - :string
+  - :dnsSOARName
+  216:
+  - :uint16
+  - :dnsSRVPriority
+  217:
+  - :uint16
+  - :dnsSRVWeight
+  218:
+  - :uint16
+  - :dnsSRVPort
+  219:
+  - :uint16
+  - :dnsSRVTarget
+  223:
+  - :uint32
+  - :tcpUrgTotalCount
+  226:
+  - :uint16
+  - :dnsID
+  244:
+  - :string
+  - :sslCertSerialNumber
+  245:
+  - :string
+  - :sslObjectType
+  246:
+  - :string
+  - :sslObjectValue
+  247:
+  - :string
+  - :sslCertValidityNotBefore
+  248:
+  - :string
+  - :sslCertValidityNotAfter
+  249:
+  - :string
+  - :sslPublicKeyAlgorithm
+  250:
+  - :string
+  - :sslPublicKeyLength
+  287:
+  - :uint8
+  - :rtpPayloadType
+  288:
+  - :uint8
+  - :reverseRtpPayloadType
+  289:
+  - :uint64
+  - :mptcpInitialDataSequenceNumber
+  290:
+  - :uint32
+  - :mptcpReceiverToken
+  291:
+  - :uint16
+  - :mptcpMaximumSegmentSize
+  292:
+  - :uint8
+  - :mptcpAddressID
+  294:
+  - :string
+  - :sslServerName
+  295:
+  - :string
+  - :sslCertificateHash
+  293:
+  - :uint8
+  - :mptcpFlags
+  500:
+  - :uint32
+  - :smallPacketCount
+  501:
+  - :uint32
+  - :nonEmptyPacketCount
+  502:
+  - :uint64
+  - :dataByteCount
+  503:
+  - :uint64
+  - :averageInterarrivalTime
+  504:
+  - :uint64
+  - :standardDeviationInterarrivalTime
+  505:
+  - :uint16
+  - :firstNonEmptyPacketSize
+  506:
+  - :uint16
+  - :maxPacketSize
+  507:
+  - :uint8
+  - :firstEightNonEmptyPacketDirections
+  508:
+  - :uint8
+  - :standardDeviationPayloadLength
+  510:
+  - :uint32
+  - :largePacketCount
+  16398:
+  - :uint8
+  - :reverseInitialTCPFlags
+  16399:
+  - :uint8
+  - :reverseUnionTCPFlags
+  16402:
+  - :string
+  - :reversePayload
+  16419:
+  - :uint8
+  - :reversePayloadEntropy
+  16420:
+  - :string
+  - :reverseOsName
+  16421:
+  - :string
+  - :reverseOsVersion
+  16422:
+  - :string
+  - :reverseFirstPacketBanner
+  16423:
+  - :string
+  - :reverseSecondPacketBanner
+  16424:
+  - :uint16
+  - :reverseFlowAttributes
+  16491:
+  - :string
+  - :reverseOsFingerPrint
+  16671:
+  - :uint8
+  - :reverseRtpPayloadType
+  16884:
+  - :uint32
+  - :reverseSmallPacketCount
+  16885:
+  - :uint32
+  - :reverseNonEmptyPacketCount
+  16886:
+  - :uint64
+  - :reverseDataByteCount
+  16887:
+  - :uint64
+  - :reverseAverageInterarrivalTime
+  16888:
+  - :uint64
+  - :reverseStandardDeviationInterarrivalTime
+  16889:
+  - :uint16
+  - :reverseFirstNonEmptyPacketSize
+  16890:
+  - :uint16
+  - :reverseMaxPacketSize
+  16892:
+  - :uint16
+  - :reverseStandardDeviationPayloadLength
+  16894:
+  - :uint32
+  - :reverseLargePacketCount
 6876:
   880:
   - :uint8

data/logstash-codec-netflow.gemspec

@@ -1,7 +1,7 @@
 Gem::Specification.new do |s|
 
   s.name            = 'logstash-codec-netflow'
-  s.version         = '3.7.1'
+  s.version         = '3.8.0'
   s.licenses        = ['Apache License (2.0)']
   s.summary         = "Reads Netflow v5 and Netflow v9 data"
   s.description     = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program"

data/spec/codecs/netflow_spec.rb

@@ -2014,8 +2014,126 @@ describe LogStash::Codecs::Netflow do
     end
   end
 
+  context "IPFIX YAF basic with applabel" do
+    # These samples have been generated with:
+    # /usr/local/bin/yaf --silk --ipfix=udp --live=pcap --out=host02 --ipfix-port=2055 --in=eth0 --applabel --verbose --mac --verbose --max-payload 384
+    let(:data) do
+      packets = []
+      packets << IO.read(File.join(File.dirname(__FILE__), "ipfix_test_yaf_tpls_option_tpl.dat"), :mode => "rb")
+      packets << IO.read(File.join(File.dirname(__FILE__), "ipfix_test_yaf_tpl45841.dat"), :mode => "rb")
+      packets << IO.read(File.join(File.dirname(__FILE__), "ipfix_test_yaf_data45841.dat"), :mode => "rb")
+      packets << IO.read(File.join(File.dirname(__FILE__), "ipfix_test_yaf_data45873.dat"), :mode => "rb")
+      packets << IO.read(File.join(File.dirname(__FILE__), "ipfix_test_yaf_data53248.dat"), :mode => "rb")
+    end
+
+    let(:json_events) do
+      events = []
+      events << <<-END
+      {
+        "netflow": {
+          "destinationIPv4Address": "172.16.32.100",
+          "octetTotalCount": 132,
+          "destinationTransportPort": 53,
+          "vlanId": 0,
+          "reversePacketTotalCount": 2,
+          "reverseFlowDeltaMilliseconds": 1,
+          "sourceIPv4Address": "172.16.32.201",
+          "reverseVlanId": 0,
+          "reverseIpClassOfService": 0,
+          "reverseOctetTotalCount": 200,
+          "reverseFlowAttributes": 0,
+          "ipClassOfService": 0,
+          "version": 10,
+          "flowEndReason": 1,
+          "protocolIdentifier": 17,
+          "silkAppLabel": 53,
+          "sourceTransportPort": 46086,
+          "packetTotalCount": 2,
+          "flowEndMilliseconds": "2016-12-25T12:58:35.819Z",
+          "flowStartMilliseconds": "2016-12-25T12:58:35.818Z",
+          "flowAttributes": 1
+        },
+        "@timestamp": "2016-12-25T13:03:38.000Z",
+        "@version": "1"
+      }
+      END
+
+      events << <<-END
+      {
+        "netflow": {
+          "destinationTransportPort": 9997,
+          "reversePacketTotalCount": 2,
+          "reverseFlowDeltaMilliseconds": 0,
+          "sourceIPv4Address": "172.16.32.100",
+          "reverseTcpSequenceNumber": 3788795034,
+          "reverseVlanId": 0,
+          "reverseOctetTotalCount": 92,
+          "ipClassOfService": 2,
+          "reverseInitialTCPFlags": 18,
+          "tcpSequenceNumber": 340533701,
+          "silkAppLabel": 0,
+          "sourceTransportPort": 63499,
+          "flowEndMilliseconds": "2016-12-25T12:58:34.346Z",
+          "flowAttributes": 0,
+          "destinationIPv4Address": "172.16.32.215",
+          "octetTotalCount": 172,
+          "vlanId": 0,
+          "reverseIpClassOfService": 0,
+          "reverseFlowAttributes": 0,
+          "unionTCPFlags": 17,
+          "version": 10,
+          "flowEndReason": 3,
+          "protocolIdentifier": 6,
+          "initialTCPFlags": 194,
+          "reverseUnionTCPFlags": 17,
+          "packetTotalCount": 4,
+          "flowStartMilliseconds": "2016-12-25T12:58:33.345Z"
+        },
+        "@timestamp": "2016-12-25T12:58:38.000Z",
+        "@version": "1"
+      }
+      END
+
+      events << <<-END
+      {
+        "netflow": {
+          "droppedPacketTotalCount": 0,
+          "exporterIPv4Address": "172.16.32.201",
+          "ignoredPacketTotalCount": 58,
+          "meanPacketRate": 6,
+          "flowTableFlushEventCount": 39,
+          "flowTablePeakCount": 58,
+          "version": 10,
+          "exportedFlowRecordTotalCount": 31,
+          "systemInitTimeMilliseconds": 1482670712000,
+          "notSentPacketTotalCount": 0,
+          "exportingProcessId": 0,
+          "meanFlowRate": 0,
+          "expiredFragmentCount": 0,
+          "assembledFragmentCount": 0,
+          "packetTotalCount": 1960
+        },
+        "@timestamp": "2016-12-25T13:03:33.000Z",
+        "@version": "1"
+      }
+      END
+      events.map{|event| event.gsub(/\s+/, "")}
+    end
+
+    it "should decode raw data" do
+      expect(decode.size).to eq(3)
+      expect(decode[0].get("[netflow][silkAppLabel]")).to eq(53)
+      expect(decode[1].get("[netflow][initialTCPFlags]")).to eq(194)
+      expect(decode[2].get("[netflow][flowTablePeakCount]")).to eq(58)
+    end
 
+    it "should serialize to json" do
+      expect(JSON.parse(decode[0].to_json)).to eq(JSON.parse(json_events[0]))
+      expect(JSON.parse(decode[1].to_json)).to eq(JSON.parse(json_events[1]))
+      expect(JSON.parse(decode[2].to_json)).to eq(JSON.parse(json_events[2]))
+    end
 
+  end
 
 end
 

metadata

@@ -1,22 +1,22 @@
 --- !ruby/object:Gem::Specification
 name: logstash-codec-netflow
 version: !ruby/object:Gem::Version
-  version: 3.7.1
+  version: 3.8.0
 platform: ruby
 authors:
 - Elastic
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2017-11-07 00:00:00.000000000 Z
+date: 2017-11-11 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - '>='
       - !ruby/object:Gem::Version
         version: '1.60'
-    - - "<="
+    - - <=
       - !ruby/object:Gem::Version
         version: '2.99'
   name: logstash-core-plugin-api
@@ -24,16 +24,16 @@ dependencies:
   type: :runtime
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - '>='
       - !ruby/object:Gem::Version
         version: '1.60'
-    - - "<="
+    - - <=
       - !ruby/object:Gem::Version
         version: '2.99'
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - '>='
       - !ruby/object:Gem::Version
         version: 1.5.0
   name: bindata
@@ -41,13 +41,13 @@ dependencies:
   type: :runtime
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - '>='
       - !ruby/object:Gem::Version
         version: 1.5.0
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - '>='
       - !ruby/object:Gem::Version
         version: 1.0.0
   name: logstash-devutils
@@ -55,12 +55,10 @@ dependencies:
   type: :development
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - '>='
       - !ruby/object:Gem::Version
         version: 1.0.0
-description: This gem is a Logstash plugin required to be installed on top of the
-  Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This
-  gem is not a stand-alone program
+description: This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program
 email: info@elastic.co
 executables: []
 extensions: []
@@ -95,6 +93,11 @@ files:
 - spec/codecs/ipfix_test_vmware_vds_data266.dat
 - spec/codecs/ipfix_test_vmware_vds_data266_267.dat
 - spec/codecs/ipfix_test_vmware_vds_tpl.dat
+- spec/codecs/ipfix_test_yaf_data45841.dat
+- spec/codecs/ipfix_test_yaf_data45873.dat
+- spec/codecs/ipfix_test_yaf_data53248.dat
+- spec/codecs/ipfix_test_yaf_tpl45841.dat
+- spec/codecs/ipfix_test_yaf_tpls_option_tpl.dat
 - spec/codecs/netflow5.dat
 - spec/codecs/netflow5_test_invalid01.dat
 - spec/codecs/netflow5_test_invalid02.dat
@@ -151,17 +154,17 @@ require_paths:
 - lib
 required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
-  - - ">="
+  - - '>='
     - !ruby/object:Gem::Version
       version: '0'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
-  - - ">="
+  - - '>='
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
 rubyforge_project:
-rubygems_version: 2.6.11
+rubygems_version: 2.4.8
 signing_key:
 specification_version: 4
 summary: Reads Netflow v5 and Netflow v9 data
@@ -180,6 +183,11 @@ test_files:
 - spec/codecs/ipfix_test_vmware_vds_data266.dat
 - spec/codecs/ipfix_test_vmware_vds_data266_267.dat
 - spec/codecs/ipfix_test_vmware_vds_tpl.dat
+- spec/codecs/ipfix_test_yaf_data45841.dat
+- spec/codecs/ipfix_test_yaf_data45873.dat
+- spec/codecs/ipfix_test_yaf_data53248.dat
+- spec/codecs/ipfix_test_yaf_tpl45841.dat
+- spec/codecs/ipfix_test_yaf_tpls_option_tpl.dat
 - spec/codecs/netflow5.dat
 - spec/codecs/netflow5_test_invalid01.dat
 - spec/codecs/netflow5_test_invalid02.dat