logstash-codec-netflow 3.0.0 → 3.1.0

data/lib/logstash/codecs/netflow/util.rb

@@ -105,7 +105,7 @@ class Netflow5PDU < BinData::Record
   end
 end
 
-class TemplateFlowset < BinData::Record
+class NetflowTemplateFlowset < BinData::Record
   endian :big
   array  :templates, :read_until => lambda { array.num_bytes == flowset_length - 4 } do
     uint16 :template_id
@@ -117,7 +117,7 @@ class TemplateFlowset < BinData::Record
   end
 end
 
-class OptionFlowset < BinData::Record
+class NetflowOptionFlowset < BinData::Record
   endian :big
   array  :templates, :read_until => lambda { flowset_length - 4 - array.num_bytes <= 2 } do
     uint16 :template_id
@@ -147,9 +147,63 @@ class Netflow9PDU < BinData::Record
     uint16 :flowset_id, :assert => lambda { [0, 1, *(256..65535)].include?(flowset_id) }
     uint16 :flowset_length, :assert => lambda { flowset_length > 4 }
     choice :flowset_data, :selection => :flowset_id do
-      template_flowset 0
-      option_flowset   1
-      string           :default, :read_length => lambda { flowset_length - 4 }
+      netflow_template_flowset 0
+      netflow_option_flowset   1
+      string                   :default, :read_length => lambda { flowset_length - 4 }
+    end
+  end
+end
+
+class IpfixTemplateFlowset < BinData::Record
+  endian :big
+  array  :templates, :read_until => lambda { flowset_length - 4 - array.num_bytes <= 2 } do
+    uint16 :template_id
+    uint16 :field_count
+    array  :fields, :initial_length => :field_count do
+      bit1   :enterprise
+      bit15  :field_type
+      uint16 :field_length
+      uint32 :enterprise_id, :onlyif => lambda { enterprise != 0 }
+    end
+  end
+  # skip :length => lambda { flowset_length - 4 - set.num_bytes } ?
+end
+
+class IpfixOptionFlowset < BinData::Record
+  endian :big
+  array  :templates, :read_until => lambda { flowset_length - 4 - array.num_bytes <= 2 } do
+    uint16 :template_id
+    uint16 :field_count
+    uint16 :scope_count, :assert => lambda { scope_count > 0 }
+    array  :scope_fields, :initial_length => lambda { scope_count } do
+      bit1   :enterprise
+      bit15  :field_type
+      uint16 :field_length
+      uint32 :enterprise_id, :onlyif => lambda { enterprise != 0 }
+    end
+    array  :option_fields, :initial_length => lambda { field_count - scope_count } do
+      bit1   :enterprise
+      bit15  :field_type
+      uint16 :field_length
+      uint32 :enterprise_id, :onlyif => lambda { enterprise != 0 }
+    end
+  end
+end
+
+class IpfixPDU < BinData::Record
+  endian :big
+  uint16 :version
+  uint16 :pdu_length
+  uint32 :unix_sec
+  uint32 :flow_seq_num
+  uint32 :observation_domain_id
+  array  :records, :read_until => lambda { array.num_bytes == pdu_length - 16 } do
+    uint16 :flowset_id, :assert => lambda { [2, 3, *(256..65535)].include?(flowset_id) }
+    uint16 :flowset_length, :assert => lambda { flowset_length > 4 }
+    choice :flowset_data, :selection => :flowset_id do
+      ipfix_template_flowset 2
+      ipfix_option_flowset   3
+      string                 :default, :read_length => lambda { flowset_length - 4 }
     end
   end
 end

data/logstash-codec-netflow.gemspec

@@ -1,9 +1,9 @@
 Gem::Specification.new do |s|
 
   s.name            = 'logstash-codec-netflow'
-  s.version         = '3.0.0'
+  s.version         = '3.1.0'
   s.licenses        = ['Apache License (2.0)']
-  s.summary         = "The netflow codec is for decoding Netflow v5/v9 flows."
+  s.summary         = "The netflow codec is for decoding Netflow v5/v9/v10 (IPFIX) flows."
   s.description     = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program"
   s.authors         = ["Elastic"]
   s.email           = 'info@elastic.co'

data/spec/codecs/netflow_spec.rb

@@ -434,6 +434,251 @@ describe LogStash::Codecs::Netflow do
       expect(decode[0].get("[netflow][scope_system]")).to eq(0)
       expect(decode[0].get("[netflow][total_flows_exp]")).to eq(1)
     end
+  end
+
+  context "IPFIX" do
+    let(:data) do
+      # this netflow raw data was produced with softflowd and captured with netcat
+      # softflowd -D -i eth0 -v 10 -t maxlife=1 -n 127.0.01:8765
+      # nc -k -4 -u -l 127.0.0.1 8765 > ipfix.dat
+      data = []
+      data << IO.read(File.join(File.dirname(__FILE__), "ipfix.dat"), :mode => "rb")
+    end
+
+    let(:json_events) do
+      events = []
+      events << <<-END
+        {
+          "@timestamp": "2015-05-13T11:20:26.000Z",
+          "netflow": {
+            "version": 10,
+            "meteringProcessId": 2679,
+            "systemInitTimeMilliseconds": 1431516013506,
+            "selectorAlgorithm": 1,
+            "samplingPacketInterval": 1,
+            "samplingPacketSpace": 0
+          },
+          "@version": "1"
+        }
+      END
 
+      events << <<-END
+        {
+          "@timestamp": "2015-05-13T11:20:26.000Z",
+          "netflow": {
+            "version": 10,
+            "sourceIPv4Address": "192.168.253.1",
+            "destinationIPv4Address": "192.168.253.128",
+            "octetDeltaCount": 260,
+            "packetDeltaCount": 5,
+            "ingressInterface": 0,
+            "egressInterface": 0,
+            "sourceTransportPort": 60560,
+            "destinationTransportPort": 22,
+            "protocolIdentifier": 6,
+            "tcpControlBits": 16,
+            "ipVersion": 4,
+            "ipClassOfService": 0,
+            "icmpTypeCodeIPv4": 0,
+            "vlanId": 0,
+            "flowStartSysUpTime": 0,
+            "flowEndSysUpTime": 12726
+          },
+          "@version": "1"
+        }
+      END
+
+      events << <<-END
+        {
+          "@timestamp": "2015-05-13T11:20:26.000Z",
+          "netflow": {
+            "version": 10,
+            "sourceIPv4Address": "192.168.253.128",
+            "destinationIPv4Address": "192.168.253.1",
+            "octetDeltaCount": 1000,
+            "packetDeltaCount": 6,
+            "ingressInterface": 0,
+            "egressInterface": 0,
+            "sourceTransportPort": 22,
+            "destinationTransportPort": 60560,
+            "protocolIdentifier": 6,
+            "tcpControlBits": 24,
+            "ipVersion": 4,
+            "ipClassOfService": 0,
+            "icmpTypeCodeIPv4": 0,
+            "vlanId": 0,
+            "flowStartSysUpTime": 0,
+            "flowEndSysUpTime": 12726
+          },
+          "@version": "1"
+        }
+      END
+
+      events << <<-END
+        {
+          "@timestamp": "2015-05-13T11:20:26.000Z",
+          "netflow": {
+            "version": 10,
+            "sourceIPv4Address": "192.168.253.2",
+            "destinationIPv4Address": "192.168.253.132",
+            "octetDeltaCount": 601,
+            "packetDeltaCount": 2,
+            "ingressInterface": 0,
+            "egressInterface": 0,
+            "sourceTransportPort": 53,
+            "destinationTransportPort": 35262,
+            "protocolIdentifier": 17,
+            "tcpControlBits": 0,
+            "ipVersion": 4,
+            "ipClassOfService": 0,
+            "icmpTypeCodeIPv4": 0,
+            "vlanId": 0,
+            "flowStartSysUpTime": 1104,
+            "flowEndSysUpTime": 1142
+          },
+          "@version": "1"
+        }
+      END
+
+      events << <<-END
+        {
+          "@timestamp": "2015-05-13T11:20:26.000Z",
+          "netflow": {
+            "version": 10,
+            "sourceIPv4Address": "192.168.253.132",
+            "destinationIPv4Address": "192.168.253.2",
+            "octetDeltaCount": 148,
+            "packetDeltaCount": 2,
+            "ingressInterface": 0,
+            "egressInterface": 0,
+            "sourceTransportPort": 35262,
+            "destinationTransportPort": 53,
+            "protocolIdentifier": 17,
+            "tcpControlBits": 0,
+            "ipVersion": 4,
+            "ipClassOfService": 0,
+            "icmpTypeCodeIPv4": 0,
+            "vlanId": 0,
+            "flowStartSysUpTime": 1104,
+            "flowEndSysUpTime": 1142
+          },
+          "@version": "1"
+        }
+      END
+
+      events << <<-END
+        {
+          "@timestamp": "2015-05-13T11:20:26.000Z",
+          "netflow": {
+            "version": 10,
+            "sourceIPv4Address": "54.214.9.161",
+            "destinationIPv4Address": "192.168.253.132",
+            "octetDeltaCount": 5946,
+            "packetDeltaCount": 14,
+            "ingressInterface": 0,
+            "egressInterface": 0,
+            "sourceTransportPort": 443,
+            "destinationTransportPort": 49935,
+            "protocolIdentifier": 6,
+            "tcpControlBits": 26,
+            "ipVersion": 4,
+            "ipClassOfService": 0,
+            "icmpTypeCodeIPv4": 0,
+            "vlanId": 0,
+            "flowStartSysUpTime": 1142,
+            "flowEndSysUpTime": 2392
+          },
+          "@version": "1"
+        }
+      END
+
+      events << <<-END
+        {
+          "@timestamp": "2015-05-13T11:20:26.000Z",
+          "netflow": {
+            "version": 10,
+            "sourceIPv4Address": "192.168.253.132",
+            "destinationIPv4Address": "54.214.9.161",
+            "octetDeltaCount": 2608,
+            "packetDeltaCount": 13,
+            "ingressInterface": 0,
+            "egressInterface": 0,
+            "sourceTransportPort": 49935,
+            "destinationTransportPort": 443,
+            "protocolIdentifier": 6,
+            "tcpControlBits": 26,
+            "ipVersion": 4,
+            "ipClassOfService": 0,
+            "icmpTypeCodeIPv4": 0,
+            "vlanId": 0,
+            "flowStartSysUpTime": 1142,
+            "flowEndSysUpTime": 2392
+          },
+          "@version": "1"
+        }
+      END
+
+      events.map{|event| event.gsub(/\s+/, "")}
+    end
+
+    it "should decode raw data" do
+      expect(decode.size).to eq(7)
+
+      expect(decode[0].get("[netflow][version]")).to eq(10)
+      expect(decode[0].get("[netflow][systemInitTimeMilliseconds]")).to eq(1431516013506)
+
+      expect(decode[1].get("[netflow][version]")).to eq(10)
+      expect(decode[1].get("[netflow][sourceIPv4Address]")).to eq("192.168.253.1")
+      expect(decode[1].get("[netflow][destinationIPv4Address]")).to eq("192.168.253.128")
+      expect(decode[1].get("[netflow][sourceTransportPort]")).to eq(60560)
+      expect(decode[1].get("[netflow][destinationTransportPort]")).to eq(22)
+      expect(decode[1].get("[netflow][protocolIdentifier]")).to eq(6)
+      expect(decode[1].get("[netflow][tcpControlBits]")).to eq(16)
+
+      expect(decode[2].get("[netflow][version]")).to eq(10)
+      expect(decode[2].get("[netflow][sourceIPv4Address]")).to eq("192.168.253.128")
+      expect(decode[2].get("[netflow][destinationIPv4Address]")).to eq("192.168.253.1")
+      expect(decode[2].get("[netflow][sourceTransportPort]")).to eq(22)
+      expect(decode[2].get("[netflow][destinationTransportPort]")).to eq(60560)
+      expect(decode[2].get("[netflow][protocolIdentifier]")).to eq(6)
+      expect(decode[2].get("[netflow][tcpControlBits]")).to eq(24)
+
+      expect(decode[3].get("[netflow][sourceIPv4Address]")).to eq("192.168.253.2")
+      expect(decode[3].get("[netflow][destinationIPv4Address]")).to eq("192.168.253.132")
+      expect(decode[3].get("[netflow][sourceTransportPort]")).to eq(53)
+      expect(decode[3].get("[netflow][destinationTransportPort]")).to eq(35262)
+      expect(decode[3].get("[netflow][protocolIdentifier]")).to eq(17)
+
+      expect(decode[4].get("[netflow][sourceIPv4Address]")).to eq("192.168.253.132")
+      expect(decode[4].get("[netflow][destinationIPv4Address]")).to eq("192.168.253.2")
+      expect(decode[4].get("[netflow][sourceTransportPort]")).to eq(35262)
+      expect(decode[4].get("[netflow][destinationTransportPort]")).to eq(53)
+      expect(decode[4].get("[netflow][protocolIdentifier]")).to eq(17)
+
+      expect(decode[5].get("[netflow][sourceIPv4Address]")).to eq("54.214.9.161")
+      expect(decode[5].get("[netflow][destinationIPv4Address]")).to eq("192.168.253.132")
+      expect(decode[5].get("[netflow][sourceTransportPort]")).to eq(443)
+      expect(decode[5].get("[netflow][destinationTransportPort]")).to eq(49935)
+      expect(decode[5].get("[netflow][protocolIdentifier]")).to eq(6)
+      expect(decode[5].get("[netflow][tcpControlBits]")).to eq(26)
+
+      expect(decode[6].get("[netflow][sourceIPv4Address]")).to eq("192.168.253.132")
+      expect(decode[6].get("[netflow][destinationIPv4Address]")).to eq("54.214.9.161")
+      expect(decode[6].get("[netflow][sourceTransportPort]")).to eq(49935)
+      expect(decode[6].get("[netflow][destinationTransportPort]")).to eq(443)
+      expect(decode[6].get("[netflow][protocolIdentifier]")).to eq(6)
+      expect(decode[6].get("[netflow][tcpControlBits]")).to eq(26)
+    end
+
+    it "should serialize to json" do
+      expect(JSON.parse(decode[0].to_json)).to eq(JSON.parse(json_events[0]))
+      expect(JSON.parse(decode[1].to_json)).to eq(JSON.parse(json_events[1]))
+      expect(JSON.parse(decode[2].to_json)).to eq(JSON.parse(json_events[2]))
+      expect(JSON.parse(decode[3].to_json)).to eq(JSON.parse(json_events[3]))
+      expect(JSON.parse(decode[4].to_json)).to eq(JSON.parse(json_events[4]))
+      expect(JSON.parse(decode[5].to_json)).to eq(JSON.parse(json_events[5]))
+      expect(JSON.parse(decode[6].to_json)).to eq(JSON.parse(json_events[6]))
+    end
   end
+
 end

metadata

@@ -1,60 +1,58 @@
 --- !ruby/object:Gem::Specification
 name: logstash-codec-netflow
 version: !ruby/object:Gem::Version
-  version: 3.0.0
+  version: 3.1.0
 platform: ruby
 authors:
 - Elastic
-autorequire: 
+autorequire:
 bindir: bin
 cert_chain: []
-date: 2016-05-06 00:00:00.000000000 Z
+date: 2016-05-17 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
-  name: logstash-core-plugin-api
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ~>
       - !ruby/object:Gem::Version
         version: '2.0'
-  type: :runtime
+  name: logstash-core-plugin-api
   prerelease: false
+  type: :runtime
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ~>
       - !ruby/object:Gem::Version
         version: '2.0'
 - !ruby/object:Gem::Dependency
-  name: bindata
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - '>='
       - !ruby/object:Gem::Version
         version: 1.5.0
-  type: :runtime
+  name: bindata
   prerelease: false
+  type: :runtime
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - '>='
       - !ruby/object:Gem::Version
         version: 1.5.0
 - !ruby/object:Gem::Dependency
-  name: logstash-devutils
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - '>='
       - !ruby/object:Gem::Version
         version: '0'
-  type: :development
+  name: logstash-devutils
   prerelease: false
+  type: :development
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - '>='
       - !ruby/object:Gem::Version
         version: '0'
-description: This gem is a Logstash plugin required to be installed on top of the
-  Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This
-  gem is not a stand-alone program
+description: This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program
 email: info@elastic.co
 executables: []
 extensions: []
@@ -67,9 +65,12 @@ files:
 - NOTICE.TXT
 - README.md
 - lib/logstash/codecs/netflow.rb
+- lib/logstash/codecs/netflow/iana2yaml.rb
+- lib/logstash/codecs/netflow/ipfix.yaml
 - lib/logstash/codecs/netflow/netflow.yaml
 - lib/logstash/codecs/netflow/util.rb
 - logstash-codec-netflow.gemspec
+- spec/codecs/ipfix.dat
 - spec/codecs/netflow5.dat
 - spec/codecs/netflow5_test_invalid01.dat
 - spec/codecs/netflow5_test_invalid02.dat
@@ -89,27 +90,28 @@ licenses:
 metadata:
   logstash_plugin: 'true'
   logstash_group: codec
-post_install_message: 
+post_install_message:
 rdoc_options: []
 require_paths:
 - lib
 required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
-  - - ">="
+  - - '>='
     - !ruby/object:Gem::Version
       version: '0'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
-  - - ">="
+  - - '>='
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubyforge_project: 
-rubygems_version: 2.5.1
-signing_key: 
+rubyforge_project:
+rubygems_version: 2.4.5
+signing_key:
 specification_version: 4
-summary: The netflow codec is for decoding Netflow v5/v9 flows.
+summary: The netflow codec is for decoding Netflow v5/v9/v10 (IPFIX) flows.
 test_files:
+- spec/codecs/ipfix.dat
 - spec/codecs/netflow5.dat
 - spec/codecs/netflow5_test_invalid01.dat
 - spec/codecs/netflow5_test_invalid02.dat