logstash-codec-netflow 2.1.1 → 3.0.0

data/lib/logstash/codecs/netflow/util.rb

@@ -105,19 +105,19 @@ class Netflow5PDU < BinData::Record
   end
 end
 
-class NetflowTemplateFlowset < BinData::Record
+class TemplateFlowset < BinData::Record
   endian :big
   array  :templates, :read_until => lambda { array.num_bytes == flowset_length - 4 } do
     uint16 :template_id
     uint16 :field_count
-    array  :record_fields, :initial_length => :field_count do
+    array  :fields, :initial_length => :field_count do
       uint16 :field_type
       uint16 :field_length
     end
   end
 end
 
-class NetflowOptionFlowset < BinData::Record
+class OptionFlowset < BinData::Record
   endian :big
   array  :templates, :read_until => lambda { flowset_length - 4 - array.num_bytes <= 2 } do
     uint16 :template_id
@@ -147,63 +147,9 @@ class Netflow9PDU < BinData::Record
     uint16 :flowset_id, :assert => lambda { [0, 1, *(256..65535)].include?(flowset_id) }
     uint16 :flowset_length, :assert => lambda { flowset_length > 4 }
     choice :flowset_data, :selection => :flowset_id do
-      netflow_template_flowset 0
-      netflow_option_flowset   1
-      string                   :default, :read_length => lambda { flowset_length - 4 }
-    end
-  end
-end
-
-class IpfixTemplateFlowset < BinData::Record
-  endian :big
-  array  :templates, :read_until => lambda { flowset_length - 4 - array.num_bytes <= 2 } do
-    uint16 :template_id
-    uint16 :field_count
-    array  :record_fields, :initial_length => :field_count do
-      bit1   :enterprise
-      bit15  :field_type
-      uint16 :field_length
-      uint32 :enterprise_id, :onlyif => lambda { enterprise != 0 }
-    end
-  end
-  # skip :length => lambda { flowset_length - 4 - set.num_bytes } ?
-end
-
-class IpfixOptionFlowset < BinData::Record
-  endian :big
-  array  :templates, :read_until => lambda { flowset_length - 4 - array.num_bytes <= 2 } do
-    uint16 :template_id
-    uint16 :field_count
-    uint16 :scope_count, :assert => lambda { scope_count > 0 }
-    array  :scope_fields, :initial_length => lambda { scope_count } do
-      bit1   :enterprise
-      bit15  :field_type
-      uint16 :field_length
-      uint32 :enterprise_id, :onlyif => lambda { enterprise != 0 }
-    end
-    array  :option_fields, :initial_length => lambda { field_count - scope_count } do
-      bit1   :enterprise
-      bit15  :field_type
-      uint16 :field_length
-      uint32 :enterprise_id, :onlyif => lambda { enterprise != 0 }
-    end
-  end
-end
-
-class IpfixPDU < BinData::Record
-  endian :big
-  uint16 :version
-  uint16 :pdu_length
-  uint32 :unix_sec
-  uint32 :flow_seq_num
-  uint32 :observation_domain_id
-  array  :records, :read_until => lambda { array.num_bytes == pdu_length - 16 } do
-    uint16 :flowset_id, :assert => lambda { [2, 3, *(256..65535)].include?(flowset_id) }
-    uint16 :flowset_length, :assert => lambda { flowset_length > 4 }
-    choice :flowset_data, :selection => :flowset_id do
-      ipfix_template_flowset 2
-      ipfix_option_flowset   3
-      string                 :default, :read_length => lambda { flowset_length - 4 }
+      template_flowset 0
+      option_flowset   1
+      string           :default, :read_length => lambda { flowset_length - 4 }
     end
   end
 end

data/logstash-codec-netflow.gemspec

@@ -1,9 +1,9 @@
 Gem::Specification.new do |s|
 
   s.name            = 'logstash-codec-netflow'
-  s.version         = '2.1.1'
+  s.version         = '3.0.0'
   s.licenses        = ['Apache License (2.0)']
-  s.summary         = "The netflow codec is for decoding Netflow v5/v9/IPFIX flows."
+  s.summary         = "The netflow codec is for decoding Netflow v5/v9 flows."
   s.description     = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program"
   s.authors         = ["Elastic"]
   s.email           = 'info@elastic.co'
@@ -20,9 +20,8 @@ Gem::Specification.new do |s|
   s.metadata = { "logstash_plugin" => "true", "logstash_group" => "codec" }
 
   # Gem dependencies
-  s.add_runtime_dependency "logstash-core-plugin-api", "~> 1.0"
+  s.add_runtime_dependency "logstash-core-plugin-api", "~> 2.0"
   s.add_runtime_dependency 'bindata', ['>= 1.5.0']
   s.add_development_dependency 'logstash-devutils'
 end
 
-

data/spec/codecs/netflow_spec.rb

@@ -104,19 +104,19 @@ describe LogStash::Codecs::Netflow do
     it "should decode raw data" do
       expect(decode.size).to eq(2)
 
-      expect(decode[0]["[netflow][version]"]).to eq(5)
-      expect(decode[0]["[netflow][ipv4_src_addr]"]).to eq("10.0.2.2")
-      expect(decode[0]["[netflow][ipv4_dst_addr]"]).to eq("10.0.2.15")
-      expect(decode[0]["[netflow][l4_src_port]"]).to eq(54435)
-      expect(decode[0]["[netflow][l4_dst_port]"]).to eq(22)
-      expect(decode[0]["[netflow][tcp_flags]"]).to eq(16)
+      expect(decode[0].get("[netflow][version]")).to eq(5)
+      expect(decode[0].get("[netflow][ipv4_src_addr]")).to eq("10.0.2.2")
+      expect(decode[0].get("[netflow][ipv4_dst_addr]")).to eq("10.0.2.15")
+      expect(decode[0].get("[netflow][l4_src_port]")).to eq(54435)
+      expect(decode[0].get("[netflow][l4_dst_port]")).to eq(22)
+      expect(decode[0].get("[netflow][tcp_flags]")).to eq(16)
 
-      expect(decode[1]["[netflow][version]"]).to eq(5)
-      expect(decode[1]["[netflow][ipv4_src_addr]"]).to eq("10.0.2.15")
-      expect(decode[1]["[netflow][ipv4_dst_addr]"]).to eq("10.0.2.2")
-      expect(decode[1]["[netflow][l4_src_port]"]).to eq(22)
-      expect(decode[1]["[netflow][l4_dst_port]"]).to eq(54435)
-      expect(decode[1]["[netflow][tcp_flags]"]).to eq(24)
+      expect(decode[1].get("[netflow][version]")).to eq(5)
+      expect(decode[1].get("[netflow][ipv4_src_addr]")).to eq("10.0.2.15")
+      expect(decode[1].get("[netflow][ipv4_dst_addr]")).to eq("10.0.2.2")
+      expect(decode[1].get("[netflow][l4_src_port]")).to eq(22)
+      expect(decode[1].get("[netflow][l4_dst_port]")).to eq(54435)
+      expect(decode[1].get("[netflow][tcp_flags]")).to eq(24)
     end
 
     it "should serialize to json" do
@@ -193,7 +193,7 @@ describe LogStash::Codecs::Netflow do
 
     it "should decode raw data" do
       expect(decode.size).to eq(7)
-      expect(decode[0]["[netflow][version]"]).to eq(9)
+      expect(decode[0].get("[netflow][version]")).to eq(9)
     end
 
     it "should serialize to json" do
@@ -235,8 +235,8 @@ describe LogStash::Codecs::Netflow do
     end
 
     it "should decode the mac address" do
-      expect(decode[1]["[netflow][in_src_mac]"]).to eq("00:50:56:c0:00:01")
-      expect(decode[1]["[netflow][in_dst_mac]"]).to eq("00:0c:29:70:86:09")
+      expect(decode[1].get("[netflow][in_src_mac]")).to eq("00:50:56:c0:00:01")
+      expect(decode[1].get("[netflow][in_dst_mac]")).to eq("00:0c:29:70:86:09")
     end
 
     it "should serialize to json" do
@@ -292,7 +292,7 @@ describe LogStash::Codecs::Netflow do
 
     it "should decode raw data" do
       expect(decode.size).to eq(14)
-      expect(decode[1]["[netflow][version]"]).to eq(9)
+      expect(decode[1].get("[netflow][version]")).to eq(9)
     end
 
     it "should serialize to json" do
@@ -379,8 +379,8 @@ describe LogStash::Codecs::Netflow do
 
     it "should decode raw data" do
       expect(decode.size).to eq(9)
-      expect(decode[1]["[netflow][l4_src_port]"]).to eq(123)
-      expect(decode[8]["[netflow][l4_src_port]"]).to eq(22)
+      expect(decode[1].get("[netflow][l4_src_port]")).to eq(123)
+      expect(decode[8].get("[netflow][l4_src_port]")).to eq(22)
     end
 
     it "should serialize to json" do
@@ -431,254 +431,9 @@ describe LogStash::Codecs::Netflow do
     end
 
     it "should decode raw data" do
-      expect(decode[0]["[netflow][scope_system]"]).to eq(0)
-      expect(decode[0]["[netflow][total_flows_exp]"]).to eq(1)
+      expect(decode[0].get("[netflow][scope_system]")).to eq(0)
+      expect(decode[0].get("[netflow][total_flows_exp]")).to eq(1)
     end
-  end
-
-  context "IPFIX" do
-    let(:data) do
-      # this netflow raw data was produced with softflowd and captured with netcat
-      # softflowd -D -i eth0 -v 10 -t maxlife=1 -n 127.0.01:8765
-      # nc -k -4 -u -l 127.0.0.1 8765 > ipfix.dat
-      data = []
-      data << IO.read(File.join(File.dirname(__FILE__), "ipfix.dat"), :mode => "rb")
-    end
-
-    let(:json_events) do
-      events = []
-      events << <<-END
-        {
-          "@timestamp": "2015-05-13T11:20:26.000Z",
-          "netflow": {
-            "version": 10,
-            "meteringProcessId": 2679,
-            "systemInitTimeMilliseconds": 1431516013506,
-            "selectorAlgorithm": 1,
-            "samplingPacketInterval": 1,
-            "samplingPacketSpace": 0
-          },
-          "@version": "1"
-        }
-      END
 
-      events << <<-END
-        {
-          "@timestamp": "2015-05-13T11:20:26.000Z",
-          "netflow": {
-            "version": 10,
-            "sourceIPv4Address": "192.168.253.1",
-            "destinationIPv4Address": "192.168.253.128",
-            "octetDeltaCount": 260,
-            "packetDeltaCount": 5,
-            "ingressInterface": 0,
-            "egressInterface": 0,
-            "sourceTransportPort": 60560,
-            "destinationTransportPort": 22,
-            "protocolIdentifier": 6,
-            "tcpControlBits": 16,
-            "ipVersion": 4,
-            "ipClassOfService": 0,
-            "icmpTypeCodeIPv4": 0,
-            "vlanId": 0,
-            "flowStartSysUpTime": 0,
-            "flowEndSysUpTime": 12726
-          },
-          "@version": "1"
-        }
-      END
-
-      events << <<-END
-        {
-          "@timestamp": "2015-05-13T11:20:26.000Z",
-          "netflow": {
-            "version": 10,
-            "sourceIPv4Address": "192.168.253.128",
-            "destinationIPv4Address": "192.168.253.1",
-            "octetDeltaCount": 1000,
-            "packetDeltaCount": 6,
-            "ingressInterface": 0,
-            "egressInterface": 0,
-            "sourceTransportPort": 22,
-            "destinationTransportPort": 60560,
-            "protocolIdentifier": 6,
-            "tcpControlBits": 24,
-            "ipVersion": 4,
-            "ipClassOfService": 0,
-            "icmpTypeCodeIPv4": 0,
-            "vlanId": 0,
-            "flowStartSysUpTime": 0,
-            "flowEndSysUpTime": 12726
-          },
-          "@version": "1"
-        }
-      END
-
-      events << <<-END
-        {
-          "@timestamp": "2015-05-13T11:20:26.000Z",
-          "netflow": {
-            "version": 10,
-            "sourceIPv4Address": "192.168.253.2",
-            "destinationIPv4Address": "192.168.253.132",
-            "octetDeltaCount": 601,
-            "packetDeltaCount": 2,
-            "ingressInterface": 0,
-            "egressInterface": 0,
-            "sourceTransportPort": 53,
-            "destinationTransportPort": 35262,
-            "protocolIdentifier": 17,
-            "tcpControlBits": 0,
-            "ipVersion": 4,
-            "ipClassOfService": 0,
-            "icmpTypeCodeIPv4": 0,
-            "vlanId": 0,
-            "flowStartSysUpTime": 1104,
-            "flowEndSysUpTime": 1142
-          },
-          "@version": "1"
-        }
-      END
-
-      events << <<-END
-        {
-          "@timestamp": "2015-05-13T11:20:26.000Z",
-          "netflow": {
-            "version": 10,
-            "sourceIPv4Address": "192.168.253.132",
-            "destinationIPv4Address": "192.168.253.2",
-            "octetDeltaCount": 148,
-            "packetDeltaCount": 2,
-            "ingressInterface": 0,
-            "egressInterface": 0,
-            "sourceTransportPort": 35262,
-            "destinationTransportPort": 53,
-            "protocolIdentifier": 17,
-            "tcpControlBits": 0,
-            "ipVersion": 4,
-            "ipClassOfService": 0,
-            "icmpTypeCodeIPv4": 0,
-            "vlanId": 0,
-            "flowStartSysUpTime": 1104,
-            "flowEndSysUpTime": 1142
-          },
-          "@version": "1"
-        }
-      END
-
-      events << <<-END
-        {
-          "@timestamp": "2015-05-13T11:20:26.000Z",
-          "netflow": {
-            "version": 10,
-            "sourceIPv4Address": "54.214.9.161",
-            "destinationIPv4Address": "192.168.253.132",
-            "octetDeltaCount": 5946,
-            "packetDeltaCount": 14,
-            "ingressInterface": 0,
-            "egressInterface": 0,
-            "sourceTransportPort": 443,
-            "destinationTransportPort": 49935,
-            "protocolIdentifier": 6,
-            "tcpControlBits": 26,
-            "ipVersion": 4,
-            "ipClassOfService": 0,
-            "icmpTypeCodeIPv4": 0,
-            "vlanId": 0,
-            "flowStartSysUpTime": 1142,
-            "flowEndSysUpTime": 2392
-          },
-          "@version": "1"
-        }
-      END
-
-      events << <<-END
-        {
-          "@timestamp": "2015-05-13T11:20:26.000Z",
-          "netflow": {
-            "version": 10,
-            "sourceIPv4Address": "192.168.253.132",
-            "destinationIPv4Address": "54.214.9.161",
-            "octetDeltaCount": 2608,
-            "packetDeltaCount": 13,
-            "ingressInterface": 0,
-            "egressInterface": 0,
-            "sourceTransportPort": 49935,
-            "destinationTransportPort": 443,
-            "protocolIdentifier": 6,
-            "tcpControlBits": 26,
-            "ipVersion": 4,
-            "ipClassOfService": 0,
-            "icmpTypeCodeIPv4": 0,
-            "vlanId": 0,
-            "flowStartSysUpTime": 1142,
-            "flowEndSysUpTime": 2392
-          },
-          "@version": "1"
-        }
-      END
-
-      events.map{|event| event.gsub(/\s+/, "")}
-    end
-
-    it "should decode raw data" do
-      expect(decode.size).to eq(7)
-
-      expect(decode[0]["[netflow][version]"]).to eq(10)
-      expect(decode[0]["[netflow][systemInitTimeMilliseconds]"]).to eq(1431516013506)
-
-      expect(decode[1]["[netflow][version]"]).to eq(10)
-      expect(decode[1]["[netflow][sourceIPv4Address]"]).to eq("192.168.253.1")
-      expect(decode[1]["[netflow][destinationIPv4Address]"]).to eq("192.168.253.128")
-      expect(decode[1]["[netflow][sourceTransportPort]"]).to eq(60560)
-      expect(decode[1]["[netflow][destinationTransportPort]"]).to eq(22)
-      expect(decode[1]["[netflow][protocolIdentifier]"]).to eq(6)
-      expect(decode[1]["[netflow][tcpControlBits]"]).to eq(16)
-
-      expect(decode[2]["[netflow][version]"]).to eq(10)
-      expect(decode[2]["[netflow][sourceIPv4Address]"]).to eq("192.168.253.128")
-      expect(decode[2]["[netflow][destinationIPv4Address]"]).to eq("192.168.253.1")
-      expect(decode[2]["[netflow][sourceTransportPort]"]).to eq(22)
-      expect(decode[2]["[netflow][destinationTransportPort]"]).to eq(60560)
-      expect(decode[2]["[netflow][protocolIdentifier]"]).to eq(6)
-      expect(decode[2]["[netflow][tcpControlBits]"]).to eq(24)
-
-      expect(decode[3]["[netflow][sourceIPv4Address]"]).to eq("192.168.253.2")
-      expect(decode[3]["[netflow][destinationIPv4Address]"]).to eq("192.168.253.132")
-      expect(decode[3]["[netflow][sourceTransportPort]"]).to eq(53)
-      expect(decode[3]["[netflow][destinationTransportPort]"]).to eq(35262)
-      expect(decode[3]["[netflow][protocolIdentifier]"]).to eq(17)
-
-      expect(decode[4]["[netflow][sourceIPv4Address]"]).to eq("192.168.253.132")
-      expect(decode[4]["[netflow][destinationIPv4Address]"]).to eq("192.168.253.2")
-      expect(decode[4]["[netflow][sourceTransportPort]"]).to eq(35262)
-      expect(decode[4]["[netflow][destinationTransportPort]"]).to eq(53)
-      expect(decode[4]["[netflow][protocolIdentifier]"]).to eq(17)
-
-      expect(decode[5]["[netflow][sourceIPv4Address]"]).to eq("54.214.9.161")
-      expect(decode[5]["[netflow][destinationIPv4Address]"]).to eq("192.168.253.132")
-      expect(decode[5]["[netflow][sourceTransportPort]"]).to eq(443)
-      expect(decode[5]["[netflow][destinationTransportPort]"]).to eq(49935)
-      expect(decode[5]["[netflow][protocolIdentifier]"]).to eq(6)
-      expect(decode[5]["[netflow][tcpControlBits]"]).to eq(26)
-
-      expect(decode[6]["[netflow][sourceIPv4Address]"]).to eq("192.168.253.132")
-      expect(decode[6]["[netflow][destinationIPv4Address]"]).to eq("54.214.9.161")
-      expect(decode[6]["[netflow][sourceTransportPort]"]).to eq(49935)
-      expect(decode[6]["[netflow][destinationTransportPort]"]).to eq(443)
-      expect(decode[6]["[netflow][protocolIdentifier]"]).to eq(6)
-      expect(decode[6]["[netflow][tcpControlBits]"]).to eq(26)
-    end
-
-    it "should serialize to json" do
-      expect(JSON.parse(decode[0].to_json)).to eq(JSON.parse(json_events[0]))
-      expect(JSON.parse(decode[1].to_json)).to eq(JSON.parse(json_events[1]))
-      expect(JSON.parse(decode[2].to_json)).to eq(JSON.parse(json_events[2]))
-      expect(JSON.parse(decode[3].to_json)).to eq(JSON.parse(json_events[3]))
-      expect(JSON.parse(decode[4].to_json)).to eq(JSON.parse(json_events[4]))
-      expect(JSON.parse(decode[5].to_json)).to eq(JSON.parse(json_events[5]))
-      expect(JSON.parse(decode[6].to_json)).to eq(JSON.parse(json_events[6]))
-    end
   end
-
 end

metadata

@@ -1,58 +1,60 @@
 --- !ruby/object:Gem::Specification
 name: logstash-codec-netflow
 version: !ruby/object:Gem::Version
-  version: 2.1.1
+  version: 3.0.0
 platform: ruby
 authors:
 - Elastic
-autorequire:
+autorequire: 
 bindir: bin
 cert_chain: []
-date: 2016-07-08 00:00:00.000000000 Z
+date: 2016-05-06 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
+  name: logstash-core-plugin-api
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.0'
-  name: logstash-core-plugin-api
-  prerelease: false
+        version: '2.0'
   type: :runtime
+  prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.0'
+        version: '2.0'
 - !ruby/object:Gem::Dependency
+  name: bindata
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: 1.5.0
-  name: bindata
-  prerelease: false
   type: :runtime
+  prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: 1.5.0
 - !ruby/object:Gem::Dependency
+  name: logstash-devutils
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
-  name: logstash-devutils
-  prerelease: false
   type: :development
+  prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
-description: This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program
+description: This gem is a Logstash plugin required to be installed on top of the
+  Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This
+  gem is not a stand-alone program
 email: info@elastic.co
 executables: []
 extensions: []
@@ -65,12 +67,9 @@ files:
 - NOTICE.TXT
 - README.md
 - lib/logstash/codecs/netflow.rb
-- lib/logstash/codecs/netflow/iana2yaml.rb
-- lib/logstash/codecs/netflow/ipfix.yaml
 - lib/logstash/codecs/netflow/netflow.yaml
 - lib/logstash/codecs/netflow/util.rb
 - logstash-codec-netflow.gemspec
-- spec/codecs/ipfix.dat
 - spec/codecs/netflow5.dat
 - spec/codecs/netflow5_test_invalid01.dat
 - spec/codecs/netflow5_test_invalid02.dat
@@ -90,28 +89,27 @@ licenses:
 metadata:
   logstash_plugin: 'true'
   logstash_group: codec
-post_install_message:
+post_install_message: 
 rdoc_options: []
 require_paths:
 - lib
 required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
-  - - '>='
+  - - ">="
     - !ruby/object:Gem::Version
       version: '0'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
-  - - '>='
+  - - ">="
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubyforge_project:
-rubygems_version: 2.4.8
-signing_key:
+rubyforge_project: 
+rubygems_version: 2.5.1
+signing_key: 
 specification_version: 4
-summary: The netflow codec is for decoding Netflow v5/v9/IPFIX flows.
+summary: The netflow codec is for decoding Netflow v5/v9 flows.
 test_files:
-- spec/codecs/ipfix.dat
 - spec/codecs/netflow5.dat
 - spec/codecs/netflow5_test_invalid01.dat
 - spec/codecs/netflow5_test_invalid02.dat