logstash-codec-netflow 0.1.0

checksums.yaml

@@ -0,0 +1,15 @@
+---
+!binary "U0hBMQ==":
+  metadata.gz: !binary |-
+    MTc3NTE4OWRjM2RiYzc5MDhlNjViMTQ1MzZiNWFiMmIyODcwOGYwZg==
+  data.tar.gz: !binary |-
+    MjEzYTdhMDVkNzcwYjE1MWI5ZmY4MzFjZGFhYjQ1YmEzYWE1NzAyYQ==
+SHA512:
+  metadata.gz: !binary |-
+    YTg3N2Q1NTVmMzcwOWY4YWYxM2MxMWI1YTIyYmY1NjQzOTQxMDgwYzZkYjAy
+    Nzk3ZjI5NTJjMzA4MjAyMWQ0YjFjNmU0NDViOTI1MmEyNWI4ZGMyNjJkYzdj
+    ZWU1MzhkMzAxZjM2Y2EyZmVlNzUwYmQzZjkwYjdmMDMzYmZkZTE=
+  data.tar.gz: !binary |-
+    OWJkODUxNDkxZDJhYTQ2YWY5Yzk3MDljMTBjNDkzOGJkZDZlZGFhY2E4NWE3
+    MGNkYTk0ZDMxODc3NGZmNTg2YTVjNjRkYzk3NTYyNzJhOWI4OTE2ZjM3MGRh
+    ZTk0MWIyNWI2Nzg5NjQ4NjQ5OGRkZDk0ZjIzNGJmNWMyN2Y0Y2Q=

data/.gitignore

@@ -0,0 +1,3 @@
+*.gem
+Gemfile.lock
+.bundle

data/Gemfile

@@ -0,0 +1,4 @@
+source 'http://rubygems.org'
+gem 'rake'
+gem 'gem_publisher'
+gem 'archive-tar-minitar'

data/LICENSE

@@ -0,0 +1,13 @@
+Copyright (c) 2012-2014 Elasticsearch <http://www.elasticsearch.org>
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.

data/Rakefile

@@ -0,0 +1,6 @@
+@files=[]
+
+task :default do
+  system("rake -T")
+end
+

data/lib/logstash/codecs/netflow.rb

@@ -0,0 +1,263 @@
+# encoding: utf-8
+require "logstash/filters/base"
+require "logstash/namespace"
+require "logstash/timestamp"
+
+# The "netflow" codec is for decoding Netflow v5/v9 flows.
+class LogStash::Codecs::Netflow < LogStash::Codecs::Base
+  config_name "netflow"
+  milestone 1
+
+  # Netflow v9 template cache TTL (minutes)
+  config :cache_ttl, :validate => :number, :default => 4000
+
+  # Specify into what field you want the Netflow data.
+  config :target, :validate => :string, :default => "netflow"
+
+  # Specify which Netflow versions you will accept.
+  config :versions, :validate => :array, :default => [5, 9]
+
+  # Override YAML file containing Netflow field definitions
+  #
+  # Each Netflow field is defined like so:
+  #
+  #    ---
+  #    id:
+  #    - default length in bytes
+  #    - :name
+  #    id:
+  #    - :uintN or :ip4_addr or :ip6_addr or :mac_addr or :string
+  #    - :name
+  #    id:
+  #    - :skip
+  #
+  # See <https://github.com/logstash/logstash/tree/v%VERSION%/lib/logstash/codecs/netflow/netflow.yaml> for the base set.
+  config :definitions, :validate => :path
+
+  public
+  def initialize(params={})
+    super(params)
+    @threadsafe = false
+  end
+
+  public
+  def register
+    require "logstash/codecs/netflow/util"
+    @templates = Vash.new()
+
+    # Path to default Netflow v9 field definitions
+    filename = ::File.expand_path('netflow/netflow.yaml', ::File.dirname(__FILE__))
+
+    begin
+      @fields = YAML.load_file(filename)
+    rescue Exception => e
+      raise "#{self.class.name}: Bad syntax in definitions file #{filename}"
+    end
+
+    # Allow the user to augment/override/rename the supported Netflow fields
+    if @definitions
+      raise "#{self.class.name}: definitions file #{@definitions} does not exists" unless File.exists?(@definitions)
+      begin
+        @fields.merge!(YAML.load_file(@definitions))
+      rescue Exception => e
+        raise "#{self.class.name}: Bad syntax in definitions file #{@definitions}"
+      end
+    end
+  end # def register
+
+  public
+  def decode(payload, &block)
+    header = Header.read(payload)
+
+    unless @versions.include?(header.version)
+      @logger.warn("Ignoring Netflow version v#{header.version}")
+      return
+    end
+
+    if header.version == 5
+      flowset = Netflow5PDU.read(payload)
+    elsif header.version == 9
+      flowset = Netflow9PDU.read(payload)
+    else
+      @logger.warn("Unsupported Netflow version v#{header.version}")
+      return
+    end
+
+    flowset.records.each do |record|
+      if flowset.version == 5
+        event = LogStash::Event.new
+
+        # FIXME Probably not doing this right WRT JRuby?
+        #
+        # The flowset header gives us the UTC epoch seconds along with
+        # residual nanoseconds so we can set @timestamp to that easily
+        event.timestamp = LogStash::Timestamp.at(flowset.unix_sec, flowset.unix_nsec / 1000)
+        event[@target] = {}
+
+        # Copy some of the pertinent fields in the header to the event
+        ['version', 'flow_seq_num', 'engine_type', 'engine_id', 'sampling_algorithm', 'sampling_interval', 'flow_records'].each do |f|
+          event[@target][f] = flowset[f]
+        end
+
+        # Create fields in the event from each field in the flow record
+        record.each_pair do |k,v|
+          case k.to_s
+          when /_switched$/
+            # The flow record sets the first and last times to the device
+            # uptime in milliseconds. Given the actual uptime is provided
+            # in the flowset header along with the epoch seconds we can
+            # convert these into absolute times
+            millis = flowset.uptime - v
+            seconds = flowset.unix_sec - (millis / 1000)
+            micros = (flowset.unix_nsec / 1000) - (millis % 1000)
+            if micros < 0
+              seconds--
+              micros += 1000000
+            end
+            # FIXME Again, probably doing this wrong WRT JRuby?
+            event[@target][k.to_s] = Time.at(seconds, micros).utc.strftime("%Y-%m-%dT%H:%M:%S.%3NZ")
+          else
+            event[@target][k.to_s] = v
+          end
+        end
+
+        yield event
+      elsif flowset.version == 9
+        case record.flowset_id
+        when 0
+          # Template flowset
+          record.flowset_data.templates.each do |template|
+            catch (:field) do
+              fields = []
+              template.fields.each do |field|
+                entry = netflow_field_for(field.field_type, field.field_length)
+                if ! entry
+                  throw :field
+                end
+                fields += entry
+              end
+              # We get this far, we have a list of fields
+              #key = "#{flowset.source_id}|#{event["source"]}|#{template.template_id}"
+              key = "#{flowset.source_id}|#{template.template_id}"
+              @templates[key, @cache_ttl] = BinData::Struct.new(:endian => :big, :fields => fields)
+              # Purge any expired templates
+              @templates.cleanup!
+            end
+          end
+        when 1
+          # Options template flowset
+          record.flowset_data.templates.each do |template|
+            catch (:field) do
+              fields = []
+              template.option_fields.each do |field|
+                entry = netflow_field_for(field.field_type, field.field_length)
+                if ! entry
+                  throw :field
+                end
+                fields += entry
+              end
+              # We get this far, we have a list of fields
+              #key = "#{flowset.source_id}|#{event["source"]}|#{template.template_id}"
+              key = "#{flowset.source_id}|#{template.template_id}"
+              @templates[key, @cache_ttl] = BinData::Struct.new(:endian => :big, :fields => fields)
+              # Purge any expired templates
+              @templates.cleanup!
+            end
+          end
+        when 256..65535
+          # Data flowset
+          #key = "#{flowset.source_id}|#{event["source"]}|#{record.flowset_id}"
+          key = "#{flowset.source_id}|#{record.flowset_id}"
+          template = @templates[key]
+
+          if ! template
+            #@logger.warn("No matching template for flow id #{record.flowset_id} from #{event["source"]}")
+            @logger.warn("No matching template for flow id #{record.flowset_id}")
+            next
+          end
+
+          length = record.flowset_length - 4
+
+          # Template shouldn't be longer than the record and there should
+          # be at most 3 padding bytes
+          if template.num_bytes > length or ! (length % template.num_bytes).between?(0, 3)
+            @logger.warn("Template length doesn't fit cleanly into flowset", :template_id => record.flowset_id, :template_length => template.num_bytes, :record_length => length)
+            next
+          end
+
+          array = BinData::Array.new(:type => template, :initial_length => length / template.num_bytes)
+
+          records = array.read(record.flowset_data)
+
+          records.each do |r|
+            event = LogStash::Event.new(
+              LogStash::Event::TIMESTAMP => LogStash::Timestamp.at(flowset.unix_sec),
+              @target => {}
+            )
+
+            # Fewer fields in the v9 header
+            ['version', 'flow_seq_num'].each do |f|
+              event[@target][f] = flowset[f]
+            end
+
+            event[@target]['flowset_id'] = record.flowset_id
+
+            r.each_pair do |k,v|
+              case k.to_s
+              when /_switched$/
+                millis = flowset.uptime - v
+                seconds = flowset.unix_sec - (millis / 1000)
+                # v9 did away with the nanosecs field
+                micros = 1000000 - (millis % 1000)
+                event[@target][k.to_s] = Time.at(seconds, micros).utc.strftime("%Y-%m-%dT%H:%M:%S.%3NZ")
+              else
+                event[@target][k.to_s] = v
+              end
+            end
+
+            yield event
+          end
+        else
+          @logger.warn("Unsupported flowset id #{record.flowset_id}")
+        end
+      end
+    end
+  end # def filter
+
+  private
+  def uint_field(length, default)
+    # If length is 4, return :uint32, etc. and use default if length is 0
+    ("uint" + (((length > 0) ? length : default) * 8).to_s).to_sym
+  end # def uint_field
+
+  private
+  def netflow_field_for(type, length)
+    if @fields.include?(type)
+      field = @fields[type]
+      if field.is_a?(Array)
+
+        if field[0].is_a?(Integer)
+          field[0] = uint_field(length, field[0])
+        end
+
+        # Small bit of fixup for skip or string field types where the length
+        # is dynamic
+        case field[0]
+        when :skip
+          field += [nil, {:length => length}]
+        when :string
+          field += [{:length => length, :trim_padding => true}]
+        end
+
+        @logger.debug("Definition complete", :field => field)
+        [field]
+      else
+        @logger.warn("Definition should be an array", :field => field)
+        nil
+      end
+    else
+      @logger.warn("Unsupported field", :type => type, :length => length)
+      nil
+    end
+  end # def netflow_field_for
+end # class LogStash::Filters::Netflow

data/lib/logstash/codecs/netflow/netflow.yaml

@@ -0,0 +1,215 @@
+---
+1:
+- 4
+- :in_bytes
+2:
+- 4
+- :in_pkts
+3:
+- 4
+- :flows
+4:
+- :uint8
+- :protocol
+5:
+- :uint8
+- :src_tos
+6:
+- :uint8
+- :tcp_flags
+7:
+- :uint16
+- :l4_src_port
+8:
+- :ip4_addr
+- :ipv4_src_addr
+9:
+- :uint8
+- :src_mask
+10:
+- 2
+- :input_snmp
+11:
+- :uint16
+- :l4_dst_port
+12:
+- :ip4_addr
+- :ipv4_dst_addr
+13:
+- :uint8
+- :dst_mask
+14:
+- 2
+- :output_snmp
+15:
+- :ip4_addr
+- :ipv4_next_hop
+16:
+- 2
+- :src_as
+17:
+- 2
+- :dst_as
+18:
+- :ip4_addr
+- :bgp_ipv4_next_hop
+19:
+- 4
+- :mul_dst_pkts
+20:
+- 4
+- :mul_dst_bytes
+21:
+- :uint32
+- :last_switched
+22:
+- :uint32
+- :first_switched
+23:
+- 4
+- :out_bytes
+24:
+- 4
+- :out_pkts
+25:
+- :uint16
+- :min_pkt_length
+26:
+- :uint16
+- :max_pkt_length
+27:
+- :ip6_addr
+- :ipv6_src_addr
+28:
+- :ip6_addr
+- :ipv6_dst_addr
+29:
+- :uint8
+- :ipv6_src_mask
+30:
+- :uint8
+- :ipv6_dst_mask
+31:
+- :uint24
+- :ipv6_flow_label
+32:
+- :uint16
+- :icmp_type
+33:
+- :uint8
+- :mul_igmp_type
+34:
+- :uint32
+- :sampling_interval
+35:
+- :uint8
+- :sampling_algorithm
+36:
+- :uint16
+- :flow_active_timeout
+37:
+- :uint16
+- :flow_inactive_timeout
+38:
+- :uint8
+- :engine_type
+39:
+- :uint8
+- :engine_id
+40:
+- 4
+- :total_bytes_exp
+41:
+- 4
+- :total_pkts_exp
+42:
+- 4
+- :total_flows_exp
+43:
+- :skip
+44:
+- :ip4_addr
+- :ipv4_src_prefix
+45:
+- :ip4_addr
+- :ipv4_dst_prefix
+46:
+- :uint8
+- :mpls_top_label_type
+47:
+- :uint32
+- :mpls_top_label_ip_addr
+48:
+- 4
+- :flow_sampler_id
+49:
+- :uint8
+- :flow_sampler_mode
+50:
+- :uint32
+- :flow_sampler_random_interval
+51:
+- :skip
+52:
+- :uint8
+- :min_ttl
+53:
+- :uint8
+- :max_ttl
+54:
+- :uint16
+- :ipv4_ident
+55:
+- :uint8
+- :dst_tos
+56:
+- :mac_addr
+- :in_src_max
+57:
+- :mac_addr
+- :out_dst_max
+58:
+- :uint16
+- :src_vlan
+59:
+- :uint16
+- :dst_vlan
+60:
+- :uint8
+- :ip_protocol_version
+61:
+- :uint8
+- :direction
+62:
+- :ip6_addr
+- :ipv6_next_hop
+63:
+- :ip6_addr
+- :bgp_ipv6_next_hop
+64:
+- :uint32
+- :ipv6_option_headers
+64:
+- :skip
+65:
+- :skip
+66:
+- :skip
+67:
+- :skip
+68:
+- :skip
+69:
+- :skip
+80:
+- :mac_addr
+- :in_dst_mac
+81:
+- :mac_addr
+- :out_src_mac
+82:
+- :string
+- :if_name
+83:
+- :string
+- :if_desc

data/lib/logstash/codecs/netflow/util.rb

@@ -0,0 +1,212 @@
+# encoding: utf-8
+require "bindata"
+require "ipaddr"
+
+class IP4Addr < BinData::Primitive
+  endian :big
+  uint32 :storage
+
+  def set(val)
+    ip = IPAddr.new(val)
+    if ! ip.ipv4?
+      raise ArgumentError, "invalid IPv4 address '#{val}'"
+    end
+    self.storage = ip.to_i
+  end
+
+  def get
+    IPAddr.new_ntoh([self.storage].pack('N')).to_s
+  end
+end
+
+class IP6Addr < BinData::Primitive
+  endian  :big
+  uint128 :storage
+
+  def set(val)
+    ip = IPAddr.new(val)
+    if ! ip.ipv6?
+      raise ArgumentError, "invalid IPv6 address `#{val}'"
+    end
+    self.storage = ip.to_i
+  end
+
+  def get
+    IPAddr.new_ntoh((0..7).map { |i|
+      (self.storage >> (112 - 16 * i)) & 0xffff
+    }.pack('n8')).to_s
+  end
+end
+
+class MacAddr < BinData::Primitive
+  array :bytes, :type => :uint8, :initial_length => 6
+
+  def set(val)
+    ints = val.split(/:/).collect { |int| int.to_i(16) }
+    self.bytes = ints
+  end
+
+  def get
+    self.bytes.collect { |byte| byte.to_s(16) }.join(":")
+  end
+end
+
+class Header < BinData::Record
+  endian :big
+  uint16 :version
+end
+
+class Netflow5PDU < BinData::Record
+  endian :big
+  uint16 :version
+  uint16 :flow_records
+  uint32 :uptime
+  uint32 :unix_sec
+  uint32 :unix_nsec
+  uint32 :flow_seq_num
+  uint8  :engine_type
+  uint8  :engine_id
+  bit2   :sampling_algorithm
+  bit14  :sampling_interval
+  array  :records, :initial_length => :flow_records do
+    ip4_addr :ipv4_src_addr
+    ip4_addr :ipv4_dst_addr
+    ip4_addr :ipv4_next_hop
+    uint16   :input_snmp
+    uint16   :output_snmp
+    uint32   :in_pkts
+    uint32   :in_bytes
+    uint32   :first_switched
+    uint32   :last_switched
+    uint16   :l4_src_port
+    uint16   :l4_dst_port
+    skip     :length => 1
+    uint8    :tcp_flags # Split up the TCP flags maybe?
+    uint8    :protocol
+    uint8    :src_tos
+    uint16   :src_as
+    uint16   :dst_as
+    uint8    :src_mask
+    uint8    :dst_mask
+    skip     :length => 2
+  end
+end
+
+class TemplateFlowset < BinData::Record
+  endian :big
+  array  :templates, :read_until => lambda { array.num_bytes == flowset_length - 4 } do
+    uint16 :template_id
+    uint16 :field_count
+    array  :fields, :initial_length => :field_count do
+      uint16 :field_type
+      uint16 :field_length
+    end
+  end
+end
+
+class OptionFlowset < BinData::Record
+  endian :big
+  array  :templates, :read_until => lambda { flowset_length - 4 - array.num_bytes <= 2 } do
+    uint16 :template_id
+    uint16 :scope_length
+    uint16 :option_length
+    array  :scope_fields, :initial_length => lambda { scope_length / 4 } do
+      uint16 :field_type
+      uint16 :field_length
+    end
+    array  :option_fields, :initial_length => lambda { option_length / 4 } do
+      uint16 :field_type
+      uint16 :field_length
+    end
+  end
+  skip   :length => lambda { templates.length.odd? ? 2 : 0 }
+end
+
+class Netflow9PDU < BinData::Record
+  endian :big
+  uint16 :version
+  uint16 :flow_records
+  uint32 :uptime
+  uint32 :unix_sec
+  uint32 :flow_seq_num
+  uint32 :source_id
+  array  :records, :read_until => :eof do
+    uint16 :flowset_id
+    uint16 :flowset_length
+    choice :flowset_data, :selection => :flowset_id do
+      template_flowset 0
+      option_flowset   1
+      string           :default, :read_length => lambda { flowset_length - 4 }
+    end
+  end
+end
+
+# https://gist.github.com/joshaven/184837
+class Vash < Hash
+  def initialize(constructor = {})
+    @register ||= {}
+    if constructor.is_a?(Hash)
+      super()
+      merge(constructor)
+    else
+      super(constructor)
+    end
+  end
+
+  alias_method :regular_writer, :[]= unless method_defined?(:regular_writer)
+  alias_method :regular_reader, :[] unless method_defined?(:regular_reader)
+
+  def [](key)
+    sterilize(key)
+    clear(key) if expired?(key)
+    regular_reader(key)
+  end
+
+  def []=(key, *args)
+    if args.length == 2
+      value, ttl = args[1], args[0]
+    elsif args.length == 1
+      value, ttl = args[0], 60
+    else
+      raise ArgumentError, "Wrong number of arguments, expected 2 or 3, received: #{args.length+1}\n"+
+                           "Example Usage:  volatile_hash[:key]=value OR volatile_hash[:key, ttl]=value"
+    end
+    sterilize(key)
+    ttl(key, ttl)
+    regular_writer(key, value)
+  end
+
+  def merge(hsh)
+    hsh.map {|key,value| self[sterile(key)] = hsh[key]}
+    self
+  end
+
+  def cleanup!
+    now = Time.now.to_i
+    @register.map {|k,v| clear(k) if v < now}
+  end
+
+  def clear(key)
+    sterilize(key)
+    @register.delete key
+    self.delete key
+  end
+
+  private
+  def expired?(key)
+    Time.now.to_i > @register[key].to_i
+  end
+
+  def ttl(key, secs=60)
+    @register[key] = Time.now.to_i + secs.to_i
+  end
+
+  def sterile(key)
+    String === key ? key.chomp('!').chomp('=') : key.to_s.chomp('!').chomp('=').to_sym
+  end
+
+  def sterilize(key)
+    key = sterile(key)
+  end
+end
+

data/logstash-codec-netflow.gemspec

@@ -0,0 +1,26 @@
+Gem::Specification.new do |s|
+
+  s.name            = 'logstash-codec-netflow'
+  s.version         = '0.1.0'
+  s.licenses        = ['Apache License (2.0)']
+  s.summary         = "The netflow codec is for decoding Netflow v5/v9 flows."
+  s.description     = "The netflow codec is for decoding Netflow v5/v9 flows."
+  s.authors         = ["Elasticsearch"]
+  s.email           = 'richard.pijnenburg@elasticsearch.com'
+  s.homepage        = "http://logstash.net/"
+  s.require_paths = ["lib"]
+
+  # Files
+  s.files = `git ls-files`.split($\)
+
+  # Tests
+  s.test_files = s.files.grep(%r{^(test|spec|features)/})
+
+  # Special flag to let us know this is actually a logstash plugin
+  s.metadata = { "logstash_plugin" => "true", "group" => "codec" }
+
+  # Gem dependencies
+  s.add_runtime_dependency 'logstash', '>= 1.4.0', '< 2.0.0'
+  s.add_runtime_dependency 'bindata', ['>= 1.5.0']
+end
+

data/rakelib/publish.rake

@@ -0,0 +1,9 @@
+require "gem_publisher"
+
+desc "Publish gem to RubyGems.org"
+task :publish_gem do |t|
+  gem_file = Dir.glob(File.expand_path('../*.gemspec',File.dirname(__FILE__))).first
+  gem = GemPublisher.publish_if_updated(gem_file, :rubygems)
+  puts "Published #{gem}" if gem
+end
+

data/rakelib/vendor.rake

@@ -0,0 +1,169 @@
+require "net/http"
+require "uri"
+require "digest/sha1"
+
+def vendor(*args)
+  return File.join("vendor", *args)
+end
+
+directory "vendor/" => ["vendor"] do |task, args|
+  mkdir task.name
+end
+
+def fetch(url, sha1, output)
+
+  puts "Downloading #{url}"
+  actual_sha1 = download(url, output)
+
+  if actual_sha1 != sha1
+    fail "SHA1 does not match (expected '#{sha1}' but got '#{actual_sha1}')"
+  end
+end # def fetch
+
+def file_fetch(url, sha1)
+  filename = File.basename( URI(url).path )
+  output = "vendor/#{filename}"
+  task output => [ "vendor/" ] do
+    begin
+      actual_sha1 = file_sha1(output)
+      if actual_sha1 != sha1
+        fetch(url, sha1, output)
+      end
+    rescue Errno::ENOENT
+      fetch(url, sha1, output)
+    end
+  end.invoke
+
+  return output
+end
+
+def file_sha1(path)
+  digest = Digest::SHA1.new
+  fd = File.new(path, "r")
+  while true
+    begin
+      digest << fd.sysread(16384)
+    rescue EOFError
+      break
+    end
+  end
+  return digest.hexdigest
+ensure
+  fd.close if fd
+end
+
+def download(url, output)
+  uri = URI(url)
+  digest = Digest::SHA1.new
+  tmp = "#{output}.tmp"
+  Net::HTTP.start(uri.host, uri.port, :use_ssl => (uri.scheme == "https")) do |http|
+    request = Net::HTTP::Get.new(uri.path)
+    http.request(request) do |response|
+      fail "HTTP fetch failed for #{url}. #{response}" if [200, 301].include?(response.code)
+      size = (response["content-length"].to_i || -1).to_f
+      count = 0
+      File.open(tmp, "w") do |fd|
+        response.read_body do |chunk|
+          fd.write(chunk)
+          digest << chunk
+          if size > 0 && $stdout.tty?
+            count += chunk.bytesize
+            $stdout.write(sprintf("\r%0.2f%%", count/size * 100))
+          end
+        end
+      end
+      $stdout.write("\r      \r") if $stdout.tty?
+    end
+  end
+
+  File.rename(tmp, output)
+
+  return digest.hexdigest
+rescue SocketError => e
+  puts "Failure while downloading #{url}: #{e}"
+  raise
+ensure
+  File.unlink(tmp) if File.exist?(tmp)
+end # def download
+
+def untar(tarball, &block)
+  require "archive/tar/minitar"
+  tgz = Zlib::GzipReader.new(File.open(tarball))
+  # Pull out typesdb
+  tar = Archive::Tar::Minitar::Input.open(tgz)
+  tar.each do |entry|
+    path = block.call(entry)
+    next if path.nil?
+    parent = File.dirname(path)
+    
+    mkdir_p parent unless File.directory?(parent)
+
+    # Skip this file if the output file is the same size
+    if entry.directory?
+      mkdir path unless File.directory?(path)
+    else
+      entry_mode = entry.instance_eval { @mode } & 0777
+      if File.exists?(path)
+        stat = File.stat(path)
+        # TODO(sissel): Submit a patch to archive-tar-minitar upstream to
+        # expose headers in the entry.
+        entry_size = entry.instance_eval { @size }
+        # If file sizes are same, skip writing.
+        next if stat.size == entry_size && (stat.mode & 0777) == entry_mode
+      end
+      puts "Extracting #{entry.full_name} from #{tarball} #{entry_mode.to_s(8)}"
+      File.open(path, "w") do |fd|
+        # eof? check lets us skip empty files. Necessary because the API provided by
+        # Archive::Tar::Minitar::Reader::EntryStream only mostly acts like an
+        # IO object. Something about empty files in this EntryStream causes
+        # IO.copy_stream to throw "can't convert nil into String" on JRuby
+        # TODO(sissel): File a bug about this.
+        while !entry.eof?
+          chunk = entry.read(16384)
+          fd.write(chunk)
+        end
+          #IO.copy_stream(entry, fd)
+      end
+      File.chmod(entry_mode, path)
+    end
+  end
+  tar.close
+  File.unlink(tarball) if File.file?(tarball)
+end # def untar
+
+def ungz(file)
+
+  outpath = file.gsub('.gz', '')
+  tgz = Zlib::GzipReader.new(File.open(file))
+  begin
+    File.open(outpath, "w") do |out|
+      IO::copy_stream(tgz, out)
+    end
+    File.unlink(file)
+  rescue
+    File.unlink(outpath) if File.file?(outpath)
+   raise
+  end
+  tgz.close
+end
+
+desc "Process any vendor files required for this plugin"
+task "vendor" do |task, args|
+
+  @files.each do |file| 
+    download = file_fetch(file['url'], file['sha1'])
+    if download =~ /.tar.gz/
+      prefix = download.gsub('.tar.gz', '').gsub('vendor/', '')
+      untar(download) do |entry|
+        if !file['files'].nil?
+          next unless file['files'].include?(entry.full_name.gsub(prefix, ''))
+          out = entry.full_name.split("/").last
+        end
+        File.join('vendor', out)
+      end
+    elsif download =~ /.gz/
+      ungz(download)
+    end
+  end
+
+end

data/spec/codecs/netflow_spec.rb

@@ -0,0 +1 @@
+require 'spec_helper'

metadata

@@ -0,0 +1,93 @@
+--- !ruby/object:Gem::Specification
+name: logstash-codec-netflow
+version: !ruby/object:Gem::Version
+  version: 0.1.0
+platform: ruby
+authors:
+- Elasticsearch
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2014-11-05 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: logstash
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: 1.4.0
+    - - <
+      - !ruby/object:Gem::Version
+        version: 2.0.0
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: 1.4.0
+    - - <
+      - !ruby/object:Gem::Version
+        version: 2.0.0
+- !ruby/object:Gem::Dependency
+  name: bindata
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: !binary |-
+          MS41LjA=
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: !binary |-
+          MS41LjA=
+description: The netflow codec is for decoding Netflow v5/v9 flows.
+email: richard.pijnenburg@elasticsearch.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- .gitignore
+- Gemfile
+- LICENSE
+- Rakefile
+- lib/logstash/codecs/netflow.rb
+- lib/logstash/codecs/netflow/netflow.yaml
+- lib/logstash/codecs/netflow/util.rb
+- logstash-codec-netflow.gemspec
+- rakelib/publish.rake
+- rakelib/vendor.rake
+- spec/codecs/netflow_spec.rb
+homepage: http://logstash.net/
+licenses:
+- Apache License (2.0)
+metadata:
+  logstash_plugin: 'true'
+  group: codec
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.4.1
+signing_key: 
+specification_version: 4
+summary: The netflow codec is for decoding Netflow v5/v9 flows.
+test_files:
+- spec/codecs/netflow_spec.rb