fluent-plugin-netflow 0.1.2 → 0.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 1374b820e1a4d0cb22669c1a4bd2b671959cb032
-  data.tar.gz: 2539e059a991ae7b3fbb6af818b86a6e376646c8
+  metadata.gz: d5a011a605cc39a2aff47556d09b98973bd84eb9
+  data.tar.gz: 779a28e68e6cd2bcd480d74ddcaa69605111839b
 SHA512:
-  metadata.gz: 524a0902155144ee32939939617e5051f7d6618f87c4c1316476a0738839f46b6b39d317246cba239fd67102f7e061486f4dea53eed7b2c875b5581a76753b43
-  data.tar.gz: c27c449af0a538fed071427472905aececdd9daeacfa6405123137f578cc4d75be22ed260e549874aa5f6220ddefba6d0ccbbec9b4903ba1e7951a1123540bf9
+  metadata.gz: 0c9396c4d0b6f9b8f1d6fda6ca0a53bdb4c0772fb12780e5a52930e840c886f73b4b6221cf6225773053038cf532319aab0c3439797b244239a20c3816462ae1
+  data.tar.gz: 6e6e44c7685bff5996bd068b068308934245f32928ab0e334623dfadd0359e7dc14658d96a7c85dded55dd58a7c9c155f0f3e033626cf3923295870f4dcc3977

data/.gitignore

@@ -0,0 +1,24 @@
+*.gem
+*.rbc
+.bundle
+.config
+.yardoc
+Gemfile.lock
+InstalledFiles
+_yardoc
+coverage
+doc/
+lib/bundler/man
+pkg
+rdoc
+spec/reports
+test/tmp
+test/version_tmp
+tmp
+# For TextMate, emacs, vim
+*.tmproj
+tmtags
+*~
+\#*
+.\#*
+*.swp

data/.travis.yml

@@ -1,19 +1,18 @@
 language: ruby
 
 rvm:
-  - 1.9.3
   - 2.0.0
   - 2.1
+  - 2.2
+  - 2.3.0
   - ruby-head
   - rbx
 
-branches:
-  only:
-    - master
-
 matrix:
   allow_failures:
     - rvm: ruby-head
     - rvm: rbx
 
+before_install: gem update bundler
+
 script: bundle exec rake test

data/Rakefile

@@ -6,7 +6,7 @@ require 'rake/testtask'
 
 Rake::TestTask.new(:test) do |test|
   test.libs << 'lib' << 'test'
-  test.test_files = FileList['test/*.rb']
+  test.test_files = FileList['test/**/test_*.rb']
   test.verbose = true
 end
 

data/VERSION

@@ -1 +1 @@
-0.1.2
+0.2.0

data/example/fluentd.conf

@@ -0,0 +1,9 @@
+<source>
+  @type netflow
+  bind 127.0.0.1
+  tag  example.netflow
+</source>
+
+<match example.netflow>
+  @type stdout
+</match>

data/fluent-plugin-netflow.gemspec

@@ -20,4 +20,5 @@ Gem::Specification.new do |gem|
   gem.add_dependency "fluentd", [">= 0.10.17", "< 2"]
   gem.add_dependency "bindata", "~> 2.1"
   gem.add_development_dependency "rake", ">= 0.9.2"
+  gem.add_development_dependency "test-unit", "~> 3.0"
 end

data/lib/fluent/plugin/in_netflow.rb

@@ -15,21 +15,18 @@
 #    See the License for the specific language governing permissions and
 #    limitations under the License.
 #
+require 'cool.io'
+require 'fluent/plugin/socket_util'
+require 'fluent/plugin/parser_netflow'
+
 module Fluent
   class NetflowInput < Input
     Plugin.register_input('netflow', self)
 
-    def initialize
-      super
-      require 'cool.io'
-      require 'fluent/plugin/socket_util'
-      require 'fluent/plugin/parser_netflow'
-    end
-
-    config_param :port, :integer, :default => 5140
-    config_param :bind, :string, :default => '0.0.0.0'
+    config_param :port, :integer, default: 5140
+    config_param :bind, :string, default: '0.0.0.0'
     config_param :tag, :string
-    config_param :protocol_type, :default => :udp do |val|
+    config_param :protocol_type, default: :udp do |val|
       case val.downcase
       when 'udp'
         :udp
@@ -62,8 +59,8 @@ module Fluent
 
     def run
       @loop.run
-    rescue
-      log.error "unexpected error", :error=>$!.to_s
+    rescue => e
+      log.error "unexpected error", error_class: e.class, error: e.message
       log.error_backtrace
     end
 
@@ -82,7 +79,7 @@ module Fluent
         router.emit(@tag, time, record)
       }
     rescue => e
-      log.warn data.dump, :error => e.to_s
+      log.warn "unexpected error on parsing", data: data.dump, error_class: e.class, error: e.message
       log.warn_backtrace
     end
 
@@ -109,8 +106,9 @@ module Fluent
       def on_readable
         msg, addr = @io.recvfrom_nonblock(4096)
         @callback.call(addr[3], msg)
-      rescue
-        # TODO log?
+      rescue => e
+        log.error "unexpected error on reading from socket", error_class: e.class, error: e.message
+        log.error_backtrace
       end
     end
   end

data/lib/fluent/plugin/netflow_records.rb

@@ -0,0 +1,160 @@
+require "bindata"
+
+module Fluent
+  class TextParser
+    class NetflowParser < Parser
+      class IP4Addr < BinData::Primitive
+        endian :big
+        uint32 :storage
+
+        def set(val)
+          ip = IPAddr.new(val)
+          if ! ip.ipv4?
+            raise ArgumentError, "invalid IPv4 address '#{val}'"
+          end
+          self.storage = ip.to_i
+        end
+
+        def get
+          IPAddr.new_ntoh([self.storage].pack('N')).to_s
+        end
+      end
+
+      class IP6Addr < BinData::Primitive
+        endian  :big
+        uint128 :storage
+
+        def set(val)
+          ip = IPAddr.new(val)
+          if ! ip.ipv6?
+            raise ArgumentError, "invalid IPv6 address `#{val}'"
+          end
+          self.storage = ip.to_i
+        end
+
+        def get
+          IPAddr.new_ntoh((0..7).map { |i|
+              (self.storage >> (112 - 16 * i)) & 0xffff
+            }.pack('n8')).to_s
+        end
+      end
+
+      class MacAddr < BinData::Primitive
+        array :bytes, type: :uint8, initial_length: 6
+
+        def set(val)
+          ints = val.split(/:/).collect { |int| int.to_i(16) }
+          self.bytes = ints
+        end
+
+        def get
+          self.bytes.collect { |byte| byte.value.to_s(16).rjust(2,'0') }.join(":")
+        end
+      end
+
+      class MplsLabel < BinData::Primitive
+        bit20 :label
+        bit3  :exp
+        bit1  :bottom
+        def set(val)
+          self.label = val >> 4
+          self.exp = (val & 0b1111) >> 1
+          self.bottom = val & 0b1
+        end
+        def get
+          self.label
+        end
+      end
+
+      class Header < BinData::Record
+        endian :big
+        uint16 :version
+      end
+
+      class Netflow5PDU < BinData::Record
+        endian :big
+        uint16 :version
+        uint16 :flow_records
+        uint32 :uptime
+        uint32 :unix_sec
+        uint32 :unix_nsec
+        uint32 :flow_seq_num
+        uint8  :engine_type
+        uint8  :engine_id
+        bit2   :sampling_algorithm
+        bit14  :sampling_interval
+        array  :records, initial_length: :flow_records do
+          ip4_addr :ipv4_src_addr
+          ip4_addr :ipv4_dst_addr
+          ip4_addr :ipv4_next_hop
+          uint16   :input_snmp
+          uint16   :output_snmp
+          uint32   :in_pkts
+          uint32   :in_bytes
+          uint32   :first_switched
+          uint32   :last_switched
+          uint16   :l4_src_port
+          uint16   :l4_dst_port
+          skip     length: 1
+          uint8    :tcp_flags # Split up the TCP flags maybe?
+          uint8    :protocol
+          uint8    :src_tos
+          uint16   :src_as
+          uint16   :dst_as
+          uint8    :src_mask
+          uint8    :dst_mask
+          skip     length: 2
+        end
+      end
+
+      class TemplateFlowset < BinData::Record
+        endian :big
+        array  :templates, read_until: lambda { array.num_bytes == flowset_length - 4 } do
+          uint16 :template_id
+          uint16 :field_count
+          array  :fields, initial_length: :field_count do
+            uint16 :field_type
+            uint16 :field_length
+          end
+        end
+      end
+
+      class OptionFlowset < BinData::Record
+        endian :big
+        array  :templates, read_until: lambda { flowset_length - 4 - array.num_bytes <= 2 } do
+          uint16 :template_id
+          uint16 :scope_length
+          uint16 :option_length
+          array  :scope_fields, initial_length: lambda { scope_length / 4 } do
+            uint16 :field_type
+            uint16 :field_length
+          end
+          array  :option_fields, initial_length: lambda { option_length / 4 } do
+            uint16 :field_type
+            uint16 :field_length
+          end
+        end
+        skip   length: lambda { templates.length.odd? ? 2 : 0 }
+      end
+
+      class Netflow9PDU < BinData::Record
+        endian :big
+        uint16 :version
+        uint16 :flow_records
+        uint32 :uptime
+        uint32 :unix_sec
+        uint32 :flow_seq_num
+        uint32 :source_id
+        array  :records, read_until: :eof do
+          uint16 :flowset_id
+          uint16 :flowset_length
+          choice :flowset_data, selection: :flowset_id do
+            template_flowset 0
+            option_flowset   1
+            string           :default, read_length: lambda { flowset_length - 4 }
+          end
+        end
+      end
+    end
+  end
+end

data/lib/fluent/plugin/parser_netflow.rb

@@ -1,18 +1,26 @@
-require "bindata"
 require "ipaddr"
 require 'yaml'
 
 require 'fluent/parser'
 
+require_relative 'netflow_records'
+require_relative 'vash'
+
 module Fluent
   class TextParser
     # port from logstash's netflow parser
     class NetflowParser < Parser
       Plugin.register_parser('netflow', self)
 
-      config_param :cache_ttl, :integer, :default => 4000
-      config_param :versions, :array, :default => [5, 9]
-      config_param :definitions, :string, :default => nil
+      config_param :switched_times_from_uptime, :bool, default: false
+      config_param :cache_ttl, :integer, default: 4000
+      config_param :versions, :array, default: [5, 9]
+      config_param :definitions, :string, default: nil
+
+      # Cisco NetFlow Export Datagram Format
+      # http://www.cisco.com/c/en/us/td/docs/net_mgmt/netflow_collection_engine/3-6/user/guide/format.html
+      # Cisco NetFlow Version 9 Flow-Record Format
+      # http://www.cisco.com/en/US/technologies/tk648/tk362/technologies_white_paper09186a00800a3db9.html
 
       def configure(conf)
         super
@@ -23,8 +31,8 @@ module Fluent
 
         begin
           @fields = YAML.load_file(filename)
-        rescue Exception => e
-          raise "Bad syntax in definitions file #{filename}"
+        rescue => e
+          raise "Bad syntax in definitions file #{filename}", error_class: e.class, error: e.message
         end
 
         # Allow the user to augment/override/rename the supported Netflow fields
@@ -32,8 +40,8 @@ module Fluent
           raise "definitions file #{@definitions} does not exists" unless File.exist?(@definitions)
           begin
             @fields.merge!(YAML.load_file(@definitions))
-          rescue Exception => e
-            raise "Bad syntax in definitions file #{@definitions}"
+          rescue => e
+            raise "Bad syntax in definitions file #{@definitions}", error_class: e.class, error: e.message
           end
         end
         # Path to default Netflow v9 scope field definitions
@@ -41,427 +49,291 @@ module Fluent
 
         begin
           @scope_fields = YAML.load_file(filename)
-        rescue Exception => e
-          raise "Bad syntax in scope definitions file #{filename}"
+        rescue => e
+          raise "Bad syntax in scope definitions file #{filename}", error_class: e.class, error: e.message
         end
       end
 
-      def call(payload)
-        header = Header.read(payload)
-        unless @versions.include?(header.version)
-          $log.warn "Ignoring Netflow version v#{header.version}"
-          return
-        end
-
-        if header.version == 5
-          flowset = Netflow5PDU.read(payload)
-        elsif header.version == 9
+      def call(payload, &block)
+        version,_ = payload[0,2].unpack('n')
+        case version
+        when 5
+          forV5(payload, block)
+        when 9
+          # TODO: implement forV9
           flowset = Netflow9PDU.read(payload)
+          handle_v9(flowset, block)
         else
-          $log.warn "Unsupported Netflow version v#{header.version}"
-          return
+          $log.warn "Unsupported Netflow version v#{version}: #{version.class}"
         end
+      end
 
-        flowset.records.each do |record|
-          if flowset.version == 5
-            event = {}
-
-            # FIXME Probably not doing this right WRT JRuby?
-            #
-            # The flowset header gives us the UTC epoch seconds along with
-            # residual nanoseconds so we can set @timestamp to that easily
-            time = flowset.unix_sec
-
-            # Copy some of the pertinent fields in the header to the event
-            ['version', 'flow_seq_num', 'engine_type', 'engine_id', 'sampling_algorithm', 'sampling_interval', 'flow_records'].each do |f|
-              event[f] = flowset[f]
-            end
-
-            # Create fields in the event from each field in the flow record
-            record.each_pair do |k,v|
-              case k.to_s
-              when /_switched$/
-                # The flow record sets the first and last times to the device
-                # uptime in milliseconds. Given the actual uptime is provided
-                # in the flowset header along with the epoch seconds we can
-                # convert these into absolute times
-                millis = flowset.uptime - v
-                seconds = flowset.unix_sec - (millis / 1000)
-                micros = (flowset.unix_nsec / 1000) - (millis % 1000)
-                if micros < 0
-                  seconds--
-                    micros += 1000000
-                end
-
-                # FIXME Again, probably doing this wrong WRT JRuby?
-                event[k.to_s] = Time.at(seconds, micros).utc.strftime("%Y-%m-%dT%H:%M:%S.%3NZ")
-              else
-                event[k.to_s] = v
-              end
-            end
-
-            yield time, event
-          elsif flowset.version == 9
-            case record.flowset_id
-            when 0
-              # Template flowset
-              record.flowset_data.templates.each do |template|
-                catch (:field) do
-                  fields = []
-                  template.fields.each do |field|
-                    entry = netflow_field_for(field.field_type, field.field_length, @fields)
-                    if !entry
-                      throw :field
-                    end
-                    fields += entry
-                  end
-                  # We get this far, we have a list of fields
-                  #key = "#{flowset.source_id}|#{event["source"]}|#{template.template_id}"
-                  key = "#{flowset.source_id}|#{template.template_id}"
-                  @templates[key, @cache_ttl] = BinData::Struct.new(:endian => :big, :fields => fields)
-                  # Purge any expired templates
-                  @templates.cleanup!
-                end
-              end
-            when 1
-              # Options template flowset
-              record.flowset_data.templates.each do |template|
-                catch (:field) do
-                  fields = []
-                  template.scope_fields.each do |field|
-                    entry = netflow_field_for(field.field_type, field.field_length, @scope_fields)
-                    if ! entry
-                      throw :field
-                    end
-                    fields += entry
-                  end
-                  template.option_fields.each do |field|
-                    entry = netflow_field_for(field.field_type, field.field_length, @fields)
-                    if ! entry
-                      throw :field
-                    end
-                    fields += entry
-                  end
-                  # We get this far, we have a list of fields
-                  #key = "#{flowset.source_id}|#{event["source"]}|#{template.template_id}"
-                  key = "#{flowset.source_id}|#{template.template_id}"
-                  @templates[key, @cache_ttl] = BinData::Struct.new(:endian => :big, :fields => fields)
-                  # Purge any expired templates
-                  @templates.cleanup!
-                end
-              end
-            when 256..65535
-              # Data flowset
-              #key = "#{flowset.source_id}|#{event["source"]}|#{record.flowset_id}"
-              key = "#{flowset.source_id}|#{record.flowset_id}"
-              template = @templates[key]
-              if ! template
-                #$log.warn("No matching template for flow id #{record.flowset_id} from #{event["source"]}")
-                $log.warn("No matching template for flow id #{record.flowset_id}")
-                next
-              end
-
-              length = record.flowset_length - 4
+      private
 
-              # Template shouldn't be longer than the record and there should
-              # be at most 3 padding bytes
-              if template.num_bytes > length or ! (length % template.num_bytes).between?(0, 3)
-                $log.warn("Template length doesn't fit cleanly into flowset", :template_id => record.flowset_id, :template_length => template.num_bytes, :record_length => length)
-                next
-              end
+      def ipv4_addr_to_string(uint32)
+        "#{(uint32 & 0xff000000) >> 24}.#{(uint32 & 0x00ff0000) >> 16}.#{(uint32 & 0x0000ff00) >> 8}.#{uint32 & 0x000000ff}"
+      end
 
-              array = BinData::Array.new(:type => template, :initial_length => length / template.num_bytes)
-
-              records = array.read(record.flowset_data)
-              records.each do |r|
-                time = flowset.unix_sec
-                event = {}
-
-                # Fewer fields in the v9 header
-                ['version', 'flow_seq_num'].each do |f|
-                  event[f] = flowset[f]
-                end
-
-                event['flowset_id'] = record.flowset_id
-
-                r.each_pair do |k,v|
-                  case k.to_s
-                  when /_switched$/
-                    millis = flowset.uptime - v
-                    seconds = flowset.unix_sec - (millis / 1000)
-                    # v9 did away with the nanosecs field
-                    micros = 1000000 - (millis % 1000)
-                    event[k.to_s] = Time.at(seconds, micros).utc.strftime("%Y-%m-%dT%H:%M:%S.%3NZ")
-                  else
-                    event[k.to_s] = v
-                  end
-                end
-
-                yield time, event
-              end
-            else
-              $log.warn("Unsupported flowset id #{record.flowset_id}")
-            end
-          end
+      def msec_from_boot_to_time(msec, uptime, current_unix_time, current_nsec)
+        millis = uptime - msec
+        seconds = current_unix_time - (millis / 1000)
+        micros = (current_nsec / 1000) - ((millis % 1000) * 1000)
+        if micros < 0
+          seconds -= 1
+          micros += 1000000
         end
+        Time.at(seconds, micros)
       end
 
-      private
-
-      def uint_field(length, default)
-        # If length is 4, return :uint32, etc. and use default if length is 0
-        ("uint" + (((length > 0) ? length : default) * 8).to_s).to_sym
+      def format_for_switched(time)
+        time.utc.strftime("%Y-%m-%dT%H:%M:%S.%3NZ")
       end
 
-      def netflow_field_for(type, length, field_definitions)
-        if field_definitions.include?(type)
-          field = field_definitions[type]
-          if field.is_a?(Array)
+      NETFLOW_V5_HEADER_FORMAT = 'nnNNNNnn'
+      NETFLOW_V5_HEADER_BYTES  = 24
+      NETFLOW_V5_RECORD_FORMAT = 'NNNnnNNNNnnnnnnnxx'
+      NETFLOW_V5_RECORD_BYTES  = 48
+
+      # V5 header
+      # uint16 :version        # n
+      # uint16 :flow_records   # n
+      # uint32 :uptime         # N
+      # uint32 :unix_sec       # N
+      # uint32 :unix_nsec      # N
+      # uint32 :flow_seq_num   # N
+      # uint8  :engine_type    # n -> 0xff00
+      # uint8  :engine_id      #   -> 0x00ff
+      # bit2   :sampling_algorithm # n -> 0b1100000000000000
+      # bit14  :sampling_interval  #   -> 0b0011111111111111
+
+      # V5 records
+      # array  :records, initial_length: :flow_records do
+      #   ip4_addr :ipv4_src_addr # uint32 N
+      #   ip4_addr :ipv4_dst_addr # uint32 N
+      #   ip4_addr :ipv4_next_hop # uint32 N
+      #   uint16   :input_snmp    # n
+      #   uint16   :output_snmp   # n
+      #   uint32   :in_pkts       # N
+      #   uint32   :in_bytes      # N
+      #   uint32   :first_switched # N
+      #   uint32   :last_switched  # N
+      #   uint16   :l4_src_port    # n
+      #   uint16   :l4_dst_port    # n
+      #   skip     length: 1  # n -> (ignored)
+      #   uint8    :tcp_flags #   -> 0x00ff
+      #   uint8    :protocol  # n -> 0xff00
+      #   uint8    :src_tos   #   -> 0x00ff
+      #   uint16   :src_as   # n
+      #   uint16   :dst_as   # n
+      #   uint8    :src_mask # n -> 0xff00
+      #   uint8    :dst_mask #   -> 0x00ff
+      #   skip     length: 2 # xx
+      # end
+      def forV5(payload, block)
+        version, flow_records, uptime, unix_sec, unix_nsec, flow_seq_num, engine, sampling = payload.unpack(NETFLOW_V5_HEADER_FORMAT)
+        engine_type = (engine & 0xff00) >> 8
+        engine_id = engine & 0x00ff
+        sampling_algorithm = (sampling & 0b1100000000000000) >> 14
+        sampling_interval = sampling & 0b0011111111111111
+
+        time = Time.at(unix_sec, unix_nsec / 1000).to_i # TODO: Fluent::EventTime
+
+        records_bytes = payload.bytesize - NETFLOW_V5_HEADER_BYTES
+
+        if records_bytes / NETFLOW_V5_RECORD_BYTES != flow_records
+          $log.warn "bytesize mismatch, records_bytes:#{records_bytes}, records:#{flow_records}"
+          return
+        end
 
-            if field[0].is_a?(Integer)
-              field[0] = uint_field(length, field[0])
-            end
+        format_full = NETFLOW_V5_RECORD_FORMAT * flow_records
+        objects = payload[NETFLOW_V5_HEADER_BYTES, records_bytes].unpack(format_full)
+
+        while objects.size > 0
+          src_addr, dst_addr, next_hop, input_snmp, output_snmp,
+          in_pkts, in_bytes, first_switched, last_switched, l4_src_port, l4_dst_port,
+          tcp_flags_16, protocol_src_tos, src_as, dst_as, src_dst_mask = objects.shift(16)
+          record = {
+            "version" => version,
+            "uptime"  => uptime,
+            "flow_records" => flow_records,
+            "flow_seq_num" => flow_seq_num,
+            "engine_type"  => engine_type,
+            "engine_id"    => engine_id,
+            "sampling_algorithm" => sampling_algorithm,
+            "sampling_interval"  => sampling_interval,
+
+            "ipv4_src_addr" => ipv4_addr_to_string(src_addr),
+            "ipv4_dst_addr" => ipv4_addr_to_string(dst_addr),
+            "ipv4_next_hop" => ipv4_addr_to_string(next_hop),
+            "input_snmp"  => input_snmp,
+            "output_snmp" => output_snmp,
+            "in_pkts"  => in_pkts,
+            "in_bytes" => in_bytes,
+            "first_switched" => first_switched,
+            "last_switched"  => last_switched,
+            "l4_src_port" => l4_src_port,
+            "l4_dst_port" => l4_dst_port,
+            "tcp_flags" => tcp_flags_16 & 0x00ff,
+            "protocol" => (protocol_src_tos & 0xff00) >> 8,
+            "src_tos"  => (protocol_src_tos & 0x00ff),
+            "src_as"   => src_as,
+            "dst_as"   => dst_as,
+            "src_mask" => (src_dst_mask & 0xff00) >> 8,
+            "dst_mask" => (src_dst_mask & 0x00ff)
+          }
+          unless @switched_times_from_uptime
+            record["first_switched"] = format_for_switched(msec_from_boot_to_time(record["first_switched"], uptime, unix_sec, unix_nsec))
+            record["last_switched"]  = format_for_switched(msec_from_boot_to_time(record["last_switched"] , uptime, unix_sec, unix_nsec))
+          end
 
-            # Small bit of fixup for skip or string field types where the length
-            # is dynamic
-            case field[0]
-            when :skip
-              field += [nil, {:length => length}]
-            when :string
-              field += [{:length => length, :trim_padding => true}]
-            end
+          block.call(time, record)
+        end
+      end
 
-            [field]
+      def handle_v9(flowset, block)
+        flowset.records.each do |record|
+          case record.flowset_id
+          when 0
+            handle_v9_flowset_template(flowset, record)
+          when 1
+            handle_v9_flowset_options_template(flowset, record)
+          when 256..65535
+            handle_v9_flowset_data(flowset, record, block)
           else
-            $log.warn("Definition should be an array", :field => field)
-            nil
+            $log.warn "Unsupported flowset id #{record.flowset_id}"
           end
-        else
-          $log.warn("Unsupported field", :type => type, :length => length)
-          nil
         end
       end
 
-      class IP4Addr < BinData::Primitive
-        endian :big
-        uint32 :storage
-
-        def set(val)
-          ip = IPAddr.new(val)
-          if ! ip.ipv4?
-            raise ArgumentError, "invalid IPv4 address '#{val}'"
+      def handle_v9_flowset_template(flowset, record)
+        record.flowset_data.templates.each do |template|
+          catch (:field) do
+            fields = []
+            template.fields.each do |field|
+              entry = netflow_field_for(field.field_type, field.field_length, @fields)
+              if !entry
+                throw :field
+              end
+              fields += entry
+            end
+            # We get this far, we have a list of fields
+            key = "#{flowset.source_id}|#{template.template_id}"
+            @templates[key, @cache_ttl] = BinData::Struct.new(endian: :big, fields: fields)
+            # Purge any expired templates
+            @templates.cleanup!
           end
-          self.storage = ip.to_i
-        end
-
-        def get
-          IPAddr.new_ntoh([self.storage].pack('N')).to_s
         end
       end
 
-      class IP6Addr < BinData::Primitive
-        endian  :big
-        uint128 :storage
-
-        def set(val)
-          ip = IPAddr.new(val)
-          if ! ip.ipv6?
-            raise ArgumentError, "invalid IPv6 address `#{val}'"
+      def handle_v9_flowset_options_template(flowset, record)
+        record.flowset_data.templates.each do |template|
+          catch (:field) do
+            fields = []
+            template.scope_fields.each do |field|
+              entry = netflow_field_for(field.field_type, field.field_length, @scope_fields)
+              if ! entry
+                throw :field
+              end
+              fields += entry
+            end
+            template.option_fields.each do |field|
+              entry = netflow_field_for(field.field_type, field.field_length, @fields)
+              if ! entry
+                throw :field
+              end
+              fields += entry
+            end
+            # We get this far, we have a list of fields
+            key = "#{flowset.source_id}|#{template.template_id}"
+            @templates[key, @cache_ttl] = BinData::Struct.new(endian: :big, fields: fields)
+            # Purge any expired templates
+            @templates.cleanup!
           end
-          self.storage = ip.to_i
-        end
-
-        def get
-          IPAddr.new_ntoh((0..7).map { |i|
-              (self.storage >> (112 - 16 * i)) & 0xffff
-            }.pack('n8')).to_s
         end
       end
 
-      class MacAddr < BinData::Primitive
-        array :bytes, :type => :uint8, :initial_length => 6
+      FIELDS_FOR_COPY_V9 = ['version', 'flow_seq_num']
 
-        def set(val)
-          ints = val.split(/:/).collect { |int| int.to_i(16) }
-          self.bytes = ints
+      def handle_v9_flowset_data(flowset, record, block)
+        key = "#{flowset.source_id}|#{record.flowset_id}"
+        template = @templates[key]
+        if ! template
+          $log.warn("No matching template for flow id #{record.flowset_id}")
+          return
         end
 
-        def get
-          self.bytes.collect { |byte| byte.value.to_s(16).rjust(2,'0') }.join(":")
-        end
-      end
+        length = record.flowset_length - 4
 
-      class MplsLabel < BinData::Primitive
-        bit20 :label
-        bit3  :exp
-        bit1  :bottom
-        def set(val)
-          self.label = val >> 4
-          self.exp = (val & 0b1111) >> 1
-          self.bottom = val & 0b1
-        end
-        def get
-          self.label
+        # Template shouldn't be longer than the record and there should
+        # be at most 3 padding bytes
+        if template.num_bytes > length or ! (length % template.num_bytes).between?(0, 3)
+          $log.warn "Template length doesn't fit cleanly into flowset",
+                    template_id: record.flowset_id, template_length: template.num_bytes, record_length: length
+          return
         end
-      end
 
-      class Header < BinData::Record
-        endian :big
-        uint16 :version
-      end
+        array = BinData::Array.new(type: template, initial_length: length / template.num_bytes)
 
-      class Netflow5PDU < BinData::Record
-        endian :big
-        uint16 :version
-        uint16 :flow_records
-        uint32 :uptime
-        uint32 :unix_sec
-        uint32 :unix_nsec
-        uint32 :flow_seq_num
-        uint8  :engine_type
-        uint8  :engine_id
-        bit2   :sampling_algorithm
-        bit14  :sampling_interval
-        array  :records, :initial_length => :flow_records do
-          ip4_addr :ipv4_src_addr
-          ip4_addr :ipv4_dst_addr
-          ip4_addr :ipv4_next_hop
-          uint16   :input_snmp
-          uint16   :output_snmp
-          uint32   :in_pkts
-          uint32   :in_bytes
-          uint32   :first_switched
-          uint32   :last_switched
-          uint16   :l4_src_port
-          uint16   :l4_dst_port
-          skip     :length => 1
-          uint8    :tcp_flags # Split up the TCP flags maybe?
-          uint8    :protocol
-          uint8    :src_tos
-          uint16   :src_as
-          uint16   :dst_as
-          uint8    :src_mask
-          uint8    :dst_mask
-          skip     :length => 2
-        end
-      end
+        records = array.read(record.flowset_data)
+        records.each do |r|
+          time = flowset.unix_sec
+          event = {}
 
-      class TemplateFlowset < BinData::Record
-        endian :big
-        array  :templates, :read_until => lambda { array.num_bytes == flowset_length - 4 } do
-          uint16 :template_id
-          uint16 :field_count
-          array  :fields, :initial_length => :field_count do
-            uint16 :field_type
-            uint16 :field_length
+          # Fewer fields in the v9 header
+          FIELDS_FOR_COPY_V9.each do |f|
+            event[f] = flowset[f]
           end
-        end
-      end
 
-      class OptionFlowset < BinData::Record
-        endian :big
-        array  :templates, :read_until => lambda { flowset_length - 4 - array.num_bytes <= 2 } do
-          uint16 :template_id
-          uint16 :scope_length
-          uint16 :option_length
-          array  :scope_fields, :initial_length => lambda { scope_length / 4 } do
-            uint16 :field_type
-            uint16 :field_length
-          end
-          array  :option_fields, :initial_length => lambda { option_length / 4 } do
-            uint16 :field_type
-            uint16 :field_length
+          event['flowset_id'] = record.flowset_id
+
+          r.each_pair do |k,v|
+            case k.to_s
+            when /_switched$/
+              millis = flowset.uptime - v
+              seconds = flowset.unix_sec - (millis / 1000)
+              # v9 did away with the nanosecs field
+              micros = 1000000 - (millis % 1000)
+              event[k.to_s] = Time.at(seconds, micros).utc.strftime("%Y-%m-%dT%H:%M:%S.%3NZ")
+            else
+              event[k.to_s] = v
+            end
           end
+
+          block.call(time, event)
         end
-        skip   :length => lambda { templates.length.odd? ? 2 : 0 }
       end
 
-      class Netflow9PDU < BinData::Record
-        endian :big
-        uint16 :version
-        uint16 :flow_records
-        uint32 :uptime
-        uint32 :unix_sec
-        uint32 :flow_seq_num
-        uint32 :source_id
-        array  :records, :read_until => :eof do
-          uint16 :flowset_id
-          uint16 :flowset_length
-          choice :flowset_data, :selection => :flowset_id do
-            template_flowset 0
-            option_flowset   1
-            string           :default, :read_length => lambda { flowset_length - 4 }
-          end
-        end
+      def uint_field(length, default)
+        # If length is 4, return :uint32, etc. and use default if length is 0
+        ("uint" + (((length > 0) ? length : default) * 8).to_s).to_sym
       end
 
-      # https://gist.github.com/joshaven/184837
-      class Vash < Hash
-        def initialize(constructor = {})
-          @register ||= {}
-          if constructor.is_a?(Hash)
-            super()
-            merge(constructor)
-          else
-            super(constructor)
-          end
-        end
+      def netflow_field_for(type, length, field_definitions)
+        if field_definitions.include?(type)
+          field = field_definitions[type]
+          if field.is_a?(Array)
 
-        alias_method :regular_writer, :[]= unless method_defined?(:regular_writer)
-        alias_method :regular_reader, :[] unless method_defined?(:regular_reader)
+            if field[0].is_a?(Integer)
+              field[0] = uint_field(length, field[0])
+            end
 
-        def [](key)
-          sterilize(key)
-          clear(key) if expired?(key)
-          regular_reader(key)
-        end
+            # Small bit of fixup for skip or string field types where the length
+            # is dynamic
+            case field[0]
+            when :skip
+              field += [nil, {length: length}]
+            when :string
+              field += [{length: length, trim_padding: true}]
+            end
 
-        def []=(key, *args)
-          if args.length == 2
-            value, ttl = args[1], args[0]
-          elsif args.length == 1
-            value, ttl = args[0], 60
+            [field]
           else
-            raise ArgumentError, "Wrong number of arguments, expected 2 or 3, received: #{args.length+1}\n"+
-              "Example Usage:  volatile_hash[:key]=value OR volatile_hash[:key, ttl]=value"
+            $log.warn "Definition should be an array", field: field
+            nil
           end
-          sterilize(key)
-          ttl(key, ttl)
-          regular_writer(key, value)
-        end
-
-        def merge(hsh)
-          hsh.map {|key,value| self[sterile(key)] = hsh[key]}
-          self
-        end
-
-        def cleanup!
-          now = Time.now.to_i
-          @register.map {|k,v| clear(k) if v < now}
-        end
-
-        def clear(key)
-          sterilize(key)
-          @register.delete key
-          self.delete key
-        end
-
-        private
-
-        def expired?(key)
-          Time.now.to_i > @register[key].to_i
-        end
-
-        def ttl(key, secs=60)
-          @register[key] = Time.now.to_i + secs.to_i
-        end
-
-        def sterile(key)
-          String === key ? key.chomp('!').chomp('=') : key.to_s.chomp('!').chomp('=').to_sym
-        end
-
-        def sterilize(key)
-          key = sterile(key)
+        else
+          $log.warn "Unsupported field", type: type, length: length
+          nil
         end
       end
     end