nfrb 0.0.1.alpha

data/.document

@@ -0,0 +1,5 @@
+lib/**/*.rb
+bin/*
+- 
+features/**/*.feature
+LICENSE.txt

data/Gemfile

@@ -0,0 +1,15 @@
+source "http://rubygems.org"
+# Add dependencies required to use your gem here.
+# Example:
+#   gem "activesupport", ">= 2.3.5"
+
+# Add dependencies to develop your gem here.
+# Include everything needed to run rake, tests, features, etc.
+group :development do
+  gem "shoulda", ">= 0"
+  gem "bundler", "~> 1.0.0"
+  gem "jeweler", "~> 1.6.4"
+  gem "rcov", ">= 0"
+  gem "rdoc", ">= 2.4.2"
+  gem "rake-compiler", ">= 0"
+end

data/Gemfile.lock

@@ -0,0 +1,27 @@
+GEM
+  remote: http://rubygems.org/
+  specs:
+    git (1.2.5)
+    jeweler (1.6.4)
+      bundler (~> 1.0)
+      git (>= 1.2.5)
+      rake
+    json (1.6.3)
+    rake (0.9.2)
+    rake-compiler (0.7.9)
+      rake
+    rcov (0.9.11)
+    rdoc (3.11)
+      json (~> 1.4)
+    shoulda (2.11.3)
+
+PLATFORMS
+  ruby
+
+DEPENDENCIES
+  bundler (~> 1.0.0)
+  jeweler (~> 1.6.4)
+  rake-compiler
+  rcov
+  rdoc (>= 2.4.2)
+  shoulda

data/LICENSE.txt

@@ -0,0 +1,32 @@
+The nfrb gem is derived from nfdump code and is distributed under the BSD 
+license:
+
+Copyright (c) 2009, Peter Haag
+Copyright (c) 2004-2008, SWITCH - Teleinformatikdienste fuer Lehre und Forschung
+Copyright (c) 2011, Davide Guerri <davide.guerri@gmail.com>
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without 
+modification, are permitted provided that the following conditions are met:
+
+ * Redistributions of source code must retain the above copyright notice, 
+   this list of conditions and the following disclaimer.
+ * Redistributions in binary form must reproduce the above copyright notice, 
+   this list of conditions and the following disclaimer in the documentation 
+   and/or other materials provided with the distribution.
+ * Neither the name of SWITCH nor the names of its contributors may be 
+   used to endorse or promote products derived from this software without 
+   specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 
+AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 
+IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 
+ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE 
+LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 
+CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 
+SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 
+INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 
+CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 
+ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 
+POSSIBILITY OF SUCH DAMAGE.
+  

data/README.rdoc

@@ -0,0 +1,56 @@
+= nfrb
+
+Nfrb is a very simple yet fast gem that can be used to parse nfcapd files.
+
+== Usage
+
+=== Installation
+   
+   # git clone git://github.com/dguerri/nfrb.git
+   # cd nfrb
+   # bundle install
+   # rake install
+
+=== Sample code
+
+nfrb-bech.rb:
+
+   require 'nfrb'
+   
+   raise ArgumentError, "Missing nfcapd files" if ARGV.count < 1
+   
+   nfparser = NfRb::NfReader.new
+        
+   i=0
+   start_t = Time.now
+   
+   nfparser.process_files(ARGV) do |flow|
+   # puts flow.inspect(); 
+     i+=1 
+   end
+   
+   stop_t = Time.now
+   
+   puts "#{i} flows processed in #{stop_t - start_t} seconds."
+   puts "Flows per seconds = #{i / (stop_t - start_t)}"
+
+=== Sample output benchmark (3.4 Ghz Intel Core i7)
+
+   # bundle exec ruby nfrb-bech.rb nfcapd.201112181720 
+   325602 flows processed in 0.67114 seconds.
+   Flows per seconds = 488513.404029311
+
+== Contributing to nfrb
+ 
+* Check out the latest master to make sure the feature hasn't been implemented or the bug hasn't been fixed yet
+* Check out the issue tracker to make sure someone already hasn't requested it and/or contributed it
+* Fork the project
+* Start a feature/bugfix branch
+* Commit and push until you are happy with your contribution
+* Make sure to add tests for it. This is important so I don't break it in a future version unintentionally.
+* Please try not to mess with the Rakefile, version, or history. If you want to have your own version, or is otherwise necessary, that is fine, but please isolate to its own commit so I can cherry-pick around it.
+
+== Copyright
+
+Copyright (c) 2011 Davide Guerri. See LICENSE.txt for further details.
+

data/Rakefile

@@ -0,0 +1,58 @@
+# encoding: utf-8
+
+require 'rubygems'
+require 'bundler'
+begin
+  Bundler.setup(:default, :development)
+rescue Bundler::BundlerError => e
+  $stderr.puts e.message
+  $stderr.puts "Run `bundle install` to install missing gems"
+  exit e.status_code
+end
+require 'rake'
+
+require 'rake/extensiontask'
+Rake::ExtensionTask.new("rb_nfrb")
+
+require 'jeweler'
+require './lib/nfrb/version.rb'
+Jeweler::Tasks.new do |gem|
+  # gem is a Gem::Specification... see http://docs.rubygems.org/read/chapter/20 for more options
+  gem.name = "nfrb"
+  gem.homepage = "http://github.com/dguerri/nfrb"
+  gem.license = "BSD license"
+  gem.summary = %Q{Nfrb: nfcapd files parser.}
+  gem.description = %Q{Nfrb is a very simple yet fast gem that can be used to parse nfcapd files.}
+  gem.email = "davide.guerri@gmail.com"
+  gem.authors = ["Davide Guerri"]
+  gem.version = NfRb::Version::STRING
+  # dependencies defined in Gemfile
+end
+Jeweler::RubygemsDotOrgTasks.new
+
+require 'rake/testtask'
+Rake::TestTask.new(:test) do |test|
+  test.libs << 'lib' << 'test'
+  test.pattern = 'test/**/test_*.rb'
+  test.verbose = true
+end
+
+require 'rcov/rcovtask'
+Rcov::RcovTask.new do |test|
+  test.libs << 'test'
+  test.pattern = 'test/**/test_*.rb'
+  test.verbose = true
+  test.rcov_opts << '--exclude "gems/*"'
+end
+
+task :default => :build
+
+require 'rdoc/task'
+RDoc::Task.new do |rdoc|
+  version = File.exist?('VERSION') ? File.read('VERSION') : ""
+
+  rdoc.rdoc_dir = 'rdoc'
+  rdoc.title = "nfrb #{version}"
+  rdoc.rdoc_files.include('README*')
+  rdoc.rdoc_files.include('lib/**/*.rb')
+end

data/ext/rb_nfrb/README.files

@@ -0,0 +1,14 @@
+The following files are from nfdump v 1.6.4
+
+fts_compat.c 
+fts_compat.h 
+lzoconf.h
+lzodefs.h
+minilzo.h
+nf_common.h
+nffile.c
+nffile.h
+nfx.c
+nfx.h
+util.c
+util.h

data/ext/rb_nfrb/config.h

@@ -0,0 +1,37 @@
+#ifndef NFRB_CONFIG_H
+#define NFRB_CONFIG_H 1
+
+// TODO: The following values must be calculated...
+
+/* Define to 1 if you have the <dirent.h> header file, and it defines `DIR'.
+   */
+#define HAVE_DIRENT_H 1
+
+/* Define to 1 if you have the <stdint.h> header file. */
+#define HAVE_STDINT_H 1
+
+/* The size of `int', as computed by sizeof. */
+#define SIZEOF_INT 4
+
+/* The size of `long', as computed by sizeof. */
+#define SIZEOF_LONG 8
+
+/* The size of `long long', as computed by sizeof. */
+#define SIZEOF_LONG_LONG 8
+
+/* The size of `ptrdiff_t', as computed by sizeof. */
+#define SIZEOF_PTRDIFF_T 8
+
+/* The size of `short', as computed by sizeof. */
+#define SIZEOF_SHORT 2
+
+/* The size of `size_t', as computed by sizeof. */
+#define SIZEOF_SIZE_T 8
+
+/* The size of `void *', as computed by sizeof. */
+#define SIZEOF_VOID_P 8
+
+/* The size of `__int64', as computed by sizeof. */
+#define SIZEOF___INT64 0
+
+#endif

data/ext/rb_nfrb/dnc/nffile_inline.c

@@ -0,0 +1,579 @@
+/*
+ *  Copyright (c) 2009, Peter Haag
+ *  Copyright (c) 2004-2008, SWITCH - Teleinformatikdienste fuer Lehre und Forschung
+ *  All rights reserved.
+ *  
+ *  Redistribution and use in source and binary forms, with or without 
+ *  modification, are permitted provided that the following conditions are met:
+ *  
+ *   * Redistributions of source code must retain the above copyright notice, 
+ *     this list of conditions and the following disclaimer.
+ *   * Redistributions in binary form must reproduce the above copyright notice, 
+ *     this list of conditions and the following disclaimer in the documentation 
+ *     and/or other materials provided with the distribution.
+ *   * Neither the name of SWITCH nor the names of its contributors may be 
+ *     used to endorse or promote products derived from this software without 
+ *     specific prior written permission.
+ *  
+ *  THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 
+ *  AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 
+ *  IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 
+ *  ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE 
+ *  LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 
+ *  CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 
+ *  SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 
+ *  INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 
+ *  CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 
+ *  ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 
+ *  POSSIBILITY OF SUCH DAMAGE.
+ *  
+ *  $Author: haag $
+ *
+ *  $Id: nffile_inline.c 40 2009-12-16 10:41:44Z haag $
+ *
+ *  $LastChangedRevision: 40 $
+ *	
+ */
+
+/* 
+ * nffile_inline.c is needed for daemon code as well as normal stdio code 
+ * therefore a generic LogError is defined, which maps to the 
+ * approriate logging channel - either stderr or syslog
+ */
+void LogError(char *format, ...);
+
+static inline int CheckBufferSpace(nffile_t *nffile, size_t required);
+
+static inline void AppendToBuffer(nffile_t *nffile, void *record, size_t required);
+
+static inline void ExpandRecord_v2(common_record_t *input_record, extension_info_t *extension_info, master_record_t *output_record );
+
+static void PackRecord(master_record_t *master_record, nffile_t *nffile);
+
+static inline int CheckBufferSpace(nffile_t *nffile, size_t required) {
+
+#ifdef DEVEL
+//	printf("Buffer Size %u\n", nffile->block_header->size);
+#endif
+	// flush current buffer to disc
+	if ( (nffile->block_header->size + required )  > WRITE_BUFFSIZE ) {
+
+		// this should never happen, but catch it anyway
+		if ( required > WRITE_BUFFSIZE ) {
+			LogError("Required buffer size %zu too big for output buffer!" , required);
+			return 0;
+		}
+
+		if ( WriteBlock(nffile) <= 0 ) {
+			LogError("Failed to write output buffer to disk: '%s'" , strerror(errno));
+			return 0;
+		} 
+	}
+
+	return 1;
+} // End of CheckBufferSpace
+
+
+/*
+ * Expand file record into master record for further processing
+ * LP64 CPUs need special 32bit operations as it is not guarateed, that 64bit
+ * values are aligned 
+ */
+static inline void ExpandRecord_v2(common_record_t *input_record, extension_info_t *extension_info, master_record_t *output_record ) {
+extension_map_t *extension_map = extension_info->map;
+uint32_t	i, *u;
+size_t		size;
+void		*p = (void *)input_record;
+
+	// set map ref
+	output_record->map_ref = extension_map;
+	
+	// Copy common data block
+	size = COMMON_RECORD_DATA_SIZE;
+	memcpy((void *)output_record, p, size);
+	p = (void *)input_record->data;
+
+	// Required extension 1 - IP addresses
+	if ( (input_record->flags & FLAG_IPV6_ADDR) != 0 )	{ // IPv6
+		// IPv6
+		memcpy((void *)output_record->v6.srcaddr, p, 4 * sizeof(uint64_t));	
+		p = (void *)((pointer_addr_t)p + 4 * sizeof(uint64_t));
+	} else { 	
+		// IPv4
+		u = (uint32_t *)p;
+		output_record->v6.srcaddr[0] = 0;
+		output_record->v6.srcaddr[1] = 0;
+		output_record->v4.srcaddr 	 = u[0];
+
+		output_record->v6.dstaddr[0] = 0;
+		output_record->v6.dstaddr[1] = 0;
+		output_record->v4.dstaddr 	 = u[1];
+		p = (void *)((pointer_addr_t)p + 2 * sizeof(uint32_t));
+	}
+
+	// Required extension 2 - packet counter
+	if ( (input_record->flags & FLAG_PKG_64 ) != 0 ) { 
+		// 64bit packet counter
+		value64_t	l, *v = (value64_t *)p;
+		l.val.val32[0] = v->val.val32[0];
+		l.val.val32[1] = v->val.val32[1];
+		output_record->dPkts = l.val.val64;
+		p = (void *)((pointer_addr_t)p + sizeof(uint64_t));
+	} else {	
+		// 32bit packet counter
+		output_record->dPkts = *((uint32_t *)p);
+		p = (void *)((pointer_addr_t)p + sizeof(uint32_t));
+	}
+
+	// Required extension 3 - byte counter
+	if ( (input_record->flags & FLAG_BYTES_64 ) != 0 ) { 
+		// 64bit byte counter
+		value64_t	l, *v = (value64_t *)p;
+		l.val.val32[0] = v->val.val32[0];
+		l.val.val32[1] = v->val.val32[1];
+		output_record->dOctets = l.val.val64;
+		p = (void *)((pointer_addr_t)p + sizeof(uint64_t));
+	} else {	
+		// 32bit bytes counter
+		output_record->dOctets = *((uint32_t *)p);
+		p = (void *)((pointer_addr_t)p + sizeof(uint32_t));
+	}
+
+	// preset one single flow
+	output_record->aggr_flows = 1;
+
+	// Process optional extensions
+	i=0;
+	while ( extension_map->ex_id[i] ) {
+		switch (extension_map->ex_id[i++]) {
+			// 0 - 3 should never be in an extension table so - ignore it
+			case 0:
+			case 1:
+			case 2:
+			case 3:
+				break;
+			case EX_IO_SNMP_2: {
+				tpl_ext_4_t *tpl = (tpl_ext_4_t *)p;
+				output_record->input  = tpl->input;
+				output_record->output = tpl->output;
+				p = (void *)tpl->data;
+				} break;
+			case EX_IO_SNMP_4: {
+				tpl_ext_5_t *tpl = (tpl_ext_5_t *)p;
+				output_record->input  = tpl->input;
+				output_record->output = tpl->output;
+				p = (void *)tpl->data;
+				} break;
+			case EX_AS_2: {
+				tpl_ext_6_t *tpl = (tpl_ext_6_t *)p;
+				output_record->srcas = tpl->src_as;
+				output_record->dstas = tpl->dst_as;
+				p = (void *)tpl->data;
+				} break;
+			case EX_AS_4: {
+				tpl_ext_7_t *tpl = (tpl_ext_7_t *)p;
+				output_record->srcas = tpl->src_as;
+				output_record->dstas = tpl->dst_as;
+				p = (void *)tpl->data;
+				} break;
+			case EX_MULIPLE: {
+				tpl_ext_8_t *tpl = (tpl_ext_8_t *)p;
+				// use a 32 bit int to copy all 4 fields
+				output_record->any = tpl->any;
+				p = (void *)tpl->data;
+				} break;
+			case EX_NEXT_HOP_v4: {
+				tpl_ext_9_t *tpl = (tpl_ext_9_t *)p;
+				output_record->ip_nexthop.v6[0] = 0;
+				output_record->ip_nexthop.v6[1] = 0;
+				output_record->ip_nexthop.v4	= tpl->nexthop;
+				p = (void *)tpl->data;
+				ClearFlag(output_record->flags, FLAG_IPV6_NH);
+				} break;
+			case EX_NEXT_HOP_v6: {
+				tpl_ext_10_t *tpl = (tpl_ext_10_t *)p;
+				output_record->ip_nexthop.v6[0] = tpl->nexthop[0];
+				output_record->ip_nexthop.v6[1] = tpl->nexthop[1];
+				p = (void *)tpl->data;
+				SetFlag(output_record->flags, FLAG_IPV6_NH);
+				} break;
+			case EX_NEXT_HOP_BGP_v4: {
+				tpl_ext_11_t *tpl = (tpl_ext_11_t *)p;
+				output_record->bgp_nexthop.v6[0] = 0;
+				output_record->bgp_nexthop.v6[1] = 0;
+				output_record->bgp_nexthop.v4	= tpl->bgp_nexthop;
+				ClearFlag(output_record->flags, FLAG_IPV6_NHB);
+				p = (void *)tpl->data;
+				} break;
+			case EX_NEXT_HOP_BGP_v6: {
+				tpl_ext_12_t *tpl = (tpl_ext_12_t *)p;
+				output_record->bgp_nexthop.v6[0] = tpl->bgp_nexthop[0];
+				output_record->bgp_nexthop.v6[1] = tpl->bgp_nexthop[1];
+				p = (void *)tpl->data;
+				SetFlag(output_record->flags, FLAG_IPV6_NHB);
+				} break;
+			case EX_VLAN: {
+				tpl_ext_13_t *tpl = (tpl_ext_13_t *)p;
+				output_record->src_vlan = tpl->src_vlan;
+				output_record->dst_vlan = tpl->dst_vlan;
+				p = (void *)tpl->data;
+				} break;
+			case EX_OUT_PKG_4: {
+				tpl_ext_14_t *tpl = (tpl_ext_14_t *)p;
+				output_record->out_pkts = tpl->out_pkts;
+				p = (void *)tpl->data;
+				} break;
+			case EX_OUT_PKG_8: {
+				tpl_ext_15_t v, *tpl = (tpl_ext_15_t *)p;
+				v.v[0] = tpl->v[0];
+				v.v[1] = tpl->v[1];
+				output_record->out_pkts = v.out_pkts;
+				p = (void *)tpl->data;
+				} break;
+			case EX_OUT_BYTES_4: {
+				tpl_ext_16_t *tpl = (tpl_ext_16_t *)p;
+				output_record->out_bytes = tpl->out_bytes;
+				p = (void *)tpl->data;
+				} break;
+			case EX_OUT_BYTES_8: {
+				tpl_ext_17_t v,*tpl = (tpl_ext_17_t *)p;
+				v.v[0] = tpl->v[0];
+				v.v[1] = tpl->v[1];
+				output_record->out_bytes = v.out_bytes;
+				p = (void *)tpl->data;
+				} break;
+			case EX_AGGR_FLOWS_4: {
+				tpl_ext_18_t *tpl = (tpl_ext_18_t *)p;
+				output_record->aggr_flows = tpl->aggr_flows;
+				p = (void *)tpl->data;
+				} break;
+			case EX_AGGR_FLOWS_8: {
+				tpl_ext_19_t v, *tpl = (tpl_ext_19_t *)p;
+				v.v[0] = tpl->v[0];
+				v.v[1] = tpl->v[1];
+				output_record->aggr_flows = v.aggr_flows;
+				p = (void *)tpl->data;
+				} break;
+			case EX_MAC_1: {
+				tpl_ext_20_t v, *tpl = (tpl_ext_20_t *)p;
+				v.v1[0] = tpl->v1[0];
+				v.v1[1] = tpl->v1[1];
+				output_record->in_src_mac = v.in_src_mac;
+
+				v.v2[0] = tpl->v2[0];
+				v.v2[1] = tpl->v2[1];
+				output_record->out_dst_mac = v.out_dst_mac;
+				p = (void *)tpl->data;
+				} break;
+			case EX_MAC_2: {
+				tpl_ext_21_t v, *tpl = (tpl_ext_21_t *)p;
+				v.v1[0] = tpl->v1[0];
+				v.v1[1] = tpl->v1[1];
+				output_record->in_dst_mac = v.in_dst_mac;
+				v.v2[0] = tpl->v2[0];
+				v.v2[1] = tpl->v2[1];
+				output_record->out_src_mac = v.out_src_mac;
+				p = (void *)tpl->data;
+				} break;
+			case EX_MPLS: {
+				tpl_ext_22_t *tpl = (tpl_ext_22_t *)p;
+				int j;
+				for (j=0; j<10; j++ ) {
+					output_record->mpls_label[j] = tpl->mpls_label[j];
+				}
+				p = (void *)tpl->data;
+			} break;
+			case EX_ROUTER_IP_v4: {
+				tpl_ext_23_t *tpl = (tpl_ext_23_t *)p;
+				output_record->ip_router.v6[0] = 0;
+				output_record->ip_router.v6[1] = 0;
+				output_record->ip_router.v4	= tpl->router_ip;
+				p = (void *)tpl->data;
+				ClearFlag(output_record->flags, FLAG_IPV6_EXP);
+				} break;
+			case EX_ROUTER_IP_v6: {
+				tpl_ext_24_t *tpl = (tpl_ext_24_t *)p;
+				output_record->ip_router.v6[0] = tpl->router_ip[0];
+				output_record->ip_router.v6[1] = tpl->router_ip[1];
+				p = (void *)tpl->data;
+				SetFlag(output_record->flags, FLAG_IPV6_EXP);
+				} break;
+			case EX_ROUTER_ID: {
+				tpl_ext_25_t *tpl = (tpl_ext_25_t *)p;
+				output_record->engine_type = tpl->engine_type;
+				output_record->engine_id   = tpl->engine_id;
+				p = (void *)tpl->data;
+				} break;
+		}
+	}
+	
+} // End of ExpandRecord_v2
+
+static void PackRecord(master_record_t *master_record, nffile_t *nffile) {
+extension_map_t *extension_map = master_record->map_ref;
+uint32_t required =  COMMON_RECORD_DATA_SIZE + extension_map->extension_size;
+size_t	 size;
+void	 *p;
+int		i;
+
+	// check size of packets and bytes
+	if ( master_record->dPkts >  0xffffffffLL ) {
+		master_record->flags |= FLAG_PKG_64;
+		required += 8;
+	} else {
+		master_record->flags &= ~FLAG_PKG_64;
+		required += 4;
+	}
+
+	if ( master_record->dOctets >  0xffffffffLL ) {
+		master_record->flags |= FLAG_BYTES_64;
+		required += 8;
+	} else {
+		master_record->flags &= ~FLAG_BYTES_64;
+		required += 4;
+	}
+	if ( (master_record->flags & FLAG_IPV6_ADDR) != 0 )	// IPv6
+		required += 32;
+	else
+		required += 8;
+
+	master_record->size = required;
+
+	// flush current buffer to disc if not enough space
+	if ( !CheckBufferSpace(nffile, required) ) {
+		return;
+	}
+
+	// enough buffer space available at this point
+	p = nffile->buff_ptr;
+
+	// write common record
+	size = COMMON_RECORD_DATA_SIZE;
+	memcpy(p, (void *)master_record, size);
+	p = (void *)((pointer_addr_t)p + size);
+
+	// Required extension 1 - IP addresses
+	if ( (master_record->flags & FLAG_IPV6_ADDR) != 0 )	{ // IPv6
+		// IPv6
+		memcpy(p, (void *)master_record->v6.srcaddr, 4 * sizeof(uint64_t));	
+		p = (void *)((pointer_addr_t)p + 4 * sizeof(uint64_t));
+	} else { 	
+		// IPv4
+		uint32_t *u = (uint32_t *)p;
+		u[0] = master_record->v4.srcaddr;
+		u[1] = master_record->v4.dstaddr;
+		p = (void *)((pointer_addr_t)p + 2 * sizeof(uint32_t));
+	}
+
+	// Required extension 2 - packet counter
+	if ( (master_record->flags & FLAG_PKG_64 ) != 0 ) { 
+		// 64bit packet counter
+		value64_t	l, *v = (value64_t *)p;
+		l.val.val64 = master_record->dPkts;
+		v->val.val32[0] = l.val.val32[0];
+		v->val.val32[1] = l.val.val32[1];
+		p = (void *)((pointer_addr_t)p + sizeof(uint64_t));
+	} else {	
+		// 32bit packet counter
+		*((uint32_t *)p) = master_record->dPkts;
+		p = (void *)((pointer_addr_t)p + sizeof(uint32_t));
+	}
+
+	// Required extension 3 - byte counter
+	if ( (master_record->flags & FLAG_BYTES_64 ) != 0 ) { 
+		// 64bit byte counter
+		value64_t	l, *v = (value64_t *)p;
+		l.val.val64 = master_record->dOctets;
+		v->val.val32[0] = l.val.val32[0];
+		v->val.val32[1] = l.val.val32[1];
+		p = (void *)((pointer_addr_t)p + sizeof(uint64_t));
+	} else {	
+		// 32bit bytes counter
+		*((uint32_t *)p) = master_record->dOctets;
+		p = (void *)((pointer_addr_t)p + sizeof(uint32_t));
+	}
+
+	// Process optional extensions
+	i=0;
+	while ( extension_map->ex_id[i] ) {
+		switch (extension_map->ex_id[i++]) {
+			// 0 - 3 should never be in an extension table so - ignore it
+			case 0:
+			case 1:
+			case 2:
+			case 3:
+				break;
+			case EX_IO_SNMP_2: { // input/output SNMP 2 byte
+				tpl_ext_4_t *tpl = (tpl_ext_4_t *)p;
+				tpl->input  = master_record->input;
+				tpl->output = master_record->output;
+				p = (void *)tpl->data;
+				} break;
+			case EX_IO_SNMP_4: { // input/output SNMP 4 byte
+				tpl_ext_5_t *tpl = (tpl_ext_5_t *)p;
+				tpl->input  = master_record->input;
+				tpl->output = master_record->output;
+				p = (void *)tpl->data;
+				} break;
+			case EX_AS_2: { // srcas/dstas 2 byte
+				tpl_ext_6_t *tpl = (tpl_ext_6_t *)p;
+				tpl->src_as = master_record->srcas;
+				tpl->dst_as = master_record->dstas;
+				p = (void *)tpl->data;
+				} break;
+			case EX_AS_4: { // srcas/dstas 4 byte
+				tpl_ext_7_t *tpl = (tpl_ext_7_t *)p;
+				tpl->src_as = master_record->srcas;
+				tpl->dst_as = master_record->dstas;
+				p = (void *)tpl->data;
+				} break;
+			case EX_MULIPLE: {
+				tpl_ext_8_t *tpl = (tpl_ext_8_t *)p;
+				// use a 32 bit int to copy all 4 fields
+				tpl->any = master_record->any;
+				p = (void *)tpl->data;
+				} break;
+			case EX_NEXT_HOP_v4: {
+				tpl_ext_9_t *tpl = (tpl_ext_9_t *)p;
+				tpl->nexthop = master_record->ip_nexthop.v4;
+				p = (void *)tpl->data;
+				} break;
+			case EX_NEXT_HOP_v6: {
+				tpl_ext_10_t *tpl = (tpl_ext_10_t *)p;
+				tpl->nexthop[0] = master_record->ip_nexthop.v6[0];
+				tpl->nexthop[1] = master_record->ip_nexthop.v6[1];
+				p = (void *)tpl->data;
+				} break;
+			case EX_NEXT_HOP_BGP_v4: {
+				tpl_ext_11_t *tpl = (tpl_ext_11_t *)p;
+				tpl->bgp_nexthop = master_record->bgp_nexthop.v4;
+				p = (void *)tpl->data;
+				} break;
+			case EX_NEXT_HOP_BGP_v6: {
+				tpl_ext_12_t *tpl = (tpl_ext_12_t *)p;
+				tpl->bgp_nexthop[0] = master_record->bgp_nexthop.v6[0];
+				tpl->bgp_nexthop[1] = master_record->bgp_nexthop.v6[1];
+				p = (void *)tpl->data;
+				} break;
+			case EX_VLAN: {
+				tpl_ext_13_t *tpl = (tpl_ext_13_t *)p;
+				tpl->src_vlan = master_record->src_vlan;
+				tpl->dst_vlan = master_record->dst_vlan;
+				p = (void *)tpl->data;
+				} break;
+			case EX_OUT_PKG_4: {
+				tpl_ext_14_t *tpl = (tpl_ext_14_t *)p;
+				tpl->out_pkts = master_record->out_pkts;
+				p = (void *)tpl->data;
+				} break;
+			case EX_OUT_PKG_8: {
+				tpl_ext_15_t v, *tpl = (tpl_ext_15_t *)p;
+				v.out_pkts = master_record->out_pkts;
+				tpl->v[0] = v.v[0];
+				tpl->v[1] = v.v[1];
+				p = (void *)tpl->data;
+				} break;
+			case EX_OUT_BYTES_4: {
+				tpl_ext_16_t *tpl = (tpl_ext_16_t *)p;
+				tpl->out_bytes = master_record->out_bytes;
+				p = (void *)tpl->data;
+				} break;
+			case EX_OUT_BYTES_8: {
+				tpl_ext_17_t v, *tpl = (tpl_ext_17_t *)p;
+				v.out_bytes = master_record->out_bytes;
+				tpl->v[0] = v.v[0];
+				tpl->v[1] = v.v[1];
+				p = (void *)tpl->data;
+				} break;
+			case EX_AGGR_FLOWS_4: {
+				tpl_ext_18_t *tpl = (tpl_ext_18_t *)p;
+				tpl->aggr_flows = master_record->aggr_flows;
+				p = (void *)tpl->data;
+				} break;
+			case EX_AGGR_FLOWS_8: {
+				tpl_ext_19_t v, *tpl = (tpl_ext_19_t *)p;
+				v.aggr_flows = master_record->aggr_flows;
+				tpl->v[0] = v.v[0];
+				tpl->v[1] = v.v[1];
+				p = (void *)tpl->data;
+				} break;
+			case EX_MAC_1: {
+				tpl_ext_20_t v, *tpl = (tpl_ext_20_t *)p;
+				v.in_src_mac = master_record->in_src_mac;
+				tpl->v1[0] = v.v1[0];
+				tpl->v1[1] = v.v1[1];
+				v.out_dst_mac = master_record->out_dst_mac;
+				tpl->v2[0] = v.v2[0];
+				tpl->v2[1] = v.v2[1];
+				p = (void *)tpl->data;
+				} break;
+			case EX_MAC_2: {
+				tpl_ext_21_t v, *tpl = (tpl_ext_21_t *)p;
+				v.in_dst_mac = master_record->in_dst_mac;
+				tpl->v1[0] = v.v1[0];
+				tpl->v1[1] = v.v1[1];
+				v.out_src_mac = master_record->out_src_mac;
+				tpl->v2[0] = v.v2[0];
+				tpl->v2[1] = v.v2[1];
+				p = (void *)tpl->data;
+				} break;
+			case EX_MPLS: {
+				tpl_ext_22_t *tpl = (tpl_ext_22_t *)p;
+				int j;
+				for (j=0; j<10; j++ ) {
+					tpl->mpls_label[j] = master_record->mpls_label[j];
+				}
+				p = (void *)tpl->data;
+				} break;
+			case EX_ROUTER_IP_v4: {
+				tpl_ext_23_t *tpl = (tpl_ext_23_t *)p;
+				tpl->router_ip = master_record->ip_router.v4;
+				p = (void *)tpl->data;
+				} break;
+			case EX_ROUTER_IP_v6: {
+				tpl_ext_24_t *tpl = (tpl_ext_24_t *)p;
+				tpl->router_ip[0] = master_record->ip_router.v6[0];
+				tpl->router_ip[1] = master_record->ip_router.v6[1];
+				p = (void *)tpl->data;
+				} break;
+			case EX_ROUTER_ID: {
+				tpl_ext_25_t *tpl = (tpl_ext_25_t *)p;
+				tpl->engine_type = master_record->engine_type;
+				tpl->engine_id   = master_record->engine_id;
+				p = (void *)tpl->data;
+				} break;
+
+		}
+	}
+
+	nffile->block_header->size 		+= required;
+	nffile->block_header->NumRecords++;
+#ifdef DEVEL
+	if ( ((pointer_addr_t)p - (pointer_addr_t)nffile->buff_ptr) != required ) {
+		fprintf(stderr, "Packrecord: size missmatch: required: %i, written: %li!\n", 
+			required, (long)((ptrdiff_t)p - (ptrdiff_t)nffile->buff_ptr));
+		exit(255);
+	}
+#endif
+	nffile->buff_ptr = p;
+
+} // End of PackRecord
+
+static inline void AppendToBuffer(nffile_t *nffile, void *record, size_t required) {
+
+	// flush current buffer to disc
+	if ( !CheckBufferSpace(nffile, required)) {
+		return;
+	}
+
+	// enough buffer space available at this point
+	memcpy(nffile->buff_ptr, record, required);
+
+	// update stat
+	nffile->block_header->NumRecords++;
+	nffile->block_header->size += required;
+
+	// advance write pointer
+	nffile->buff_ptr = (void *)((pointer_addr_t)nffile->buff_ptr + required);
+
+} // End of AppendToBuffer