nfrb 0.0.1.alpha

data/ext/rb_nfrb/nfrb.c

@@ -0,0 +1,368 @@
+/* 
+ * This file is part of the nfrb gem for Ruby.
+ * Copyright (C) 2011 Davide Guerri
+ *
+ * This code is largely derived from nfreader.c of nfdump suite.
+ *
+ */
+	 
+#include <ruby.h>
+
+#include <stdio.h>
+#include <unistd.h>
+#include <stdlib.h>
+#include <stdarg.h>
+#include <errno.h>
+#include <time.h>
+#include <string.h>
+#include <ctype.h>
+#include <sys/types.h>
+#include <sys/socket.h>
+#include <netinet/in.h>
+#include <arpa/inet.h>
+#include <sys/stat.h>
+#include <fcntl.h>
+
+#include "nffile.h"
+#include "nfx.h"
+#include "util.h"
+
+#include "config.h"
+
+#if ( SIZEOF_VOID_P == 8 )
+typedef uint64_t    pointer_addr_t;
+#else
+typedef uint32_t    pointer_addr_t;
+#endif
+
+#include "dnc/nffile_inline.c"
+
+VALUE source_address_sym;
+VALUE destination_address_sym;
+VALUE first_seen_sym;
+VALUE last_seen_sym;
+VALUE msec_first_seen_sym;
+VALUE msec_last_seen_sym;
+VALUE protocol_sym;
+VALUE source_port_sym;
+VALUE destination_port_sym;
+VALUE tcp_flags_sym;
+VALUE packets_sym;
+VALUE bytes_sym;
+VALUE forwarding_status_sym;
+VALUE tos_sym;
+VALUE input_interface_sym;
+VALUE output_interface_sym;
+VALUE destination_as_sym;
+VALUE source_as_sym;
+VALUE source_mask_sym;
+VALUE destination_mask_sym;
+VALUE destination_tos_sym;
+VALUE direction_sym;
+VALUE next_hop_sym;
+VALUE bgp_next_hop_sym;
+VALUE source_vlan_sym;
+VALUE destination_vlan_sym;
+
+typedef struct nfreader_s {
+    extension_map_list_t ext_maps;
+} nfreader_t;
+
+
+static void nfreader_free(void *ptr) {
+    nfreader_t *nfreader_prt;
+    FreeExtensionMaps(&nfreader_prt->ext_maps);
+}
+
+static int process_file(extension_map_list_t *ext_maps, nffile_t *nffile) {
+	master_record_t	master_record;
+	common_record_t *flow_record = NULL;
+	int done, ret, i, j, id;
+	char source_ip[40], destination_ip[40], nexthop_ip[40];
+	VALUE hash_v;
+
+	hash_v = rb_hash_new();
+
+	done = 0;
+	while (!done) {
+		// get next data block from file
+		ret = ReadBlock(nffile);
+
+		switch (ret) {
+			case NF_CORRUPT:
+			case NF_ERROR:
+				return ret; // Corrupt datafile
+			case NF_EOF: {
+				done = 1;
+				continue;
+			} break; // not really needed
+		}
+
+		if (nffile->block_header->id == Large_BLOCK_Type) {
+			// skip
+			continue;
+		}
+
+		if (nffile->block_header->id != DATA_BLOCK_TYPE_2) {
+			// Can't process block type "nffile->block_header->id". Skip block...
+			continue;
+		}
+
+		flow_record = nffile->buff_ptr;
+		for (i=0; i < nffile->block_header->NumRecords; i++) {
+			if (flow_record->type == CommonRecordType) {
+				uint32_t map_id = flow_record->ext_map;
+				if (ext_maps->slot[map_id] == NULL) {
+					// Corrupt data file! No such extension map id: "flow_record->ext_map". Skip record...
+				} else {
+					ExpandRecord_v2(flow_record, ext_maps->slot[flow_record->ext_map], &master_record);
+
+					// update number of flows matching a given map
+					ext_maps->slot[map_id]->ref_count++;
+
+					// Prepare an hash for the ruby block paramater
+					if ( (master_record.flags & FLAG_IPV6_ADDR ) != 0 ) { // IPv6
+						master_record.v6.srcaddr[0] = htonll(master_record.v6.srcaddr[0]);
+						master_record.v6.srcaddr[1] = htonll(master_record.v6.srcaddr[1]);
+						master_record.v6.dstaddr[0] = htonll(master_record.v6.dstaddr[0]);
+						master_record.v6.dstaddr[1] = htonll(master_record.v6.dstaddr[1]);
+						inet_ntop(AF_INET6, master_record.v6.srcaddr, source_ip, sizeof(source_ip));
+						inet_ntop(AF_INET6, master_record.v6.dstaddr, destination_ip, sizeof(destination_ip));
+					} else {	// IPv4
+						master_record.v4.srcaddr = htonl(master_record.v4.srcaddr);
+						master_record.v4.dstaddr = htonl(master_record.v4.dstaddr);
+						inet_ntop(AF_INET, &master_record.v4.srcaddr, source_ip, sizeof(source_ip));
+						inet_ntop(AF_INET, &master_record.v4.dstaddr, destination_ip, sizeof(destination_ip));
+					}
+					source_ip[40-1] = 0;
+					destination_ip[40-1] = 0;
+
+					// netflow common record fields
+					rb_hash_aset(hash_v, source_address_sym, rb_tainted_str_new2(source_ip));
+					rb_hash_aset(hash_v, destination_address_sym, rb_tainted_str_new2(destination_ip));
+					rb_hash_aset(hash_v, first_seen_sym, INT2NUM(master_record.first));
+					rb_hash_aset(hash_v, last_seen_sym, INT2NUM(master_record.last));
+					rb_hash_aset(hash_v, msec_first_seen_sym, INT2NUM(master_record.msec_first));
+					rb_hash_aset(hash_v, msec_last_seen_sym, INT2NUM(master_record.msec_last));
+					rb_hash_aset(hash_v, protocol_sym, INT2FIX(master_record.prot));
+					rb_hash_aset(hash_v, source_port_sym, INT2FIX(master_record.srcport));
+					rb_hash_aset(hash_v, destination_port_sym, INT2FIX(master_record.dstport));
+					rb_hash_aset(hash_v, tcp_flags_sym, INT2FIX(master_record.tcp_flags));
+					rb_hash_aset(hash_v, packets_sym, INT2NUM((unsigned long long) master_record.dPkts));
+					rb_hash_aset(hash_v, bytes_sym, INT2NUM((unsigned long long) master_record.dOctets));
+					rb_hash_aset(hash_v, forwarding_status_sym, INT2FIX(master_record.fwd_status));
+					rb_hash_aset(hash_v, tos_sym, INT2FIX(master_record.tos));
+
+					// netflow extension fields
+					rb_hash_aset(hash_v, input_interface_sym , Qnil);
+					rb_hash_aset(hash_v, output_interface_sym, Qnil);
+					rb_hash_aset(hash_v, destination_as_sym  , Qnil);
+					rb_hash_aset(hash_v, source_as_sym       , Qnil);
+					rb_hash_aset(hash_v, source_mask_sym     , Qnil);
+					rb_hash_aset(hash_v, destination_mask_sym, Qnil);
+					rb_hash_aset(hash_v, destination_tos_sym , Qnil);
+					rb_hash_aset(hash_v, direction_sym       , Qnil);
+					rb_hash_aset(hash_v, next_hop_sym        , Qnil);
+					rb_hash_aset(hash_v, bgp_next_hop_sym    , Qnil);
+					rb_hash_aset(hash_v, source_vlan_sym     , Qnil);
+					rb_hash_aset(hash_v, destination_vlan_sym, Qnil);
+
+					j=0;
+					while ( (id = master_record.map_ref->ex_id[j++]) != 0 ) {
+						switch(id) {
+							case EX_IO_SNMP_2:
+								rb_hash_aset(hash_v, input_interface_sym, INT2FIX(master_record.input));
+								rb_hash_aset(hash_v, output_interface_sym, INT2FIX(master_record.output));
+								break;
+							case EX_IO_SNMP_4:
+								rb_hash_aset(hash_v, input_interface_sym, INT2NUM(master_record.input));
+								rb_hash_aset(hash_v, output_interface_sym, INT2NUM(master_record.output));
+								break;
+							case EX_AS_2:
+								rb_hash_aset(hash_v, destination_as_sym, INT2FIX(master_record.dstas));
+								rb_hash_aset(hash_v, source_as_sym, INT2FIX(master_record.input));
+								break;
+							case EX_AS_4:
+								rb_hash_aset(hash_v, destination_as_sym, INT2NUM(master_record.dstas));
+								rb_hash_aset(hash_v, source_as_sym, INT2NUM(master_record.input));
+								break;
+							case EX_MULIPLE:
+								rb_hash_aset(hash_v, source_mask_sym, INT2NUM(master_record.src_mask));
+								rb_hash_aset(hash_v, destination_mask_sym, INT2NUM(master_record.dst_mask));
+								rb_hash_aset(hash_v, destination_tos_sym, INT2FIX(master_record.dst_tos));
+								rb_hash_aset(hash_v, direction_sym, INT2FIX(master_record.dir));
+								break;
+							case EX_NEXT_HOP_v4:
+								master_record.ip_nexthop.v4=htonl(master_record.ip_nexthop.v4);
+								nexthop_ip[0] = 0;
+								inet_ntop(AF_INET, &master_record.ip_nexthop.v4, nexthop_ip, sizeof(nexthop_ip));
+								nexthop_ip[40-1] = 0;
+								rb_hash_aset(hash_v, next_hop_sym, rb_tainted_str_new2(nexthop_ip));
+								break;
+							case EX_NEXT_HOP_v6:
+								nexthop_ip[0] = 0;
+								master_record.ip_nexthop.v6[0] = htonll(master_record.ip_nexthop.v6[0]);
+								master_record.ip_nexthop.v6[1] = htonll(master_record.ip_nexthop.v6[1]);
+								inet_ntop(AF_INET6, master_record.ip_nexthop.v6, nexthop_ip, sizeof(nexthop_ip));
+								nexthop_ip[40-1] = 0;
+								rb_hash_aset(hash_v, next_hop_sym, rb_tainted_str_new2(nexthop_ip));
+								break;
+							case EX_NEXT_HOP_BGP_v4:
+								master_record.ip_nexthop.v4=htonl(master_record.bgp_nexthop.v4);
+								nexthop_ip[0] = 0;
+								inet_ntop(AF_INET, &master_record.bgp_nexthop.v4, nexthop_ip, sizeof(nexthop_ip));
+								nexthop_ip[40-1] = 0;
+								rb_hash_aset(hash_v, bgp_next_hop_sym, rb_tainted_str_new2(nexthop_ip));
+								break;
+							case EX_NEXT_HOP_BGP_v6:
+								nexthop_ip[0] = 0;
+								master_record.bgp_nexthop.v6[0] = htonll(master_record.bgp_nexthop.v6[0]);
+								master_record.bgp_nexthop.v6[1] = htonll(master_record.bgp_nexthop.v6[1]);
+								inet_ntop(AF_INET6, master_record.bgp_nexthop.v6, nexthop_ip, sizeof(nexthop_ip));
+								nexthop_ip[40-1] = 0;
+								rb_hash_aset(hash_v, bgp_next_hop_sym, rb_tainted_str_new2(nexthop_ip));
+								break;
+							case EX_VLAN:
+								rb_hash_aset(hash_v, source_vlan_sym, INT2FIX(master_record.src_vlan));
+								rb_hash_aset(hash_v, destination_vlan_sym, INT2FIX(master_record.dst_vlan));
+								break;
+							default:
+							    ;
+								// Not implemented
+						}
+					}
+
+					// Yield to the ruby block
+					rb_yield(hash_v);
+				}
+
+			} else if (flow_record->type == ExtensionMapType) {
+				extension_map_t *map = (extension_map_t *)flow_record;
+				Insert_Extension_Map(ext_maps, map);
+			} // else Skip unknown record type flow_record->type
+			// Advance pointer by number of bytes for netflow record
+			flow_record = (common_record_t *)((pointer_addr_t)flow_record + flow_record->size);
+		} // for all records
+	} // while
+
+    return 0;
+
+} // End of process_data
+
+static VALUE rb_process_file(VALUE self, VALUE filename_v) {
+    nfreader_t *nfreader_prt = NULL;
+    nffile_t *nffile = NULL;
+    int ret;
+
+    Check_Type(filename_v, T_STRING);
+
+    Data_Get_Struct(self, nfreader_t, nfreader_prt);
+    InitExtensionMaps(&nfreader_prt->ext_maps);
+
+	nffile = OpenFile(RSTRING(filename_v)->ptr, NULL);
+	if (!nffile)
+	    rb_raise(rb_eIOError, "problem opening file '%s'", RSTRING(filename_v)->ptr);
+
+    ret = process_file(&nfreader_prt->ext_maps, nffile);
+
+	CloseFile(nffile);
+	DisposeFile(nffile);
+
+	PackExtensionMapList(&nfreader_prt->ext_maps);
+
+    return Qnil;
+}
+
+static VALUE rb_process_files(VALUE self, VALUE filenames_a) {
+    nfreader_t *nfreader_prt = NULL;
+    nffile_t *nffile = NULL;
+    VALUE filename_v;
+    long i;
+    int ret;
+
+    Check_Type(filenames_a, T_ARRAY);
+
+	if (!rb_block_given_p())
+		rb_raise(rb_eArgError, "a block is required");
+
+    Data_Get_Struct(self, nfreader_t, nfreader_prt);
+    InitExtensionMaps(&nfreader_prt->ext_maps);
+
+    for (i=0 ; i < RARRAY_LEN(filenames_a) ; i++) {
+	    filename_v = rb_ary_entry(filenames_a, i);
+	    Check_Type(filename_v, T_STRING);
+
+	    nffile = OpenFile(RSTRING(filename_v)->ptr, NULL);
+	    if (!nffile)
+		    rb_raise(rb_eIOError, "problem opening file '%s'", RSTRING(filename_v)->ptr);
+
+        ret = process_file(&nfreader_prt->ext_maps, nffile);
+
+	    CloseFile(nffile);
+	    DisposeFile(nffile);
+    }
+
+	PackExtensionMapList(&nfreader_prt->ext_maps);
+
+    return Qnil;
+}
+
+
+static VALUE rb_nfreader_init(VALUE self) {
+    nfreader_t *nfreader_prt = NULL;
+
+    Data_Get_Struct(self, nfreader_t, nfreader_prt);
+
+    return self;
+}
+
+
+VALUE rb_nfreader_new(VALUE nfreader_class) {
+    nfreader_t *nfreader_prt = NULL;
+
+    VALUE nfreader_data = Data_Make_Struct(nfreader_class, nfreader_t, NULL, NULL, nfreader_prt);
+
+    rb_obj_call_init(nfreader_data, 0, NULL);
+
+    return nfreader_data;
+}
+
+
+void Init_rb_nfrb() {
+	VALUE nfrbModule = rb_define_module("NfRb");
+
+	VALUE rb_nfreader_class = rb_define_class_under(nfrbModule, "NfReader", rb_cObject);
+	rb_define_singleton_method(rb_nfreader_class, "new", rb_nfreader_new, 0);
+	rb_define_method(rb_nfreader_class, "initialize", rb_nfreader_init, 0);
+	rb_define_method(rb_nfreader_class, "process_file", rb_process_file, 1);
+	rb_define_method(rb_nfreader_class, "process_files", rb_process_files, 1);
+
+    // Symbols inits
+    source_address_sym      = ID2SYM(rb_intern("source_address"));
+    destination_address_sym = ID2SYM(rb_intern("destination_address"));
+    first_seen_sym          = ID2SYM(rb_intern("first_seen"));
+    last_seen_sym           = ID2SYM(rb_intern("last_seen"));
+    msec_first_seen_sym     = ID2SYM(rb_intern("msec_first_seen"));
+    msec_last_seen_sym      = ID2SYM(rb_intern("msec_last_seen"));
+    protocol_sym            = ID2SYM(rb_intern("protocol"));
+    source_port_sym         = ID2SYM(rb_intern("source_port"));
+    destination_port_sym    = ID2SYM(rb_intern("destination_port"));
+    tcp_flags_sym           = ID2SYM(rb_intern("tcp_flags"));
+    packets_sym             = ID2SYM(rb_intern("packets"));
+    bytes_sym               = ID2SYM(rb_intern("bytes"));
+    forwarding_status_sym   = ID2SYM(rb_intern("forwarding_status"));
+    tos_sym                 = ID2SYM(rb_intern("tos"));
+    input_interface_sym     = ID2SYM(rb_intern("input_interface"));
+    output_interface_sym    = ID2SYM(rb_intern("output_interface"));
+    destination_as_sym      = ID2SYM(rb_intern("destination_as"));
+    source_as_sym           = ID2SYM(rb_intern("source_as"));
+    source_mask_sym         = ID2SYM(rb_intern("source_mask"));
+    destination_mask_sym    = ID2SYM(rb_intern("destination_mask"));
+    destination_tos_sym     = ID2SYM(rb_intern("destination_tos"));
+    direction_sym           = ID2SYM(rb_intern("direction"));
+    next_hop_sym            = ID2SYM(rb_intern("next_hop"));
+    bgp_next_hop_sym        = ID2SYM(rb_intern("bgp_next_hop"));
+    source_vlan_sym         = ID2SYM(rb_intern("source_vlan"));
+    destination_vlan_sym    = ID2SYM(rb_intern("destination_vlan"));
+
+}

data/ext/rb_nfrb/nfx.c

@@ -0,0 +1,636 @@
+/*
+ *  Copyright (c) 2009, Peter Haag
+ *  Copyright (c) 2004-2008, SWITCH - Teleinformatikdienste fuer Lehre und Forschung
+ *  All rights reserved.
+ *  
+ *  Redistribution and use in source and binary forms, with or without 
+ *  modification, are permitted provided that the following conditions are met:
+ *  
+ *   * Redistributions of source code must retain the above copyright notice, 
+ *     this list of conditions and the following disclaimer.
+ *   * Redistributions in binary form must reproduce the above copyright notice, 
+ *     this list of conditions and the following disclaimer in the documentation 
+ *     and/or other materials provided with the distribution.
+ *   * Neither the name of SWITCH nor the names of its contributors may be 
+ *     used to endorse or promote products derived from this software without 
+ *     specific prior written permission.
+ *  
+ *  THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 
+ *  AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 
+ *  IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 
+ *  ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE 
+ *  LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 
+ *  CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 
+ *  SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 
+ *  INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 
+ *  CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 
+ *  ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 
+ *  POSSIBILITY OF SUCH DAMAGE.
+ *  
+ *  $Author: haag $
+ *
+ *  $Id: nfx.c 58 2010-02-26 12:26:07Z haag $
+ *
+ *  $LastChangedRevision: 58 $
+ *	
+ */
+
+#include "config.h"
+
+#include <sys/types.h>
+#include <unistd.h>
+#include <stdio.h>
+#include <errno.h>
+#include <unistd.h>
+#include <stdlib.h>
+#include <ctype.h>
+#include <string.h>
+#include <syslog.h>
+#include <stdarg.h>
+
+#ifdef HAVE_STDINT_H
+#include <stdint.h>
+#endif
+
+#ifndef DEVEL
+#   define dbg_printf(...) /* printf(__VA_ARGS__) */
+#else
+#   define dbg_printf(...) printf(__VA_ARGS__)
+#endif
+
+#include "nf_common.h"
+#include "nffile.h"
+#include "util.h"
+#include "nfx.h"
+
+/* global vars */
+
+/*
+ * see nffile.h for detailed extension description
+ */
+extension_descriptor_t extension_descriptor[] = {
+	// fill indices 0 - 3
+	{ COMMON_BLOCK_ID,		0,	 0, 1,   "Required extension: Common record"},
+	{ EX_IPv4v6,			0,	 0, 1,   "Required extension: IPv4/IPv6 src/dst address"},
+	{ EX_PACKET_4_8,		0,	 0, 1,   "Required extension: 4/8 byte input packets"},
+	{ EX_BYTE_4_8,			0,	 0, 1,   "Required extension: 4/8 byte input bytes"},
+
+	// the optional extension
+	{ EX_IO_SNMP_2, 		4, 	 1, 1,   "2 byte input/output interface index"},
+	{ EX_IO_SNMP_4, 		8, 	 1, 1,   "4 byte input/output interface index"},
+	{ EX_AS_2, 				4, 	 2, 1,   "2 byte src/dst AS number"},
+	{ EX_AS_4, 				8, 	 2, 1,   "4 byte src/dst AS number"},
+	{ EX_MULIPLE, 			4, 	 3, 0,   "dst tos, direction, src/dst mask"}, 
+	{ EX_NEXT_HOP_v4,		4,	 4, 0,   "IPv4 next hop"},
+	{ EX_NEXT_HOP_v6,		16,	 4, 0,   "IPv6 next hop"},
+	{ EX_NEXT_HOP_BGP_v4,	4,	 5, 0,   "IPv4 BGP next IP"},
+	{ EX_NEXT_HOP_BGP_v6,	16,	 5, 0,   "IPv6 BGP next IP"},
+	{ EX_VLAN,				4,	 6, 0,   "src/dst vlan id"},
+	{ EX_OUT_PKG_4,			4,	 7, 0,   "4 byte output packets"},
+	{ EX_OUT_PKG_8,			8,	 7, 0,   "8 byte output packets"},
+	{ EX_OUT_BYTES_4,		4,	 8, 0,   "4 byte output bytes"},
+	{ EX_OUT_BYTES_8,		8,	 8, 0,   "8 byte output bytes"},
+	{ EX_AGGR_FLOWS_4,		4,	 9, 0,   "4 byte aggregated flows"},
+	{ EX_AGGR_FLOWS_8,		8,	 9, 0,   "8 byte aggregated flows"},
+	{ EX_MAC_1,				16,	10, 0,   "in src/out dst mac address"},
+	{ EX_MAC_2,				16,	11, 0,   "in dst/out src mac address"},
+	{ EX_MPLS,				40,	12, 0,   "MPLS Labels"},
+	{ EX_ROUTER_IP_v4,		4,	13, 0,   "IPv4 router IP addr"},
+	{ EX_ROUTER_IP_v6,		16,	13, 0,   "IPv6 router IP addr"},
+	{ EX_ROUTER_ID,			4,	14, 0,   "router ID"},
+
+	// last entry
+	{ 0,	0,	0, 0,	NULL }
+};
+
+uint32_t Max_num_extensions;
+
+void FixExtensionMap(extension_map_t *map);
+
+void InitExtensionMaps(extension_map_list_t *extension_map_list ) {
+int i;
+	memset((void *)extension_map_list->slot, 0, MAX_EXTENSION_MAPS * sizeof(extension_info_t *));
+	memset((void *)extension_map_list->page, 0, MAX_EXTENSION_MAPS * sizeof(extension_info_t *));
+
+	extension_map_list->next_free = 0;
+	extension_map_list->max_used  = -1;
+
+	Max_num_extensions = 0;
+	i = 1;
+	while ( extension_descriptor[i++].id ) 
+		Max_num_extensions++;
+
+} // End of InitExtensionMaps
+
+void FreeExtensionMaps(extension_map_list_t *extension_map_list) {
+int	i;
+
+	if ( extension_map_list == NULL ) 
+		return;
+	
+	// free all maps
+	for ( i=0; i <= extension_map_list->max_used; i++ ) {
+		if ( extension_map_list->slot[i] ) {
+			if ( extension_map_list->slot[i]->map ) {
+				free(extension_map_list->slot[i]->map);
+				extension_map_list->slot[i]->map = NULL;
+			}
+			free(extension_map_list->slot[i]);
+			extension_map_list->slot[i] = NULL;
+		}
+		
+	}
+
+	// free all paged maps
+	for ( i=0; i < extension_map_list->next_free; i++ ) {
+		if ( extension_map_list->page[i] ) {
+			if ( extension_map_list->page[i]->map ) {
+				free(extension_map_list->page[i]->map);
+				extension_map_list->page[i]->map = NULL;
+			}
+			free(extension_map_list->page[i]);
+			extension_map_list->page[i] = NULL;
+		}
+	}
+
+	InitExtensionMaps(extension_map_list);
+
+} // End of FreeExtensionMaps
+
+int Insert_Extension_Map(extension_map_list_t *extension_map_list, extension_map_t *map) {
+uint32_t next_free = extension_map_list->next_free;
+uint16_t map_id;
+
+	map_id = map->map_id == INIT_ID ? 0 : map->map_id & EXTENSION_MAP_MASK;
+	map->map_id = map_id;
+	dbg_printf("Insert Extension Map:\n");
+#ifdef DEVEL
+	PrintExtensionMap(map);
+#endif
+	// is this slot free
+	if ( extension_map_list->slot[map_id] ) {
+		int i, map_found;
+		dbg_printf("Map %d already exists\n", map_id);
+		// no - check if same map already in slot
+		if ( extension_map_list->slot[map_id]->map->size == map->size ) {
+			// existing map and new map have the same size 
+			dbg_printf("New map same size:\n");
+
+			// we must compare the maps
+			i = 0;
+			while ( extension_map_list->slot[map_id]->map->ex_id[i] && (extension_map_list->slot[map_id]->map->ex_id[i] == map->ex_id[i]) ) 
+				i++;
+
+			// if last entry == 0 => last map entry => maps are the same
+			if ( extension_map_list->slot[map_id]->map->ex_id[i] == 0 ) {
+				dbg_printf("Same map => nothing to do\n");
+				// same map
+				return 0;
+			} 
+			dbg_printf("Different map => continue\n");
+		}
+
+		dbg_printf("Search for map in extension page\n");
+		map_found = -1;
+		// new map is different but has same id - search for map in page list
+		for ( i = 0 ; i < next_free; i++ ) {
+			int j;
+			j = 0;
+			if ( extension_map_list->page[i]->map->size == map->size ) {
+				while ( extension_map_list->page[i]->map->ex_id[j] && (extension_map_list->page[i]->map->ex_id[j] == map->ex_id[j]) ) 
+					j++;
+			}
+			if ( extension_map_list->page[i]->map->ex_id[j] == 0 ) {
+				dbg_printf("Map found in page slot %i\n", i);
+				map_found = i;
+			}
+		}
+		if ( map_found >= 0 ) {
+			extension_info_t *tmp;
+			dbg_printf("Move map from page slot %i to slot %i\n", map_found ,map_id);
+	
+			// exchange the two maps
+			tmp = extension_map_list->slot[map_id];
+			extension_map_list->slot[map_id] = extension_map_list->page[map_found];
+			extension_map_list->slot[map_id]->map->map_id = map_id;
+
+			extension_map_list->page[map_found] = tmp;
+			extension_map_list->page[map_found]->map->map_id = map_found;
+			return 1;
+			
+		} else {
+			dbg_printf("Map not found in extension page\n");
+			// map not found - move it to the extension page to a currently free slot
+			if ( next_free < MAX_EXTENSION_MAPS ) {
+				dbg_printf("Move existing map from slot %d to page slot %d\n",map_id, next_free);
+				extension_map_list->page[next_free]   	= extension_map_list->slot[map_id];
+				extension_map_list->page[next_free]->map->map_id = next_free;
+				extension_map_list->slot[map_id] 		= NULL;
+				// ready to fill new slot
+			} else {
+				fprintf(stderr, "Extension map list exhausted - too many extension maps ( > %d ) to process;\n", MAX_EXTENSION_MAPS);
+				exit(255);
+			}
+		}
+	}
+
+	FixExtensionMap(map);
+
+	// add new entry to slot
+	extension_map_list->slot[map_id]	= (extension_info_t *)calloc(1,sizeof(extension_info_t));
+	if ( !extension_map_list->slot[map_id] ) {
+		fprintf(stderr, "calloc() error in %s line %d: %s\n", __FILE__, __LINE__, strerror(errno) );
+		exit(255);
+	}
+	extension_map_list->slot[map_id]->map   = (extension_map_t *)malloc((ssize_t)map->size);
+	if ( !extension_map_list->slot[map_id]->map ) {
+		fprintf(stderr, "malloc() error in %s line %d: %s\n", __FILE__, __LINE__, strerror(errno) );
+		exit(255);
+	}
+	memcpy((void *)extension_map_list->slot[map->map_id]->map, (void *)map, map->size);
+
+	extension_map_list->slot[map_id]->ref_count = 0;
+
+	if ( map_id > extension_map_list->max_used ) {
+		extension_map_list->max_used = map_id;
+	}
+
+	// Update next_free page slot, if it's used now
+	while ( extension_map_list->page[next_free] && (next_free < MAX_EXTENSION_MAPS))
+		next_free++;
+	extension_map_list->next_free = next_free;
+
+	// if all slots are exhausted next_free is now MAX_EXTENSION_MAPS. The next time an empty slot is needed, it will properly fail.
+	dbg_printf("Installed map in slot %d. Next free page slot: %d\n", map_id, next_free);
+	
+	//map changed
+	return 1;
+
+} // End of Insert_Extension_Map
+
+void PackExtensionMapList(extension_map_list_t *extension_map_list) {
+int i, free_slot;
+
+	dbg_printf("Pack extensions maps\n");
+	// compact extension map list - close gaps
+	free_slot = -1;
+	for ( i=0; i <= extension_map_list->max_used; i++ ) {
+		dbg_printf("Check slot: %i, ref: %u\n", i, extension_map_list->slot[i] ? extension_map_list->slot[i]->ref_count : 0);
+		if ( extension_map_list->slot[i] != NULL && extension_map_list->slot[i]->ref_count == 0 ) {
+			// Destroy slot, if no flows referenced this map
+			free(extension_map_list->slot[i]->map);
+			free(extension_map_list->slot[i]);
+			extension_map_list->slot[i] = NULL;
+			dbg_printf("Free slot: %i\n", i);
+		}
+		if ( extension_map_list->slot[i] == NULL && free_slot == -1 ) {
+			// remember this free slot
+			dbg_printf("Remember free slot at %i\n", i);
+			free_slot = i;
+		} else if ( free_slot != -1 && extension_map_list->slot[i] != NULL ) {
+			int j;
+			// move this slot down to compact the list
+			extension_map_list->slot[free_slot] = extension_map_list->slot[i];
+			extension_map_list->slot[free_slot]->map->map_id = free_slot;
+			extension_map_list->slot[i] = NULL;
+			dbg_printf("Move slot %i down to %i\n", i, free_slot);
+
+			// search for next free slot - latest slot[i] is free now
+			for ( j = free_slot + 1; j <= i; j++ ) {
+				if ( extension_map_list->slot[j] == NULL ) {
+					free_slot = j;
+					dbg_printf("Next free slot found at %i\n", free_slot);
+					break;
+				}
+			}
+		} else {
+			dbg_printf("Fell through\n");
+		}
+	}
+
+	// get max index - set index to map
+	i = 0;
+	while ( extension_map_list->slot[i] != NULL && i < MAX_EXTENSION_MAPS ) {
+		dbg_printf("Slot: %i, ref: %u\n", i, extension_map_list->slot[i]->ref_count);
+		i++;
+	}
+
+	if ( i == MAX_EXTENSION_MAPS ) {
+		// ups! - should not really happen - so we are done for now
+		if ( extension_map_list->next_free == 0 ) {
+			// map slots full but no maps im page list - we are done
+			return;
+		} else {
+			// we can't handle this event for now - too many maps - but MAX_EXTENSION_MAPS should be more than enough
+			fprintf(stderr, "Critical error in %s line %d: %s\n", __FILE__, __LINE__, "Out of maps!" );
+			exit(255);
+		}
+	}
+
+	// this points to the next free slot
+	free_slot = i;
+
+	for ( i=0; i < extension_map_list->next_free; i++ ) {
+		if ( free_slot < MAX_EXTENSION_MAPS ) {
+			if ( extension_map_list->page[i]->ref_count ) {
+				dbg_printf("Move page %u to slot %u\n", i, free_slot);
+				extension_map_list->slot[free_slot] = extension_map_list->page[i];
+				extension_map_list->slot[free_slot]->map->map_id = free_slot;
+				extension_map_list->page[i] = NULL;
+				free_slot++;
+			} else {
+				dbg_printf("Skip page %u. Zero ref count \n", i);
+			}
+		} else {
+			// we can't handle this event for now, but should not happen anyway
+			fprintf(stderr, "Critical error in %s line %d: %s\n", __FILE__, __LINE__, "Out of maps!" );
+			exit(255);
+		}
+	}
+
+	extension_map_list->max_used = free_slot - 1;
+	dbg_printf("Packed maps: %i\n", free_slot);
+
+#ifdef DEVEL
+	// Check maps
+	i = 0;
+	while ( extension_map_list->slot[i] != NULL && i < MAX_EXTENSION_MAPS ) {
+		if ( extension_map_list->slot[i]->map->map_id != i ) 
+			printf("*** Map ID missmatch in slot: %i, id: %u\n", i, extension_map_list->slot[i]->map->map_id);
+		i++;
+	}
+#endif
+
+} // End of PackExtensionMapList
+
+void SetupExtensionDescriptors(char *options) {
+int i, *mask;
+char *p, *q, *s;
+
+	Max_num_extensions = 0;
+	i = 1;
+	while ( extension_descriptor[i++].id ) 
+		Max_num_extensions++;
+
+	mask = (int *)calloc(Max_num_extensions+1, sizeof(int));
+	if ( !mask ) {
+		fprintf(stderr, "malloc() error in %s line %d: %s\n", __FILE__, __LINE__, strerror(errno) );
+		exit(255);
+	}
+
+	s = (char *)malloc(strlen(options));
+	if ( !s ) {
+		fprintf(stderr, "malloc() error in %s line %d: %s\n", __FILE__, __LINE__, strerror(errno) );
+		exit(255);
+	}
+	q = s;
+	*q = '\0';
+	p = options;
+	while ( *p ) {
+		if ( !isspace(*p) )
+			*q++ = *p;
+		p++;
+	}
+	*q = '\0';
+
+	p = s;
+	while ( p && *p ) {
+		int sign;
+		q = strchr(p, ',');
+		if ( q )
+			*q++ = '\0';
+		
+		// get possible sign
+		sign = 1;
+		if ( *p == '-' ) {
+			sign = -1;
+			p++;
+		}
+		if ( *p == '+' ) {
+			sign = 1;
+			p++;
+		}
+
+		if ( strcmp(p, "all") == 0 ) {
+			for (i=4; i<= Max_num_extensions; i++ ) 
+				extension_descriptor[i].enabled = sign == 1 ? : 0;
+		} else {
+			switch ( *p ) {
+				case '\0':
+					fprintf(stderr, "Extension format error: Unexpected end of format.\n");
+					exit(255);
+					break;
+				case '*': 
+					for (i=4; i<= Max_num_extensions; i++ ) 
+						extension_descriptor[i].enabled = sign == 1 ? : 0;
+					break;
+				default: {
+					int i = strtol(p, NULL, 10);
+					if ( i == 0 ) {
+						fprintf(stderr, "Extension format error: Unexpected string: %s.\n", p);
+						exit(255);
+					}
+					if ( i > Max_num_extensions ) {
+						fprintf(stderr, "Extension format error: Invalid extension: %i\n", i);
+						exit(255);
+					}
+					mask[i] = sign;
+				}
+					
+			}
+		}
+		p = q;
+	}
+	for (i=4; i<= Max_num_extensions; i++ ) {
+		int ui = extension_descriptor[i].user_index;
+
+		// mask[ui] == 0 means no input from user -> default behaviour or already overwritten by '*' 
+		if ( mask[ui] < 0 ) {
+			extension_descriptor[i].enabled = 0;
+		}
+		if ( mask[ui] > 0 ) {
+			extension_descriptor[i].enabled = 1;
+		}
+		if ( extension_descriptor[i].enabled ) {
+			dbg_printf("Add extension: %s\n", extension_descriptor[i].description);
+			syslog(LOG_INFO, "Add extension: %s", extension_descriptor[i].description);
+		}
+	}
+
+	free(mask);
+
+} // End of SetupExtensionDescriptors
+
+void PrintExtensionMap(extension_map_t *map) {
+int i;
+
+	printf("Extension Map:\n");
+	printf("  Map ID   = %u\n", map->map_id);
+	printf("  Map Size = %u\n", map->size);
+	printf("  Ext Size = %u\n", map->extension_size);
+	i=0;
+	while (map->ex_id[i]) {
+		int id = map->ex_id[i++];
+		printf("  Index %3i, ext %3u = %s\n", extension_descriptor[id].user_index, id, extension_descriptor[id].description );
+	}
+	printf("\n");
+
+} // End of PrintExtensionMap
+
+int VerifyExtensionMap(extension_map_t *map) {
+int i, failed, extension_size, max_elements;
+
+	failed = 0;
+	if (( map->size & 0x3 ) != 0 ) {
+		printf("Verify map id %i: WARNING: map size %i not aligned!\n", map->map_id, map->size);
+		failed = 1;
+	}
+
+	if ( ((int)map->size - (int)sizeof(extension_map_t)) <= 0 ) {
+		printf("Verify map id %i: ERROR: map size %i too small!\n", map->map_id, map->size);
+		failed = 1;
+		return 0;
+	}
+
+	max_elements = (map->size - sizeof(extension_map_t)) / sizeof(uint16_t);
+	extension_size = 0;
+	i=0;
+    while (map->ex_id[i] && i <= max_elements) {
+        int id = map->ex_id[i];
+		if ( id > Max_num_extensions ) {
+			printf("Verify map id %i: ERROR: element id %i out of range [%i]!\n", map->map_id, id, Max_num_extensions);
+			failed = 1;
+		}
+        extension_size += extension_descriptor[id].size;
+        i++;
+    }
+
+	if ( (extension_size != map->extension_size ) ) {
+		printf("Verify map id %i: ERROR extension size: Expected %i, Map reports: %i!\n",  map->map_id,
+			extension_size, map->extension_size);
+		failed = 1;
+	}
+	if ( (i != max_elements ) && ((max_elements-i) != 1) ) {
+		// off by 1 is the opt alignment
+		printf("Verify map id %i: ERROR: Expected %i elements in map, but found %i!\n", map->map_id, max_elements, i);
+		failed = 1;
+	}
+
+	return failed;
+
+} // End of VerifyExtensionMap
+
+void FixExtensionMap(extension_map_t *map) {
+int i, extension_size, max_elements;
+
+	if (( map->size & 0x3 ) != 0 ) {
+		printf("PANIC! - Verify map id %i: WARNING: map size %i not aligned!\n", map->map_id, map->size);
+		exit(255);
+	}
+
+	if ( ((int)map->size - (int)sizeof(extension_map_t)) <= 0 ) {
+		printf("PANIC! - Verify map id %i: ERROR: map size %i too small!\n", map->map_id, map->size);
+		exit(255);
+	}
+
+	max_elements = (map->size - sizeof(extension_map_t)) / sizeof(uint16_t);
+	extension_size = 0;
+	i=0;
+    while (map->ex_id[i] && i <= max_elements) {
+        int id = map->ex_id[i];
+		if ( id > Max_num_extensions ) {
+			printf("PANIC! - Verify map id %i: ERROR: element id %i out of range [%i]!\n", map->map_id, id, Max_num_extensions);
+		}
+        extension_size += extension_descriptor[id].size;
+        i++;
+    }
+
+	// silently fix extension size bug of nfdump <= 1.6.2 ..
+	if ( (extension_size != map->extension_size ) ) {
+#ifdef DEVEL
+		printf("FixExtension map extension size from %i to %i\n", map->extension_size, extension_size);
+#endif
+		map->extension_size = extension_size;
+	}
+
+	if ( (i != max_elements ) && ((max_elements-i) != 1) ) {
+		// off by 1 is the opt alignment
+		printf("Verify map id %i: ERROR: Expected %i elements in map, but found %i!\n", map->map_id, max_elements, i);
+	}
+
+} // End of FixExtensionMap
+
+
+void DumpExMaps(char *filename) {
+int i, done;
+nffile_t	*nffile;
+common_record_t *flow_record;
+uint32_t skipped_blocks;
+uint64_t total_bytes;
+
+	Max_num_extensions = 0;
+	i = 1;
+	while ( extension_descriptor[i++].id ) 
+		Max_num_extensions++;
+
+	printf("\nDump all extension maps:\n");
+	printf("========================\n");
+
+	nffile = OpenFile(filename, NULL);
+	if ( !nffile ) {
+		return;
+	}
+
+	done = 0;
+	while ( !done ) {
+	int i, ret;
+
+		// get next data block from file
+		ret = ReadBlock(nffile);
+
+		switch (ret) {
+			case NF_CORRUPT:
+			case NF_ERROR:
+				if ( ret == NF_CORRUPT ) 
+					LogError("Corrupt data file '%s': '%s'\n",filename);
+				else 
+					LogError("Read error in file '%s': %s\n",filename, strerror(errno) );
+				done = 1;
+				continue;
+				break;
+				// fall through - get next file in chain
+			case NF_EOF:
+				done = 1;
+				continue;
+				break;
+	
+			default:
+				// successfully read block
+				total_bytes += ret;
+		}
+
+		if ( nffile->block_header->id != DATA_BLOCK_TYPE_2 ) {
+			skipped_blocks++;
+			continue;
+		}
+
+		// block type = 2
+
+		flow_record = (common_record_t *)nffile->buff_ptr;
+		for ( i=0; i < nffile->block_header->NumRecords; i++ ) {
+			if ( flow_record->type == ExtensionMapType ) {
+				extension_map_t *map = (extension_map_t *)flow_record;
+				VerifyExtensionMap(map);
+				PrintExtensionMap(map);
+			}
+
+			// Advance pointer by number of bytes for netflow record
+			flow_record = (common_record_t *)((pointer_addr_t)flow_record + flow_record->size);	
+		}
+	}
+
+	CloseFile(nffile);
+	DisposeFile(nffile);
+
+} // End of DumpExMaps
+