geoipdb 0.1.4

data/.document

@@ -0,0 +1,5 @@
+lib/**/*.rb
+bin/*
+- 
+features/**/*.feature
+LICENSE.txt

data/.rspec

@@ -0,0 +1 @@
+--color

data/Gemfile

@@ -0,0 +1,13 @@
+source "http://rubygems.org"
+# Add dependencies required to use your gem here.
+# Example:
+#   gem "activesupport", ">= 2.3.5"
+
+# Add dependencies to develop your gem here.
+# Include everything needed to run rake, tests, features, etc.
+group :development do
+  gem "rspec", "~> 2.1.0"
+  gem "bundler", "~> 1.0.0"
+  gem "jeweler", "~> 1.5.1"
+  gem "rcov", ">= 0"
+end

data/LICENSE.txt

@@ -0,0 +1,20 @@
+Copyright (c) 2010 Eugen Martin
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.markdown

@@ -0,0 +1,18 @@
+# geoipdb: fast (in memory!) geo location db.
+
+Fast (>3 Mio queries/sec!!!) GeoIpDb implementation for Ruby using C-Extensions. 
+
+  * Returns a GeoLocation to a given IP. 
+  * Reads Data from CSV-Files and uses internal binary caching.
+
+## Usage
+
+    db = GeoIpDb.init "city_codes.csv", "ip_city.txt", "ip_city.cache"
+    location_hash = db.city_by_ip("178.0.0.1")
+    # => {"name"=>"eschborn", "country"=>"deu", "lng"=>8.55, "lat"=>50.133333} 
+
+== Copyright
+
+Copyright (c) 2010 Eugen Martin. See LICENSE.txt for
+further details.
+

data/Rakefile

@@ -0,0 +1,48 @@
+require 'rubygems'
+require 'bundler'
+begin
+  Bundler.setup(:default, :development)
+rescue Bundler::BundlerError => e
+  $stderr.puts e.message
+  $stderr.puts "Run `bundle install` to install missing gems"
+  exit e.status_code
+end
+require 'rake'
+
+require 'jeweler'
+Jeweler::Tasks.new do |gem|
+  # gem is a Gem::Specification... see http://docs.rubygems.org/read/chapter/20 for more options
+  gem.name = "geoipdb"
+  gem.homepage = "http://github.com/olgen/geoipdb"
+  gem.license = "MIT"
+  gem.summary = %Q{Fast (>3 Mio queries/sec!!!) GeoIpDb implementation for Ruby using C-Extensions.}
+  gem.description = %Q{Returns a GeoLocation to a given IP. Reads Data from CSV-Files and uses internal binary caching.}
+  gem.email = "eugeniusmartinus@googlemail.com"
+  gem.authors = ["Eugen Martin"]
+  gem.extensions   = ['ext/extconf.rb']
+  gem.require_path = "ext"  
+end
+Jeweler::RubygemsDotOrgTasks.new
+
+require 'rspec/core'
+require 'rspec/core/rake_task'
+RSpec::Core::RakeTask.new(:spec) do |spec|
+  spec.pattern = FileList['spec/**/*_spec.rb']
+end
+
+RSpec::Core::RakeTask.new(:rcov) do |spec|
+  spec.pattern = 'spec/**/*_spec.rb'
+  spec.rcov = true
+end
+
+task :default => :spec
+
+require 'rake/rdoctask'
+Rake::RDocTask.new do |rdoc|
+  version = File.exist?('VERSION') ? File.read('VERSION') : ""
+
+  rdoc.rdoc_dir = 'rdoc'
+  rdoc.title = "geoipdb #{version}"
+  rdoc.rdoc_files.include('README*')
+  rdoc.rdoc_files.include('lib/**/*.rb')
+end

data/VERSION

@@ -0,0 +1 @@
+0.1.4

data/ext/build.sh

@@ -0,0 +1 @@
+gcc -Wall -o test test.c ipdb.c ipdb.h

data/ext/extconf.rb

@@ -0,0 +1,5 @@
+require "mkmf"
+
+extension_name = "geoipdb"
+dir_config(extension_name)
+create_makefile(extension_name)

data/ext/geoipdb.c

@@ -0,0 +1,109 @@
+#include "ipdb.h"
+
+#include "ruby.h"
+
+/** 
+Ruby Wrapper
+*/
+
+typedef struct geipdb {
+  IPDB *db;
+} geoipdb;
+
+static VALUE cGeoIpDb;
+
+VALUE method_hello_world(VALUE self);
+
+// free the memory used by the db, called by the Ruby-GC
+void geoipdb_free(geoipdb *gi) {
+  printf("Freeing memory for the GeoIpDb.. \n");
+  if(gi == NULL)
+    return;
+  if (gi->db != NULL){
+    if(gi->db->cities != NULL ){
+      printf("..freeing cities \n");
+      free(gi->db->cities);
+      gi->db->cities = NULL;
+    }
+    if(gi->db->ranges != NULL ){
+      printf("..freeing ranges \n");
+      free(gi->db->ranges);
+      gi->db->ranges = NULL;
+    }
+    if(gi->db != NULL){
+      printf("..freeing ipdb \n");      
+      free(gi->db);      
+    }
+  }
+}
+
+
+VALUE ipdb_init(VALUE self, VALUE cities_file_name, VALUE ranges_file_name, VALUE cache_file_name) {
+  geoipdb *gi;
+  
+  Check_Type(cities_file_name, T_STRING);
+  Check_Type(ranges_file_name, T_STRING);
+  Check_Type(cache_file_name, T_STRING);
+  
+  char *cities_csv_file = RSTRING(cities_file_name)->ptr;
+  char *ranges_csv_file = RSTRING(ranges_file_name)->ptr;
+  char *cache_file =      RSTRING(cache_file_name)->ptr;  
+  
+  gi = malloc(sizeof(geoipdb));
+  
+  gi->db= init_db(cities_csv_file, ranges_csv_file, cache_file);
+  
+  if(gi->db == NULL)
+  {
+    if(DEBUG)
+      printf("Could not init DB!\n");
+    /*
+      TODO: Add geoipdb_free in this case.. though not important for production...
+    */
+    return Qnil;
+  }else{
+    if(DEBUG)
+      printf("\nDB Init completed!\n");
+    return(Data_Wrap_Struct(cGeoIpDb, 0, geoipdb_free, gi));    
+  }
+}
+
+
+VALUE city_to_hash(City * city){
+  VALUE hash = rb_hash_new();       
+  // return a ruby-hash with the needed data
+  rb_hash_aset(hash, rb_str_new2("name"), rb_str_new2(city->name));  
+  rb_hash_aset(hash, rb_str_new2("country"), rb_str_new2(city->country_iso2));      
+  rb_hash_aset(hash, rb_str_new2("lat"), rb_float_new(city->lat));  
+  rb_hash_aset(hash, rb_str_new2("lng"), rb_float_new(city->lng));      
+  rb_hash_aset(hash, rb_str_new2("city_code"), INT2FIX(city->city_code));        
+  return hash;
+}
+
+VALUE ipdb_city_by_ip(VALUE self, VALUE ip_string){
+  char *ip = RSTRING(ip_string)->ptr;
+  geoipdb *gi;
+  
+  Data_Get_Struct(self, geoipdb, gi);
+  City * city = city_by_ip(gi->db, ip);  
+  if(city == NULL)
+  {
+    // printf("Could not find city for ip: %s\n", ip);
+    return Qnil;
+  }else{
+    // printf("Found city for ip: %s\n", ip);
+    // print_city(city);
+    return city_to_hash(city);
+    // create a hash to return to ruby:
+  }
+}
+
+
+
+void Init_geoipdb(void)
+{
+  //Erstelle die Klasse cGeoIpDb als Subklasse von Object
+  cGeoIpDb = rb_define_class( "GeoIpDb", rb_cObject);
+  rb_define_singleton_method( cGeoIpDb, "init",        ipdb_init, 3); 
+  rb_define_method(           cGeoIpDb, "city_by_ip",  ipdb_city_by_ip, 1);  
+}

data/ext/ipdb.c

@@ -0,0 +1,604 @@
+#include "ipdb.h"
+
+#include <search.h>
+#include <stdio.h>
+#include <string.h>
+#include <stdlib.h>
+#include <sys/time.h>
+
+
+
+
+const char country_iso2_codes[253][3] = { "--","ap","eu","ad","ae","af","ag","ai","al","am","an",
+	"ao","aq","ar","as","at","au","aw","az","ba","bb",
+	"bd","be","bf","bg","bh","bi","bj","bm","bn","bo",
+	"br","bs","bt","bv","bw","by","bz","ca","cc","cd",
+	"cf","cg","ch","ci","ck","cl","cm","cn","co","cr",
+	"cu","cv","cx","cy","cz","de","dj","dk","dm","do",
+	"dz","ec","ee","eg","eh","er","es","et","fi","fj",
+	"fk","fm","fo","fr","fx","ga","gb","gd","ge","gf",
+	"gh","gi","gl","gm","gn","gp","gq","gr","gs","gt",
+	"gu","gw","gy","hk","hm","hn","hr","ht","hu","id",
+	"ie","il","in","io","iq","ir","is","it","jm","jo",
+	"jp","ke","kg","kh","ki","km","kn","kp","kr","kw",
+	"ky","kz","la","lb","lc","li","lk","lr","ls","lt",
+	"lu","lv","ly","ma","mc","md","mg","mh","mk","ml",
+	"mm","mn","mo","mp","mq","mr","ms","mt","mu","mv",
+	"mw","mx","my","mz","na","nc","ne","nf","ng","ni",
+	"nl","no","np","nr","nu","nz","om","pa","pe","pf",
+	"pg","ph","pk","pl","pm","pn","pr","ps","pt","pw",
+	"py","qa","re","ro","ru","rw","sa","sb","sc","sd",
+	"se","sg","sh","si","sj","sk","sl","sm","sn","so",
+	"sr","st","sv","sy","sz","tc","td","tf","tg","th",
+	"tj","tk","tm","tn","to","tl","tr","tt","tv","tw",
+	"tz","ua","ug","um","us","uy","uz","va","vc","ve",
+	"vg","vi","vn","vu","wf","ws","ye","yt","rs","za",
+	"zm","me","zw","a1","a2","o1","ax","gg","im","je",
+  "bl","mf"};
+
+static const unsigned num_countries = (unsigned)(sizeof(country_iso2_codes)/sizeof(country_iso2_codes[0]));
+
+const char country_iso3_codes[253][4] = { "--","ap","eu","and","are","afg","atg","aia","alb","arm","ant",
+	"ago","aq","arg","asm","aut","aus","abw","aze","bih","brb",
+	"bgd","bel","bfa","bgr","bhr","bdi","ben","bmu","brn","bol",
+	"bra","bhs","btn","bv","bwa","blr","blz","can","cc","cod",
+	"caf","cog","che","civ","cok","chl","cmr","chn","col","cri",
+	"cub","cpv","cx","cyp","cze","deu","dji","dnk","dma","dom",
+	"dza","ecu","est","egy","esh","eri","esp","eth","fin","fji",
+	"flk","fsm","fro","fra","fx","gab","gbr","grd","geo","guf",
+	"gha","gib","grl","gmb","gin","glp","gnq","grc","gs","gtm",
+	"gum","gnb","guy","hkg","hm","hnd","hrv","hti","hun","idn",
+	"irl","isr","ind","io","irq","irn","isl","ita","jam","jor",
+	"jpn","ken","kgz","khm","kir","com","kna","prk","kor","kwt",
+	"cym","kaz","lao","lbn","lca","lie","lka","lbr","lso","ltu",
+	"lux","lva","lby","mar","mco","mda","mdg","mhl","mkd","mli",
+	"mmr","mng","mac","mnp","mtq","mrt","msr","mlt","mus","mdv",
+	"mwi","mex","mys","moz","nam","ncl","ner","nfk","nga","nic",
+	"nld","nor","npl","nru","niu","nzl","omn","pan","per","pyf",
+	"png","phl","pak","pol","spm","pcn","pri","pse","prt","plw",
+	"pry","qat","reu","rou","rus","rwa","sau","slb","syc","sdn",
+	"swe","sgp","shn","svn","sjm","svk","sle","smr","sen","som",
+	"sur","stp","slv","syr","swz","tca","tcd","tf","tgo","tha",
+	"tjk","tkl","tkm","tun","ton","tls","tur","tto","tuv","twn",
+	"tza","ukr","uga","um","usa","ury","uzb","vat","vct","ven",
+	"vgb","vir","vnm","vut","wlf","wsm","yem","yt","srb","zaf",
+	"zmb","mne","zwe","a1","a2","o1","ala","ggy","imn","jey",
+  "blm","maf"};
+
+
+void 
+print_range(const IpRange* e){
+  printf( "from: %lu, to:%lu ->City-idx: %i \n",e->from, e->to,e->city_index );    
+}
+
+void 
+print_ranges(IPDB * db){
+  int i;
+  for(i = 0; i < db->ranges_count; ++i)
+  {
+    print_range(&(db->ranges[i]));
+  }
+}
+
+void 
+print_city(const City * e){
+  if(e == NULL)
+  {
+    return;
+  }
+  printf( "City: code:%i, name:%s, country: %s, lat: %10.7f, lng: %10.7f  \n",e->city_code, e->name, e->country_iso3, e->lat, e->lng );    
+}
+
+void 
+print_cities(IPDB * db){
+  int i;
+  for(i = 0; i < db->cities_count; ++i)
+  {
+    print_city(&(db->cities[i]));
+  }
+}
+
+
+
+void print_stats(IPDB * db){
+  printf("DB STATS: \n");
+  printf("\tCities: %i\n", db->cities_count);
+  printf("\tRanges: %i\n", db->ranges_count);  
+}
+
+double 
+get_time(struct timeval *tim){
+  gettimeofday(tim, NULL);
+  return tim->tv_sec+(tim->tv_usec/1000000.0);
+}
+
+
+unsigned long
+ip_to_int(const char *addr){
+	unsigned int    c, octet, t;
+	unsigned long   ipnum;
+	int             i = 3;
+
+	octet = ipnum = 0;
+	while ((c = *addr++)) {
+		if (c == '.') {
+			if (octet > 255)
+				return 0;
+			ipnum <<= 8;
+			ipnum += octet;
+			i--;
+			octet = 0;
+		} else {
+			t = octet;
+			octet <<= 3;
+			octet += t;
+			octet += t;
+			c -= '0';
+			if (c > 9)
+				return 0;
+			octet += c;
+		}
+	}
+	if ((octet > 255) || (i != 0))
+		return 0;
+	ipnum <<= 8;
+	return ipnum + octet;
+}
+
+
+// Function to compare 
+// - either two ip-ranges: i.e.: a(from...to) <=> b(from...to)
+// - or a ip(i.e. range without to) and an ip-range: i.e. a(from...NULL) <=> b(from...to); a(from...to) <=> b(from ... NULL)
+int compare_ranges(const void *fa, const void *fb) {
+  if(fa == NULL)
+  {
+    printf("FA IS NULL!!!\n");
+    return 0;
+  }
+  if(fb == NULL)
+  {
+    printf("FB IS NULL!!!\n");
+    return 0;
+  }
+
+
+  const IpRange *a = (IpRange *) fa;
+  const IpRange *b = (IpRange *) fb;  
+
+  // printf("\tComparing: a:");
+  // print_range(a);
+  // printf(" with b:");
+  // print_range(b);
+  // printf("\n");
+
+  if(a->from>0 && a->to>0 && b->from>0 && b->to>0){ //regular case: both entries are ranges
+    if(a->to  <  b->from)    {
+      return -1;
+    }else if(a->from > b->to){
+      return +1;
+    }else{
+      return 0;
+    }
+  }else if(a->to == 0 && b->to>0){//a is a search_object
+    if(a->from  <  b->from)    {
+      return -1;
+    }else if(a->from > b->to){
+      return +1;
+    }else{
+      return 0;
+    }
+  }else if(b->to == 0 && a->to>0){//b is a search_object
+    if(b->from  <  a->from){
+      return -1;
+    }else if(b->from > a->to){
+      return +1;
+    }else{
+      return 0;
+    }
+  }else if(a->to == 0 && b->to == 0){ //both are search objects - this should not happen!
+    return a->from - b->from;
+  }
+  return 0;
+}
+
+
+int 
+compare_cities(const void *a, const void *b){
+  const City city_a = *(City*)a;
+  const City city_b = * (City*) b;
+  // sort cities by city_code
+  return city_a.city_code - city_b.city_code;
+}
+
+void
+sort_cities(IPDB * db){
+  if(DEBUG)
+    printf("Sorting %i Cities in db...\n", db->cities_count);
+  
+  struct timeval tim;
+  double t1 = get_time(&tim);  
+  
+  qsort(db->cities,db->cities_count,sizeof(City), compare_cities);  
+  if(DEBUG)
+    printf("\n Sorting cities needed %.6lf seconds\n", get_time(&tim)-t1);  
+}
+
+
+int // returns a city-inde
+city_index_by_code(IPDB * db, uint16 city_code){
+  City *search, *result;
+  search = malloc(sizeof(City));
+  search->city_code = city_code;
+  result = (City*) bsearch(search, db->cities, db->cities_count, sizeof(City), compare_cities);
+  
+  if(search != NULL)
+    free(search);
+  
+  if(result == NULL)
+  {
+    if(DEBUG)
+      printf("Could not find searched city with code: %i \n", city_code);
+    return -1;
+  }else{
+    // printf("Found result: \t");
+    // print_city(result);
+    int index;
+    index = (result - db->cities);
+    // printf("Found index: %i\n", index);
+    return index;
+  }
+}
+
+
+City * 
+city_by_ip(IPDB *db, char *ip){
+  IpRange * search, *result;
+  search = (IpRange *)malloc(sizeof(IpRange));
+  void * res;
+
+  // print_stats(db);
+  
+  if(db == NULL)
+  {
+    printf("ERROR: DB ist NULL! ");
+    return NULL;
+  }
+  
+  if(db->ranges_count == 0)
+  {
+    printf("ERROR: DB has no Ranges Data. Can not search!\n");
+    return NULL;
+  }
+    
+  search->from = ip_to_int(ip);
+  search->to=0;
+  search->city_index = 0;
+  if(DEBUG)
+    printf("Searching for: ip=%s, ipnum=%lu \n", ip, search->from);
+  // result = (IpRange*) bsearch(search, db->ranges, db->ranges_count-2, sizeof(IpRange), compare_ranges);    
+  res = bsearch(search, db->ranges, db->ranges_count, sizeof(IpRange), compare_ranges);
+  if(search != NULL)
+    free(search);
+    
+  if(res == NULL)
+  { 
+    if(DEBUG)
+    printf("ERROR: Could not find the IP: %s! THIS SHOULD NOT HAPPEN!\n", ip);
+    return NULL;
+  }else{
+    result = (IpRange*) res;
+    if(DEBUG){
+      printf("Found Range: \t");    
+      print_range(result);  
+    }
+  }
+  
+  if(db->cities_count == 0)
+  {
+    if(DEBUG)
+      printf("ERROR: DB has no City Data. Can not search!\n");
+    return NULL;
+  }
+  
+  
+  if(result->city_index >0 && result->city_index < db->cities_count)
+  {
+    // address the city directly via the array-idx
+    return &(db->cities[result->city_index]);
+  } else {
+    if(DEBUG)
+      printf("ERROR: Could not find city with index: %i - THIS SHOULD NOT HAPPEN!\n", result->city_index);
+    return NULL; 
+  }
+}
+
+
+// read ip-ranges from csv file, of format:
+// from_ip|to_ip|city_code
+void 
+read_ranges_csv(IPDB * db){  
+  struct timeval tim;
+  double t1 = get_time(&tim);
+
+  db->ranges = malloc(sizeof(IpRange) * db->max_ranges_count);
+  
+  if(DEBUG)
+    printf("Parsing RANGES-CSV-file: %s\n", db->ranges_csv_file);
+  FILE * f = fopen(db->ranges_csv_file, "rt");
+  if(f == NULL)
+  {
+    if(DEBUG)
+      printf("Could not open the CSV-file: %s", db->ranges_csv_file);
+    return;
+  }
+  char line[256];
+  char* from = NULL;
+  char* to = NULL;
+  char* city_code = NULL;
+  int invalid_cities_count = 0;
+  int city_index;
+  
+  IpRange* entry;
+  db->ranges_count = 0;
+  while (fgets(line,sizeof(line),f) && db->ranges_count < db->max_ranges_count){
+    if(DEBUG && db->ranges_count % 1000000 == 0)
+      printf("Worked lines: %i\n", db->ranges_count);
+
+    from =      strtok(line, RANGES_DELIM);
+    to =        strtok(NULL, RANGES_DELIM);
+    city_code = strtok(NULL, RANGES_DELIM);
+    city_index = city_index_by_code(db, atoi(city_code));
+    if(city_index < 0)
+    { 
+      if(DEBUG)
+        printf("Could not find city for code: %i", atoi(city_code));
+      invalid_cities_count ++;
+      continue;
+    }else{
+      entry = &(db->ranges[db->ranges_count]);
+    
+      entry->from = ip_to_int(from);
+      entry->to = ip_to_int(to);
+      // entry->city_code = atoi(city_code);
+    
+      entry->city_index = city_index;
+      // printf("Line: %s", line);
+      // printf("from: %u,to: %u, city_code:%i \n",entry->from,entry->to,entry->city_code);
+      // db->ranges[i] = *entry;
+      // printf("working record nr: %li\n", db->ranges_count);
+      db->ranges_count++;
+    }
+  } 
+  if(invalid_cities_count)
+  {
+    printf("Found invalid cities: %i", invalid_cities_count);
+  }
+  printf("\n Parsing of %i records needed %.6lf seconds\n", db->ranges_count, get_time(&tim)-t1);  
+}
+
+
+
+//translate country iso3 to iso2
+char * 
+iso2_code(char* iso3){
+  int i = 0;
+  for( i = 0; i < num_countries; i++)
+  {
+    // printf("Trying cmp of %s and %s...", iso3, country_iso3_codes[i]);
+    if(  strcmp(country_iso3_codes[i],iso3)==0)
+    {
+      return (char*) country_iso2_codes[i];
+    }
+  }
+  // printf("Could not find iso2 code for iso3: %s, using: '%s' \n", iso3, country_iso2_codes[0]);
+  return (char*) country_iso2_codes[0];
+}
+
+//read city-data from csv-file of format:
+// COUNTRY,REGION,CITY-NAME,METRO-CODE,CITY-CODE,LATITUDE,LONGITUDE
+void
+read_cities_csv(IPDB * db){
+  struct timeval tim;
+  double t1 = get_time(&tim);
+
+  db->cities_count = 0;
+  db->cities = malloc(sizeof(City) * db->max_cities_count);
+  
+  if(DEBUG)
+    printf("Parsing Cities-CSV-file: %s\n", db->cities_csv_file);
+  FILE * f = fopen(db->cities_csv_file, "rt");
+  if(f == NULL)
+  {
+    if(DEBUG)
+      printf("Could not open the Cities-CSV-file: %s", db->cities_csv_file);
+    return;
+  }
+  char line[256];
+  char  *country, *region, *name,*metro_code,*city_code,*lat,*lng ;
+  int i = 0;
+  City* entry;
+
+  while (fgets(line,sizeof(line),f) && db->cities_count < db->max_cities_count){
+    i++;
+    if(i == 1)
+      continue;//skip the header
+
+    if(DEBUG && i % 1000000 == 0)
+    {
+      printf("Worked lines: %i\n", i);
+    }
+   // printf("Line: %s", line);
+   // COUNTRY,REGION,CITY-NAME,METRO-CODE,CITY-CODE,LATITUDE,LONGITUDE    
+    country =    strtok(line, CITIES_DELIM);
+    region =     strtok(NULL, CITIES_DELIM);
+    name =       strtok(NULL, CITIES_DELIM);
+    metro_code = strtok(NULL, CITIES_DELIM);    
+    city_code =  strtok(NULL, CITIES_DELIM);
+    lat =        strtok(NULL, CITIES_DELIM);
+    lng =        strtok(NULL, CITIES_DELIM);    
+    
+    entry = &(db->cities[db->cities_count]);
+    
+    strncpy(entry->country_iso3, country, strlen(country));
+
+    // entry->country_iso2 = iso2_code(entry->country_iso3);    
+    strncpy(entry->country_iso2, iso2_code(country), 2);
+    strncpy(entry->name, name, strlen(name));
+        
+    entry->city_code =    atoi(city_code);
+    entry->lat =          atof(lat);
+    entry->lng =          atof(lng);
+    db->cities_count++;
+  }
+  if(DEBUG)
+    printf("\n Parsing of %i records needed %.6lf seconds\n", db->cities_count, get_time(&tim)-t1);
+}
+
+
+
+/**
+cache-file is an exact binary copy of the ranges+cities-arrays from memory,
+the layout goes like this:
+  db->cities_count [4 Bytes]
+  db->ranges_count [4 Bytes]
+
+  db->cities [sizeof(City)=24 x db->ranges_count Bytes]
+  db->ranges [sizeof(IpRange)=24 x db->ranges_count Bytes]
+*/
+void  
+write_cache_file(IPDB * db){
+  struct timeval tim;
+  double t1 = get_time(&tim);
+  int objects_written;
+  
+  FILE * f;
+  f = fopen(db->cache_file_name, "w");
+  if(f==NULL){
+    if(DEBUG)
+      printf("Could not open Cache-File: %s", db->cache_file_name);
+    return;
+  }
+  if(DEBUG){
+    printf("Dumping %i records to cache-file: %s\n\n", db->ranges_count, db->cache_file_name);
+  
+    //write the record length at file header
+    printf("Writing DB-Header of length: %li\n",sizeof(db->ranges_count));
+  
+    printf("RecordLength: %li\n",sizeof(IpRange));  
+    printf("FieldLength: %li\n",sizeof(db->ranges[0].from));    
+  }
+  //write the header: i.e.: numbers of records
+  fwrite(&(db->cities_count), sizeof(db->cities_count),1,f);  
+  fwrite(&(db->ranges_count), sizeof(db->ranges_count),1,f);
+  
+  if(DEBUG)
+    printf("Writing Contents with %i cities, a %li bytes each, should = %li \n", db->cities_count, sizeof(City), db->cities_count * sizeof(City));  
+  //write the actual data: all the ranges-array-buffer:
+  objects_written = fwrite(db->cities, sizeof(City), db->cities_count, f);
+  if(DEBUG)
+    printf("Writing Contents with %i ranges, a %li bytes each, should = %li \n", db->ranges_count, sizeof(IpRange), db->ranges_count * sizeof(IpRange));  
+  //write the actual data: all the ranges-array-buffer:
+  objects_written += fwrite(db->ranges, sizeof(IpRange), db->ranges_count, f);
+  
+
+  fclose(f);
+  if(DEBUG)  
+    printf("\n Writing CacheFile of %i objects needed %.6lf seconds\n", objects_written, get_time(&tim)-t1);  
+}
+
+int 
+read_cache_file(IPDB * db){ 
+  struct timeval tim;
+  double t1 = get_time(&tim);  
+  FILE * f;
+  f = fopen(db->cache_file_name, "r");
+  if(f==NULL){
+    if(DEBUG)
+      printf("Could not open Cache-File: %s", db->cache_file_name);
+    return 0;
+  }
+  int cities_header_read = fread(&(db->cities_count), sizeof(db->cities_count),1,f);
+  int ranges_header_read = fread(&(db->ranges_count), sizeof(db->ranges_count),1,f);
+
+  
+  if(cities_header_read == 0 || ranges_header_read == 0 || db->cities_count == 0 || db->ranges_count ==0 )
+  {
+    printf("Could not read Cities-Header from Cache-File: %s", db->cache_file_name);
+    return 0;
+  }
+  if(DEBUG)  
+    printf("Reading DB-Header from Cache-File: %s, with %i cities and %i ranges\n",db->cache_file_name, db->cities_count, db->ranges_count);  
+
+  int objects_read = 0;
+  if(DEBUG)
+    printf("Allocating: %lu for cities-array \n", sizeof(City)*(db->cities_count));
+  db->cities = malloc(sizeof(City) * db->cities_count);
+  objects_read += fread(db->cities, sizeof(City),db->cities_count,f);
+  if(DEBUG)
+    printf("Allocating: %lu for ranges-array \n", sizeof(IpRange)*(db->ranges_count));
+  db->ranges = malloc(sizeof(IpRange) * db->ranges_count);
+  objects_read += fread(db->ranges, sizeof(IpRange),db->ranges_count,f);
+  
+
+  fclose(f);
+  if(DEBUG)  
+    printf("Reading cacheFile of %i objects needed %.6lf seconds\n", objects_read, get_time(&tim)-t1);    
+  return objects_read;
+}
+
+void 
+benchmark_search(IPDB * db,int count){
+  printf("(Naiv) benchmark of the City-Search-Function with %i counts \n", count);
+  struct timeval tim;  
+  double t1 = get_time(&tim);
+  int i;
+  City * city;
+
+  for(i=0;i<count; i++){
+     city = city_by_ip(db, "278.50.47.0");
+  }
+  double delta = get_time(&tim)-t1;
+  
+  printf("\n\nSearch: %.6lf seconds elapsed, i.e. %.6lf Ops/Second \n", delta, count / delta);  
+}
+
+IPDB * init_db(char * cities_csv_file, char * ranges_csv_file, char * cache_file_name){
+  IPDB *db;
+  db = (IPDB*)malloc(sizeof(IPDB));
+  if (db == NULL) //no memory left
+    return NULL;
+  db->cities = NULL;
+  db->ranges = NULL;
+  db->cache_file_name = cache_file_name;
+
+  db->cities_csv_file = cities_csv_file;
+  db->max_cities_count = MAX_CITIES_COUNT;
+  db->ranges_csv_file = ranges_csv_file;
+  db->max_ranges_count = MAX_RANGES_COUNT;
+    
+
+  if(USE_CACHE && read_cache_file(db) > 0){
+    if(DEBUG)
+      printf("Loaded DB from Cache-File with %i records \n", db->ranges_count);
+  }else{
+    if(DEBUG)
+      printf("Initializing IPDB from CSV-file: %s \n", db->ranges_csv_file);    
+    read_cities_csv(db);
+    // print_cities(db);
+    if(db->cities_count == 0)
+    {
+      return NULL;
+    }
+    sort_cities(db);
+    read_ranges_csv(db);
+    // //TODO: sort ranges
+    if(db!=NULL && db->ranges_count > 0 && USE_CACHE)
+    {
+      if(DEBUG)
+        printf("Got %i records from CSV-file, writing to cache...\n", db->ranges_count);    
+      write_cache_file(db);
+    }
+  }
+  return db;  
+}
+

data/ext/ipdb.h

@@ -0,0 +1,69 @@
+
+#ifdef INT_2_BYTES
+typedef char int8;
+typedef int int16;
+typedef unsigned int uint16;
+typedef long int32;
+#else
+typedef char int8;
+typedef short int16;
+typedef unsigned short uint16;
+typedef int int32;
+#endif
+
+
+#define RANGES_DELIM "|"
+#define CITIES_DELIM ","
+#define MAX_CITIES_COUNT 100000 //Usually we have about 50 000 Cities
+#define MAX_RANGES_COUNT 10000000 //Usually we have about 6 Mio IP-Ranges
+
+
+#define USE_CACHE 1
+#define DEBUG 0
+
+typedef struct{
+  unsigned long from;
+  unsigned long to;
+  // uint16 city_code; //city codes are not larger than 2**16 = 65536
+  uint16 city_index; //index of the city in the cities-array
+} IpRange;
+
+typedef struct{
+  uint16 city_code; //city codes are not larger than 2**16 = 65536
+  char name[32];
+  double lat;
+  double lng;
+  
+  char country_iso3[4];
+  char country_iso2[3];  
+} City;
+
+typedef struct{
+  char *ranges_csv_file; //the CSV-file with ip-ranges-to-city-code-mappings
+  unsigned int max_ranges_count;
+  unsigned int ranges_count;
+
+  char *cities_csv_file;
+  unsigned int cities_count;  
+  unsigned int max_cities_count;
+
+  
+  char *cache_file_name; // a binary file to store the whole db.....
+  IpRange * ranges;
+  City * cities;
+
+} IPDB;
+
+
+// "publicly" visible functions
+IPDB * 
+init_db(char * cities_csv_file, char * ranges_csv_file, char * cache_file_name);
+
+City * 
+city_by_ip(IPDB *db, char *ip);
+
+void 
+print_city(const City * e);
+
+void 
+benchmark_search(IPDB * db,int count);

data/ext/test.c

@@ -0,0 +1,42 @@
+#include "ipdb.h"
+
+#include <search.h>
+#include <stdio.h>
+#include <string.h>
+#include <stdlib.h>
+#include <sys/time.h>
+
+
+
+int main() {
+  IPDB * db = init_db("data/cities.csv", "data/ip_ranges.csv", "data/ipdb.cache");
+  // IPDB * db = init_db(NULL, NULL, NULL);  
+  
+
+  // City * city;
+  // 
+  // // city = city_by_ip(db, "1.16.0.0");  
+  // int i;
+  int count = 10000000;
+  
+  benchmark_search(db, count);
+  
+  // for(i = 0; i < count; ++i)
+  // {
+  //   if(i % 100000 == 0)
+  //     printf("Working: i= %i \n", i);
+  //     
+  //   city = city_by_ip(db, "91.44.93.35");
+  // }
+  
+
+
+
+  
+ //  
+  int * response;
+  scanf("%i",response);
+
+  return 0;
+
+}

data/geoipdb.gemspec

@@ -0,0 +1,71 @@
+# Generated by jeweler
+# DO NOT EDIT THIS FILE DIRECTLY
+# Instead, edit Jeweler::Tasks in Rakefile, and run 'rake gemspec'
+# -*- encoding: utf-8 -*-
+
+Gem::Specification.new do |s|
+  s.name = %q{geoipdb}
+  s.version = "0.1.4"
+
+  s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
+  s.authors = ["Eugen Martin"]
+  s.date = %q{2011-01-13}
+  s.description = %q{Returns a GeoLocation to a given IP. Reads Data from CSV-Files and uses internal binary caching.}
+  s.email = %q{eugeniusmartinus@googlemail.com}
+  s.extensions = ["ext/extconf.rb"]
+  s.extra_rdoc_files = [
+    "LICENSE.txt",
+    "README.markdown"
+  ]
+  s.files = [
+    ".document",
+    ".rspec",
+    "Gemfile",
+    "LICENSE.txt",
+    "README.markdown",
+    "Rakefile",
+    "VERSION",
+    "ext/build.sh",
+    "ext/extconf.rb",
+    "ext/geoipdb.c",
+    "ext/ipdb.c",
+    "ext/ipdb.h",
+    "ext/test.c",
+    "geoipdb.gemspec",
+    "lib/geoipdb.rb",
+    "spec/geoipdb_spec.rb",
+    "spec/spec_helper.rb"
+  ]
+  s.homepage = %q{http://github.com/olgen/geoipdb}
+  s.licenses = ["MIT"]
+  s.require_paths = ["ext"]
+  s.rubygems_version = %q{1.3.7}
+  s.summary = %q{Fast (>3 Mio queries/sec!!!) GeoIpDb implementation for Ruby using C-Extensions.}
+  s.test_files = [
+    "spec/geoipdb_spec.rb",
+    "spec/spec_helper.rb"
+  ]
+
+  if s.respond_to? :specification_version then
+    current_version = Gem::Specification::CURRENT_SPECIFICATION_VERSION
+    s.specification_version = 3
+
+    if Gem::Version.new(Gem::VERSION) >= Gem::Version.new('1.2.0') then
+      s.add_development_dependency(%q<rspec>, ["~> 2.1.0"])
+      s.add_development_dependency(%q<bundler>, ["~> 1.0.0"])
+      s.add_development_dependency(%q<jeweler>, ["~> 1.5.1"])
+      s.add_development_dependency(%q<rcov>, [">= 0"])
+    else
+      s.add_dependency(%q<rspec>, ["~> 2.1.0"])
+      s.add_dependency(%q<bundler>, ["~> 1.0.0"])
+      s.add_dependency(%q<jeweler>, ["~> 1.5.1"])
+      s.add_dependency(%q<rcov>, [">= 0"])
+    end
+  else
+    s.add_dependency(%q<rspec>, ["~> 2.1.0"])
+    s.add_dependency(%q<bundler>, ["~> 1.0.0"])
+    s.add_dependency(%q<jeweler>, ["~> 1.5.1"])
+    s.add_dependency(%q<rcov>, [">= 0"])
+  end
+end
+

data/sample_data/cities.csv

@@ -0,0 +1,7 @@
+COUNTRY,REGION,CITY-NAME,METRO-CODE,CITY-CODE,LATITUDE,LONGITUDE
+***,***,?,0,7,0,0
+afg,?,?,0,1,33,1
+afg,no region,herat,-1,2,34.3452,62.
+afg,no region,kabul,-1,3,34.5167,69.1833
+afg,no region,kandahar,-1,4,31.6111,65.702
+afg,no region,mazar-e sharif,-1,5,36.7041,67.1096

data/sample_data/citiess_corrupt.csv

@@ -0,0 +1,8 @@
+COUNTRY,REGION,CITY-NAME,METRO-CODE,CITY-CODE,LATITUDE,LONGITUDE
+***,***,?,0,7,0,0
+afg,?,?,0,1,33,1
+afg,no region,herat,-1,2,34.3452,62.
+
+afg,no , , , , region,kabul,-1,3,34.5167,69.1833
+afg,no region,, , kandahar,-1,4,31.6111,65.702
+afg,no region,maza , ,r-e sharif,-1,5,36.7041,67.1096

data/sample_data/ip_ranges.csv

@@ -0,0 +1,11 @@
+start_ip|end_ip|field 13|
+0.0.0.0|0.0.0.255|0|
+0.0.1.0|0.255.255.255|1|
+1.0.0.0|1.0.0.255|2|
+1.0.1.0|1.1.0.255|3|
+1.1.1.0|1.1.1.255|4|
+1.1.2.0|1.2.2.255|5|
+1.2.3.0|1.2.3.255|5|
+1.2.4.0|1.3.255.255|2|
+1.4.0.0|1.4.0.255|5|
+1.4.1.0|1.8.255.255|3|

data/sample_data/ip_ranges_corrupt.csv

@@ -0,0 +1,20 @@
+
+
+
+start_ip|end_ip|field 13|
+0.0.0.0|0.0.0.255|0|
+
+0.0.1.0|0.255.255.255|1|
+
+1.0.0.0|1.0.0.255|2|
+
+1.0.1.0|1.1.0.255|3|
+
+asfdasdf asdf asfasdfasdf§$%&/
+
+1, | | 7 |.1.1.0|1.1.1.255|4|
+1.1.2.0|1.2.2.255|5|
+1.2.3.0|1.2.3.255|5|
+1.2.4.0|1.3.255.255|2|
+1.4.0.0|1.4.0.255|5|
+1.4.1.0|1.8.255.255|3|

data/spec/geoipdb_spec.rb

@@ -0,0 +1,37 @@
+require File.expand_path(File.dirname(__FILE__)+'/spec_helper')
+
+
+describe "Geoipdb" do
+  def init_db
+    @db = GeoIpDb.init './sample_data/cities.csv', './sample_data/ip_ranges.csv', @cache_file
+  end
+  
+  before :each do
+    @cache_file = 'sample_data/ipdb.cache'
+  end
+  
+  it "should init correctly with sample data and create the cache-file" do
+    init_db
+    @db.should_not be_nil
+    File.exist?(@cache_file).should be_true
+  end
+  
+  it "sould find the sample cities correcty" do 
+    init_db
+    #afg,no region,kabul,-1,3,34.5167,69.1833
+    @db.city_by_ip("1.1.0.254").should == {'city_code'=>3, 'name'=>'kabul', 'country'=>'af', 'lat'=>34.5167, 'lng'=>69.1833}
+  end
+  
+  it "should not throw an exception fault if data is corrupt" do
+    @db = GeoIpDb.init './sample_data/cities_corrupt.csv', './sample_data/ip_ranges_corrupt.csv', @cache_file    
+  end
+
+  it "should not init a db object if data files are missing" do
+    GeoIpDb.init( './sample_data/bla.csv', './sample_data/blubb.csv', @cache_file    ).should be_nil
+  end
+
+  
+  after :each do
+    File.unlink @cache_file if File.exist? @cache_file
+  end
+end

data/spec/spec_helper.rb

@@ -0,0 +1,12 @@
+$LOAD_PATH.unshift(File.join(File.dirname(__FILE__), '..', 'ext'))
+$LOAD_PATH.unshift(File.dirname(__FILE__))
+require 'rspec'
+require 'geoipdb'
+
+# Requires supporting files with custom matchers and macros, etc,
+# in ./support/ and its subdirectories.
+Dir["#{File.dirname(__FILE__)}/support/**/*.rb"].each {|f| require f}
+
+RSpec.configure do |config|
+  
+end

metadata

@@ -0,0 +1,150 @@
+--- !ruby/object:Gem::Specification 
+name: geoipdb
+version: !ruby/object:Gem::Version 
+  hash: 19
+  prerelease: 
+  segments: 
+  - 0
+  - 1
+  - 4
+  version: 0.1.4
+platform: ruby
+authors: 
+- Eugen Martin
+autorequire: 
+bindir: bin
+cert_chain: []
+
+date: 2011-01-14 00:00:00 +01:00
+default_executable: 
+dependencies: 
+- !ruby/object:Gem::Dependency 
+  type: :development
+  version_requirements: &id001 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ~>
+      - !ruby/object:Gem::Version 
+        hash: 11
+        segments: 
+        - 2
+        - 1
+        - 0
+        version: 2.1.0
+  requirement: *id001
+  prerelease: false
+  name: rspec
+- !ruby/object:Gem::Dependency 
+  type: :development
+  version_requirements: &id002 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ~>
+      - !ruby/object:Gem::Version 
+        hash: 23
+        segments: 
+        - 1
+        - 0
+        - 0
+        version: 1.0.0
+  requirement: *id002
+  prerelease: false
+  name: bundler
+- !ruby/object:Gem::Dependency 
+  type: :development
+  version_requirements: &id003 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ~>
+      - !ruby/object:Gem::Version 
+        hash: 1
+        segments: 
+        - 1
+        - 5
+        - 1
+        version: 1.5.1
+  requirement: *id003
+  prerelease: false
+  name: jeweler
+- !ruby/object:Gem::Dependency 
+  type: :development
+  version_requirements: &id004 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        hash: 3
+        segments: 
+        - 0
+        version: "0"
+  requirement: *id004
+  prerelease: false
+  name: rcov
+description: Returns a GeoLocation to a given IP. Reads Data from CSV-Files and uses internal binary caching.
+email: eugeniusmartinus@googlemail.com
+executables: []
+
+extensions: 
+- ext/extconf.rb
+extra_rdoc_files: 
+- LICENSE.txt
+- README.markdown
+files: 
+- .document
+- .rspec
+- Gemfile
+- LICENSE.txt
+- README.markdown
+- Rakefile
+- VERSION
+- ext/build.sh
+- ext/extconf.rb
+- ext/geoipdb.c
+- ext/ipdb.c
+- ext/ipdb.h
+- ext/test.c
+- geoipdb.gemspec
+- lib/geoipdb.rb
+- sample_data/cities.csv
+- sample_data/citiess_corrupt.csv
+- sample_data/ip_ranges.csv
+- sample_data/ip_ranges_corrupt.csv
+- spec/geoipdb_spec.rb
+- spec/spec_helper.rb
+has_rdoc: true
+homepage: http://github.com/olgen/geoipdb
+licenses: 
+- MIT
+post_install_message: 
+rdoc_options: []
+
+require_paths: 
+- ext
+required_ruby_version: !ruby/object:Gem::Requirement 
+  none: false
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      hash: 3
+      segments: 
+      - 0
+      version: "0"
+required_rubygems_version: !ruby/object:Gem::Requirement 
+  none: false
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      hash: 3
+      segments: 
+      - 0
+      version: "0"
+requirements: []
+
+rubyforge_project: 
+rubygems_version: 1.4.2
+signing_key: 
+specification_version: 3
+summary: Fast (>3 Mio queries/sec!!!) GeoIpDb implementation for Ruby using C-Extensions.
+test_files: 
+- spec/geoipdb_spec.rb
+- spec/spec_helper.rb