geo_normalize 0.1.0

data/ext/geo_normalize/extconf.rb

@@ -0,0 +1,4 @@
+require 'mkmf'
+extension_name = 'geo_normalize'
+dir_config(extension_name)
+create_makefile(extension_name)

data/ext/geo_normalize/geo_normalize.c

@@ -0,0 +1,92 @@
+#include <stdlib.h>
+#include <string.h>
+#include <iconv.h>
+#include <errno.h>
+
+static char *geo_charmap[] = {
+	NULL , NULL , NULL , NULL , NULL , NULL , NULL , NULL , NULL , NULL , NULL , NULL , NULL , NULL , NULL , NULL ,
+	NULL , NULL , NULL , NULL , NULL , NULL , NULL , NULL , NULL , NULL , NULL , NULL , NULL , NULL , NULL , NULL ,
+	NULL , NULL , NULL , NULL , NULL , NULL , NULL , NULL , NULL , NULL , NULL , NULL , NULL , NULL , NULL , NULL ,
+	"0"  , "1"  , "2"  , "3"  , "4"  , "5"  , "6"  , "7"  , "8"  , "9"  , NULL , NULL , NULL , NULL , NULL , NULL ,
+	NULL , "a"  , "b"  , "c"  , "d"  , "e"  , "f"  , "g"  , "h"  , "i"  , "j"  , "k"  , "l"  , "m"  , "n"  , "o"  ,
+	"p"  , "q"  , "r"  , "s"  , "t"  , "u"  , "v"  , "w"  , "x"  , "y"  , "z"  , NULL , NULL , NULL , NULL , NULL ,
+	NULL , "a"  , "b"  , "c"  , "d"  , "e"  , "f"  , "g"  , "h"  , "i"  , "j"  , "k"  , "l"  , "m"  , "n"  , "o"  ,
+	"p"  , "q"  , "r"  , "s"  , "t"  , "u"  , "v"  , "w"  , "x"  , "y"  , "z"  , NULL , NULL , NULL , NULL , NULL ,
+	NULL , NULL , NULL , NULL , NULL , NULL , NULL , NULL , NULL , NULL , NULL , NULL , NULL , NULL , NULL , NULL ,
+	NULL , NULL , NULL , NULL , NULL , NULL , NULL , NULL , NULL , NULL , NULL , NULL , NULL , NULL , NULL , NULL ,
+	NULL , NULL , NULL , NULL , NULL , NULL , NULL , NULL , NULL , NULL , NULL , NULL , NULL , NULL , NULL , NULL ,
+	NULL , NULL , NULL , NULL , NULL , NULL , NULL , NULL , NULL , NULL , NULL , NULL , NULL , NULL , NULL , NULL ,
+	"a"  , "a"  , "a"  , "a"  , "a"  , "a"  , "ae" , "c"  , "e"  , "e"  , "e"  , "e"  , "i"  , "i"  , "i"  , "i"  ,
+	"d"  , "n"  , "o"  , "o"  , "o"  , "o"  , "o"  , "x"  , "o"  , "u"  , "u"  , "u"  , "u"  , "y"  , "th" , "ss" ,
+	"a"  , "a"  , "a"  , "a"  , "a"  , "a"  , "ae" , "c"  , "e"  , "e"  , "e"  , "e"  , "i"  , "i"  , "i"  , "i"  ,
+	"o"  , "n"  , "o"  , "o"  , "o"  , "o"  , "o"  , NULL , "o"  , "u"  , "u"  , "u"  , "u"  , "y"  , "th" , "y"  ,
+	};
+
+/*
+ * This takes a UTF-8 string like "Rivière-du-Loup" and, optionally,
+ * a pre-allocated UTF-8 to ISO-8859-1//IGNORE conversion descriptor,
+ * and returns a normalized ascii version "riviereduloup"
+ *
+ * It's indended for processing maxmind geo data as well as geo-targetting rules before
+ * doing strcmp(), to tolerate variations in accent/whitespace/punctuation/capitalization
+ *
+ * Returns a freshly-malloc()ed string on success, NULL on failure.  Be sure to free()
+ * result when you no longer need it
+ */
+char *
+geo_normalize(char *in, iconv_t cd)
+{
+	iconv_t local_cd = NULL;
+	char *inptr, *out, *outptr, *latin, *latinptr;
+	char *c, *r;
+	size_t inlen, latinlen, conv;
+
+	if (in == NULL)
+		return NULL;
+	inlen = strlen(in);
+
+	if (cd == NULL) {
+		local_cd = iconv_open("ISO-8859-1//IGNORE", "UTF-8");
+		if (local_cd == (iconv_t) -1)
+			return NULL;
+		cd = local_cd;
+	}
+
+	// Do a single allocation to be used for:
+	// [normalized output]\x00[iconv output]\x00
+	out = malloc((inlen*4)+2);
+	if (out == NULL) {
+		if (local_cd != NULL)
+			iconv_close(local_cd);
+		return NULL;
+	}
+	latin = out + (inlen*2) + 1;
+	latinlen = (inlen*2) + 1;
+
+	inptr = in;
+	latinptr = latin;
+	conv = iconv(cd, &inptr, &inlen, &latinptr, &latinlen);
+	if (local_cd != NULL)
+		iconv_close(local_cd);
+
+	if (conv == (size_t) -1 && errno == E2BIG) {
+		// Failed
+		free(out);
+		return NULL;
+	}
+
+	// Scan latin and normalize into out
+	outptr = out;
+	for (c = latin; c < latinptr; c++) {
+		if ((r = geo_charmap[(unsigned char)*c]) != NULL) {
+			while (*r != 0) {
+				*outptr = *r;
+				r++;
+				outptr++;
+			}
+		}
+	}
+	*outptr = 0;
+
+	return out;
+}

data/ext/geo_normalize/ruby_geo_normalize.c

@@ -0,0 +1,36 @@
+#include <ruby.h>
+#include <iconv.h>
+
+VALUE GeoNormalize = Qnil;
+
+void Init_geo_normalize();
+VALUE method_geo_normalize(VALUE self, VALUE str);
+char *geo_normalize(char *str, iconv_t cd);
+
+void Init_geo_normalize() {
+    GeoNormalize = rb_define_module("GeoNormalize");
+    rb_define_singleton_method(GeoNormalize, "normalize", method_geo_normalize, 1);
+}
+
+VALUE method_geo_normalize(VALUE self, VALUE str) {
+    if (TYPE(str) != T_STRING)
+	rb_raise(rb_eArgError, "normalize needs a string parameter");
+
+    size_t length = RSTRING_LEN(str);
+    char *buffer = malloc(length+1);
+    memcpy(buffer, RSTRING_PTR(str), length);
+    buffer[length] = 0;
+
+    char *result = geo_normalize(buffer, NULL);
+
+    free(buffer);
+
+    if (result == NULL)
+	rb_raise(rb_eStandardError, "geo_normalize call failed");
+
+    VALUE ret = rb_str_new2(result);
+
+    free(result);
+
+    return ret;
+}

metadata

@@ -0,0 +1,64 @@
+--- !ruby/object:Gem::Specification
+name: geo_normalize
+version: !ruby/object:Gem::Version
+  version: 0.1.0
+  prerelease: 
+platform: ruby
+authors:
+- Jeremie Lasalle Ratelle
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2013-05-23 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: rake-compiler
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+description: 
+email: 
+executables: []
+extensions:
+- ext/geo_normalize/extconf.rb
+extra_rdoc_files: []
+files:
+- ext/geo_normalize/geo_normalize.c
+- ext/geo_normalize/ruby_geo_normalize.c
+- ext/geo_normalize/extconf.rb
+homepage: 
+licenses: []
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 1.8.25
+signing_key: 
+specification_version: 3
+summary: GeoIP city name normalization
+test_files: []