geo_coder 0.1.0

data/src/libsqlite3_geocoder/Makefile

@@ -0,0 +1,22 @@
+all: libsqlite3_geocoder.so
+CC=gcc -fPIC
+libsqlite3_geocoder.so: extension.o wkb_compress.o util.o metaphon.o 
+	$(CC) -lsqlite3 -lm -I/usr/include -shared $^ -o $@ 
+test: wkb_compress.c
+	$(CC) -DTEST -o wkb_compress $^
+clean:
+	rm -f *.o *.so wkb_compress
+
+libsqlite3_geocoder.so: extension.o wkb_compress.o util.o metaphon.o levenshtein.o
+	$(CC) -lsqlite3 -lm -I/usr/include -shared $^ -o $@ 
+
+test: test_wkb_compress test_levenshtein
+
+test_wkb_compress: wkb_compress.c
+	$(CC) -DTEST -o wkb_compress $^
+
+test_levenshtein: levenshtein.c
+	$(CC) -DTEST -o levenshtein $^
+
+clean:
+	rm -f *.o *.so wkb_compress levenshtein

data/src/libsqlite3_geocoder/Makefile.nix

@@ -0,0 +1,15 @@
+all: libsqlite3_geocoder.so
+
+libsqlite3_geocoder.so: extension.o wkb_compress.o util.o metaphon.o levenshtein.o
+	$(CC) -shared $^ -o $@
+
+test: test_wkb_compress test_levenshtein
+
+test_wkb_compress: wkb_compress.c
+	$(CC) -DTEST -o wkb_compress $^
+
+test_levenshtein: levenshtein.c
+	$(CC) -DTEST -o levenshtein $^
+
+clean:
+	rm -f *.o *.so wkb_compress levenshtein

data/src/libsqlite3_geocoder/Makefile.redhat

@@ -0,0 +1,15 @@
+all: libsqlite3_geocoder.so
+CFLAGS=-fPIC
+libsqlite3_geocoder.so: extension.o wkb_compress.o util.o metaphon.o levenshtein.o
+	$(CC) $(CFLAGS)  -shared $^ -o $@
+	
+test: test_wkb_compress test_levenshtein
+
+test_wkb_compress: wkb_compress.c
+	$(CC) -DTEST -o wkb_compress $^
+
+test_levenshtein: levenshtein.c
+	$(CC) -DTEST -o levenshtein $^
+
+clean:
+	rm -f *.o *.so wkb_compress levenshtein

data/src/libsqlite3_geocoder/extension.c

@@ -0,0 +1,121 @@
+# include <sqlite3ext.h>
+# include <stdio.h>
+# include <string.h>
+# include <assert.h>
+# include <math.h>
+
+# include "extension.h"
+
+static SQLITE_EXTENSION_INIT1;
+
+static void
+sqlite3_metaphone (sqlite3_context *context, int argc, sqlite3_value **argv) {
+    const unsigned char *input = sqlite3_value_text(argv[0]);
+    int max_phones = 0;
+    char *output; 
+    int len;
+    if (sqlite3_value_type(argv[0]) == SQLITE_NULL) {
+        sqlite3_result_null(context);
+        return;
+    }
+    if (argc > 1)
+        max_phones = sqlite3_value_int(argv[1]);
+    if (max_phones <= 0)
+        max_phones = strlen(input);
+    output = sqlite3_malloc((max_phones+1)*sizeof(char));
+    len = metaphone(input, output, max_phones); 
+    sqlite3_result_text(context, output, len, sqlite3_free);
+}
+
+static void
+sqlite3_levenshtein (sqlite3_context *context, int argc, sqlite3_value **argv) {
+    const unsigned char *s1 = sqlite3_value_text(argv[0]),
+                        *s2 = sqlite3_value_text(argv[1]);
+    double dist;
+    if (sqlite3_value_type(argv[0]) == SQLITE_NULL ||
+        sqlite3_value_type(argv[1]) == SQLITE_NULL) {
+        sqlite3_result_null(context);
+        return;
+    }
+    dist = levenshtein_distance(s1, s2);
+    sqlite3_result_double(context, dist);
+}
+
+static void
+sqlite3_digit_suffix (sqlite3_context *context,
+                           int argc, sqlite3_value **argv) {
+    if (sqlite3_value_type(argv[0]) == SQLITE_NULL) {
+        sqlite3_result_null(context);
+        return;
+    }
+    const unsigned char *input = sqlite3_value_text(argv[0]);
+    char *output = sqlite3_malloc((strlen(input)+1) * sizeof(char));
+    size_t len = digit_suffix(input, output);
+    sqlite3_result_text(context, output, len, sqlite3_free);
+}
+
+static void
+sqlite3_nondigit_prefix (sqlite3_context *context,
+                           int argc, sqlite3_value **argv) {
+    if (sqlite3_value_type(argv[0]) == SQLITE_NULL) {
+        sqlite3_result_null(context);
+        return;
+    }
+    const unsigned char *input = sqlite3_value_text(argv[0]);
+    char *output = sqlite3_malloc((strlen(input)+1) * sizeof(char));
+    size_t len = nondigit_prefix(input, output);
+    sqlite3_result_text(context, output, len, sqlite3_free);
+}
+
+
+static void
+sqlite3_compress_wkb_line (sqlite3_context *context,
+                           int argc, sqlite3_value **argv) {
+    if (sqlite3_value_type(argv[0]) == SQLITE_NULL) {
+        sqlite3_result_null(context);
+        return;
+    }
+    unsigned long input_len = sqlite3_value_bytes(argv[0]);
+    const void *input = sqlite3_value_blob(argv[0]);
+    unsigned long output_len = ceil((input_len-9)/8.0) * 4;
+    unsigned long len = 0;
+    void *output = sqlite3_malloc(output_len);
+    len = compress_wkb_line(output, input, input_len); 
+    assert(len == output_len);
+    sqlite3_result_blob(context, output, len, sqlite3_free);
+}
+
+static void
+sqlite3_uncompress_wkb_line (sqlite3_context *context,
+                           int argc, sqlite3_value **argv) {
+    unsigned long input_len = sqlite3_value_bytes(argv[0]);
+    const void *input = sqlite3_value_blob(argv[0]);
+    unsigned long output_len = input_len*2+9;
+    unsigned long len = 0;
+    void *output = sqlite3_malloc(output_len);
+    len = uncompress_wkb_line(output, input, input_len);
+    assert(len == output_len);
+    sqlite3_result_blob(context, output, len, sqlite3_free);
+}
+
+int sqlite3_extension_init (sqlite3 * db, char **pzErrMsg,
+                            const sqlite3_api_routines *pApi) {
+    SQLITE_EXTENSION_INIT2(pApi);
+    
+    sqlite3_create_function(db, "metaphone", 1, SQLITE_ANY,
+                            NULL, sqlite3_metaphone, NULL, NULL);
+    sqlite3_create_function(db, "metaphone", 2, SQLITE_ANY,
+                            NULL, sqlite3_metaphone, NULL, NULL);
+    
+    sqlite3_create_function(db, "levenshtein", 2, SQLITE_ANY,
+                            NULL, sqlite3_levenshtein, NULL, NULL);
+    sqlite3_create_function(db, "compress_wkb_line", 1, SQLITE_ANY,
+                            NULL, sqlite3_compress_wkb_line, NULL, NULL);
+    sqlite3_create_function(db, "uncompress_wkb_line", 1, SQLITE_ANY,
+                            NULL, sqlite3_uncompress_wkb_line, NULL, NULL);
+    sqlite3_create_function(db, "digit_suffix", 1, SQLITE_ANY,
+                            NULL, sqlite3_digit_suffix, NULL, NULL);
+    sqlite3_create_function(db, "nondigit_prefix", 1, SQLITE_ANY,
+                            NULL, sqlite3_nondigit_prefix, NULL, NULL);
+    return 0;
+}

data/src/libsqlite3_geocoder/extension.h

@@ -0,0 +1,13 @@
+#ifndef SQLITE3_GEOCODER
+#define SQLITE3_GEOCODER
+
+#include <stdint.h>
+
+int metaphone(const char *Word, char *Metaph, int max_phones);
+double levenshtein_distance (const unsigned char *s1, const unsigned char *s2);
+signed int rindex_nondigit (const char *string);
+signed int nondigit_prefix (const char *input, char *output);
+uint32_t compress_wkb_line (void *dest, const void *src, uint32_t len);
+uint32_t uncompress_wkb_line (void *dest, const void *src, uint32_t len);
+
+#endif

data/src/libsqlite3_geocoder/levenshtein.c

@@ -0,0 +1,42 @@
+# include <string.h>
+# define STRLEN_MAX 256
+# define min(x, y) ((x) < (y) ? (x) : (y))
+# define max(x, y) ((x) > (y) ? (x) : (y))
+# define NO_CASE (~(unsigned char)32)
+# define eql(x, y) (((x) & NO_CASE) == ((y) & NO_CASE))
+
+static int d[STRLEN_MAX][STRLEN_MAX]; // this isn't thread safe
+
+double levenshtein_distance (const unsigned char *s1, const unsigned char *s2) {
+    const size_t len1 = min(strlen(s1), STRLEN_MAX-1),
+                 len2 = min(strlen(s2), STRLEN_MAX-1);
+    int cost, i, j;
+
+    for (i = 1; i <= len1; ++i) d[i][0] = i;
+    for (i = 1; i <= len2; ++i) d[0][i] = i;
+    for (i = 1; i <= len1; ++i) {
+        for (j = 1; j <= len2; ++j) {
+            cost = (eql(s1[i-1], s2[j-1]) ? 0 : 1);
+            d[i][j] = min(min(
+                        d[i-1][j  ] + 1,              /* deletion */
+                        d[i  ][j-1] + 1),             /* insertion */
+                        d[i-1][j-1] + cost);          /* substitution */
+            if (i > 1 && j > 1 && eql(s1[i-1], s2[j-2]) && eql(s1[i-2], s2[j-1])) {
+               d[i][j] = min( d[i][j],
+                              d[i-2][j-2] + cost );   /* transposition */
+            }
+        }
+    }
+    return (d[len1][len2] / (double) max(len1, len2));
+}
+
+#ifdef TEST
+#include <stdio.h>
+
+int main (int argc, char **argv) {
+    if (argc < 3) return -1;
+    printf("%.1f%%\n", levenshtein_distance(argv[1],argv[2]) * 100);
+    return 0;
+}
+
+#endif

data/src/libsqlite3_geocoder/metaphon.c

@@ -0,0 +1,278 @@
+/* +++Customized by SDE for sqlite3 use 09-Mar-2009 */
+/* +++File obtained from http://www.shedai.net/c/new/METAPHON.C */
+/* +++Date previously modified: 05-Jul-1997 */
+
+/*
+**  METAPHON.C - Phonetic string matching
+**
+**  The Metaphone algorithm was developed by Lawrence Phillips. Like the
+**  Soundex algorithm, it compares words that sound alike but are spelled
+**  differently. Metaphone was designed to overcome difficulties encountered
+**  with Soundex.
+**
+**  This implementation was written by Gary A. Parker and originally published
+**  in the June/July, 1991 (vol. 5 nr. 4) issue of C Gazette. As published,
+**  this code was explicitly placed in the public domain by the author.
+*/
+
+#include <ctype.h>
+#include <string.h> /* strlen() */
+#include <stdio.h>
+#define malloc(x) sqlite3_malloc((x))
+#define free(x) sqlite3_free((x))
+
+/*
+**  Character coding array
+*/
+
+static char vsvfn[26] = {
+      1,16,4,16,9,2,4,16,9,2,0,2,2,2,1,4,0,2,4,4,1,0,0,0,8,0};
+/*    A  B C  D E F G  H I J K L M N O P Q R S T U V W X Y Z      */
+
+/*
+**  Macros to access the character coding array
+*/
+
+#define vowel(x)  (vsvfn[(x) - 'A'] & 1)  /* AEIOU    */
+#define same(x)   (vsvfn[(x) - 'A'] & 2)  /* FJLMNR   */
+#define varson(x) (vsvfn[(x) - 'A'] & 4)  /* CGPST    */
+#define frontv(x) (vsvfn[(x) - 'A'] & 8)  /* EIY      */
+#define noghf(x)  (vsvfn[(x) - 'A'] & 16) /* BDH      */
+
+int metaphone(const char *Word, char *Metaph, int max_phones) {
+    char *n, *n_start, *n_end;    /* Pointers to string               */
+    char *metaph_start = Metaph, *metaph_end;    
+    /* Pointers to metaph         */
+    int ntrans_len = strlen(Word)+4;
+    char *ntrans = (char *)malloc(sizeof(char) * ntrans_len);
+    /* Word with uppercase letters      */
+    int KSflag;                   /* State flag for X translation     */
+
+    /* SDE -- special case: if the word starts with a number, just
+     * copy the leading digits and return. This means we don't
+     * metaphone cardinal number suffixes (i.e. "st","nd","rd") */
+    int leading_digit = isdigit(*Word);
+    /* SDE -- check for a leading semivowel. needed because
+     * the copy in ntrans gets destroyed by the metaphone process. */
+    char leading_semivowel = '\0';
+
+    /*
+     ** Copy word to internal buffer, dropping non-alphabetic characters
+     ** and converting to upper case.
+     */
+    for (n = ntrans + 1, n_end = ntrans + ntrans_len - 2;
+            *Word && n < n_end; ++Word)
+    {
+        /* SDE -- see previous comment */
+        if (leading_digit && isalpha(*Word))
+            break;
+        /* SDE -- copy numbers as well, for geocoding street names */
+        /* was: if (isalpha(*Word)) */
+        if (isalnum(*Word)) 
+            *n++ = toupper(*Word);
+    }
+
+    if (n == ntrans + 1) {
+        free(ntrans);
+        Metaph[0]='\0';
+        return 0;           /* Return if zero characters        */
+    }
+    else  n_end = n;          /* Set end of string pointer        */
+
+    /*
+     ** Pad with '\0's, front and rear
+     */
+
+    *n++ = '\0';
+    *n   = '\0';
+    n    = ntrans;
+    *n++ = '\0';
+    
+    /* SDE: check for leading semivowel here */
+    if (ntrans[1] == 'W' || ntrans[1] == 'Y')
+        leading_semivowel = ntrans[1];
+
+    /*
+     ** Check for PN, KN, GN, WR, WH, and X at start
+     */
+
+    switch (*n)
+    {
+        case 'P':
+        case 'K':
+        case 'G':
+            if ('N' == *(n + 1))
+                *n++ = '\0';
+            break;
+
+        case 'A':
+            if ('E' == *(n + 1))
+                *n++ = '\0';
+            break;
+
+        case 'W':
+            if ('R' == *(n + 1))
+                *n++ = '\0';
+            else if ('H' == *(n + 1))
+            {
+                *(n + 1) = *n;
+                *n++ = '\0';
+            }
+            break;
+
+        case 'X':
+            *n = 'S';
+            break;
+    }
+
+    /*
+     ** Now loop through the string, stopping at the end of the string
+     ** or when the computed Metaphone code is max_phones characters long.
+     */
+
+    KSflag = 0;              /* State flag for KStranslation     */
+    for (metaph_end = Metaph + max_phones, n_start = n;
+            n <= n_end && Metaph < metaph_end; ++n)
+    {
+        if (KSflag)
+        {
+            KSflag = 0;
+            *Metaph++ = *n;
+        }
+        else
+        {
+            /* SDE -- special case: copy numbers verbatim */
+            if (isdigit(*n)) {
+                *Metaph++ = *n;
+                continue;
+            }
+
+            /* Drop duplicates except for CC    */
+            if (*(n - 1) == *n && *n != 'C')
+                continue;
+
+            /* Check for F J L M N R  or first letter vowel */
+
+            if (same(*n) || (n == n_start && vowel(*n)))
+                *Metaph++ = *n;
+            else switch (*n)
+            {
+                case 'B':
+                    if (n < n_end || *(n - 1) != 'M')
+                        *Metaph++ = *n;
+                    break;
+
+                case 'C':
+                    if (*(n - 1) != 'S' || !frontv(*(n + 1)))
+                    {
+                        if ('I' == *(n + 1) && 'A' == *(n + 2))
+                            *Metaph++ = 'X';
+                        else if (frontv(*(n + 1)))
+                            *Metaph++ = 'S';
+                        else if ('H' == *(n + 1))
+                            *Metaph++ = ((n == n_start &&
+                                        !vowel(*(n + 2))) ||
+                                    'S' == *(n - 1)) ? 'K' : 'X';
+                        else  *Metaph++ = 'K';
+                    }
+                    break;
+
+                case 'D':
+                    *Metaph++ = ('G' == *(n + 1) && frontv(*(n + 2))) ?
+                        'J' : 'T';
+                    break;
+
+                case 'G':
+                    if ((*(n + 1) != 'H' || vowel(*(n + 2))) &&
+                            (*(n + 1) != 'N' || ((n + 1) < n_end &&
+                                                 (*(n + 2) != 'E' || *(n + 3) != 'D'))) &&
+                            (*(n - 1) != 'D' || !frontv(*(n + 1))))
+                    {
+                        *Metaph++ = (frontv(*(n + 1)) &&
+                                *(n + 2) != 'G') ? 'J' : 'K';
+                    }
+                    else if ('H' == *(n + 1) && !noghf(*(n - 3)) &&
+                            *(n - 4) != 'H')
+                    {
+                        *Metaph++ = 'F';
+                    }
+                    break;
+
+                case 'H':
+                    if (!varson(*(n - 1)) && (!vowel(*(n - 1)) ||
+                                vowel(*(n + 1))))
+                    {
+                        *Metaph++ = 'H';
+                    }
+                    break;
+
+                case 'K':
+                    if (*(n - 1) != 'C')
+                        *Metaph++ = 'K';
+                    break;
+
+                case 'P':
+                    *Metaph++ = ('H' == *(n + 1)) ? 'F' : 'P';
+                    break;
+
+                case 'Q':
+                    *Metaph++ = 'K';
+                    break;
+
+                case 'S':
+                    *Metaph++ = ('H' == *(n + 1) || ('I' == *(n + 1) &&
+                                ('O' == *(n + 2) || 'A' == *(n + 2)))) ?
+                        'X' : 'S';
+                    break;
+
+                case 'T':
+                    if ('I' == *(n + 1) && ('O' == *(n + 2) ||
+                                'A' == *(n + 2)))
+                    {
+                        *Metaph++ = 'X';
+                    }
+                    else if ('H' == *(n + 1))
+                        /* SDE: was:
+                           *Metaph++ = 'O';
+                           but that's WRONG. */
+                        *Metaph++ = '0';
+                    else if (*(n + 1) != 'C' || *(n + 2) != 'H')
+                        *Metaph++ = 'T';
+                    break;
+
+                case 'V':
+                    *Metaph++ = 'F';
+                    break;
+
+                case 'W':
+                case 'Y':
+                    if (vowel(*(n + 1)))
+                        *Metaph++ = *n;
+                    break;
+
+                case 'X':
+                    if (n == n_start)
+                        *Metaph++ = 'S';
+                    else
+                    {
+                        *Metaph++ = 'K';
+                        KSflag = 1;
+                    }
+                    break;
+
+                case 'Z':
+                    *Metaph++ = 'S';
+                    break;
+            }
+        }
+    }
+
+    /* SDE: special case: if word consists solely of W or Y, use that. */
+    if (Metaph == metaph_start && leading_semivowel)
+        *Metaph++ = leading_semivowel;
+
+    *Metaph = '\0';
+    free(ntrans);
+    return strlen(metaph_start);
+}
+