geo_coder 0.1.0

data/Gemfile

@@ -0,0 +1,12 @@
+source "http://rubygems.org"
+  gem "sqlite3-ruby", "= 1.2.5"
+  gem "text"
+
+# Add dependencies to develop your gem here.
+# Include everything needed to run rake, tests, features, etc.
+group :development do
+  gem "bundler", "~> 1.0.0"
+  gem "jeweler", "~> 1.6.4"
+  gem "rcov" 
+  gem "rspec"
+end

data/Gemfile.lock

@@ -0,0 +1,32 @@
+GEM
+  remote: http://rubygems.org/
+  specs:
+    diff-lcs (1.1.3)
+    git (1.2.5)
+    jeweler (1.6.4)
+      bundler (~> 1.0)
+      git (>= 1.2.5)
+      rake
+    rake (0.9.2)
+    rcov (0.9.10)
+    rspec (2.6.0)
+      rspec-core (~> 2.6.0)
+      rspec-expectations (~> 2.6.0)
+      rspec-mocks (~> 2.6.0)
+    rspec-core (2.6.4)
+    rspec-expectations (2.6.0)
+      diff-lcs (~> 1.1.2)
+    rspec-mocks (2.6.0)
+    sqlite3-ruby (1.2.5)
+    text (0.2.0)
+
+PLATFORMS
+  ruby
+
+DEPENDENCIES
+  bundler (~> 1.0.0)
+  jeweler (~> 1.6.4)
+  rcov
+  rspec
+  sqlite3-ruby (= 1.2.5)
+  text

data/History.txt

@@ -0,0 +1,6 @@
+=== 1.0.0 / 2009-06-02
+
+* 1 major enhancement
+
+  * Birthday!
+

data/Makefile

@@ -0,0 +1,13 @@
+all:
+	make -C src install
+	gem build gemspec
+
+test: all
+	ruby -Ilib tests/run.rb
+
+install: all
+	gem install *.gem
+
+clean:
+	make -C src clean
+	rm *.gem

data/Manifest.txt

@@ -0,0 +1,18 @@
+History.txt
+Manifest.txt
+README.rdoc
+Rakefile
+lib/geocoder/us/database.rb
+lib/geocoder/us/numbers.rb
+lib/geocoder/us/address.rb
+lib/geocoder/us/constants.rb
+tests/database.rb
+tests/numbers.rb
+tests/generate.rb
+tests/run.rb
+tests/address.rb
+tests/benchmark.rb
+tests/constants.rb
+tests/data/address-sample.csv
+tests/data/locations.csv
+tests/data/db-test.csv

data/README.rdoc

@@ -0,0 +1,197 @@
+= Geocoder::US
+
+Geocoder::US 2.0 is a software package designed to geocode US street
+addresses.  Although it is primarily intended for use with the US Census
+Bureau's free TIGER/Line dataset, it uses an abstract US address data model
+that can be employed with other sources of US street address range data.
+
+Geocoder::US 2.0 implements a Ruby interface to parse US street addresses, and
+perform fuzzy lookup against an SQLite 3 database. Geocoder::US is designed to
+return the best matches found, with geographic coordinates interpolated from
+the street range dataset. Geocoder::US will fill in missing information, and
+it knows about standard and common non-standard postal abbreviations, ordinal
+versus cardinal numbers, and more.
+
+Geocoder::US 2.0 is shipped with a free US ZIP code data set, compiled from
+public domain sources.
+
+== Synopsis
+
+  >> require 'geocoder/us'
+  >> db = Geocoder::US::Database.new("/opt/tiger/geocoder.db")
+  >> p db.geocode("1600 Pennsylvania Av, Washington DC")
+
+  [{:pretyp=>"", :street=>"Pennsylvania", :sufdir=>"NW", :zip=>"20502",
+    :lon=>-77.037528, :number=>"1600", :fips_county=>"11001", :predir=>"",
+    :precision=>:range, :city=>"Washington", :lat=>38.898746, :suftyp=>"Ave",
+    :state=>"DC", :prequal=>"", :sufqual=>"", :score=>0.906, :prenum=>""}]
+
+== Prerequisites
+
+To build Geocoder::US, you will need gcc/g++, make, bash or equivalent, the
+standard *NIX 'unzip' utility, and the SQLite 3 executable and development
+files installed on your system.
+
+To use the Ruby interface, you will need the 'Text' gem installed from
+rubyforge. To run the tests, you will also need the 'fastercsv' gem.
+
+Additionally, you will need a custom build of the 'sqlite3-ruby' gem that
+supports loading extension modules in SQLite. You can get a patched version of
+this gem from http://github.com/schuyler/sqlite3-ruby/. Until the sqlite3-ruby
+maintainers roll in the relevant patch, you will need *this* version.
+
+*NOTE*: If you do not have /usr/include/sqlite3ext.h installed, then your
+sqlite3 binaries are probably not configured to support dynamic extension
+loading. If not, you *must* compile and install SQLite from source, or rebuild
+your system packages. This is not believed to be a problem on Debian/Ubuntu,
+but is known to be a problem with Red Hat/CentOS.
+
+*NOTE*: If you *do* have to install from source, make sure that the
+source-installed 'sqlite3' program is in your path before proceeding (and not
+the system-installed version), using `which sqlite3`. Also, be sure that you've
+added your source install prefix (usually /usr/local) to /etc/ld.so.conf (or
+its moral equivalent) and that you've run /sbin/ldconfig.
+
+== Thread safety
+
+SQLite 3 is not designed for concurrent use of a single database handle across
+multiple threads. Therefore, to prevent segfaults, Geocoder::US::Database
+implements a global mutex that wraps all database access. The use of this mutex
+will ensure stability in multi-threaded applications, but incurs a performance
+penalty. However, since the database is read-only from Ruby, there's no reason
+in principle why multi-threaded apps can't each have their own database handle.
+
+To disable the mutex for better performance, you can do the following:
+
+ * Read the following and make sure you understand them:
+    * http://www.sqlite.org/faq.html#q6
+    * http://www.sqlite.org/cvstrac/wiki?p=MultiThreading
+ * Make sure you have compiled SQLite 3 with thread safety enabled.
+ * Instantiate a separate Geocoder::US::Database object for *each* thread
+   in your Ruby script, and pass :threadsafe => true to new() to disable mutex
+   synchronization.
+
+Per the SQLite 3 documentation, do *not* attempt to retain a
+Geocoder::US::Database object across a fork! "Problems will result if you do."
+
+== Building Geocoder::US
+
+Unpack the source and run 'make'. This will compile the SQLite 3 extension
+needed by Geocoder::US, the Shapefile import utility, and the Geocoder-US
+gem.
+
+You can run 'make install' as root to install the gem systemwide.
+
+== Generating a Geocoder::US Database
+
+Build the package from source as described above. Generating the database
+involves three basic steps:
+
+* Import the Shapefile data into an SQLite database.
+* Build the database indexes.
+* Optionally, rebuild the database to cluster indexed rows.
+
+We will presume that you are building a Geocoder::US database from TIGER/Line,
+and that you have obtained the complete set of TIGER/Line ZIP files, and put
+the entire tree in /opt/tiger. Please adjust these instructions as needed.
+
+A full TIGER/Line database import takes ten hours to run on a normal Amazon
+EC2 instance, and takes up a little over 5 gigabytes after all is said and
+done.  You will need to have at least 12 gigabytes of free disk space *after*
+downloading the TIGER/Line dataset, if you are building the full database. 
+
+=== Import TIGER/Line
+
+From inside the Geocoder::US source tree, run the following:
+
+  $ bin/tiger_import /opt/tiger/geocoder.db /opt/tiger
+
+This will unpack each TIGER/Line ZIP file to a temporary directory, and
+perform the extract/transform/load sequence to incrementally build the
+database. The process takes about 10-12 hours on a normal Amazon EC2 instance,
+or about 5 CPU hours flat out on a modern PC. Note that not all TIGER/Line
+source files contain address range information, so you will see error messages
+for some counties, but this is normal.
+
+If you only want to import specific counties, you can pipe a list of
+TIGER/Line county directories to tiger_import on stdin. For example,
+the following will install just the data for the state of Delaware:
+
+  $ ls -d /opt/tiger/10_DELAWARE/1* | bin/tiger_import ~/delaware.db
+
+The tiger_import process uses a binary utility, shp2sqlite, which is derived
+from shp2pgsql, which ships with PostGIS. The shp2sqlite utility converts
+.shp and .dbf files into SQL suitable for import into SQLite. This SQL
+is then piped into the sqlite3 command line tool, where it is loaded into
+temporary tables, and then a set of static SQL statements (kept in the sql/
+directory) are used to transform this data and import it into the database
+itself.
+
+== Build metaphones using Ruby metaphone
+
+run bin/rebuild_metaphones /opt/tiger/geocoder.db 
+
+This creates the metaphones using Ruby's metaphone function and will produce better geocoding results.
+
+=== Build the indexes
+
+After the database import is complete, you will want to construct the database
+indexes:
+
+  $ bin/build_indexes /opt/tiger/geocoder.db
+
+This process takes 25 minutes on an EC2 instance (8 CPU minutes), but it's a
+*lot* faster than building the indexes incrementally during the import
+process. Basically, this process simply feeds SQL statements to the sqlite3
+utility to construct the indexes on the existing database.
+
+=== Cluster the database tables (optional)
+
+As a final optional step, you can cluster the database tables according to
+their indexes, which will make the database smaller, and lookups faster. This
+process will take an hour or two, and may be a micro-optimization.
+
+  $ bin/rebuild_cluster /opt/tiger/geocoder.db
+
+You will need as much free disk space to run rebuild_cluster as the database
+takes up, because the process essentially reconstructs the database in a new
+file, and then it renames the new database over top of the old.
+
+== Running the unit tests
+
+From within the source tree, you can run the following:
+
+  $ ruby tests/run.rb
+
+This tests the libraries, except for the database routines. If you have a
+database built, you can run the test harness like so:
+
+  $ ruby tests/run.rb /opt/tiger/geocoder.db
+
+The full test suite may take 30 or so seconds to run completely.
+
+== License
+
+Geocoder::US 2.0 was based on earlier work by Schuyler Erle on
+a Perl module of the same name. You can find it at
+http://search.cpan.org/~sderle/.
+
+Geocoder::US 2.0 was written by Schuyler Erle, of Entropy Free LLC,
+with the gracious support of FortiusOne, Inc. Please send bug reports,
+patches, kudos, etc. to patches at geocoder.us.
+
+Copyright (c) 2009 FortiusOne, Inc.
+
+    This program is free software: you can redistribute it and/or modify
+    it under the terms of the GNU General Public License as published by
+    the Free Software Foundation, either version 3 of the License, or
+    (at your option) any later version.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU General Public License for more details.
+
+    You should have received a copy of the GNU General Public License
+    along with this program.  If not, see <http://www.gnu.org/licenses/>.
+

data/Rakefile

@@ -0,0 +1,53 @@
+# encoding: utf-8
+
+require 'rubygems'
+require 'bundler'
+begin
+  Bundler.setup(:default, :development)
+rescue Bundler::BundlerError => e
+  $stderr.puts e.message
+  $stderr.puts "Run `bundle install` to install missing gems"
+  exit e.status_code
+end
+require 'rake'
+
+require 'jeweler'
+Jeweler::Tasks.new do |gem|
+  # gem is a Gem::Specification... see http://docs.rubygems.org/read/chapter/20 for more options
+  gem.name = "geo_coder"
+  gem.homepage = "http://github.com/kornypoet/geo_coder"
+  gem.license = "MIT"
+  gem.summary = "Geocoder based upon the Geocommons Geocoder."
+  gem.description = "Geocode a text address."
+  gem.email = "dempsey.travis@gmail.com"
+  gem.authors = ["Travis Dempsey"]
+  # dependencies defined in Gemfile
+end
+Jeweler::RubygemsDotOrgTasks.new
+
+require 'rake/testtask'
+Rake::TestTask.new(:test) do |test|
+  test.libs << 'lib' << 'test'
+  test.pattern = 'test/**/test_*.rb'
+  test.verbose = true
+end
+
+require 'rcov/rcovtask'
+Rcov::RcovTask.new do |test|
+  test.libs << 'test'
+  test.pattern = 'test/**/test_*.rb'
+  test.verbose = true
+  test.rcov_opts << '--exclude "gems/*"'
+end
+
+task :default => :test
+
+require 'rdoc/task'
+Rake::RDocTask.new do |rdoc|
+  version = File.exist?('VERSION') ? File.read('VERSION') : ""
+
+  rdoc.rdoc_dir = 'rdoc'
+  rdoc.title = "geo_coder #{version}"
+  rdoc.rdoc_files.include('README*')
+  rdoc.rdoc_files.include('lib/**/*.rb')
+end

data/TODO.txt

@@ -0,0 +1,8 @@
+1. Check interpolate measure: scale longitude or not?
+5. Intersections...
+    - import ALL linestrings (even those with without ranges)
+    - throw away internal points on lines that don't have ranges
+7. Documentation (*)
+8. Make SQLite memory cache size an option to the Database constructor
+9. Precision and accuracy measure
+10. Street line set back

data/VERSION

@@ -0,0 +1 @@
+0.1.0

data/bin/build_indexes

@@ -0,0 +1,8 @@
+#!/bin/bash
+
+BASE=$(dirname $0)
+PATH=$PATH:$BASE/bin
+SQL="$BASE/../sql"
+
+# Just run the SQL that constructs the indexes.
+sqlite3 $1 < ${SQL}/index.sql

data/bin/rebuild_cluster

@@ -0,0 +1,22 @@
+#!/bin/bash
+
+BASE=$(dirname $0)
+PATH=$PATH:$BASE/bin
+SQL="$BASE/../sql"
+
+OLD_DB=$1
+DATABASE=${OLD_DB}.$$
+
+[ -r $DATABASE ] && echo "$DATABASE already exists." && exit -1
+[ ! -r $OLD_DB ] && echo "Can't read $OLD_DB." && exit -1
+
+# Create a shiny new database, attach the old one,
+#   extract the data from it, and then index that.
+#   Finally, overwrite the old database with the new one.
+( cat ${SQL}/create.sql && \
+  echo "ATTACH DATABASE '${OLD_DB}' AS old;" && \
+  cat ${SQL}/cluster.sql && \
+  echo "DETACH DATABASE old;" && \
+  cat ${SQL}/index.sql && \
+  echo "ANALYZE;" ) | sqlite3 $DATABASE \
+  && mv $DATABASE $OLD_DB

data/bin/rebuild_metaphones

@@ -0,0 +1,23 @@
+#!/usr/bin/ruby
+
+require 'rubygems'
+require 'sqlite3'
+require 'text'
+
+    @db = SQLite3::Database.new("../geocoderdata/geocoder.db")
+    @db.create_function("metaphone", 2) do |func, string, len|
+      test = string.to_s.gsub(/\W/o, "")
+      if test =~ /^(\d+)/o
+        mph = $1
+      elsif test =~ /^([wy])$/io
+        mph = $1
+      else
+        mph = Text::Metaphone.metaphone test
+      end
+      func.result = mph[0...len.to_i]
+    end
+    sql = "update place set city_phone = metaphone(city,5)"
+    
+    @db.execute sql
+    
+    @db.close

data/bin/tiger_import

@@ -0,0 +1,59 @@
+#!/bin/bash
+
+TMP="/tmp/tiger-import.$$"
+SHPS="edges"
+DBFS="featnames addr"
+BASE=$(dirname $0)
+PATH=$PATH:$BASE
+SQL="$BASE/../sql"
+HELPER_LIB="$BASE/../lib/geocoder/us/sqlite3.so"
+DATABASE=$1
+shift
+
+mkdir -p $TMP || exit 1
+
+# Initialize the database if it doesn't exist.
+[ ! -r $DATABASE ] && cat ${SQL}/{create,place}.sql | sqlite3 $DATABASE
+ 
+# Marshal the county directories to import.
+#
+# If no directory was given on the command-line, read a list from STDIN.
+if [ x"$1" = x"" ]; then
+    cat
+else
+    # Otherwise, find all of the contents of each state directory.
+    ls -d $1/[0-9]* | while read state; do
+        ls -d ${state}/[0-9]*
+    done
+fi | while read county; do
+    echo "--- $county"
+    # Unpack the county files into the temp directory.
+    for file in $SHPS $DBFS; do
+	ZIP=$(ls ${county}/*_${file}.zip 2>/dev/null)
+	SHP=$(ls ${county}/*_${file}.* 2>/dev/null)
+	if [ x"$ZIP" != x"" ]; then
+	    unzip -q $ZIP -d $TMP
+	elif [ x"$SHP" != x"" ]; then
+	    ln -s $SHP $TMP
+	fi
+    done
+    # Generate an SQL stream to feed into the sqlite3 binary.
+    # Start by loading the helper libs and initializing the temporary tables
+    #   that will hold the TIGER data before ETL.
+    (echo ".load $HELPER_LIB" && \
+     cat ${SQL}/setup.sql && \
+     for file in $SHPS; do
+       # Convert each Shapefile into SQL statements.
+       shp2sqlite -aS ${TMP}/*_${file}.shp tiger_${file}
+     done && \
+     for file in $DBFS; do
+       # Convert each DBF into SQL statements likewise.
+       shp2sqlite -an ${TMP}/*_${file}.dbf tiger_${file}
+     done && \
+     cat ${SQL}/convert.sql) | sqlite3 $DATABASE
+    # Finally, do the transform/load phase (convert.sql)
+    # and clean up the temporary files.
+    rm -f $TMP/*
+done 2>&1 | tee import-$$.log
+rm -rf $TMP
+