geo_coder 0.1.0

data/lib/geocoder/us/import.rb

@@ -0,0 +1,181 @@
+require 'tmpdir'
+require 'geocoder/us/database'
+
+require 'rubygems'
+require 'geo_ruby'
+require 'zip/zip'
+
+class Geocoder::US::Import < Geocoder::US::Database
+  @tables = {}
+
+  def self.tables
+    @tables
+  end
+
+  def tables
+    self.class.tables
+  end
+
+  def initialize (filename, options)
+    options[:create] = true
+    super(filename, options)
+    @sqlpath = options[:sql]
+    create_tables
+  end
+
+  def log (*args)
+    $stderr.print *args
+  end
+
+  def spin
+    @spin ||= 0
+    log "|/-\\"[@spin/100..@spin/100]+"\010" if @spin % 100 == 0
+    @spin += 1
+    @spin %= 400
+  end
+
+  def execute_batch (*args)
+    @db.execute_batch(*args) 
+  end
+
+  def execute_script (file)
+    if File.expand_path(file) != file
+      file = File.join(@sqlpath, file)
+    end    
+    execute_batch File.open(file).read
+  end
+
+  def load_features (file)
+    dataset = GeoRuby::Shp4r::ShpFile.open(file)
+    fields  = dataset.fields.map {|f| f.name}
+    dataset.each do |record|
+      attrs = fields.map {|f| record.data[f]}
+      geom = record.geometry
+      geom = geom.geometries[0] \
+        if geom.kind_of? GeoRuby::SimpleFeatures::GeometryCollection
+      points = geom.points.map {|pt| [pt.x, pt.y].map {|i| (i*1000000).to_i}}
+      coords = points.flatten.pack("V*")
+      yield attrs, coords
+    end
+  end
+
+  def insert_data (st, table, attrs)
+    unless st
+     values = placeholders_for attrs
+     st = @db.prepare("INSERT INTO #{table} VALUES (#{values});")
+    end
+    st.execute(attrs)
+  end
+
+  def insert_shapefile (file, table)
+    st = nil
+    load_features(file) do |attrs, geom|
+      attrs << SQLite3::Blob.new(geom) if geom
+      insert_data st, table, attrs
+    end
+  end
+
+  def insert_dbf (file, table)
+    st = nil
+    GeoRuby::Shp4r::Dbf::Reader.open(file) do |dbf|
+      fields  = dbf.fields.map {|f| f.name}
+      dbf.rows.each do |record|
+        attrs = fields.map {|f| record[f]}
+        insert_data st, table, attrs
+      end
+    end
+  end
+
+  def insert_csv (file, table, delimiter="|")
+    st = nil
+    File.open(file).readlines.each do |line|
+      attrs = line.chomp.split(delimiter)
+      insert_data st, table, attrs
+    end
+  end
+
+  def make_temp_dir (cleanup=true)
+    path = File.join(Dir.tmpdir, "geocoder-#{$$}")
+    FileUtils.mkdir_p path
+    if block_given?
+      begin
+        yield path
+      ensure
+        FileUtils.rm_r(path) if cleanup
+      end
+    else
+      path
+    end
+  end
+
+  def unpack_zip (file, path)
+    # log "- unpacking #{file}"
+    Zip::ZipFile.open(file).each do |entry|
+      target = File.join(path, entry.name)
+      # log "  - #{target}"
+      entry.extract target
+    end
+  end
+
+  def import_zip (zipfile, table)
+    make_temp_dir do |tmpdir|
+      unpack_zip zipfile, tmpdir
+      basename = File.join(tmpdir, File.basename(zipfile))[0..-5]
+      shpfile = basename + ".shp"
+      dbffile = basename + ".dbf"
+      if File.exists? shpfile
+        log "#{table} "
+        insert_shapefile shpfile, table
+      elsif File.exists? dbffile
+        log "#{table} "
+        insert_dbf dbffile, table
+      else
+        log "\nNOT FOUND: #{shpfile}\n"
+      end
+    end 
+  end
+
+  def import_path (path)
+    log "\n#{path}: "
+    execute_script "setup.sql"
+    @db.transaction do
+      tables.each do |table, glob|
+        file = Dir[File.join(path, glob)][0]
+        next unless file
+        if file =~ /\.zip$/io
+          import_zip file, table
+        else
+          import_shapefile file, table
+        end
+      end
+    end
+    execute_script "convert.sql"
+  end
+
+  def import_tree (root)
+    if !Dir[File.join(root, tables.values[0])].empty?
+        import_path root
+    else
+      Dir[File.join(root, "*")].sort.each do |file|
+        import_tree file if File.directory? file
+      end
+    end
+  end
+
+  def create_tables
+    uninit = false
+    begin
+      @db.execute("SELECT 0 FROM place")
+    rescue SQLite3::SQLException
+      uninit = true
+    end
+    if uninit
+      log "creating tables\n"
+      execute_script "create.sql"
+      post_create
+    end
+  end
+
+  def post_create
+  end
+end

data/lib/geocoder/us/import/tiger.rb

@@ -0,0 +1,13 @@
+require 'geocoder/us/import'
+
+class Geocoder::US::Import::TIGER < Geocoder::US::Import
+  @tables = {:tiger_edges     => "*_edges.zip", 
+             :tiger_featnames => "*_featnames.zip",
+             :tiger_addr      => "*_addr.zip"}
+  def post_create
+    log "importing places"
+    @db.transaction do
+    #  insert_csv File.join(@sqlpath, "place.csv"), "place"
+    end
+  end
+end

data/lib/geocoder/us/numbers.rb

@@ -0,0 +1,58 @@
+module Geocoder
+end
+
+module Geocoder::US
+  # The NumberMap class provides a means for mapping ordinal
+  # and cardinal number words to digits and back.
+  class NumberMap < Hash
+    attr_accessor :regexp
+    def self.[] (array)
+      nmap = self.new({})
+      array.each {|item| nmap << item } 
+      nmap.build_match
+      nmap
+    end
+    def initialize (array)
+      @count = 0
+    end
+    def build_match
+      @regexp = Regexp.new(
+        '\b(' + keys.flatten.join("|") + ')\b',
+        Regexp::IGNORECASE)
+    end
+    def clean (key)
+      key.is_a?(String) ? key.downcase.gsub(/\W/o, "") : key
+    end
+    def <<(item)
+      store clean(item), @count
+      store @count, item
+      @count += 1
+    end
+    def [] (key)
+      super(clean(key))
+    end
+  end
+
+  # The Cardinals constant maps digits to cardinal number words and back.
+  Cardinals = NumberMap[%w[
+    zero one two three four five six seven eight nine ten
+    eleven twelve thirteen fourteen fifteen sixteen seventeen
+    eighteen nineteen
+  ]]
+  Cardinal_Tens = %w[ twenty thirty forty fifty sixty seventy eighty ninety ]
+  Cardinal_Tens.each {|tens|
+    Cardinals << tens
+    (1..9).each {|n| Cardinals << tens + "-" + Cardinals[n]}
+  }
+
+  # The Ordinals constant maps digits to ordinal number words and back.
+  Ordinals = NumberMap[%w[
+    zeroth first second third fourth fifth sixth seventh eighth ninth
+    tenth eleventh twelfth thirteenth fourteenth fifteenth sixteenth
+    seventeenth eighteenth nineteenth
+  ]]
+  Cardinal_Tens.each {|tens|
+    Ordinals << tens.gsub("y","ieth")
+    (1..9).each {|n| Ordinals << tens + "-" + Ordinals[n]}
+  }
+end

data/navteq/README

@@ -0,0 +1,4 @@
+The navteq_import script in this directory is designed to be used with Navteq's
+local_streets layer. It works basically like tiger_import, except that you
+provide either a list of .zip files containing the local_streets.* files on the
+command line, or via standard input.

data/navteq/convert.sql

@@ -0,0 +1,37 @@
+BEGIN;
+CREATE INDEX navteq_link_id on local_streets (link_id);
+
+CREATE TEMPORARY TABLE linezip AS
+    SELECT DISTINCT tlid, zip FROM (
+        SELECT link_id AS tlid, r_postcode AS zip FROM local_streets
+           WHERE addr_type IS NOT NULL AND st_name IS NOT NULL
+           AND r_postcode IS NOT NULL
+        UNION
+        SELECT link_id AS tlid, l_postcode AS zip FROM local_streets
+           WHERE addr_type IS NOT NULL AND st_name IS NOT NULL
+           AND l_postcode IS NOT NULL
+    ) AS whatever;
+
+INSERT INTO feature
+    SELECT l.tlid, st_nm_base, metaphone(st_nm_base,5), st_nm_pref, st_typ_bef,
+           NULL, st_nm_suff, st_typ_aft, NULL, 'P', zip
+        FROM linezip l, local_streets f
+        WHERE l.tlid=f.link_id AND st_name IS NOT NULL;
+
+INSERT OR IGNORE INTO edge
+    SELECT l.tlid, compress_wkb_line(the_geom) FROM
+        (SELECT DISTINCT tlid FROM linezip) AS l, local_streets f
+        WHERE l.tlid=f.link_id AND st_name IS NOT NULL;
+
+INSERT INTO range
+    SELECT link_id, digit_suffix(l_refaddr), digit_suffix(l_nrefaddr),
+           nondigit_prefix(l_refaddr), l_postcode, 'L'
+    FROM linezip l, local_streets f
+    WHERE l.tlid=f.link_id AND l_refaddr IS NOT NULL
+    UNION
+    SELECT link_id, digit_suffix(r_refaddr), digit_suffix(r_nrefaddr),
+           nondigit_prefix(r_refaddr), r_postcode, 'R'
+    FROM linezip l, local_streets f
+    WHERE l.tlid=f.link_id AND r_refaddr IS NOT NULL;
+
+END;

data/navteq/navteq_import

@@ -0,0 +1,39 @@
+#!/bin/bash
+
+TMP="/tmp/navteq-import.$$"
+SHPS="local_streets"
+DBFS=""
+BASE=$(dirname $0)
+PATH=$PATH:$BASE/../bin
+SQL="$BASE/../sql"
+HELPER_LIB="$BASE/../lib/geocoder/us/sqlite3.so"
+DATABASE=$1
+shift
+
+mkdir -p $TMP || exit 1
+
+[ ! -r $DATABASE ] && cat ${SQL}/create.sql ${SQL}/place.sql | sqlite3 $DATABASE
+ 
+if [ x"$1" = x"" ]; then
+    cat
+else
+    ls $@
+fi | while read county; do
+    echo "--- $county"
+    if [ -r ${county%.zip}.zip ]; then
+        unzip -q $(ls ${county}.zip) -d $TMP
+    else
+        cp ${county%.*}.* $TMP
+    fi
+    (echo ".load $HELPER_LIB" && \
+     cat ${BASE}/prepare.sql && \
+     for file in $SHPS; do
+       shp2sqlite -aS $(ls ${TMP}/${file}.shp) ${file}
+     done && \
+     for file in $DBFS; do
+       shp2sqlite -an $(ls ${TMP}/${file}.dbf) ${file}
+     done && \
+     cat ${BASE}/convert.sql) | sqlite3 $DATABASE
+    rm -f $TMP/*
+done 2>&1 | tee import-$$.log
+rm -rf $TMP

data/navteq/prepare.sql

@@ -0,0 +1,92 @@
+PRAGMA temp_store=MEMORY;
+PRAGMA journal_mode=MEMORY;
+PRAGMA synchronous=OFF;
+PRAGMA cache_size=250000;
+PRAGMA count_changes=0;
+BEGIN;
+CREATE TABLE "local_streets" (gid integer PRIMARY KEY,
+"the_geom" blob,
+"link_id" integer,
+"st_name" varchar(80),
+"feat_id" integer,
+"st_langcd" varchar(3),
+"num_stnmes" integer,
+"st_nm_pref" varchar(2),
+"st_typ_bef" varchar(30),
+"st_nm_base" varchar(35),
+"st_nm_suff" varchar(2),
+"st_typ_aft" varchar(30),
+"st_typ_att" varchar(1),
+"addr_type" varchar(1),
+"l_refaddr" varchar(10),
+"l_nrefaddr" varchar(10),
+"l_addrsch" varchar(1),
+"l_addrform" varchar(1),
+"r_refaddr" varchar(10),
+"r_nrefaddr" varchar(10),
+"r_addrsch" varchar(1),
+"r_addrform" varchar(1),
+"ref_in_id" integer,
+"nref_in_id" integer,
+"n_shapepnt" integer,
+"func_class" varchar(1),
+"speed_cat" varchar(1),
+"fr_spd_lim" integer,
+"to_spd_lim" integer,
+"to_lanes" integer,
+"from_lanes" integer,
+"enh_geom" varchar(1),
+"lane_cat" varchar(1),
+"divider" varchar(1),
+"dir_travel" varchar(1),
+"l_area_id" integer,
+"r_area_id" integer,
+"l_postcode" varchar(11),
+"r_postcode" varchar(11),
+"l_numzones" integer,
+"r_numzones" integer,
+"num_ad_rng" integer,
+"ar_auto" varchar(1),
+"ar_bus" varchar(1),
+"ar_taxis" varchar(1),
+"ar_carpool" varchar(1),
+"ar_pedest" varchar(1),
+"ar_trucks" varchar(1),
+"ar_traff" varchar(1),
+"ar_deliv" varchar(1),
+"ar_emerveh" varchar(1),
+"paved" varchar(1),
+"private" varchar(1),
+"frontage" varchar(1),
+"bridge" varchar(1),
+"tunnel" varchar(1),
+"ramp" varchar(1),
+"tollway" varchar(1),
+"poiaccess" varchar(1),
+"contracc" varchar(1),
+"roundabout" varchar(1),
+"interinter" varchar(1),
+"undeftraff" varchar(1),
+"ferry_type" varchar(1),
+"multidigit" varchar(1),
+"maxattr" varchar(1),
+"spectrfig" varchar(1),
+"indescrib" varchar(1),
+"manoeuvre" varchar(1),
+"dividerleg" varchar(1),
+"inprocdata" varchar(1),
+"full_geom" varchar(1),
+"urban" varchar(1),
+"route_type" varchar(1),
+"dironsign" varchar(1),
+"explicatbl" varchar(1),
+"nameonrdsn" varchar(1),
+"postalname" varchar(1),
+"stalename" varchar(1),
+"vanityname" varchar(1),
+"junctionnm" varchar(1),
+"exitname" varchar(1),
+"scenic_rt" varchar(1),
+"scenic_nm" varchar(1));
+--SELECT AddGeometryColumn('','local_streets','the_geom','-1','MULTILINESTRING',2);
+END;

data/sql/cluster.sql

@@ -0,0 +1,16 @@
+.echo on
+-- turn off various pragmas to make SQLite faster
+PRAGMA temp_store=MEMORY;
+PRAGMA journal_mode=OFF;
+PRAGMA synchronous=OFF;
+PRAGMA cache_size=500000;
+PRAGMA count_changes=0;
+BEGIN TRANSACTION;
+-- order the contents of each table by their indexes to reduce
+--   the number of disk pages that need to be read on each query.
+INSERT INTO place SELECT * FROM old.place ORDER BY zip, priority;
+INSERT INTO edge SELECT * FROM old.edge ORDER BY tlid;
+INSERT INTO feature SELECT * FROM old.feature ORDER BY street_phone, zip;
+INSERT INTO feature_edge SELECT * FROM old.feature_edge ORDER BY fid;
+INSERT INTO range SELECT * FROM old.range ORDER BY tlid;
+COMMIT;