RubyGems - geo_coder - Versions diffs - 0.1.0 - Mend

geo_coder 0.1.0

Files changed (119) hide show

data/Gemfile +12 -0
data/Gemfile.lock +32 -0
data/History.txt +6 -0
data/Makefile +13 -0
data/Manifest.txt +18 -0
data/README.rdoc +197 -0
data/Rakefile +53 -0
data/TODO.txt +8 -0
data/VERSION +1 -0
data/bin/build_indexes +8 -0
data/bin/rebuild_cluster +22 -0
data/bin/rebuild_metaphones +23 -0
data/bin/tiger_import +59 -0
data/demos/demo/app/ext/geocodewrap.rb +84 -0
data/demos/demo/app/views/index.builder +13 -0
data/demos/demo/app/views/index.erb +71 -0
data/demos/demo/config.ru +12 -0
data/demos/demo/config/bootstraps.rb +130 -0
data/demos/demo/config/geoenvironment.rb +25 -0
data/demos/demo/geocoder_helper.rb +12 -0
data/demos/demo/geocom_geocode.rb +10 -0
data/demos/demo/main.rb +3 -0
data/demos/demo/rakefile.rb +17 -0
data/demos/demo/tmp/restart.txt +0 -0
data/demos/simpledemo/views/index.builder +13 -0
data/demos/simpledemo/views/index.erb +69 -0
data/demos/simpledemo/ws.rb +83 -0
data/doc/Makefile +7 -0
data/doc/html4css1.css +279 -0
data/doc/lookup.rst +193 -0
data/doc/parsing.rst +125 -0
data/doc/voidspace.css +147 -0
data/geo_coder.gemspec +172 -0
data/lib/geocoder/us.rb +21 -0
data/lib/geocoder/us/address.rb +290 -0
data/lib/geocoder/us/constants.rb +670 -0
data/lib/geocoder/us/database.rb +745 -0
data/lib/geocoder/us/import.rb +181 -0
data/lib/geocoder/us/import/tiger.rb +13 -0
data/lib/geocoder/us/numbers.rb +58 -0
data/navteq/README +4 -0
data/navteq/convert.sql +37 -0
data/navteq/navteq_import +39 -0
data/navteq/prepare.sql +92 -0
data/sql/cluster.sql +16 -0
data/sql/convert.sql +80 -0
data/sql/create.sql +37 -0
data/sql/index.sql +12 -0
data/sql/place.csv +104944 -0
data/sql/place.sql +104948 -0
data/sql/setup.sql +78 -0
data/src/Makefile +13 -0
data/src/README +14 -0
data/src/liblwgeom/Makefile +75 -0
data/src/liblwgeom/box2d.c +54 -0
data/src/liblwgeom/lex.yy.c +4799 -0
data/src/liblwgeom/liblwgeom.h +1405 -0
data/src/liblwgeom/lwalgorithm.c +946 -0
data/src/liblwgeom/lwalgorithm.h +52 -0
data/src/liblwgeom/lwcircstring.c +759 -0
data/src/liblwgeom/lwcollection.c +541 -0
data/src/liblwgeom/lwcompound.c +118 -0
data/src/liblwgeom/lwcurvepoly.c +86 -0
data/src/liblwgeom/lwgeom.c +886 -0
data/src/liblwgeom/lwgeom_api.c +2201 -0
data/src/liblwgeom/lwgparse.c +1219 -0
data/src/liblwgeom/lwgunparse.c +1054 -0
data/src/liblwgeom/lwline.c +525 -0
data/src/liblwgeom/lwmcurve.c +125 -0
data/src/liblwgeom/lwmline.c +137 -0
data/src/liblwgeom/lwmpoint.c +138 -0
data/src/liblwgeom/lwmpoly.c +141 -0
data/src/liblwgeom/lwmsurface.c +129 -0
data/src/liblwgeom/lwpoint.c +439 -0
data/src/liblwgeom/lwpoly.c +579 -0
data/src/liblwgeom/lwsegmentize.c +1047 -0
data/src/liblwgeom/lwutil.c +369 -0
data/src/liblwgeom/measures.c +861 -0
data/src/liblwgeom/postgis_config.h +93 -0
data/src/liblwgeom/ptarray.c +847 -0
data/src/liblwgeom/vsprintf.c +179 -0
data/src/liblwgeom/wktparse.h +126 -0
data/src/liblwgeom/wktparse.lex +74 -0
data/src/liblwgeom/wktparse.tab.c +2353 -0
data/src/liblwgeom/wktparse.tab.h +145 -0
data/src/liblwgeom/wktparse.y +385 -0
data/src/libsqlite3_geocoder/Makefile +22 -0
data/src/libsqlite3_geocoder/Makefile.nix +15 -0
data/src/libsqlite3_geocoder/Makefile.redhat +15 -0
data/src/libsqlite3_geocoder/extension.c +121 -0
data/src/libsqlite3_geocoder/extension.h +13 -0
data/src/libsqlite3_geocoder/levenshtein.c +42 -0
data/src/libsqlite3_geocoder/metaphon.c +278 -0
data/src/libsqlite3_geocoder/util.c +37 -0
data/src/libsqlite3_geocoder/wkb_compress.c +54 -0
data/src/metaphone/Makefile +7 -0
data/src/metaphone/README +49 -0
data/src/metaphone/extension.c +37 -0
data/src/metaphone/metaphon.c +251 -0
data/src/shp2sqlite/Makefile +37 -0
data/src/shp2sqlite/Makefile.nix +36 -0
data/src/shp2sqlite/Makefile.redhat +35 -0
data/src/shp2sqlite/dbfopen.c +1595 -0
data/src/shp2sqlite/getopt.c +695 -0
data/src/shp2sqlite/getopt.h +127 -0
data/src/shp2sqlite/shapefil.h +500 -0
data/src/shp2sqlite/shp2sqlite.c +1974 -0
data/src/shp2sqlite/shpopen.c +1894 -0
data/tests/address.rb +236 -0
data/tests/benchmark.rb +20 -0
data/tests/constants.rb +57 -0
data/tests/data/address-sample.csv +52 -0
data/tests/data/db-test.csv +57 -0
data/tests/data/locations.csv +4 -0
data/tests/database.rb +137 -0
data/tests/generate.rb +34 -0
data/tests/numbers.rb +46 -0
data/tests/run.rb +11 -0
metadata +237 -0

data/demos/demo/app/ext/geocodewrap.rb ADDED Viewed

@@ -0,0 +1,84 @@
+require 'rubygems'
+require 'geocoder/us/database'
+require 'logger'
+module Sinatra
+  module GeocodeWrap
+    attr_accessor :db
+    def self.registered(app)
+      options = {:cache_size => 100000}
+       @@db = Geocoder::US::Database.new("/Users/katechapman/usgeocode.db", options)
+       stats = Logger.new("geocoderstats.log", 10, 1024000)
+       app.get '/' do
+     	   unless params[:address].nil?
+           begin
+             @records = @@db.geocode params[:address]
+             stats.debug "Geocoded: 1, Failed: 0, Geocoded At: " << DateTime.now.to_s
+   	       rescue Exception => e
+   	         stats.debug "Geocoded: 1, Failed: 1, Geocoded At: " << DateTime.now.to_s
+   	         puts e.message
+   	       end
+   	      end
+	        case params[:format]
+	        when /xml/
+	          builder :index
+	        when /atom/
+	          builder :atom
+	        when /json/
+	          @records.to_json
+ 	        else
+	          erb :index
+	        end
+        end
+       app.post '/batch' do
+         failed_codes = 0
+         total_codes = 0
+         puts Time.now
+       	if params[:uploaded_csv].nil?
+          csv_file = request.env["rack.input"].read
+       	  csv = FasterCSV.parse(csv_file, :row_sep => "*", :col_sep => "|")
+       else
+       	  FileUtils.mkdir_p('uploads/')
+          FileUtils.mv(params[:uploaded_csv][:tempfile].path, "uploads/#{params[:uploaded_csv][:filename]}")
+          csv_file = open("uploads/#{params[:uploaded_csv][:filename]}")
+          @filename = params[:uploaded_csv][:filename].gsub(/\.csv/,"")
+          csv = FasterCSV.parse(csv_file)
+       end
+       	  headers = csv[0]
+       	  @records = csv.collect do |record|
+       	    total_codes += 1
+       	  next if record == headers
+            begin
+       	      result = @@db.geocode record[1]
+       	      if result.empty?
+       	         result[0] = {:lon => nil, :lat => nil, :precision => 'unmatched', :score => 0}
+       	         failed_codes += 1
+       	      end
+       	      result.first.merge(headers[0] => record[0])
+            rescue Exception => e
+              failed_codes += 1
+       	      puts e.message
+              next
+       	     end
+             end.compact
+            puts Time.now
+            stats.debug "Geocoded: " << total_codes.to_s << ", Failed: " << failed_codes.to_s << ",Geocoded At: " << DateTime.now.to_s
+            case params[:format]
+       	    when /xml/
+       	     builder :index
+       	    when /atom/
+       	     builder :atom
+       	    when /json/
+       	     @records.to_json
+       	    else
+       	     erb :index
+       	    end
+          end
+        end
+      end
+  register GeocodeWrap
+end

data/demos/demo/app/views/index.builder ADDED Viewed

@@ -0,0 +1,13 @@
+xml.locations do
+  unless @records.nil?
+    @records.each do |record|
+      xml.location do
+        xml.score format("%.2f", record[:score]*100)
+        %w{lat lon number prefix pretyp predir prequal street suftyp sufdir sufqual city state zip}.each do |field|
+          xml.tag! field, record[field.to_sym]
+        end
+      end
+    end
+  end
+end

data/demos/demo/app/views/index.erb ADDED Viewed

@@ -0,0 +1,71 @@
+<!DOCTYPE html
+    PUBLIC "-//W3C//DTD XHTML 1.0 Strict//EN"
+        "http://www.w3.org/TR/xhtml1/DTD/xhtml1-strict.dtd">
+<html xmlns="http://www.w3.org/1999/xhtml" lang="en" xml:lang="en">
+<head>
+    <style type="text/css">
+    html {font-family: Arial, sans-serif;}
+    table {font-size: 8pt; border-collapse: collapse;}
+    td { border: 1px solid black; padding: .25em .5em .25em .5em; }
+    </style>
+</head>
+<body onload="">
+  <p><b>Geocoder Demo</b></p>
+  <p>
+    <form>
+      <label>Enter an address:</label> <input type="text" name="address" value="<%= params[:address] %>" size="80">
+      <input type="submit" value="Geocode" />
+    </form>
+    <form action="batch" method="POST" enctype="multipart/form-data" accept-charset="utf-8">
+      <label>Upload a CSV:</label> <input type="file" name="uploaded_csv" id="uploaded_csv">
+      <input type="submit" value="Batch Geocode" />
+    </form>
+  </p>
+  <% unless @records.nil? %>
+  <table>
+    <tr>
+      <td>Match</td>
+      <td>Precision</td>
+      <td>Lat</td>
+      <td>Lon</td>
+      <td>#</td>
+      <td>Qual</td>
+      <td>Dir</td>
+      <td>Type</td>
+      <td>Street</td>
+      <td>Type</td>
+      <td>Dir</td>
+      <td>Qual</td>
+      <td>City</td>
+      <td>St</td>
+      <td>ZIP</td>
+      <td>&nbsp;</td>
+    </tr>
+    <% for record in @records %>
+    <tr>
+      <td><%= format("%.2f", record[:score]*100) %>%</td>
+      <td><%= record[:precision].to_s %></td>
+      <td><%= record[:lat].to_s %></td>
+      <td><%= record[:lon].to_s %></td>
+      <td><%= record[:prefix] if record[:prefix] %><%= record[:number] %></td>
+      <td><%= record[:pretyp] %></td>
+      <td><%= record[:predir] %></td>
+      <td><%= record[:prequal] %></td>
+      <td><%= record[:street] %></td>
+      <td><%= record[:suftyp] %></td>
+      <td><%= record[:sufdir] %></td>
+      <td><%= record[:sufqual] %></td>
+      <td><%= record[:city] %></td>
+      <td><%= record[:state] %></td>
+      <td><%= record[:zip] %></td>
+      <td><a href="http://maps.google.com/maps?q=<%=record[:lat]%>,<%=record[:lon]%>"
+        target="_blank">map</a></td>
+      </tr>
+      <% end %>
+    </table>
+    <% end %>
+    <% unless @filename.nil? %>
+     <a href="/link.atom?filename=<%= @filename %>">Atom Feed</a>
+   <% end %>
+  </body>
+  </html>

data/demos/demo/config.ru ADDED Viewed

@@ -0,0 +1,12 @@
+require 'rubygems'
+require 'sinatra'
+Sinatra::Application.default_options.merge!(
+ :run => false,
+ :env => ENV['RACK_ENV']
+)
+require 'geocom_geocode'
+run GeocomGeocode::GeocodeServer

data/demos/demo/config/bootstraps.rb ADDED Viewed

@@ -0,0 +1,130 @@
+require 'rubygems'
+module BootStraps
+  class Framework
+    def initialize
+      @methods = {}
+    end
+    def apply_settings!(app)
+      @methods.each_pair do |method, calls|
+        calls.each do |arg_set|
+          app.send(method, *arg_set)
+        end
+      end
+    end
+    def method_missing(method, *args)
+      @methods[method] ||= []
+      @methods[method] << args
+    end
+  end
+  class DataStore
+    def connect_action(&block)
+      @connect_action = block
+    end
+    #TODO raise UndefinedConnectAction
+    def connect
+      @connect_action.call if @connect_action
+    end
+  end
+  class Configuration
+    attr_accessor :db, :global, :default_env, :vendor_dir, :lib_paths, :framework, :vendored
+    attr_reader :gems
+    def initialize
+      @framework = Framework.new
+      @gems = {}
+      @global = {}
+      @default_env = 'production'
+      @vendor_dir = File.join(root, 'vendor')
+      @lib_paths = []
+      @vendored = false
+    end
+    def env
+      ENV['RACK_ENV'] ||= default_env
+    end
+    def env=(val)
+      ENV['RACK_ENV'] = val
+    end
+    def root
+      File.join(File.expand_path(File.dirname(__FILE__)), "..")
+    end
+    def gem(*args)
+      gem = args.first
+      ver = args.last
+      @gems[gem] = ver
+      #its concievable that vendored could be changed mid config
+      use_vendor if vendored
+      Kernel.send(:gem, *args)
+      require gem
+    end
+    private
+    def use_vendor
+      Gem.clear_paths
+      prepend_gem_path!(File.join(root, 'vendor'))
+    end
+    def prepend_gem_path!(path)
+      ENV['GEM_PATH'] = path
+    end
+  end
+  class Initializer
+    @@config = Configuration.new
+    class << self
+      def configure
+        unless @@config.frozen?
+          yield @@config
+          @@config.freeze
+        end
+      end
+      def config
+        @@config
+      end
+      def boot!
+        require File.join(@@config.root, 'config', 'geoenvironment.rb')
+        require_libs
+      end
+      private
+      def require_libs
+        [
+         subdir_expansion('lib'),
+         subdir_expansion(File.join('app','ext'))
+        ].each do |p|
+          require_all(p)
+        end
+      end
+      def require_all(path)
+        Dir[path].each { |f| require f }
+      end
+      def subdir_expansion(subdir)
+        File.join(@@config.root, subdir, '**', '*.rb')
+      end
+    end
+  end
+end
+BootStraps::Initializer.boot!
+Straps = BootStraps::Initializer.config

data/demos/demo/config/geoenvironment.rb ADDED Viewed

@@ -0,0 +1,25 @@
+BootStraps::Initializer.configure do |config|
+  #Use the vendor directory
+  config.vendored = true
+  config.default_env = 'production'
+  config.gem 'sinatra'
+  config.gem 'fastercsv'
+  config.gem 'json'
+  config.framework.set :root, config.root
+  config.framework.set :environment, config.env
+  config.framework.set :raise_errors, true
+  config.framework.set :views, File.join('app','views')
+  config.framework.set :server, 'mongrel'
+  config.framework.set :static, true
+  config.framework.set :logging, true
+  config.framework.set :port, 4567
+  config.framework.set :lock, false
+end

data/demos/demo/geocoder_helper.rb ADDED Viewed

@@ -0,0 +1,12 @@
+require 'rubygems'
+require 'geocoder/us/database'
+require 'fastercsv'
+require 'json'
+def initialize
+end

data/demos/demo/geocom_geocode.rb ADDED Viewed

@@ -0,0 +1,10 @@
+require 'config/bootstraps'
+module GeocomGeocode
+  class GeocodeServer < Sinatra::Base
+    register Sinatra::GeocodeWrap
+    configure do
+      Straps.framework.apply_settings!(self)
+    end
+  end
+end

data/demos/demo/main.rb ADDED Viewed

@@ -0,0 +1,3 @@
+require 'geocom_geocode'
+GeocomGeocode::GeocodeServer.run!

data/demos/demo/rakefile.rb ADDED Viewed

@@ -0,0 +1,17 @@
+require 'rake'
+task :boot_env do
+  require 'config/bootstraps';
+end
+namespace :db do
+  task :migrate => :connect do
+    ActiveRecord::Base.logger = Logger.new(STDOUT)
+    ActiveRecord::Migration.verbose = true
+    ActiveRecord::Migrator.migrate('db/migrate/', nil)
+  end
+  task :connect => :boot_env do
+    BootStraps::Initializer.config.db.connect
+  end
+end

data/demos/demo/tmp/restart.txt ADDED Viewed

File without changes

data/demos/simpledemo/views/index.builder ADDED Viewed

@@ -0,0 +1,13 @@
+xml.locations do
+  unless @records.nil?
+    @records.each do |record|
+      xml.location do
+        xml.score format("%.2f", record[:score]*100)
+        %w{lat lon number prefix pretyp predir prequal street suftyp sufdir sufqual city state zip}.each do |field|
+          xml.tag! field, record[field.to_sym]
+        end
+      end
+    end
+  end
+end

data/demos/simpledemo/views/index.erb ADDED Viewed

@@ -0,0 +1,69 @@
+<!DOCTYPE html
+    PUBLIC "-//W3C//DTD XHTML 1.0 Strict//EN"
+        "http://www.w3.org/TR/xhtml1/DTD/xhtml1-strict.dtd">
+<html xmlns="http://www.w3.org/1999/xhtml" lang="en" xml:lang="en">
+<head>
+    <style type="text/css">
+    html {font-family: Arial, sans-serif;}
+    table {font-size: 8pt; border-collapse: collapse;}
+    td { border: 1px solid black; padding: .25em .5em .25em .5em; }
+    </style>
+</head>
+<body onload="">
+  <p><b>Geocoder Demo</b></p>
+  <p>
+    <form>
+      <label>Enter an address:</label> <input type="text" name="address" value="<%= params[:address] %>" size="80">
+      <input type="submit" value="Geocode" />
+    </form>
+    <form action="batch" method="POST" enctype="multipart/form-data" accept-charset="utf-8">
+      <label>Upload a CSV:</label> <input type="file" name="uploaded_csv" id="uploaded_csv">
+      <input type="submit" value="Batch Geocode" />
+    </form>
+  </p>
+  <% unless @records.nil? %>
+  <table>
+    <tr>
+      <td>Match</td>
+      <td>Lat</td>
+      <td>Lon</td>
+      <td>#</td>
+      <td>Qual</td>
+      <td>Dir</td>
+      <td>Type</td>
+      <td>Street</td>
+      <td>Type</td>
+      <td>Dir</td>
+      <td>Qual</td>
+      <td>City</td>
+      <td>St</td>
+      <td>ZIP</td>
+      <td>&nbsp;</td>
+    </tr>
+    <% for record in @records %>
+    <tr>
+      <td><%= format("%.2f", record[:score]*100) %>%</td>
+      <td><%= record[:lat].to_s %></td>
+      <td><%= record[:lon].to_s %></td>
+      <td><%= record[:prefix] if record[:prefix] %><%= record[:number] %></td>
+      <td><%= record[:pretyp] %></td>
+      <td><%= record[:predir] %></td>
+      <td><%= record[:prequal] %></td>
+      <td><%= record[:street] %></td>
+      <td><%= record[:suftyp] %></td>
+      <td><%= record[:sufdir] %></td>
+      <td><%= record[:sufqual] %></td>
+      <td><%= record[:city] %></td>
+      <td><%= record[:state] %></td>
+      <td><%= record[:zip] %></td>
+      <td><a href="http://maps.google.com/maps?q=<%=record[:lat]%>,<%=record[:lon]%>"
+        target="_blank">map</a></td>
+      </tr>
+      <% end %>
+    </table>
+    <% end %>
+    <% unless @filename.nil? %>
+     <a href="/link.atom?filename=<%= @filename %>">Atom Feed</a>
+   <% end %>
+  </body>
+  </html>