census_shapes 0.2.0

data/lib/generators/census_shapes/setup/templates/config/initializers/shapes_globals.rb

@@ -0,0 +1,4 @@
+CENSUS_HOST = 'ftp.census.gov'       
+CENSUS_SHAPES_PATH = 'geo/tiger/TIGER2010'
+CENSUS_STATES = YAML.load_file('lib/yaml/us_states.yml')
+CENSUS_SHAPES = YAML.load_file('lib/yaml/us_shapes.yml')

data/lib/generators/census_shapes/setup/templates/controllers/shapes_controller.rb

@@ -0,0 +1,17 @@
+class CONTROLLER_NAMEController < ApplicationController
+
+  def index
+    bbox = MODEL_NAME.bbox(params[:z],params[:x], params[:y])
+    params.merge!(:bbox=> bbox) if bbox
+    respond_to do |format|
+      format.html
+      format.json  {
+        render json: ({
+          :status=> "OK",
+          :type => "FeatureCollection",
+          :features=> MODEL_NAME.features(params)
+        })
+      }
+    end
+  end
+end

data/lib/generators/census_shapes/setup/templates/db/migrate/create_shapes.rb

@@ -0,0 +1,86 @@
+class Create<%= controller_name.titleize.gsub(' ', '').gsub(/\W/,'') %> < ActiveRecord::Migration
+  def up
+    create_table(:<%= controller_name %>) do |t|
+        t.string :type, :limit => 12
+        t.string :sumlevel, :limit => 3
+        t.string :geoid10, :limit => 15
+        t.string :name10, :limit => 100
+        t.string :namelsad10, :limit => 100
+        t.string :region10, :limit => 2
+        t.string :division10, :limit => 2
+        t.string :state10, :limit => 2
+        t.string :statens10, :limit => 8
+        t.string :statefp10, :limit => 2
+        t.string :zcta5ce10, :limit => 5
+        t.string :countyfp10, :limit => 3
+        t.string :countyns10, :limit => 8
+        t.string :cousubfp10, :limit => 5
+        t.string :cousubns10, :limit => 8
+        t.string :submcdfp10, :limit => 5
+        t.string :submcdns10, :limit => 8
+        t.string :tractce10, :limit => 6
+        t.string :blockce10, :limit => 4
+        t.string :blkgrpce10, :limit => 6
+        t.string :placefp10, :limit => 5
+        t.string :placens10, :limit => 8
+        t.string :csafp10, :limit => 3
+        t.string :cbsafp10, :limit => 5
+        t.string :metdivfp10, :limit => 5
+        t.string :cd111fp, :limit => 2
+        t.string :cdsessn, :limit => 3
+        t.string :anrcfp10, :limit => 5
+        t.string :anrcns10, :limit => 8
+        t.string :aiannhce10, :limit => 4
+        t.string :aiannhns10, :limit => 8
+        t.string :trsubce10, :limit => 3
+        t.string :trsubns10, :limit => 8
+        t.string :sldust10, :limit => 3
+        t.string :sldlst10, :limit => 3
+        t.string :vtdst10, :limit => 6
+        t.string :elsdlea10, :limit => 5
+        t.string :scsdlea10, :limit => 5
+        t.string :unsdlea10, :limit => 5
+        t.string :stusps10, :limit => 2
+        t.string :lsad10, :limit => 2
+        t.string :lsy10, :limit=> 4
+        t.string :lograde10, :limit => 2
+        t.string :higrade10, :limit => 2
+        t.string :sdtyp10, :limigt => 1
+        t.string :classfp10, :limit => 2
+        t.string :comptyp10, :limit => 1
+        t.string :aiannhr10, :limit => 1
+        t.string :aiannhfp10, :limit => 5
+        t.string :trsubfp10, :limit => 5
+        t.string :partflg10, :limit => 1
+        t.string :pcicbsa10, :limit => 1
+        t.string :pcinecta10, :limit => 1
+        t.string :mtfcc10, :limit => 5
+        t.string :memi10, :limit => 1
+        t.string :ur10, :limit => 1
+        t.string :uace10, :limit => 1
+        t.string :uatyp10, :limit => 1
+        t.string :vtdi10, :limit => 1
+        t.string :cnectafp10, :limit => 3
+        t.string :nectafp10, :limit => 5
+        t.string :nctadvfp10, :limit => 5
+        t.string :funcstat10, :limit => 1
+        t.float :aland10, :length => 8
+        t.float :awater10,  :length => 8
+        t.point :latlng, :srid=> 4326
+        t.geometry :geom, :srid=> 4326
+        t.decimal :intptlat10, :precision => 15, :scale => 12
+        t.decimal :intptlon10, :precision => 15, :scale => 12 
+    end
+    add_index :<%= controller_name %>, [:type,:geoid10], :name => "geo_index", :unique=> true
+    execute "ALTER TABLE <%= controller_name %> RENAME COLUMN id TO gid;"
+    execute "CREATE INDEX b_point ON <%= controller_name %> USING GIST (latlng);";
+    execute "CREATE INDEX b_geom ON <%= controller_name %> USING GIST (geom);";
+  end
+  
+  def down
+    drop_table :<%= controller_name %>
+  end
+end
+
+
+

data/lib/generators/census_shapes/setup/templates/lib/tasks/census_shapes.rake

@@ -0,0 +1,167 @@
+require 'net/ftp'
+require 'yaml'
+require 'progress_bar'
+
+CONFIG = YAML::load(File.open('config/database.yml'))
+DB = CONFIG[Rails.env]['database']
+USER = CONFIG[Rails.env]['username']
+HOST = CONFIG[Rails.env]['host']
+TEMPLATE = CONFIG[Rails.env]['template']
+
+namespace :census_shapes do
+  desc "Import Census Shapes" 
+  task :import  => :environment do
+    if local_path(ENV['FILEPATH'])
+      for geo in format_args(ENV['SHAPE'], msg('install'))
+        before_import_set_defaults(geo)
+        puts "Importing Census Shape #{geo['name']} (#{geo['slug']})"
+        @dbar = ProgressBar.new(52)
+        for state in CENSUS_STATES
+          path = "#{CENSUS_SHAPES_PATH}/#{geo['path']}/"
+          file = "tl_2010_#{state[1]["id"].to_s.rjust(2,'0')}_#{geo['slug'].downcase}10"
+          download_zip(path, file, geo['slug'])
+          unzip_shapefiles(file, geo['slug'])
+          import_shapefiles(file, geo['slug'], ENV['ARCHIVE'])
+          @dbar.increment!
+        end
+        after_import_update_fields
+      end
+    end
+  end
+
+  desc "Validate Geometry"
+  task :validate_geometry => :environment do
+    @dbar = ProgressBar.new(Geography.count)
+    Geography.find_in_batches(:select=> "gid, geoid10, type, ST_IsValid(geom) AS isvalid", :batch_size => 100) do |batch|
+      batch.each do |geo|
+        if geo['isvalid'] == 'f';
+          puts "Geography #{geo['type']} : gid = #{geo['gid']}, geoid10 = #{geo['geoid10']} is #{geo['isvalid']}"
+          ActiveRecord::Base.connection.execute("UPDATE CONTROLLER_NAME SET geom = '#{geo.fix_geometry['geo']}' WHERE geoid10 = '#{geo.geoid10}' AND type = '#{geo.type}'")
+          puts "Geography fixed? #{ActiveRecord::Base.connection.execute("SELECT ST_IsValid(geom) AS isvalid FROM Geographies WHERE geoid10 = '#{geo.geoid10}' AND type = '#{geo.type}'").first['isvalid'] == 't'}"
+        end
+        @dbar.increment!
+      end
+    end
+  end
+
+  desc "Repair Geometry"
+  task :repair_geometry => :environment do
+    for geo in format_args(ENV['SHAPE'], msg('repair'))
+      @dbar = ProgressBar.new(Geography.where(:type=> geo['slug'].capitalize).count)
+      Geography.find_in_batches(:select=> "gid, geoid10, name10, type, latlng, ST_AsText(geom) AS geo", :conditions => {:type => geo['slug'].capitalize}, :batch_size => 100) do |batch|
+        batch.each do |b|
+          rebuild = false
+          ids = ActiveRecord::Base.connection.execute("SELECT c.geoid10 FROM CONTROLLER_NAME AS p, CONTROLLER_NAME AS c WHERE c.geoid10 != '#{b.geoid10}' AND p.geoid10 = '#{b.geoid10}' AND c.type='#{b.type}' AND ST_DWithin(p.latlng, c.latlng, .5);").map{|id| id['geoid10']}
+          if !ids.empty?
+            intersects = Geography.where("geoid10 IN ('#{ids.join("','")}') AND type = '#{b.type}' AND ST_Intersects('#{b['geo']}', geom)")
+            if !intersects.empty?
+              polygons = b.polygons
+              intersects.each do |int| 
+                int.polygons.each_with_index do |c, i|
+                  current = ActiveRecord::Base.connection.execute("SELECT ST_Within('#{c}','#{b['geo']}') AS within").first
+                  if !polygons.include?(c) && current['within'] == "t"
+                    match = ActiveRecord::Base.connection.execute("SELECT #{polygons.map.with_index{|p, i| "ST_Intersects('#{p}', '#{c}') AS poly_#{i}"}.join(",")}").first.values.map{|aa| aa == "t" }
+                    if !match.include?(true)
+                      polygons << c
+                      rebuild = true
+                      puts "#{b.type} #{b.geoid10} has children #{int.geoid10}"
+                    end
+                  end
+                end
+              end
+              if rebuild
+                new_geom = Geography.merge(polygons)
+                if new_geom['isvalid']
+                  ActiveRecord::Base.connection.execute("UPDATE CONTROLLER_NAME SET geom = '#{new_geom['geom']}' WHERE geoid10 = '#{b.geoid10}' AND type = '#{b.type}'")
+                else
+                  ActiveRecord::Base.connection.execute("UPDATE CONTROLLER_NAME SET geom = ST_Buffer('#{new_geom['geom']}', .0000001) WHERE geoid10 = '#{b.geoid10}' AND type = '#{b.type}'")                
+                end
+              end
+            end
+          end
+          @dbar.increment!
+        end
+      end
+    end
+  end
+
+  def msg(type)
+    msg = "\nPlease specify one or more of the following Census shapes:\n\n"
+    msg += "#{CENSUS_SHAPES.map{|g| g[0]}.join(",")}\n\n"
+    if type == "install"
+      msg += "For example: rake census_shapes:install SHAPE=STATE,COUNTY\n\n"
+      msg += "Add the option 'ARCHIVE=true' to archive the zip files after importing."
+    elsif type== "validate"
+      msg += "For example: rake census_shapes:validate_geometry SHAPE=STATE,COUNTY\n\n"
+    elsif type== "repair"
+      msg += "For example: rake census_shapes:repair_geometry SHAPE=STATE,COUNTY\n\n"
+    end
+    return msg
+  end
+
+  def format_args(args, msg)
+    geos = []
+    if args.nil?
+      puts msg
+    else
+      args = args.split(",")
+      for arg in args
+        g = get_levels(arg.upcase)
+        geos << g if g
+      end
+    end
+    return geos
+  end
+
+  def get_levels(level)
+    if !level.nil? && CENSUS_SHAPES.include?(level.upcase)
+      return CENSUS_SHAPES[level] 
+    else
+      return false
+    end
+  end
+  
+  def local_path(local="tmp/shapefiles/")
+    return @local if @local
+    `mkdir -p #{local}`
+    if !File.exists?(local) && !File.directory?(local)
+      puts "'#{local}' is not a valid file path."
+      return false
+    end
+    @local = local
+    return @local
+  end
+
+  def download_zip(path, file, shape)
+    if !File.exist?("#{local_path}#{shape}/#{file}.zip") 
+      `mkdir -p #{local_path}#{shape}`
+      ftp = Net::FTP.new(CENSUS_HOST)
+      ftp.login(user = "anonymous")
+      ftp.chdir(path)
+      ftp.getbinaryfile("#{file}.zip", "#{local_path}#{shape}/#{file}.zip")
+      ftp.close
+    end
+  end
+
+  def self.unzip_shapefiles(file, shape)
+    `unzip #{local_path}#{shape}/#{file}.zip -d #{local_path}#{shape}/#{file}` if File.exist?("#{local_path}#{shape}/#{file}.zip") 
+  end
+
+  def self.import_shapefiles(file, shape, archive)
+    `shp2pgsql -W Latin1 -g geom -a -D #{local_path}#{shape}/#{file}/#{file} CONTROLLER_NAME #{TEMPLATE} | psql -h #{HOST} -U #{USER} -d #{DB}`
+    `rm -rf #{local_path}#{shape}/#{file}`
+    if archive.nil?
+      `rm #{local_path}#{shape}/#{file}.zip`
+    end
+  end
+ 
+  def before_import_set_defaults(geo)
+    ActiveRecord::Base.connection.execute("ALTER TABLE ONLY CONTROLLER_NAME ALTER COLUMN type SET DEFAULT '#{geo['slug'].capitalize}'")
+    ActiveRecord::Base.connection.execute("ALTER TABLE ONLY CONTROLLER_NAME ALTER COLUMN sumlevel SET DEFAULT '#{geo['sumlevel']}'")
+  end
+
+  def after_import_update_fields
+    ActiveRecord::Base.connection.execute("UPDATE CONTROLLER_NAME SET latlng = ST_GeomFromText('POINT(' || intptlon10 || ' ' ||  intptlat10 || ')', 4326) WHERE latlng IS NULL;")
+    ActiveRecord::Base.connection.execute("UPDATE CONTROLLER_NAME SET name10 = namelsad10 WHERE namelsad10 IS NOT NULL AND name10 IS NULL;")
+  end
+end

data/lib/generators/census_shapes/setup/templates/lib/tasks/postgis_template.rake

@@ -0,0 +1,22 @@
+namespace :postgis_template do
+  desc "Creates Postgis Template"
+	task :create do
+	  config = YAML::load(File.open('config/database.yml'))
+    template = config[Rails.env]['template']
+    user = config[Rails.env]['username']
+    host = config[Rails.env]['host']
+    postgis_path = config[Rails.env]['postgis_path']
+    
+    existing = `psql -U #{user} -h #{host} -U #{user} --list`
+    if existing.scan(template).empty?
+  	  `createdb -U #{user} #{template} -h #{host} -U #{user} -E UTF8`
+      `createlang -d #{template} plpgsql -h #{host} -U #{user}`
+      `psql -d postgres -c "UPDATE pg_database SET datistemplate='true' WHERE datname='#{template}';"`
+  	  `psql -h #{host} -U #{user} -d #{template} -f #{postgis_path}postgis.sql`
+  	  `psql -h #{host} -U #{user} -d #{template} -f #{postgis_path}spatial_ref_sys.sql`
+      `psql -d #{template} -c "GRANT ALL ON geometry_columns TO PUBLIC;"`
+      `psql -d #{template} -c "GRANT ALL ON geography_columns TO PUBLIC;"` 
+      `psql -d #{template} -c "GRANT ALL ON spatial_ref_sys TO PUBLIC;"`
+	  end
+  end
+end

data/lib/generators/census_shapes/setup/templates/lib/yaml/us_shapes.yml

@@ -0,0 +1,133 @@
+---
+STATE:
+  sumlevel: '040'
+  slug: STATE
+  name: State
+  description: state
+  path: STATE/2010
+COUNTY:
+  sumlevel: '050'
+  slug: COUNTY
+  name: County
+  description: state-county
+  path: COUNTY/2010
+COUSUB:
+  sumlevel: '060'
+  slug: COUSUB
+  name: County Subdivision
+  description: state-county-county subdivision
+  path: COUSUB/2010
+SUBMCD:
+  sumlevel: '067'
+  slug: SUBMCD
+  name: Subminor Civil Subdivision
+  description: state-county-county subdivision-subminor civil subdivision
+  path: SUBMCD/2010
+TABBLOCK:
+  sumlevel: '101'
+  slug: TABBLOCK
+  name: Block
+  description: state-county-tract-block
+  path: TABBLOCK/2010
+TRACT:
+  sumlevel: '140'
+  slug: TRACT
+  name: Tract
+  description: state-county-tract
+  path: TRACT/2010
+BG:
+  sumlevel: '150'
+  slug: BG
+  name: Block Group
+  description: state-county-tract-block group
+  path: BG/2010
+PLACE:
+  sumlevel: '160'
+  slug: PLACE
+  name: Place
+  description: state-place
+  path: PLACE/2010
+ANRC:
+  sumlevel: '230'
+  slug: ANRC
+  name: Alaska Native Regional Corporation
+  description: state-alaska native regional corporation
+  path: ANRC/2010
+AIANNH:
+  sumlevel: '280'
+  slug: AIANNH
+  name: American Indian Area/Alaska Native Area/Hawaiian Home Land
+  description: state-american indian area/alaska native area/hawaiian home land
+  path: AIANNH/2010
+AITS:
+  sumlevel: '281'
+  slug: AITS
+  name: American Indian Tribal Subdivision
+  description: state-american indian area-tribal subdivision
+  path: AITS/2010
+CBSA:
+  sumlevel: '320'
+  slug: CBSA
+  name: Metropolitan Statistical Area/Micropolitan Statistical Area
+  description: state-metropolitan statistical area/micropolitan statistical area
+  path: CBSA/2010
+METDIV:
+  sumlevel: '323'
+  slug: METDIV
+  name: Metropolitan Division
+  description: state-metropolitan statistical area/micropolitan statistical area-metropolitan division
+  path: METDIV/2010
+CSA:
+  sumlevel: '340'
+  slug: CSA
+  name: Combined Statistical Area
+  description: state-combined statistical area
+  path: CSA/2010
+CD:
+  sumlevel: '500'
+  slug: CD
+  name: Congressional District (111th)
+  description:	state-congressional district (111th)
+  path: CD/111
+SLDU:
+  sumlevel: '610'
+  slug: SLDU
+  name: State Legislative District (Upper Chamber)
+  description: state-state legislative district (upper chamber)
+  path: SLDU/2010
+SLDL:
+  sumlevel: '620'
+  slug: SLDL
+  name: State Legislative District (Lower Chamber)
+  description: state-state legislative district (lower chamber)
+  path: SLDL/2010
+VTD:
+  sumlevel: '700'
+  slug: VTD
+  name: Voting District
+  description: State-County-Voting District/Remainder
+  path: VTD/2010
+ZCTA5:
+  sumlevel: '871'
+  slug: ZCTA5
+  name: ZIP Code Tabulation Area
+  description: state-zip code tabulation area
+  path: ZCTA5/2010
+ELSD:
+  sumlevel: '950'
+  slug: ELSD
+  name: School District (Elementary)
+  description: State-School District (Elementary)/Remainder
+  path: ELSD/2010
+SCSD:
+  sumlevel: '960'
+  slug: SCSD
+  name: School District (Secondary)
+  description: State-School District (Secondary)/Remainder
+  path: SCSD/2010
+UNSD:
+  sumlevel: '970'
+  slug: UNSD
+  name: School District (Unified)
+  description: State-School District (Unified)/Remainder
+  path: UNSD/2010

data/lib/generators/census_shapes/setup/templates/lib/yaml/us_states.yml

@@ -0,0 +1,209 @@
+---
+Alabama:
+  id: 1
+  abbr: AL
+  name: Alabama
+Alaska:
+  id: 2
+  abbr: AK
+  name: Alaska
+Arizona:
+  id: 4
+  abbr: AZ
+  name: Arizona
+Arkansas:
+  id: 5
+  abbr: AR
+  name: Arkansas
+California:
+  id: 6
+  abbr: CA
+  name: California
+Colorado:
+  id: 8
+  abbr: CO
+  name: Colorado
+Connecticut:
+  id: 9
+  abbr: CT
+  name: Connecticut
+Delaware:
+  id: 10
+  abbr: DE
+  name: Delaware
+District of Columbia:
+  id: 11
+  abbr: DC
+  name: District of Columbia
+Florida:
+  id: 12
+  abbr: FL
+  name: Florida
+Georgia:
+  id: 13
+  abbr: GA
+  name: Georgia
+Hawaii:
+  id: 15
+  abbr: HI
+  name: Hawaii
+Idaho:
+  id: 16
+  abbr: ID
+  name: Idaho
+Illinois:
+  id: 17
+  abbr: IL
+  name: Illinois
+Indiana:
+  id: 18
+  abbr: IN
+  name: Indiana
+Iowa:
+  id: 19
+  abbr: IA
+  name: Iowa
+Kansas:
+  id: 20
+  abbr: KS
+  name: Kansas
+Kentucky:
+  id: 21
+  abbr: KY
+  name: Kentucky
+Louisiana:
+  id: 22
+  abbr: LA
+  name: Louisiana
+Maine:
+  id: 23
+  abbr: ME
+  name: Maine
+Maryland:
+  id: 24
+  abbr: MD
+  name: Maryland
+Massachusetts:
+  id: 25
+  abbr: MA
+  name: Massachusetts
+Michigan:
+  id: 26
+  abbr: MI
+  name: Michigan
+Minnesota:
+  id: 27
+  abbr: MN
+  name: Minnesota
+Mississippi:
+  id: 28
+  abbr: MS
+  name: Mississippi
+Missouri:
+  id: 29
+  abbr: MO
+  name: Missouri
+Montana:
+  id: 30
+  abbr: MT
+  name: Montana
+Nebraska:
+  id: 31
+  abbr: NE
+  name: Nebraska
+Nevada:
+  id: 32
+  abbr: NV
+  name: Nevada
+New Hampshire:
+  id: 33
+  abbr: NH
+  name: New Hampshire
+New Jersey:
+  id: 34
+  abbr: NJ
+  name: New Jersey
+New Mexico:
+  id: 35
+  abbr: NM
+  name: New Mexico
+New York:
+  id: 36
+  abbr: NY
+  name: New York
+North Carolina:
+  id: 37
+  abbr: NC
+  name: North Carolina
+North Dakota:
+  id: 38
+  abbr: ND
+  name: North Dakota
+Ohio:
+  id: 39
+  abbr: OH
+  name: Ohio
+Oklahoma:
+  id: 40
+  abbr: OK
+  name: Oklahoma
+Oregon:
+  id: 41
+  abbr: OR
+  name: Oregon
+Pennsylvania:
+  id: 42
+  abbr: PA
+  name: Pennsylvania
+Rhode Island:
+  id: 44
+  abbr: RI
+  name: Rhode Island
+South Carolina:
+  id: 45
+  abbr: SC
+  name: South Carolina
+South Dakota:
+  id: 46
+  abbr: SD
+  name: South Dakota
+Tennessee:
+  id: 47
+  abbr: TN
+  name: Tennessee
+Texas:
+  id: 48
+  abbr: TX
+  name: Texas
+Utah:
+  id: 49
+  abbr: UT
+  name: Utah
+Vermont:
+  id: 50
+  abbr: VT
+  name: Vermont
+Virginia:
+  id: 51
+  abbr: VA
+  name: Virginia
+Washington:
+  id: 53
+  abbr: WA
+  name: Washington
+West Virginia:
+  id: 54
+  abbr: WV
+  name: West Virginia
+Wisconsin:
+  id: 55
+  abbr: WI
+  name: Wisconsin
+Wyoming:
+  id: 56
+  abbr: WY
+  name: Wyoming
+Puerto Rico:
+  id: 72
+  abbr: PR
+  name: Puerto Rico