us_geo 1.0.0

data/lib/tasks/us_geo/us_geo.rake

@@ -0,0 +1,43 @@
+# frozen_string_literal: true
+
+namespace :us_geo do
+  namespace :import do
+    klasses = {
+      regions: USGeo::Region,
+      divisions: USGeo::Division,
+      states: USGeo::State,
+      designated_market_areas: USGeo::DesignatedMarketArea,
+      combined_statistical_areas: USGeo::CombinedStatisticalArea,
+      core_based_statistical_areas: USGeo::CoreBasedStatisticalArea,
+      metropolitan_divisions: USGeo::MetropolitanDivision,
+      counties: USGeo::County,
+      county_subdivisions: USGeo::CountySubdivision,
+      urban_areas: USGeo::UrbanArea,
+      places: USGeo::Place,
+      zctas: USGeo::Zcta,
+      zcta_counties: USGeo::ZctaCounty,
+      zcta_urban_areas: USGeo::ZctaUrbanArea,
+      zcta_places: USGeo::ZctaPlace,
+      urban_area_counties: USGeo::UrbanAreaCounty,
+      place_counties: USGeo::PlaceCounty
+    }
+    klasses.each do |name, klass|
+      desc "Import data for #{klass}"
+      task name => :environment do
+        t = Time.now
+        klass.load!
+        puts "Loaded #{klass.count} rows into #{klass.table_name} in #{(Time.now - t).round(1)}s"
+        klass.removed.find_each do |record|
+          puts("  WARNING: #{klass}.#{record.id} status changed to removed")
+        end
+      end
+
+      desc "Import data for all USGeo models"
+      task all: :environment do
+        klasses.each_key do |name|
+          Rake::Task["us_geo:import:#{name}"].invoke
+        end
+      end
+    end
+  end
+end

data/lib/us_geo/base_record.rb

@@ -0,0 +1,104 @@
+# frozen_string_literal: true
+
+require "csv"
+require "open-uri"
+
+module USGeo
+
+  class LoadError < StandardError
+  end
+
+  # Base class that all models inherit from.
+  class BaseRecord < ::ActiveRecord::Base
+
+    self.abstract_class = true
+    self.table_name_prefix = "us_geo_"
+
+    STATUS_IMPORTED = 1
+    STATUS_REMOVED = -1
+    STATUS_MANUAL = 0
+
+    validates :status, inclusion: [STATUS_IMPORTED, STATUS_REMOVED, STATUS_MANUAL]
+
+    scope :imported, -> { where(status: STATUS_IMPORTED) }
+    scope :removed, -> { where(status: STATUS_REMOVED) }
+    scope :manual, -> { where(status: STATUS_MANUAL) }
+    scope :not_removed, -> { where(status: [STATUS_IMPORTED, STATUS_MANUAL]) }
+
+    class << self
+      def load!(location = nil, gzipped: true)
+        raise NotImplementedError
+      end
+
+      protected
+
+      # Insert or update a record given the unique criteria for finding it.
+      def load_record!(criteria, &block)
+        record = find_or_initialize_by(criteria)
+        record.status = STATUS_IMPORTED
+        record.updated_at = Time.now
+        yield(record)
+        record.save!
+      end
+
+      # Mark the status of any records not updated in the block as being no longer imported.
+      def import!(&block)
+        start_time = Time.at(Time.now.to_i.floor)
+        yield
+        raise LoadError.new("No data found") unless where("updated_at >= ?", start_time).exists?
+        where("updated_at < ?", start_time).imported.update_all(status: STATUS_REMOVED)
+      end
+
+      def data_uri(path)
+        path = path.to_s if path
+        if path.start_with?("/") || path.include?(":")
+          path
+        elsif USGeo.base_data_uri.include?(":")
+          "#{USGeo.base_data_uri}/#{path}"
+        else
+          File.join(USGeo.base_data_uri, path)
+        end
+      end
+
+      def load_data_file(location, &block)
+        file = nil
+        if location.include?(":")
+          file = URI.parse(location).open(read_timeout: 5, open_timeout: 5)
+        else
+          file = File.open(location)
+        end
+        begin
+          rows = []
+          CSV.new(file, headers: true).each do |row|
+            rows << row
+            if rows.size >= 50
+              transaction { rows.each(&block) }
+              rows.clear
+            end
+          end
+          transaction { rows.each(&block) } unless rows.empty?
+        ensure
+          file.close if file && !file.closed?
+        end
+      end
+
+      # Convert square meters to square miles
+      def area_meters_to_miles(square_meters)
+        (square_meters.to_f / (1609.34 ** 2)).round(6)
+      end
+    end
+
+    def imported?
+      status == STATUS_IMPORTED
+    end
+
+    def removed?
+      status == STATUS_REMOVED
+    end
+
+    def manual?
+      status == STATUS_MANUAL
+    end
+
+  end
+end

data/lib/us_geo/combined_statistical_area.rb

@@ -0,0 +1,40 @@
+# frozen_string_literal: true
+
+module USGeo
+
+  # Combined statistical area (CSA) of multiple metropolitan areas with weak regional
+  # and economic connectoins between them.
+  class CombinedStatisticalArea < BaseRecord
+
+    include Demographics
+
+    self.primary_key = "geoid"
+
+    has_many :core_based_statistical_areas, foreign_key: :csa_geoid, inverse_of: :combined_statistical_area
+
+    validates :geoid, length: {is: 3}
+    validates :name, length: {maximum: 60}
+    validates :land_area, numericality: true, presence: true
+    validates :water_area, numericality: true, presence: true
+    validates :population, numericality: {only_integer: true}, presence: true
+    validates :housing_units, numericality: {only_integer: true}, presence: true
+
+    class << self
+      def load!(uri = nil)
+        location = data_uri(uri || "combined_statistical_areas.csv")
+        import! do
+          load_data_file(location) do |row|
+            load_record!(geoid: row["GEOID"]) do |record|
+              record.name = row["Name"]
+              record.population = row["Population"]
+              record.housing_units = row["Housing Units"]
+              record.land_area = area_meters_to_miles(row["Land Area"])
+              record.water_area = area_meters_to_miles(row["Water Area"])
+            end
+          end
+        end
+      end
+    end
+
+  end
+end

data/lib/us_geo/core_based_statistical_area.rb

@@ -0,0 +1,57 @@
+# frozen_string_literal: true
+
+module USGeo
+
+  # Core based statistical area composed of one or more counties anchored by an urban center.
+  # Includes both metropolitan (population > 50,000) and micropolitan (population > 10,000
+  # but < 50,000) areas.
+  class CoreBasedStatisticalArea < BaseRecord
+
+    include Demographics
+
+    self.primary_key = "geoid"
+    self.store_full_sti_class = false
+
+    has_many :counties, foreign_key: :cbsa_geoid, inverse_of: :core_based_statistical_area
+    has_many :metropolitan_divisions, foreign_key: :cbsa_geoid, inverse_of: :core_based_statistical_area
+    belongs_to :combined_statistical_area, foreign_key: :csa_geoid, optional: true, inverse_of: :core_based_statistical_areas
+
+    validates :geoid, length: {is: 5}
+    validates :name, length: {maximum: 60}
+    validates :land_area, numericality: true, presence: true
+    validates :water_area, numericality: true, presence: true
+    validates :population, numericality: {only_integer: true}, presence: true
+    validates :housing_units, numericality: {only_integer: true}, presence: true
+
+    class << self
+      def load!(uri = nil)
+        location = data_uri(uri || "core_based_statistical_areas.csv")
+        
+        import! do
+          load_data_file(location) do |row|
+            load_record!(geoid: row["GEOID"]) do |record|
+              record.type = (row["Population"].to_i >= 50_000 ? "MetropolitanArea" : "MicropolitanArea")
+              record.name = row["Name"]
+              record.csa_geoid = row["CSA"]
+              record.population = row["Population"]
+              record.housing_units = row["Housing Units"]
+              record.land_area = area_meters_to_miles(row["Land Area"])
+              record.water_area = area_meters_to_miles(row["Water Area"])
+              record.lat = row["Latitude"]
+              record.lng = row["Longitude"]
+            end
+          end
+        end
+      end
+    end
+
+    def metropolitan?
+      raise NotImplementedError
+    end
+    
+    def micropolitan?
+      raise NotImplementedError
+    end
+
+  end
+end

data/lib/us_geo/county.rb

@@ -0,0 +1,89 @@
+# frozen_string_literal: true
+
+module USGeo
+
+  # County or county equivalent. Counties are composed of zero or more ZCTA's and may
+  # belong to a CBSA. The county's significance withing the CBSA is indicated by the
+  # central flag which indicates if it is a central or outlying county.
+  class County < BaseRecord
+
+    include Demographics
+
+    self.primary_key = "geoid"
+
+    belongs_to :designated_market_area, foreign_key: :dma_code, optional: true, inverse_of: :counties
+    belongs_to :core_based_statistical_area, foreign_key: :cbsa_geoid, optional: true, inverse_of: :counties
+    belongs_to :metropolitan_division, foreign_key: :metropolitan_division_geoid, optional: true, inverse_of: :counties
+    belongs_to :state, foreign_key: :state_code, inverse_of: :counties
+
+    has_many :subdivisions, foreign_key: :county_geoid, inverse_of: :county, class_name: "USGeo::CountySubdivision"
+
+    has_many :zcta_counties, foreign_key: :county_geoid, inverse_of: :county, dependent: :destroy
+    has_many :zctas, through: :zcta_counties
+
+    has_many :urban_area_counties, foreign_key: :county_geoid, inverse_of: :county, dependent: :destroy
+    has_many :urban_areas, through: :urban_area_counties
+
+    has_many :place_counties, foreign_key: :county_geoid, inverse_of: :county, dependent: :destroy
+    has_many :places, through: :place_counties
+
+    validates :geoid, length: {is: 5}
+    validates :name, length: {maximum: 60}
+    validates :short_name, length: {maximum: 30}
+    validates :state_code, length: {is: 2}
+    validates :fips_class_code, length: {is: 2}
+    validates :metropolitan_division_geoid, length: {is: 5}, allow_nil: true
+    validates :cbsa_geoid, length: {is: 5}, allow_nil: true
+    validates :dma_code, length: {is: 3}, allow_nil: true
+    validates :land_area, numericality: true, allow_nil: true
+    validates :water_area, numericality: true, allow_nil: true
+    validates :population, numericality: {only_integer: true}, allow_nil: true
+    validates :housing_units, numericality: {only_integer: true}, allow_nil: true
+
+    class << self
+      def load!(uri = nil)
+        location = data_uri(uri || "counties.csv")
+
+        import! do
+          load_data_file(location) do |row|
+            load_record!(geoid: row["GEOID"]) do |record|
+              record.gnis_id = row["GNIS ID"]
+              record.name = row["Name"]
+              record.short_name = row["Short Name"]
+              record.state_code = row["State"]
+              record.cbsa_geoid = row["CBSA"]
+              record.dma_code = row["DMA"]
+              record.time_zone_name = row["Time Zone"]
+              record.fips_class_code = row["FIPS Class"]
+              record.central = (row["Central"] == "T")
+              record.population = row["Population"]
+              record.housing_units = row["Housing Units"]
+              record.land_area = area_meters_to_miles(row["Land Area"])
+              record.water_area = area_meters_to_miles(row["Water Area"])
+              record.lat = row["Latitude"]
+              record.lng = row["Longitude"]
+            end
+          end
+        end
+      end
+    end
+
+    def state_fips
+      geoid[0, 2]
+    end
+
+    def county_fips
+      geoid[2, 3]
+    end
+
+    # Return the CBSA only if it is a metropolitan area.
+    def metropolitan_area
+      core_based_statistical_area if core_based_statistical_area && core_based_statistical_area.metropolitan?
+    end
+
+    def time_zone
+      ActiveSupport::TimeZone[time_zone_name] if time_zone_name
+    end
+
+  end
+end

data/lib/us_geo/county_subdivision.rb

@@ -0,0 +1,46 @@
+# frozen_string_literal: true
+
+module USGeo
+
+  # County subdivision.
+  class CountySubdivision < BaseRecord
+
+    include Demographics
+
+    self.primary_key = "geoid"
+
+    belongs_to :county, foreign_key: :county_geoid, inverse_of: :subdivisions
+
+    validates :geoid, length: {is: 10}
+    validates :name, length: {maximum: 60}
+    validates :fips_class_code, length: {is: 2}
+    validates :land_area, numericality: true, allow_nil: true
+    validates :water_area, numericality: true, allow_nil: true
+    validates :population, numericality: {only_integer: true}, allow_nil: true
+    validates :housing_units, numericality: {only_integer: true}, allow_nil: true
+
+    class << self
+      def load!(uri = nil)
+        location = data_uri(uri || "county_subdivisions.csv")
+
+        import! do
+          load_data_file(location) do |row|
+            load_record!(geoid: row["GEOID"]) do |record|
+              record.gnis_id = row["GNIS ID"]
+              record.county_geoid = row["County GEOID"]
+              record.name = row["Name"]
+              record.fips_class_code = row["FIPS Class"]
+              record.population = row["Population"]
+              record.housing_units = row["Housing Units"]
+              record.land_area = area_meters_to_miles(row["Land Area"])
+              record.water_area = area_meters_to_miles(row["Water Area"])
+              record.lat = row["Latitude"]
+              record.lng = row["Longitude"]
+            end
+          end
+        end
+      end
+    end
+
+  end
+end

data/lib/us_geo/demographics.rb

@@ -0,0 +1,25 @@
+# frozen_string_literal: true
+
+module USGeo
+  
+  # This module is mixed into all models. Note that the area given for land and water
+  # is in square miles.
+  module Demographics
+
+    # Population per square mile.
+    def population_density
+      population.to_f / land_area if population && land_area
+    end
+
+    # Total area of both land an water in square miles
+    def total_area
+      land_area.to_f + water_area.to_f if land_area
+    end
+
+    # The fraction of the area that is composed of land instead of water.
+    def percent_land
+      land_area / total_area if land_area
+    end
+
+  end
+end

data/lib/us_geo/designated_market_area.rb

@@ -0,0 +1,30 @@
+# frozen_string_literal: true
+
+module USGeo
+
+  # Media market (DMA) as defined by the Nielsen Company.
+  class DesignatedMarketArea < BaseRecord
+
+    self.primary_key = "code"
+
+    has_many :counties, foreign_key: :dma_code, inverse_of: :designated_market_area
+
+    validates :code, length: {is: 3}
+    validates :name, length: {maximum: 60}
+
+    class << self
+      def load!(uri = nil)
+        location = data_uri(uri || "dmas.csv")
+       
+        import! do
+          load_data_file(location) do |row|
+            load_record!(code: row["Code"]) do |record|
+              record.name = row["Name"]
+            end
+          end
+        end
+      end
+    end
+
+  end
+end

data/lib/us_geo/division.rb

@@ -0,0 +1,29 @@
+# frozen_string_literal: true
+
+module USGeo
+
+  # U.S. regional division composed of states.
+  class Division < BaseRecord
+
+    belongs_to :region, inverse_of: :divisions
+    has_many :states, inverse_of: :division
+
+    validates :name, length: {maximum: 30}
+
+    class << self
+      def load!(uri = nil)
+        location = data_uri(uri || "divisions.csv")
+        
+        import! do
+          load_data_file(location) do |row|
+            load_record!(id: row["Division ID"]) do |record|
+              record.name = row["Division Name"]
+              record.region_id = row["Region ID"]
+            end
+          end
+        end
+      end
+    end
+
+  end
+end

data/lib/us_geo/engine.rb

@@ -0,0 +1,6 @@
+# frozen_string_literal: true
+
+module USGeo
+  class Engine < ::Rails::Engine
+  end
+end

data/lib/us_geo/metropolitan_area.rb

@@ -0,0 +1,18 @@
+# frozen_string_literal: true
+
+module USGeo
+
+  # Core based statistical area with a population greater then 50,000.
+  class MetropolitanArea < CoreBasedStatisticalArea
+    
+    def metropolitan?
+      true
+    end
+    
+    def micropolitan?
+      false
+    end
+    
+  end
+  
+end

data/lib/us_geo/metropolitan_division.rb

@@ -0,0 +1,42 @@
+# frozen_string_literal: true
+
+module USGeo
+
+  # Division of very large metropolitian areas into groups of approximately 2.5 million people.
+  class MetropolitanDivision < BaseRecord
+
+    include Demographics
+
+    self.primary_key = "geoid"
+
+    has_many :counties, foreign_key: :metropolitan_division_geoid, inverse_of: :metropolitan_division
+    belongs_to :core_based_statistical_area, foreign_key: :cbsa_geoid, optional: true, inverse_of: :metropolitan_divisions
+
+    validates :geoid, length: {is: 5}
+    validates :name, length: {maximum: 60}
+    validates :land_area, numericality: true, presence: true
+    validates :water_area, numericality: true, presence: true
+    validates :population, numericality: {only_integer: true}, presence: true
+    validates :housing_units, numericality: {only_integer: true},  presence: true
+
+    class << self
+      def load!(uri = nil)
+        location = data_uri(uri || "metropolitan_divisions.csv")
+
+        import! do
+          load_data_file(location) do |row|
+            load_record!(geoid: row["GEOID"]) do |record|
+              record.name = row["Name"]
+              record.cbsa_geoid = row["CBSA"]
+              record.population = row["Population"]
+              record.housing_units = row["Housing Units"]
+              record.land_area = area_meters_to_miles(row["Land Area"])
+              record.water_area = area_meters_to_miles(row["Water Area"])
+            end
+          end
+        end
+      end
+    end
+
+  end
+end

data/lib/us_geo/micropolitan_area.rb

@@ -0,0 +1,18 @@
+# frozen_string_literal: true
+
+module USGeo
+
+  # Core based statistical area with a population greater then 10,000 but less than 50,000.
+  class MicropolitanArea < CoreBasedStatisticalArea
+    
+    def metropolitan?
+      false
+    end
+    
+    def micropolitan?
+      true
+    end
+    
+  end
+  
+end

data/lib/us_geo/place.rb

@@ -0,0 +1,61 @@
+# frozen_string_literal: true
+
+module USGeo
+
+  # Division of very large metropolitian areas into groups of approximately 2.5 million people.
+  class Place < BaseRecord
+
+    include Demographics
+
+    self.primary_key = "geoid"
+
+    has_many :zcta_places, foreign_key: :place_geoid, inverse_of: :place, dependent: :destroy
+    has_many :zctas, through: :zcta_places
+
+    has_many :place_counties, foreign_key: :place_geoid, inverse_of: :place, dependent: :destroy
+    has_many :counties, through: :place_counties
+    
+    belongs_to :primary_county, foreign_key: :primary_county_geoid, class_name: "USGeo::County"
+    belongs_to :urban_area, foreign_key: :urban_area_geoid, optional: true, class_name: "USGeo::UrbanArea"
+    belongs_to :state, foreign_key: :state_code, inverse_of: :places
+
+    validates :geoid, length: {is: 7}
+    validates :state_code, length: {is: 2}
+    validates :primary_county_geoid, length: {is: 5}
+    validates :urban_area_geoid, length: {is: 5}, allow_nil: true
+    validates :name, length: {maximum: 60}
+    validates :short_name, length: {maximum: 60}
+    validates :fips_class_code, length: {is: 2}
+    validates :land_area, numericality: true, allow_nil: true
+    validates :water_area, numericality: true, allow_nil: true
+    validates :population, numericality: {only_integer: true}, allow_nil: true
+    validates :housing_units, numericality: {only_integer: true}, allow_nil: true
+
+    class << self
+      def load!(uri = nil)
+        location = data_uri(uri || "places.csv")
+       
+        import! do
+          load_data_file(location) do |row|
+            load_record!(geoid: row["GEOID"]) do |record|
+              record.gnis_id = row["GNIS ID"]
+              record.name = row["Name"]
+              record.short_name = row["Short Name"]
+              record.state_code = row["State"]
+              record.primary_county_geoid = row["County GEOID"]
+              record.urban_area_geoid = row["Urban Area GEOID"]
+              record.fips_class_code = row["FIPS Class"]
+              record.population = row["Population"]
+              record.housing_units = row["Housing Units"]
+              record.land_area = area_meters_to_miles(row["Land Area"])
+              record.water_area = area_meters_to_miles(row["Water Area"])
+              record.lat = row["Latitude"]
+              record.lng = row["Longitude"]
+            end
+          end
+        end
+      end
+    end
+
+  end
+end

data/lib/us_geo/place_county.rb

@@ -0,0 +1,28 @@
+# frozen_string_literal: true
+
+module USGeo
+
+  # Mapping of urban areas to counties they overlap with.
+  class PlaceCounty < BaseRecord
+
+    belongs_to :county, foreign_key: :county_geoid, inverse_of: :place_counties
+    belongs_to :place, foreign_key: :place_geoid, inverse_of: :place_counties
+
+    validates :county_geoid, length: {is: 5}
+    validates :place_geoid, length: {is: 7}
+
+    class << self
+      def load!(uri = nil)
+        location = data_uri(uri || "place_counties.csv")
+
+        import! do
+          load_data_file(location) do |row|
+            load_record!(place_geoid: row["Place GEOID"], county_geoid: row["County GEOID"]) do |record|
+            end
+          end
+        end
+      end
+    end
+
+  end
+end

data/lib/us_geo/region.rb

@@ -0,0 +1,28 @@
+# frozen_string_literal: true
+
+module USGeo
+
+  # U.S. region.
+  class Region < BaseRecord
+
+    has_many :divisions, inverse_of: :region
+    has_many :states, inverse_of: :region
+
+    validates :name, length: {maximum: 30}
+
+    class << self
+      def load!(uri = nil)
+        location = data_uri(uri || "divisions.csv")
+
+        import! do
+          load_data_file(location) do |row|
+            load_record!(id: row["Region ID"]) do |record|
+              record.name = row["Region Name"]
+            end
+          end
+        end
+      end
+    end
+
+  end
+end