us_zipcode 0.1.1

data/features/support/string.rb

@@ -0,0 +1,128 @@
+class String
+  
+  class << self
+    def random(count = 6, ranges = [('a'..'z'),('A'..'Z'),('0'..'9')])
+      o =  ranges.map{|i| i.to_a}.flatten;  
+      string  =  (0..(count-1)).map{ o[rand(o.length)]  }.join;
+    end
+  end
+  
+  def left(count)
+    self.slice(0,count)
+  end
+  
+  def right(count)
+    self.slice(-count,count)
+  end
+  
+  def left_trim
+    # remove leading whitespace
+    self.gsub(/^[\t\s]+/, '')
+  end
+  
+  def right_trim
+    # remove trailing whitespace
+    self.gsub(/[\t\s]+$/, '')
+  end
+  
+  def trim
+    # remove leading and trailing whitespace
+    self.left_trim.right_trim
+  end
+
+  # html = <<-stop.here_with_pipe
+  #   |<!-- Begin: comment  -->
+  #   |<script type="text/javascript">
+  # stop
+  def here_with_pipe(linefeeds = false)
+    lines = self.split("\n")
+    lines.map! {|c| c.sub!(/\s*\|/, '')}
+    new_string = lines.join(linefeeds ? "\n" : " ")
+    self.replace(new_string)
+  end
+
+  def is_alpha_numeric?
+    regex = /^[a-zA-Z0-9]+$/
+    return (self =~ regex) == 0 ? true : false
+  end
+
+  def is_email_address?
+    # //Email address
+    # //Use this version to seek out email addresses in random documents and texts.
+    # //Does not match email addresses using an IP address instead of a domain name.
+    # //Does not match email addresses on new-fangled top-level domains with more than 4 letters such as .museum.
+    # //Including these increases the risk of false positives when applying the regex to random documents.
+    # '\b[A-Z0-9._%-]+@[A-Z0-9.-]+\.[A-Z]{2,4}\b'
+    # 
+    # //Email address (anchored)
+    # //Use this anchored version to check if a valid email address was entered.
+    # //Does not match email addresses using an IP address instead of a domain name.
+    # //Does not match email addresses on new-fangled top-level domains with more than 4 letters such as .museum.
+    # //Requires the "case insensitive" option to be ON.
+    # '^[A-Z0-9._%-]+@[A-Z0-9.-]+\.[A-Z]{2,4}$'
+    # 
+    # //Email address (anchored; no consecutive dots)
+    # //Use this anchored version to check if a valid email address was entered.
+    # //Improves on the original email address regex by excluding addresses with consecutive dots such as john@aol...com
+    # //Does not match email addresses using an IP address instead of a domain name.
+    # //Does not match email addresses on new-fangled top-level domains with more than 4 letters such as .museum.
+    # //Including these increases the risk of false positives when applying the regex to random documents.
+    # '^[A-Z0-9._%-]+@(?:[A-Z0-9-]+\.)+[A-Z]{2,4}$'
+    # 
+    # //Email address (no consecutive dots)
+    # //Use this version to seek out email addresses in random documents and texts.
+    # //Improves on the original email address regex by excluding addresses with consecutive dots such as john@aol...com
+    # //Does not match email addresses using an IP address instead of a domain name.
+    # //Does not match email addresses on new-fangled top-level domains with more than 4 letters such as .museum.
+    # //Including these increases the risk of false positives when applying the regex to random documents.
+    # '\b[A-Z0-9._%-]+@(?:[A-Z0-9-]+\.)+[A-Z]{2,4}\b'
+    # 
+    # //Email address (specific TLDs)
+    # //Does not match email addresses using an IP address instead of a domain name.
+    # //Matches all country code top level domains, and specific common top level domains.
+    # '^[A-Z0-9._%-]+@[A-Z0-9.-]+\.(?:[A-Z]{2}|com|org|net|biz|info|name|aero|biz|info|jobs|museum|name)$'
+    # 
+    # //Email address: Replace with HTML link
+    # '\b(?:mailto:)?([A-Z0-9._%-]+@[A-Z0-9.-]+\.[A-Z]{2,4})\b'
+    
+    email_regex = %r{^[A-Z0-9._%-]+@[A-Z0-9.-]+\.(?:[A-Z]{2}|com|org|net|biz|info|name|aero|biz|info|jobs|museum|name)$}xi # Case insensitive
+    
+    return (self =~ email_regex) == 0 ? true : false
+  end
+  
+  def is_zipcode?
+    self =~ %r{^(\d{5})(-\d{4})?$}x ? true : false
+  end
+
+  def format_phone
+    '(' << slice(0..2) << ')' << slice(3..5) << '-' << slice(-4,4)
+  end
+
+  def is_numeric?
+    begin
+      Float(self)
+    rescue
+      false # not numeric
+    else
+      true # numeric
+    end
+  end
+
+  def sanitize
+    clean_string = self.gsub(/[^a-z0-9,! \-\(\)\:\;\.\&\$]+/i, '')
+    #p "SAN: #{clean_string}"
+    clean_string
+  end
+
+  def shorten(count = 30)
+    if self.length >= count
+      shortened = self[0, count]
+      splitted = shortened.split(/\s/)
+      words = splitted.length
+      splitted[0, words-1].join(" ") + ' ...'
+    else
+      self
+    end
+  end
+  
+end

data/features/zipcodes.feature

@@ -0,0 +1,24 @@
+Feature: Us ZipCode Gem
+  In order to manage zipcode resources
+  As a rails developer
+  I want to generate models for zipcode, county and state, and populate their tables
+
+  Scenario: Generate models and migration for zipcode, county and state
+    Given a new Rails app
+    Then I should see "us_zipcode:models" when running "rails g"
+    When I run "rails g us_zipcode:models"
+    Then I should see the following files
+      | app/models/zipcode.rb    |
+      | app/models/state.rb      |
+      | app/models/county.rb     |
+      | lib/tasks/zipcodes.rake  |
+      | db/migrate               |
+    And I should see "gem "mocha"" in file "Gemfile"
+    And I should successfully run "rake db:migrate"
+
+  Scenario: Update data for zipcodes, counties and states tables
+    Given a new migrated Rails app
+    Then I should successfully run "rake zipcodes:update"
+    And I should see 51 records in the "states" table
+    And I should see 3142 records in the "counties" table
+    And I should see 42366 records in the "zipcodes" table

data/features/zipcodes.feature~

@@ -0,0 +1,24 @@
+Feature: My Zipcode Gem
+  In order to manage zipcode resources
+  As a rails developer
+  I want to generate models for zipcode, county and state, and populate their tables
+
+  Scenario: Generate models and migration for zipcode, county and state
+    Given a new Rails app
+    Then I should see "my_zipcode_gem:models" when running "rails g"
+    When I run "rails g my_zipcode_gem:models"
+    Then I should see the following files
+      | app/models/zipcode.rb    |
+      | app/models/state.rb      |
+      | app/models/county.rb     |
+      | lib/tasks/zipcodes.rake  |
+      | db/migrate               |
+    And I should see "gem "mocha"" in file "Gemfile"
+    And I should successfully run "rake db:migrate"
+
+  Scenario: Update data for zipcodes, counties and states tables
+    Given a new migrated Rails app
+    Then I should successfully run "rake zipcodes:update"
+    And I should see 51 records in the "states" table
+    And I should see 3142 records in the "counties" table
+    And I should see 42366 records in the "zipcodes" table

data/lib/generators/us_zipcode/models_generator.rb

@@ -0,0 +1,41 @@
+module UsZipcode
+  class ModelsGenerator < Base
+    include Rails::Generators::Migration
+
+    source_root File.expand_path('../templates', __FILE__)
+
+    def initialize(*args, &block)
+      super
+      migration_template 'migration.rb', "db/migrate/create_us_zipcode_models.rb"
+    end
+
+    def generate_models
+      # puts ">>> generate_zipcodes:"
+    end
+
+    def add_gems
+      add_gem "mocha", :group => :test
+    end
+
+    def create_models
+      template 'zipcode_model.rb', "app/models/zipcode.rb"
+      template 'county_model.rb', "app/models/county.rb"
+      template 'state_model.rb', "app/models/state.rb"
+    end
+
+    # Implement the required interface for Rails::Generators::Migration.
+    # taken from http://github.com/rails/rails/blob/master/activerecord/lib/generators/active_record.rb
+    def self.next_migration_number(dirname)
+      if ActiveRecord::Base.timestamped_migrations
+        Time.now.utc.strftime("%Y%m%d%H%M%S")
+      else
+        "%.3d" % (current_migration_number(dirname) + 1)
+      end
+    end
+
+    def create_rakefile
+      template 'zipcodes.rake', "lib/tasks/zipcodes.rake"
+    end
+
+  end
+end

data/lib/generators/us_zipcode/models_generator.rb~

@@ -0,0 +1,41 @@
+module MyZipcodeGem
+  class ModelsGenerator < Base
+    include Rails::Generators::Migration
+
+    source_root File.expand_path('../templates', __FILE__)
+
+    def initialize(*args, &block)
+      super
+      migration_template 'migration.rb', "db/migrate/create_my_zipcode_gem_models.rb"
+    end
+
+    def generate_models
+      # puts ">>> generate_zipcodes:"
+    end
+
+    def add_gems
+      add_gem "mocha", :group => :test
+    end
+
+    def create_models
+      template 'zipcode_model.rb', "app/models/zipcode.rb"
+      template 'county_model.rb', "app/models/county.rb"
+      template 'state_model.rb', "app/models/state.rb"
+    end
+
+    # Implement the required interface for Rails::Generators::Migration.
+    # taken from http://github.com/rails/rails/blob/master/activerecord/lib/generators/active_record.rb
+    def self.next_migration_number(dirname)
+      if ActiveRecord::Base.timestamped_migrations
+        Time.now.utc.strftime("%Y%m%d%H%M%S")
+      else
+        "%.3d" % (current_migration_number(dirname) + 1)
+      end
+    end
+
+    def create_rakefile
+      template 'zipcodes.rake', "lib/tasks/zipcodes.rake"
+    end
+
+  end
+end

data/lib/generators/us_zipcode/templates/county_model.rb

@@ -0,0 +1,17 @@
+require 'memoist'
+class County < ActiveRecord::Base
+  extend Memoist
+
+  belongs_to :state
+  has_many :zipcodes
+  
+  validates :name, uniqueness: {:scope => :state_id, :case_sensitive => false}, presence: true
+  
+  scope :without_zipcodes, -> { joins("LEFT JOIN zipcodes ON zipcodes.county_id = counties.id").where("zipcodes.county_id IS NULL")}
+  scope :without_state, -> {where("state_id IS NULL")}
+  
+  def cities
+    zipcodes.map(&:city).sort.uniq
+  end
+  memoize :cities
+end

data/lib/generators/us_zipcode/templates/county_model.rb~

@@ -0,0 +1,17 @@
+require 'memoist'
+class County < ActiveRecord::Base
+  extend Memoist
+
+  belongs_to :state
+  has_many :zipcodes
+  
+  validates :name, uniqueness: {:scope => :state_id, :case_sensitive => false}, :presence => true
+  
+  scope :without_zipcodes, -> { joins("LEFT JOIN zipcodes ON zipcodes.county_id = counties.id").where("zipcodes.county_id IS NULL")}
+  scope :without_state, -> {where("state_id IS NULL")}
+  
+  def cities
+    zipcodes.map(&:city).sort.uniq
+  end
+  memoize :cities
+end

data/lib/generators/us_zipcode/templates/migration.rb

@@ -0,0 +1,41 @@
+class CreateUsZipcodeModels < ActiveRecord::Migration
+  def self.up
+    # Zipcodes Table
+    create_table :zipcodes do |t|
+      t.string :code
+      t.string :city
+      t.integer :state_id, foreign_key: true
+      t.integer :county_id, foreign_key: true
+      t.decimal :lat, :precision => 15, :scale => 10
+      t.decimal :lon, :precision => 15, :scale => 10
+      t.timestamps
+    end
+    add_index :zipcodes, :code
+    add_index :zipcodes, :county_id
+    add_index :zipcodes, :state_id
+    add_index :zipcodes, [:lat, :lon]
+
+    # States Table
+    create_table :states do |t|
+      t.string :abbr, :limit => 2
+      t.string :name
+      t.timestamps
+    end
+    add_index :states, :abbr
+
+    # Counties Table
+    create_table :counties do |t|
+      t.integer :state_id
+      t.string :name
+      t.timestamps
+    end
+    add_index :counties, :name
+    add_index :counties, :state_id
+  end
+
+  def self.down
+    drop_table :counties
+    drop_table :states
+    drop_table :zipcodes
+  end
+end

data/lib/generators/us_zipcode/templates/migration.rb~

@@ -0,0 +1,41 @@
+class CreateMyZipcodeGemModels < ActiveRecord::Migration
+  def self.up
+    # Zipcodes Table
+    create_table :zipcodes do |t|
+      t.string :code
+      t.string :city
+      t.integer :state_id, foreign_key: true
+      t.integer :county_id, foreign_key: true
+      t.decimal :lat, :precision => 15, :scale => 10
+      t.decimal :lon, :precision => 15, :scale => 10
+      t.timestamps
+    end
+    add_index :zipcodes, :code
+    add_index :zipcodes, :county_id
+    add_index :zipcodes, :state_id
+    add_index :zipcodes, [:lat, :lon]
+
+    # States Table
+    create_table :states do |t|
+      t.string :abbr, :limit => 2
+      t.string :name
+      t.timestamps
+    end
+    add_index :states, :abbr
+
+    # Counties Table
+    create_table :counties do |t|
+      t.integer :state_id
+      t.string :name
+      t.timestamps
+    end
+    add_index :counties, :name
+    add_index :counties, :state_id
+  end
+
+  def self.down
+    drop_table :counties
+    drop_table :states
+    drop_table :zipcodes
+  end
+end

data/lib/generators/us_zipcode/templates/state_model.rb

@@ -0,0 +1,15 @@
+require 'memoist'
+class State < ActiveRecord::Base
+  extend Memoist
+
+  has_many :zipcodes
+  has_many :counties
+
+  validates :abbr, uniqueness:  { :case_sensitive => false }, presence: true
+  validates :name, uniqueness: { :case_sensitive => false }, presence: true
+  
+  def cities
+    zipcodes.map(&:city).sort.uniq
+  end
+  memoize :cities
+end

data/lib/generators/us_zipcode/templates/state_model.rb~

@@ -0,0 +1,15 @@
+require 'memoist'
+class State < ActiveRecord::Base
+  extend Memoist
+
+  has_many :zipcodes
+  has_many :counties
+
+  validates :abbr, uniqueness:  { :case_sensitive => false }, :presence => true
+  validates :name, uniqueness: { :case_sensitive => false }, :presence => true
+  
+  def cities
+    zipcodes.map(&:city).sort.uniq
+  end
+  memoize :cities
+end

data/lib/generators/us_zipcode/templates/zipcode_model.rb

@@ -0,0 +1,30 @@
+class Zipcode < ActiveRecord::Base
+  
+  belongs_to :county
+  belongs_to :state
+  
+  validates :code, uniqueness: true, presence: true
+  validates :state_id, :county_id, :city, presence: true
+  
+  scope :without_county, -> { where("county_id IS NULL") }
+  scope :without_state, -> { where("state_id IS NULL") }
+  scope :ungeocoded, -> { where("lat IS NULL OR lon IS NULL") }
+
+  class << self
+    def find_by_city(city)
+      includes(:county).where("city like '#{city}%'")
+    end
+  end
+
+  def county_and_zip
+    "#{county.name}: #{code}"
+  end
+
+  def latlon
+    [lat, lon]
+  end
+
+  def is_geocoded?
+    (!lat.nil? && !lon.nil?)
+  end
+end

data/lib/generators/us_zipcode/templates/zipcode_model.rb~

@@ -0,0 +1,30 @@
+class Zipcode < ActiveRecord::Base
+  
+  belongs_to :county
+  belongs_to :state
+  
+  validates :code, uniqueness: true, presence: true
+  validates :state_id, :county_id, :city, :presence => true
+  
+  scope :without_county, -> { where("county_id IS NULL") }
+  scope :without_state, -> { where("state_id IS NULL") }
+  scope :ungeocoded, -> { where("lat IS NULL OR lon IS NULL") }
+
+  class << self
+    def find_by_city(city)
+      includes(:county).where("city like '#{city}%'")
+    end
+  end
+
+  def county_and_zip
+    "#{county.name}: #{code}"
+  end
+
+  def latlon
+    [lat, lon]
+  end
+
+  def is_geocoded?
+    (!lat.nil? && !lon.nil?)
+  end
+end

data/lib/generators/us_zipcode/templates/zipcodes.rake

@@ -0,0 +1,163 @@
+require 'open-uri'
+require 'csv'
+namespace :zipcodes do
+
+  desc "Update states table"
+  task :update_states => :environment do
+    puts ">>> Begin update of states table..."
+    url = "https://github.com/midwire/free_zipcode_data/raw/master/all_us_states.csv"
+    data = open(url)
+    file = nil
+    if data.is_a? StringIO
+      file = Tempfile.new('all_us_states.csv')
+      file.write(data.read)
+      file.flush
+      file.close
+    else
+      file = data
+    end
+    CSV.foreach(file.path, :headers => true) do |row|
+      puts "Updating state: [#{row['name']}]"
+      state = State.where(abbr: row['abbr']).first_or_initialize
+      state.update_attribute(:name, row['name'])
+    end
+    data.close
+    puts ">>> End update of states table..."
+  end
+
+  desc "Update counties table"
+  task :update_counties => :update_states do
+    puts ">>> Begin update of counties table..."
+    url = "https://github.com/midwire/free_zipcode_data/raw/master/all_us_counties.csv"
+    data = open(url)
+    file = nil
+    if data.is_a? StringIO
+      file = Tempfile.new('all_us_counties.csv')
+      file.write(data.read)
+      file.flush
+      file.close
+    else
+      file = data
+    end
+    CSV.foreach(file.path, :headers => true) do |row|
+      puts "Updating county: [#{row['name']}]"
+      # lookup state
+      state = State.find_by_abbr!(row['state'])
+      county = County.where(name: row['name'], state_id: state.to_param).first_or_initialize
+      county.save
+    end
+    data.close
+    puts ">>> End update of counties table..."
+  end
+
+  desc "Update zipcodes table"
+  task :update_zipcodes => :update_counties do
+    puts ">>> Begin update of zipcodes table..."
+    url = "https://github.com/midwire/free_zipcode_data/raw/master/all_us_zipcodes.csv"
+    data = open(url)
+    file = nil
+    if data.is_a? StringIO
+      file = Tempfile.new('all_us_zipcodes.csv')
+      file.write(data.read)
+      file.flush
+      file.close
+    else
+      file = data
+    end
+    CSV.foreach(file.path, :headers => true) do |row|
+      puts "Updating zipcode: [#{row['code']}], '#{row['city']}, #{row['state']}, #{row['county']}"
+      # lookup state
+      state = State.find_by_abbr!(row['state'])
+      begin
+        county = County.find_by_name_and_state_id!(row['county'], state.to_param)
+      rescue Exception => e
+        puts ">>> e: [#{e}]"
+        puts ">>>> No county found for zipcode: [#{row['code']}], '#{row['city']}, #{row['state']}, #{row['county']}... SKIPPING..."
+        next
+      end
+      zipcode = Zipcode.where(code: row['code']).first_or_initialize
+      zipcode.update_attributes!(
+        :city => row['city'].titleize,
+        :state_id => state.to_param,
+        :county_id => county.to_param,
+        :lat => row['lat'],
+        :lon => row['lon']
+      )
+    end
+    data.close
+    puts ">>> End update of zipcodes table..."
+  end
+
+  desc "Populate or update the zipcodes related tables"
+  task :update => :environment do
+    Rake::Task['zipcodes:update_zipcodes'].invoke
+  end
+
+  desc "Export US States to a .csv file"
+  task :export_states => :environment do
+    @states = State.order("name ASC")
+    csv_string = CSV.generate do |csv|
+      csv << ["abbr", "name"]
+      @states.each do |state|
+        csv << [
+          state.abbr,
+          state.name
+          ]
+      end
+    end
+    filename = "all_us_states.csv"
+    open("#{Rails.root}/db/#{filename}", 'w') do |f|
+      f.write(csv_string)
+    end
+  end
+
+  desc "Export all US Counties to a .csv file"
+  task :export_counties => :environment do
+    @counties = County.order("name ASC")
+    csv_string = CSV.generate do |csv|
+      csv << ["name", "state", "county_seat"]
+      @counties.each do |county|
+        csv << [
+          county.name,
+          county.state.abbr,
+          county.county_seat
+          ]
+      end
+    end
+    filename = "all_us_counties.csv"
+    open("#{Rails.root}/db/#{filename}", 'w') do |f|
+      f.write(csv_string)
+    end
+  end
+
+  desc "Export the zipcodes with county and state data"
+  task :export_zipcodes => :environment do
+    @zipcodes = Zipcode.order("code ASC")
+    csv_string = CSV.generate do |csv|
+      csv << ["code", "city", "state", "county", "area_code", "lat", "lon"]
+      @zipcodes.each do |zip|
+        csv << [
+          zip.code,
+          zip.city,
+          zip.state.abbr,
+          zip.county.nil? ? '' : zip.county.name,
+          zip.area_code,
+          zip.lat,
+          zip.lon
+          ]
+      end
+    end
+    filename = "all_us_zipcodes.csv"
+    open("#{Rails.root}/db/#{filename}", 'w') do |f|
+      f.write(csv_string)
+    end
+  end
+
+  desc "Export zipcodes, states and counties tables"
+  task :export => :environment do
+    Rake::Task['zipcodes:export_states'].invoke
+    Rake::Task['zipcodes:export_counties'].invoke
+    Rake::Task['zipcodes:export_zipcodes'].invoke
+  end
+
+end