bplgeo 0.0.1

data/lib/bplgeo/geonames.rb

@@ -0,0 +1,146 @@
+module Bplgeo
+  class Geonames
+    def self.bplgeo_config
+      root = Rails.root || './test/dummy'
+      env = Rails.env || 'test'
+      @bplgeo_config ||= YAML::load(ERB.new(IO.read(File.join(root, 'config', 'bplgeo.yml'))).result)[env].with_indifferent_access
+    end
+
+    def self.geonames_username
+      bplgeo_config[:geonames_username] || '<username>'
+    end
+
+    def self.get_geonames_data(geoname_id)
+      max_retry = 3
+      sleep_time = 60 # In seconds
+      retry_count = 0
+
+      hier_geo = {}
+      coords = {}
+      geonames_data = {}
+
+      begin
+        if retry_count > 0
+          sleep(sleep_time)
+        end
+        retry_count = retry_count + 1
+
+        geonames_response = Typhoeus::Request.get("http://api.geonames.org/hierarchy?username=#{self.geonames_username}&lang=en&style=FULL&geonameId=" + geoname_id)
+
+      end until (geonames_response.code != 500 || retry_count == max_retry)
+
+      unless geonames_response.code == 500
+        parsed_xml = Nokogiri::Slop(geonames_response.body)
+
+        parsed_xml.geonames.geoname.each do |geoname|
+           hier_geo[geoname.fcode.text.downcase.to_sym] = geoname.toponymName.text
+        end
+
+        #FIXME: Code4Lib lazy implementation... will get last result
+        geoname = parsed_xml.geonames.geoname.last
+        coords[:latitude] = geoname.lat.text
+        coords[:longitude] = geoname.lng.text
+        coords[:combined] = coords[:latitude] + ',' + coords[:longitude]
+          coords[:box] = {}
+          coords[:box][:west] = geoname.bbox.west.text
+          coords[:box][:north] = geoname.bbox.north.text
+          coords[:box][:east] = geoname.bbox.east.text
+          coords[:box][:south] = geoname.bbox.south.text
+
+        geonames_data[:coords] = coords
+        geonames_data[:hier_geo] = hier_geo.present? ? hier_geo : nil
+      end
+
+      return geonames_data
+    end
+
+
+    def self.geonames_id_from_geo_hash(geo_hash)
+      return nil if Bplgeo::Geonames.geonames_username == '<username>'
+      geo_hash = geo_hash.clone
+
+      max_retry = 3
+      sleep_time = 60 # In seconds
+      retry_count = 0
+
+      geonames_search_array = []
+      return_hash = {}
+
+      #Don't do both neighborhood and city!
+      if geo_hash[:neighborhood_part].present?
+        geonames_search_array << geo_hash[:neighborhood_part]
+      elsif geo_hash[:city_part].present?
+        geonames_search_array << geo_hash[:city_part]
+      end
+
+      geonames_search_array << geo_hash[:state_part] if geo_hash[:state_part].present?
+      geonames_search_array << geo_hash[:country_part] if geo_hash[:country_part].present?
+      geonames_search_string = geonames_search_array.join(', ')
+
+      match_term =  geonames_search_array.first.to_ascii.downcase.strip
+
+      begin
+        if retry_count > 0
+          sleep(sleep_time)
+        end
+        retry_count = retry_count + 1
+
+        geonames_response = Typhoeus::Request.get("http://api.geonames.org/search?username=#{self.geonames_username}&lang=en&style=FULL&q=" + CGI.escape(geonames_search_string))
+
+      end until (geonames_response.code != 500 || retry_count == max_retry)
+
+      unless geonames_response.code == 500
+
+        parsed_xml = Nokogiri::Slop(geonames_response.body)
+
+        #This is ugly and needs to be redone to achieve better recursive...
+        if parsed_xml.geonames.totalResultsCount.text == '0'
+          if neighborhood_part.present?
+            geo_hash[:neighborhood_part] = nil
+            geo_hash = geonames_id_from_geo_hash(geo_hash)
+          elsif city_part.present?
+            geo_hash[:city_part] = nil
+            geo_hash = geonames_id_from_geo_hash(geo_hash)
+          end
+
+          return geo_hash
+        end
+
+        #Exact Match
+        parsed_xml.geonames.geoname.each do |geoname|
+
+          current_term = geoname.toponymName.text.to_ascii.downcase.strip
+
+          if current_term == match_term  && return_hash.blank?
+            return_hash[:id] = geoname.geonameId.text
+            return_hash[:original_string_differs] = Bplgeo::Standardizer.parsed_and_original_check(geo_hash)
+            break
+          end
+        end
+
+        if return_hash.blank?
+          #Starts With
+          parsed_xml.geonames.geoname.each do |geoname|
+
+            current_term = geoname.toponymName.text.to_ascii.downcase.strip
+
+            if current_term.starts_with?(match_term) && return_hash.blank?
+              return_hash[:id] = geoname.geonameId.text
+              return_hash[:original_string_differs] = Bplgeo::Standardizer.parsed_and_original_check(geo_hash)
+            end
+          end
+        end
+
+      end
+
+      if geonames_response.code == 500
+        raise 'Geonames Server appears to not be responding for Geographic query: ' + term
+      end
+
+      return return_hash if return_hash.present?
+
+      return nil
+
+      end
+  end
+end

data/lib/bplgeo/parser.rb

@@ -0,0 +1,227 @@
+module Bplgeo
+  class Parser
+
+    def self.bplgeo_config
+      root = Rails.root || './test/dummy'
+      env = Rails.env || 'test'
+
+      @bplgeo_config ||= YAML::load(ERB.new(IO.read(File.join(root, 'config', 'bplgeo.yml'))).result)[env].with_indifferent_access
+    end
+
+    def self.mapquest_key
+      bplgeo_config[:mapquest_key] || '<mapquest_key>'
+    end
+
+    def self.bing_key
+      bplgeo_config[:bing_key] || '<bing_key>'
+    end
+
+    def self.timeout
+      bplgeo_config[:timeout]
+    end
+
+    #Note: Limited to only looking at United States places...
+    def self.parse_bing_api(term, parse_term_flag=false)
+      return_hash = {}
+      retry_count = 3
+
+      #Skip if no bing_key... possibly move this elsewhere?
+      return return_hash if self.bing_key == '<bing_key>'
+
+      return_hash[:original_term] = term
+
+      term = Bplgeo::Standardizer.parse_for_geographic_term(term) if parse_term_flag
+      term = Bplgeo::Standardizer.standardize_geographic_term(term)
+
+      if term.blank?
+        return {}
+      end
+
+      return_hash[:standardized_term] = term
+
+      #Bing API does badly with parentheses...
+      if term.match(/[\(\)]+/)
+        return {}
+      end
+
+      #Sometimes with building, city, state, bing is dumb and will only return state. Example: Boston Harbor, Boston, Mass.
+      #So if not a street address, pass to have google handle it for better results...
+      #Example of another bad record: South Street bridge, West Bridgewater, Mass. would give a place in Holyoke
+      if term.split(',').length >= 3 && term.match(/\d/).blank? && term.downcase.match(/ave\.*,/).blank? && term.downcase.match(/avenue\.*,/).blank? && term.downcase.match(/street\.*,/).blank? && term.downcase.match(/st\.*,/).blank? && term.downcase.match(/road\.*,/).blank? && term.downcase.match(/rd\.*,/).blank?
+        return {}
+      end
+
+      Geocoder.configure(:lookup => :bing,:api_key => self.bing_key,:timeout => self.timeout, :always_raise => :all)
+      bing_api_result = Geocoder.search(term)
+
+    rescue SocketError => e
+      retry unless (retry_count -= 1).zero?
+    else
+
+      #Use only for United States results... international results are inaccurate.
+      if bing_api_result.present? && bing_api_result.first.data["address"]["countryRegion"] == 'United States'
+
+        if bing_api_result.first.data["entityType"] == 'Neighborhood'
+          return {} #Doesn't return a city... Google handles this better.
+        end
+
+        if bing_api_result.first.data["address"]["addressLine"].present?
+          return_hash[:term_differs_from_tgn] = true
+          return_hash[:street_part] = bing_api_result.first.data["address"]["addressLine"]
+          return_hash[:coords] = {:latitude=>bing_api_result.first.data["geocodePoints"].first["coordinates"].first.to_s,
+                                       :longitude=>bing_api_result.first.data["geocodePoints"].first["coordinates"].last.to_s,
+                                       :combined=>bing_api_result.first.data["geocodePoints"].first["coordinates"].first.to_s + ',' + bing_api_result.first.data["geocodePoints"].first["coordinates"].last.to_s}
+        end
+
+        return_hash[:country_part] = bing_api_result.first.data["address"]["countryRegion"]
+
+        if return_hash[:country_part] == 'United States'
+          return_hash[:state_part] = Bplgeo::Constants::STATE_ABBR[bing_api_result.first.data["address"]["adminDistrict"]]
+        else
+          return_hash[:state_part] = bing_api_result.first.data["address"]["adminDistrict"]
+        end
+
+        return_hash[:city_part] = bing_api_result.first.data["address"]["locality"]
+      else
+        return {}
+      end
+
+      #Only return if USA for now. International results often awful.
+      return return_hash[:country_part] == 'United States' ? return_hash : {}
+    end
+
+    #Mapquest allows unlimited requests - start here?
+    def self.parse_mapquest_api(term, parse_term_flag=false)
+      return_hash = {}
+      retry_count = 3
+
+      #Skip if no bing_key... possibly move this elsewhere?
+      return return_hash if self.bing_key == '<mapquest_key>'
+
+      return_hash[:original_term] = term
+
+      term = Bplgeo::Standardizer.parse_for_geographic_term(term) if parse_term_flag
+      term = Bplgeo::Standardizer.standardize_geographic_term(term)
+
+      if term.blank?
+        return {}
+      end
+
+      return_hash[:standardized_term] = term
+
+      #Mapquest returns bad data for: Manchester, Mass.
+      if term.include?('Manchester') || term.include?('Atlanta, MI')
+        return {}
+      end
+
+      #Messed up with just neighborhoods. Example: Hyde Park (Boston, Mass.) or Hyde Park (Boston, Mass.)
+      #So if not a street address, pass to have google handle it for better results...
+      if term.split(',').length >= 3 && term.match(/\d/).blank? && term.downcase.match(/ave\.*,/).blank? && term.downcase.match(/avenue\.*,/).blank? && term.downcase.match(/street\.*,/).blank? && term.downcase.match(/st\.*,/).blank? && term.downcase.match(/road\.*,/).blank? && term.downcase.match(/rd\.*,/).blank?
+        return {}
+      end
+
+      Geocoder.configure(:lookup => :mapquest,:api_key => self.mapquest_key,:timeout => self.timeout, :always_raise => :all)
+
+      mapquest_api_result = Geocoder.search(term)
+    rescue SocketError => e
+      retry unless (retry_count -= 1).zero?
+    else
+
+
+      #If this call returned a result...
+      if mapquest_api_result.present?
+
+        if mapquest_api_result.first.data["street"].present?
+          #return_hash[:term_differs_from_tgn] = true
+          return_hash[:street_part] = mapquest_api_result.first.data["street"]
+          return_hash[:coords] = {:latitude=>mapquest_api_result.first.data['latLng']['lat'].to_s,
+                                       :longitude=>mapquest_api_result.first.data['latLng']['lng'].to_s,
+                                       :combined=>mapquest_api_result.first.data['latLng']['lat'].to_s + ',' + mapquest_api_result.first.data['latLng']['lng'].to_s}
+        end
+
+        return_hash[:country_part] = Country.new(mapquest_api_result.first.data["adminArea1"]).name
+
+        if return_hash[:country_part] == 'United States'
+          return_hash[:state_part] = Bplgeo::Constants::STATE_ABBR[mapquest_api_result.first.data["adminArea3"]] || mapquest_api_result.first.data["adminArea4"]
+        else
+          return_hash[:state_part] = mapquest_api_result.first.data["adminArea3"].gsub(' province', '')
+        end
+
+        return_hash[:city_part] = mapquest_api_result.first.data["adminArea5"]
+
+        return_hash[:city_part] = return_hash[:city_part].gsub(' City', '') #Return New York as New York City...
+      end
+
+      #Only return if USA for now. Google is better with stuff like: 'Long Binh, Vietnam'
+      #Also only return if there is a city if there were more than two terms passed in. Fixes: Roxbury, MA
+      return {} unless return_hash[:country_part] == 'United States'
+      return {} if term.split(',').length >= 2 && return_hash[:city_part].blank?
+
+      return return_hash
+    end
+
+    #Final fallback is google API. The best but we are limited to 2500 requests per day unless we pay the $10k a year premium account...
+    #Note: If google cannot find street, it will return just city/state, like for "Salem Street and Paradise Road, Swampscott, MA, 01907"
+    #Seems like it sets a partial_match=>true in the data section...
+    def self.parse_google_api(term, parse_term_flag=false)
+      return_hash = {}
+      retry_count = 3
+
+      return_hash[:original_term] = term
+
+      term = Bplgeo::Standardizer.parse_for_geographic_term(term) if parse_term_flag
+      term = Bplgeo::Standardizer.standardize_geographic_term(term)
+
+      #Soviet Union returns back a place in Kazakhstan
+      if term.blank? || term == 'Soviet Union'
+        return {}
+      end
+
+      return_hash[:standardized_term] = term
+
+      ::Geocoder.configure(:lookup => :google,:api_key => nil,:timeout => self.timeout, :always_raise => :all)
+
+      google_api_result = ::Geocoder.search(term)
+    rescue SocketError => e
+      retry unless (retry_count -= 1).zero?
+    else
+
+
+      #Check if only a partial match. To avoid errors, strip out the first part and try again...
+      #Need better way to check for street endings. See: http://pe.usps.gov/text/pub28/28apc_002.htm
+      if google_api_result.present?
+        if google_api_result.first.data['partial_match'] && term.split(',').length > 1 && !term.downcase.include?('street') && !term.downcase.include?('st.') && !term.downcase.include?('avenue') && !term.downcase.include?('ave.') && !term.downcase.include?('court') && !term.downcase.include?('dr.')
+          term = term.split(',')[1..term.split(',').length-1].join(',').strip
+          google_api_result = Geocoder.search(term)
+        end
+      end
+
+      if google_api_result.present?
+        #Types: street number, route, neighborhood, establishment, transit_station, bus_station
+        google_api_result.first.data["address_components"].each do |result|
+          if (result['types'] & ['street number', 'route', 'neighborhood', 'establishment', 'transit_station', 'bus_station']).present?
+            #return_hash[:term_differs_from_tgn] = true
+            #TODO: Not implemented for Google results right now.
+            return_hash[:street_part] = 'TODO: Not Implemented for Google Results'
+            return_hash[:coords] = {:latitude=>google_api_result.first.data['geometry']['location']['lat'].to_s,
+                                         :longitude=>google_api_result.first.data['geometry']['location']['lng'].to_s,
+                                         :combined=>google_api_result.first.data['geometry']['location']['lat'].to_s + ',' + google_api_result.first.data['geometry']['location']['lng'].to_s}
+          elsif (result['types'] & ['country']).present?
+            return_hash[:country_part] = result['long_name']
+          elsif (result['types'] & ['administrative_area_level_1']).present?
+            return_hash[:state_part] = result['long_name'].to_ascii
+          elsif (result['types'] & ['locality']).present?
+            return_hash[:city_part] = result['long_name']
+          elsif (result['types'] & ['sublocality', 'political']).length == 2
+            return_hash[:neighborhood_part] = result['long_name']
+          end
+        end
+
+        return_hash[:term_differs_from_tgn] ||= google_api_result.first.data['partial_match'] unless google_api_result.first.data['partial_match'].blank?
+      end
+
+
+      return return_hash
+    end
+  end
+end

data/lib/bplgeo/standardizer.rb

@@ -0,0 +1,213 @@
+module Bplgeo
+  class Standardizer
+
+    #Take a subject string and look for potential geographic terms.
+    def self.parse_for_geographic_term(term)
+      geo_term = ''
+
+      #Likely too long to be an address... some fields have junk with an address string...
+      if term.length > 125
+        return ''
+      end
+
+      state_abbr_list = ['Mass']
+      state_name_list = []
+
+      #Countries gem of https://github.com/hexorx/countries
+      Country.new('US').states.each do |state_abbr, state_names|
+        state_abbr_list << ' ' + state_abbr
+        state_name_list << state_names["name"]
+      end
+
+      #Parsing a subject geographic term.
+      if term.include?('--')
+        term.split('--').each_with_index do |split_term, index|
+          if state_name_list.any? { |state| split_term.include? state }
+            geo_term = term.split('--')[index..term.split('--').length-1].reverse!.join(',')
+          elsif state_abbr_list.any? { |abbr| split_term.include? abbr }
+            geo_term = split_term
+          end
+        end
+        #Other than a '--' field
+        #Experimental... example: Palmer (Mass) - history or Stores (retail trade) - Palmer, Mass
+      elsif term.include?(' - ')
+        term.split(' - ').each do |split_term|
+          if state_name_list.any? { |state| split_term.include? state } || state_abbr_list.any? { |abbr| split_term.include? abbr }
+            geo_term = split_term
+          end
+
+        end
+      else
+        if state_name_list.any? { |state| term.include? state } || state_abbr_list.any? { |abbr| term.include? abbr }
+          geo_term = term
+        end
+      end
+
+      return geo_term
+    end
+
+    #Make a string in a standard format.
+    def self.standardize_geographic_term(geo_term)
+
+      geo_term = geo_term.clone #Don't change original
+
+      #Remove common junk terms
+      Bplgeo::Constants::JUNK_TERMS.each { |term| geo_term.gsub!(term, '') }
+
+      #Strip any leading periods or commas from junk terms
+      geo_term = geo_term.gsub(/^[\.,]+/, '').strip
+
+      #Replace any semicolons with commas... possible strip them?
+      geo_term = geo_term.gsub(';', ',')
+
+      #Terms in paranthesis will cause some geographic parsers to freak out. Switch to commas instead.
+      if geo_term.match(/[\(\)]+/)
+        #Attempt to fix address if something like (word)
+        if geo_term.match(/ \(+.*\)+/)
+          #Make this replacement better?
+          geo_term = geo_term.gsub(/ *\((?=[\S ]+\))/,', ')
+          geo_term = geo_term.gsub(')', '')
+
+          #Else skip this as data returned likely will be unreliable for now... FIXME when use case occurs.
+        else
+          return nil
+        end
+      end
+
+      return geo_term
+    end
+
+    #Attempt to dedup a list of geographic areas.
+    #FIXME: Horrendous first pass.
+    #Aggresive flag removes less specific matches. IE. ['Hanoi, Vietnam' and 'Vietnam'] would return just ['Hanoi, Vietnam']
+    def self.dedup_geo(geo_list, aggresive=false)
+      geo_list = geo_list.clone
+
+       base_word_geo_list = []
+       geo_list.each do |geo_term|
+         geo_term = geo_term.gsub('(','').gsub(')','').gsub('.','').gsub(',','').gsub(';','')
+         #Remove common junk terms
+         Bplgeo::Constants::JUNK_TERMS.each { |term| geo_term.gsub!(term, '') }
+
+         geo_term = geo_term.squish
+
+         base_word_geo_list << geo_term
+       end
+
+      indexes_to_remove = []
+
+      0.upto base_word_geo_list.size-1 do |index|
+        matched_words_count = []
+        current_best_term = geo_list[index]
+        current_best_term_index = index
+
+        base_word_geo_list[index].split(' ').each { |word|
+
+          (index+1).upto base_word_geo_list.size-1 do |inner_index|
+            if base_word_geo_list[inner_index].split(' ').any? { |single_word| single_word == word }
+              matched_words_count[inner_index] ||= 0
+              matched_words_count[inner_index] = matched_words_count[inner_index] + 1
+
+            end
+          end
+        }
+
+        matched_words_count.each_with_index do |matched_count, matched_index|
+          if matched_count ==  base_word_geo_list[index].split(' ').size && ((base_word_geo_list[matched_index].split(' ').size < base_word_geo_list[index].split(' ').size && aggresive) || (base_word_geo_list[matched_index].split(' ').size == base_word_geo_list[index].split(' ').size))
+            if current_best_term.split(',').size < geo_list[matched_index].split(',').size || (current_best_term.size+1 < geo_list[matched_index].size && !geo_list[matched_index].include?('('))
+              current_best_term =  geo_list[matched_index]
+              indexes_to_remove << current_best_term_index
+              current_best_term_index = matched_index
+            else
+              indexes_to_remove << matched_index
+            end
+          end
+
+        end
+      end
+
+      indexes_to_remove.each do |removal_index|
+        geo_list[removal_index] = nil
+      end
+
+      return geo_list.compact
+    end
+
+    def self.parsed_and_original_check(geo_hash)
+      term = geo_hash[:standardized_term]
+
+      if geo_hash[:street_part].present? || geo_hash[:coords].present?
+        return true
+      end
+
+      #Keep original string if three parts at least or if there is a number in the term.
+      #TODO: Make this better!
+      if (term.split(',').length >= 3 && geo_hash[:neighborhood_part].blank?) || (term.split(',').length >= 2 && geo_hash[:city_part].blank?) || term.split(',').length >= 4 || term.match(/\d/).present?
+        return true
+      end
+
+      if geo_hash[:country_part] != 'United States'
+        if geo_hash[:city_part].blank? && geo_hash[:state_part].blank?
+          #Currently do noting
+        elsif !((geo_hash[:city_part].present? && term.to_ascii.downcase.include?(geo_hash[:city_part].to_ascii.downcase)) || (geo_hash[:state_part].present? && term.to_ascii.downcase.include?(geo_hash[:state_part].to_ascii.downcase)))
+         return true
+        end
+      end
+
+
+      return false
+    end
+
+
+
+    #Take LCSH subjects and make them standard.
+    def self.LCSHize(value)
+      #Remove ending periods ... except when an initial or etc.
+      if value.last == '.' && value[-2].match(/[^A-Z]/) && !value[-4..-1].match('etc.')
+        value = value.slice(0..-2)
+      end
+
+      #Fix when '- -' occurs
+      value = value.gsub(/-\s-/,'--')
+
+      #Fix for "em" dashes - two types?
+      value = value.gsub('—','--')
+
+      #Fix for "em" dashes - two types?
+      value = value.gsub('–','--')
+
+      #Fix for ' - ' combinations
+      value = value.gsub(' - ','--')
+
+      #Remove white space after and before  '--'
+      value = value.gsub(/\s+--/,'--')
+      value = value.gsub(/--\s+/,'--')
+
+      #Ensure first work is capitalized
+      value[0] = value.first.capitalize[0]
+
+      #Strip any white space
+      value = strip_value(value)
+
+      return value
+    end
+
+    def self.strip_value(value)
+      if(value.blank?)
+        return nil
+      else
+        if value.class == Float || value.class == Fixnum
+          value = value.to_i.to_s
+        end
+
+        # Make sure it is all UTF-8 and not character encodings or HTML tags and remove any cariage returns
+        return utf8Encode(value)
+      end
+    end
+
+    #TODO: Better name for this. Should be part of an overall helped gem.
+    def self.utf8Encode(value)
+      return HTMLEntities.new.decode(ActionView::Base.full_sanitizer.sanitize(value.to_s.gsub(/\r?\n?\t/, ' ').gsub(/\r?\n/, ' ').gsub(/<br[\s]*\/>/,' '))).strip
+    end
+  end
+end