geomash 0.2.1 → 0.3.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 9de5bd2cdc879dc6103df0df9d7b31f195c02e73
-  data.tar.gz: 6bc1355c2a93d2886c1d77a00e705894b8720418
+  metadata.gz: b181c5b1f79695f4b8ba54cf5e641c9cdaefe842
+  data.tar.gz: f6db8e134bf152b162765a86b688848f16d0753d
 SHA512:
-  metadata.gz: 6622f04afaae5560a36930441afe000b3b45cb53221b6f7fddf449b5b892bdcc59e35cf1088a50eb11b811edff662ecf43234ef1b348f48f1f6cfac18d4d5486
-  data.tar.gz: 22d728689b333961bcc8e37628188797af5c1773732a6d6a5fff2120284ddf1b18fb5eac9e8fd3bfadbf648d97aac5cb827021b796cf2efb8f600cd406ff7d83
+  metadata.gz: 0a5e74f73c036f36d0d030c2edc57fd2b026add354ee845e6f5bf6ab847f81d62dda8afe08add796b63ae0ee2b87446471a1ca083c60114d7401bc88e8476989
+  data.tar.gz: c0af8abaa745856c0e416f461e230574c869f5736f3c4aa8cb416dac102fe301eac669c1a308ff91d748d3e2be78511e3c0fb96fd8d0a7851e944573e0cd2506

data/config/geomash.yml.sample

@@ -6,6 +6,7 @@ development:
   #NOTE: Bing also fairly unreliable but mostly works...
   bing_key: <bing_key>
   timeout: 7
+  parser_cache_enabled: false #See Readme before enabling this
 test: &TEST_
   tgn_enabled: true
   geonames_username: <username>
@@ -14,6 +15,7 @@ test: &TEST_
   #NOTE: Bing also fairly unreliable but mostly works...
   bing_key: <bing_key>
   timeout: 7
+  parser_cache_enabled: false #See Readme before enabling this
 production:
   tgn_enabled: true
   geonames_username: <username>
@@ -22,3 +24,4 @@ production:
   #NOTE: Bing also fairly unreliable but mostly works...
   bing_key: <bing_key>
   timeout: 7
+  parser_cache_enabled: false #See Readme before enabling this

data/lib/geomash/autoexpire_cache_dalli.rb

@@ -0,0 +1,55 @@
+#Taken from: https://github.com/alexreisner/geocoder/blob/master/examples/autoexpire_cache_dalli.rb
+module Geomash
+  class AutoexpireCacheDalli
+    def initialize(store, ttl = 86400)
+      @store = store
+      @keys = 'GeocoderDalliClientKeys'
+      @ttl = ttl
+    end
+
+    def [](url)
+      res = @store.get(url)
+      res = YAML::load(res) if res.present?
+      res
+    end
+
+    def []=(url, value)
+      if value.nil?
+        del(url)
+      else
+        key_cache_add(url) if @store.add(url, YAML::dump(value), @ttl)
+      end
+      value
+    end
+
+    def keys
+      key_cache
+    end
+
+    def del(url)
+      key_cache_delete(url) if @store.delete(url)
+    end
+
+    private
+
+    def key_cache
+      the_keys = @store.get(@keys)
+      if the_keys.nil?
+        @store.add(@keys, YAML::dump([]))
+        []
+      else
+        YAML::load(the_keys)
+      end
+    end
+
+    def key_cache_add(key)
+      @store.replace(@keys, YAML::dump(key_cache << key))
+    end
+
+    def key_cache_delete(key)
+      tmp = key_cache
+      tmp.delete(key)
+      @store.replace(@keys, YAML::dump(tmp))
+    end
+  end
+end

data/lib/geomash/autoexpire_cache_redis.rb

@@ -0,0 +1,26 @@
+#Taken from: https://github.com/alexreisner/geocoder/blob/master/examples/autoexpire_cache_redis.rb
+module Geomash
+class AutoexpireCacheRedis
+  def initialize(store, ttl = 86400)
+    @store = store
+    @ttl = ttl
+  end
+
+  def [](url)
+    @store.[](url)
+  end
+
+  def []=(url, value)
+    @store.[]=(url, value)
+    @store.expire(url, @ttl)
+  end
+
+  def keys
+    @store.keys
+  end
+
+  def del(url)
+    @store.del(url)
+  end
+end
+end

data/lib/geomash/constants.rb

@@ -72,6 +72,7 @@ module Geomash
     }
 
     #Terms that drive geographic parsers mad...
+    #Possibly check using QA against LCSH Subject topics that aren't geographical...
     JUNK_TERMS = [
         'Cranberries',
         'History',
@@ -80,7 +81,11 @@ module Geomash
         'Pictorial works.',
         /[nN]ation/,
         'Asia',
-        '(Republic)'
+        '(Republic)',
+        'Directories',
+        'Biography',
+        #Some date removal
+        /[\d]+th [cC]entury,/
     ]
 
 

data/lib/geomash/geonames.rb

@@ -89,22 +89,27 @@ module Geomash
         end
         retry_count = retry_count + 1
 
-        geonames_response = Typhoeus::Request.get("http://api.geonames.org/search?username=#{self.geonames_username}&lang=en&style=FULL&q=#{CGI.escape(geonames_search_string)}&name_equals=#{CGI.escape(exact_name_term)}&country=#{Country.find_country_by_name(geo_hash[:country_part]).alpha2}")
+        country_code = nil
+        if geo_hash[:country_part] == 'South Korea'
+          country_code = 'KR'
+        elsif geo_hash[:country_part] == 'North Korea'
+          country_code = 'KP'
+        else
+          country_code = Country.find_country_by_name(geo_hash[:country_part]).alpha2
+        end
+        geonames_response = Typhoeus::Request.get("http://api.geonames.org/search?username=#{self.geonames_username}&lang=en&style=FULL&q=#{CGI.escape(geonames_search_string)}&name_equals=#{CGI.escape(exact_name_term)}&country=#{country_code}")
 
       end until (geonames_response.code != 500 || retry_count == max_retry)
 
       unless geonames_response.code == 500
 
-        parsed_xml = Nokogiri::Slop(geonames_response.body)
+        parsed_xml = Nokogiri::XML(geonames_response.body)
 
-        begin
-          raise "geonames status error message of: #{parsed_xml.to_s}" if parsed_xml.geonames.status
-        rescue
-          #Do nothing but FIXME to not use slop
-        end
+        raise "geonames status error message of: #{parsed_xml.to_s}" if parsed_xml.xpath("//status").present?
 
         #This is ugly and needs to be redone to achieve better recursive...
-        if parsed_xml.geonames.totalResultsCount.text == '0'
+        current_count = parsed_xml.xpath("//totalResultsCount")
+        if current_count.blank? || current_count.first.text == '0'
           if geo_hash[:neighborhood_part].present?
             geo_hash_temp = geo_hash.clone
             geo_hash_temp[:neighborhood_part] = nil
@@ -121,14 +126,14 @@ module Geomash
         end
 
         #Exact Match ... FIXME to not use Slop
-        if parsed_xml.geonames.geoname.class == Nokogiri::XML::Element
-          return_hash[:id] = parsed_xml.geonames.geoname.geonameId.text
-          return_hash[:rdf] = "http://sws.geonames.org/#{return_hash[:id]}/about.rdf"
-        elsif parsed_xml.geonames.geoname.class ==Nokogiri::XML::NodeSet
-          return_hash[:id] = parsed_xml.geonames.geoname.first.geonameId.text
+        if parsed_xml.xpath("//geonames/geoname/geonameId").present?
+          return_hash[:id] = parsed_xml.xpath("//geonames/geoname/geonameId").first.text
           return_hash[:rdf] = "http://sws.geonames.org/#{return_hash[:id]}/about.rdf"
+          return_hash[:original_string_differs] = Geomash::Standardizer.parsed_and_original_check(geo_hash)
+          return return_hash
+        else
+          return nil
         end
-        return_hash[:original_string_differs] = Geomash::Standardizer.parsed_and_original_check(geo_hash)
 
       end
 

data/lib/geomash/parser.rb

@@ -1,6 +1,11 @@
 module Geomash
   class Parser
 
+    def self.cache_enabled
+      return Geomash.config[:parser_cache_enabled] unless Geomash.config[:parser_cache_enabled].nil?
+      return false
+    end
+
     def self.mapquest_key
       Geomash.config[:mapquest_key] || '<mapquest_key>'
     end
@@ -44,12 +49,21 @@ module Geomash
         return {}
       end
 
-      Geocoder.configure(:lookup => :bing,:api_key => self.bing_key,:timeout => self.timeout, :always_raise => :all)
-      bing_api_result = Geocoder.search(term)
+      ::Geocoder.configure(:lookup => :bing,:api_key => self.bing_key,:timeout => self.timeout, :always_raise => :all)
+
+      begin
+        bing_api_result = Geocoder.search(term)
+      rescue SocketError => e
+        Geocoder::Lookup.get(:bing).cache.expire(Geocoder::Query.new(term).url) if self.cache_enabled #Expire this url
+        retry unless (retry_count -= 1).zero?
+      rescue Geocoder::OverQueryLimitError => e
+        Geocoder::Lookup.get(:bing).cache.expire(Geocoder::Query.new(term).url) if self.cache_enabled #Expire this url
+        raise e
+      rescue Exception => e
+        Geocoder::Lookup.get(:bing).cache.expire(Geocoder::Query.new(term).url) if self.cache_enabled #Expire this url
+        raise e
+      end
 
-    rescue SocketError => e
-      retry unless (retry_count -= 1).zero?
-    else
 
       #Use only for United States results... international results are inaccurate.
       if bing_api_result.present? && bing_api_result.first.data["address"]["countryRegion"] == 'United States'
@@ -113,13 +127,20 @@ module Geomash
         return {}
       end
 
-      Geocoder.configure(:lookup => :mapquest,:api_key => self.mapquest_key,:timeout => self.timeout, :always_raise => :all)
-
-      mapquest_api_result = Geocoder.search(term)
-    rescue SocketError => e
-      retry unless (retry_count -= 1).zero?
-    else
-
+      ::Geocoder.configure(:lookup => :mapquest,:api_key => self.mapquest_key,:timeout => self.timeout, :always_raise => :all)
+
+      begin
+        mapquest_api_result = Geocoder.search(term)
+      rescue SocketError => e
+        Geocoder::Lookup.get(:mapquest).cache.expire(Geocoder::Query.new(term).url) if self.cache_enabled #Expire this url
+        retry unless (retry_count -= 1).zero?
+      rescue Geocoder::OverQueryLimitError => e
+        Geocoder::Lookup.get(:mapquest).cache.expire(Geocoder::Query.new(term).url) if self.cache_enabled #Expire this url
+        raise e
+      rescue Exception => e
+        Geocoder::Lookup.get(:mapquest).cache.expire(Geocoder::Query.new(term).url) if self.cache_enabled #Expire this url
+        raise e
+      end
 
       #If this call returned a result...
       if mapquest_api_result.present?
@@ -173,11 +194,18 @@ module Geomash
       return_hash[:standardized_term] = term
 
       ::Geocoder.configure(:lookup => :google,:api_key => nil,:timeout => self.timeout, :always_raise => :all)
-
-      google_api_result = ::Geocoder.search(term)
-    rescue SocketError => e
-      retry unless (retry_count -= 1).zero?
-    else
+      begin
+        google_api_result = ::Geocoder.search(term)
+      rescue SocketError => e
+        Geocoder::Lookup.get(:google).cache.expire(Geocoder::Query.new(term).url) if self.cache_enabled #Expire this url
+        retry unless (retry_count -= 1).zero?
+      rescue Geocoder::OverQueryLimitError => e
+        Geocoder::Lookup.get(:google).cache.expire(Geocoder::Query.new(term).url) if self.cache_enabled #Expire this url
+        raise e
+      rescue Exception => e
+        Geocoder::Lookup.get(:google).cache.expire(Geocoder::Query.new(term).url) if self.cache_enabled #Expire this url
+        raise e
+      end
 
 
       #Check if only a partial match. To avoid errors, strip out the first part and try again...
@@ -202,7 +230,7 @@ module Geomash
           elsif (result['types'] & ['country']).present?
             return_hash[:country_part] = result['long_name']
           elsif (result['types'] & ['administrative_area_level_1']).present?
-            return_hash[:state_part] = result['long_name'].to_ascii
+            return_hash[:state_part] = result['long_name'].to_ascii.gsub('-city', '')
           elsif (result['types'] & ['locality']).present?
             return_hash[:city_part] = result['long_name']
           elsif (result['types'] & ['sublocality', 'political']).length == 2 || result['types'].include?('neighborhood')
@@ -213,6 +241,8 @@ module Geomash
         return_hash[:term_differs_from_tgn] ||= google_api_result.first.data['partial_match'] unless google_api_result.first.data['partial_match'].blank?
       end
 
+      #FIXME: Google free API rate limit is 5 requests / 1 second now (used to be 10). Need a better way to handle this.
+      sleep(0.1)
 
       return return_hash
     end

data/lib/geomash/standardizer.rb

@@ -11,6 +11,10 @@ module Geomash
         return ''
       end
 
+      term_split_list = term.split(/[,\-\(\(]/).reject{ |e| e.empty? }
+      term_split_list.each{ |e| e.gsub!(/[^\w\s]/, "") } #Remove punctuation
+      term_split_list.each{ |e| e.strip! } #Remove any extra remaining whitespace
+      term_split_list.reject{ |e| e.empty? }
       state_abbr_list = ['Mass']
       state_name_list = []
       country_name_list = []
@@ -24,29 +28,51 @@ module Geomash
       Country.all.each do |country_name_abbr_pair|
         country_name_list << country_name_abbr_pair.first
       end
+      country_name_list.append('South Korea') #Listed as Korea, Republic of in the gem
+      country_name_list.append('North Korea') #Listed as Korea, Democratic People's Republic Of of in the gem
 
       #Parsing a subject geographic term.
-      if term.include?('--')
-        term.split('--').each_with_index do |split_term, index|
-          if state_name_list.any? { |state| split_term.include? state } || country_name_list.any? { |country| split_term.include? country }
-            geo_term = term.split('--')[index..term.split('--').length-1].reverse!.join(',')
-          elsif state_abbr_list.any? { |abbr| split_term.include? abbr }
-            geo_term = split_term
+      if (state_name_list & term_split_list).present? || (state_abbr_list & term_split_list).present? || (country_name_list & term_split_list).present?
+        if term.include?('--')
+          term.split('--').each_with_index do |split_term, index|
+            if state_name_list.any? { |state| split_term.include? state } || country_name_list.any? { |country| split_term.include? country }
+              #Cases like Naroden Etnografski Muzeĭ (Sofia, Bulgaria)--Catalogs
+              if split_term.match(/\([^\)]+\)/)
+                geo_term = split_term.gsub('(', ',').gsub(' ,', ', ')
+                geo_term = geo_term.gsub(')', '')
+
+=begin
+            if split_term.match(/\([^\)]+,[^\)]+\)/)
+              geo_term = split_term.match(/\([^\)]+\)/).to_s
+              geo_term = geo_term[1..geo_term.length-2]
+            #Abbeville (France)--History--20th century.
+            elsif split_term.match(/\([^\)]+\)/)
+              geo_term = split_term
+=end
+              else
+                geo_term = term.split('--')[index..term.split('--').length-1].reverse!.join(',')
+              end
+
+            elsif state_abbr_list.any? { |abbr| split_term.include? abbr }
+              geo_term = split_term
+            end
           end
-        end
-        #Other than a '--' field
-        #Experimental... example: Palmer (Mass) - history or Stores (retail trade) - Palmer, Mass
-      elsif term.include?(' - ')
-        term.split(' - ').each do |split_term|
-          if state_name_list.any? { |state| split_term.include? state } || state_abbr_list.any? { |abbr| split_term.include? abbr } || country_name_list.any? { |country| split_term.include? country }
-            geo_term = split_term
+          #Other than a '--' field
+          #Experimental... example: Palmer (Mass) - history or Stores (retail trade) - Palmer, Mass
+        elsif term.include?(' - ')
+          term.split(' - ').each do |split_term|
+            if state_name_list.any? { |state| split_term.include? state } || state_abbr_list.any? { |abbr| split_term.include? abbr } || country_name_list.any? { |country| split_term.include? country }
+              geo_term = split_term
+            end
+
           end
+        else
+          #if term_split_list.length > 1
+          geo_term = term.gsub('(', ',').gsub(' ,', ', ')
+          geo_term = geo_term.gsub(')', '')
+          #end
 
         end
-      else
-        if state_name_list.any? { |state| term.include? state } || state_abbr_list.any? { |abbr| term.include? abbr } || country_name_list.any? { |country| term.include? country }
-          geo_term = term
-        end
       end
 
       return geo_term
@@ -83,6 +109,8 @@ module Geomash
         end
       end
 
+      geo_term = geo_term.squeeze(',')
+
       return geo_term
     end
 
@@ -224,6 +252,7 @@ module Geomash
 
     def self.try_with_entered_names(geo_hash)
       geo_hash_local = geo_hash.clone
+      geo_hash_local[:tgn] = nil
       if geo_hash_local[:neighborhood_part].present?
          orig_string_check = geo_hash_local[:standardized_term].gsub(',', ' ').squish.split(' ').select { |value| value.downcase.to_ascii == geo_hash_local[:neighborhood_part].downcase.to_ascii}
          geo_hash_local[:neighborhood_part] = orig_string_check.first.strip if orig_string_check.present? && orig_string_check != geo_hash_local[:neighborhood_part]