broken-geocoder 1.3.4

Sign up to get free protection for your applications and to get access to all the features.
Files changed (119) hide show
  1. checksums.yaml +7 -0
  2. data/CHANGELOG.md +467 -0
  3. data/LICENSE +20 -0
  4. data/README.md +1193 -0
  5. data/bin/geocode +5 -0
  6. data/examples/autoexpire_cache_dalli.rb +62 -0
  7. data/examples/autoexpire_cache_redis.rb +28 -0
  8. data/examples/cache_bypass.rb +48 -0
  9. data/examples/reverse_geocode_job.rb +40 -0
  10. data/lib/generators/geocoder/config/config_generator.rb +14 -0
  11. data/lib/generators/geocoder/config/templates/initializer.rb +21 -0
  12. data/lib/generators/geocoder/maxmind/geolite_city_generator.rb +28 -0
  13. data/lib/generators/geocoder/maxmind/geolite_country_generator.rb +28 -0
  14. data/lib/generators/geocoder/maxmind/templates/migration/geolite_city.rb +30 -0
  15. data/lib/generators/geocoder/maxmind/templates/migration/geolite_country.rb +17 -0
  16. data/lib/geocoder.rb +48 -0
  17. data/lib/geocoder/cache.rb +90 -0
  18. data/lib/geocoder/calculations.rb +431 -0
  19. data/lib/geocoder/cli.rb +121 -0
  20. data/lib/geocoder/configuration.rb +129 -0
  21. data/lib/geocoder/configuration_hash.rb +11 -0
  22. data/lib/geocoder/esri_token.rb +38 -0
  23. data/lib/geocoder/exceptions.rb +37 -0
  24. data/lib/geocoder/ip_address.rb +13 -0
  25. data/lib/geocoder/kernel_logger.rb +25 -0
  26. data/lib/geocoder/logger.rb +47 -0
  27. data/lib/geocoder/lookup.rb +110 -0
  28. data/lib/geocoder/lookups/baidu.rb +59 -0
  29. data/lib/geocoder/lookups/baidu_ip.rb +59 -0
  30. data/lib/geocoder/lookups/base.rb +325 -0
  31. data/lib/geocoder/lookups/bing.rb +80 -0
  32. data/lib/geocoder/lookups/dstk.rb +20 -0
  33. data/lib/geocoder/lookups/esri.rb +64 -0
  34. data/lib/geocoder/lookups/freegeoip.rb +51 -0
  35. data/lib/geocoder/lookups/geocoder_ca.rb +53 -0
  36. data/lib/geocoder/lookups/geocoder_us.rb +43 -0
  37. data/lib/geocoder/lookups/geocodio.rb +42 -0
  38. data/lib/geocoder/lookups/geoip2.rb +45 -0
  39. data/lib/geocoder/lookups/geoportail_lu.rb +65 -0
  40. data/lib/geocoder/lookups/google.rb +91 -0
  41. data/lib/geocoder/lookups/google_places_details.rb +50 -0
  42. data/lib/geocoder/lookups/google_premier.rb +47 -0
  43. data/lib/geocoder/lookups/here.rb +62 -0
  44. data/lib/geocoder/lookups/ipapi_com.rb +86 -0
  45. data/lib/geocoder/lookups/ipinfo_io.rb +55 -0
  46. data/lib/geocoder/lookups/latlon.rb +59 -0
  47. data/lib/geocoder/lookups/mapbox.rb +53 -0
  48. data/lib/geocoder/lookups/mapquest.rb +59 -0
  49. data/lib/geocoder/lookups/mapzen.rb +15 -0
  50. data/lib/geocoder/lookups/maxmind.rb +90 -0
  51. data/lib/geocoder/lookups/maxmind_geoip2.rb +69 -0
  52. data/lib/geocoder/lookups/maxmind_local.rb +65 -0
  53. data/lib/geocoder/lookups/nominatim.rb +52 -0
  54. data/lib/geocoder/lookups/okf.rb +44 -0
  55. data/lib/geocoder/lookups/opencagedata.rb +58 -0
  56. data/lib/geocoder/lookups/ovi.rb +62 -0
  57. data/lib/geocoder/lookups/pelias.rb +64 -0
  58. data/lib/geocoder/lookups/pointpin.rb +68 -0
  59. data/lib/geocoder/lookups/postcode_anywhere_uk.rb +51 -0
  60. data/lib/geocoder/lookups/smarty_streets.rb +50 -0
  61. data/lib/geocoder/lookups/telize.rb +55 -0
  62. data/lib/geocoder/lookups/test.rb +44 -0
  63. data/lib/geocoder/lookups/yandex.rb +58 -0
  64. data/lib/geocoder/models/active_record.rb +50 -0
  65. data/lib/geocoder/models/base.rb +39 -0
  66. data/lib/geocoder/models/mongo_base.rb +62 -0
  67. data/lib/geocoder/models/mongo_mapper.rb +26 -0
  68. data/lib/geocoder/models/mongoid.rb +32 -0
  69. data/lib/geocoder/query.rb +111 -0
  70. data/lib/geocoder/railtie.rb +26 -0
  71. data/lib/geocoder/request.rb +83 -0
  72. data/lib/geocoder/results/baidu.rb +79 -0
  73. data/lib/geocoder/results/baidu_ip.rb +62 -0
  74. data/lib/geocoder/results/base.rb +67 -0
  75. data/lib/geocoder/results/bing.rb +52 -0
  76. data/lib/geocoder/results/dstk.rb +6 -0
  77. data/lib/geocoder/results/esri.rb +75 -0
  78. data/lib/geocoder/results/freegeoip.rb +45 -0
  79. data/lib/geocoder/results/geocoder_ca.rb +60 -0
  80. data/lib/geocoder/results/geocoder_us.rb +39 -0
  81. data/lib/geocoder/results/geocodio.rb +70 -0
  82. data/lib/geocoder/results/geoip2.rb +62 -0
  83. data/lib/geocoder/results/geoportail_lu.rb +69 -0
  84. data/lib/geocoder/results/google.rb +139 -0
  85. data/lib/geocoder/results/google_places_details.rb +35 -0
  86. data/lib/geocoder/results/google_premier.rb +6 -0
  87. data/lib/geocoder/results/here.rb +71 -0
  88. data/lib/geocoder/results/ipapi_com.rb +45 -0
  89. data/lib/geocoder/results/ipinfo_io.rb +48 -0
  90. data/lib/geocoder/results/latlon.rb +71 -0
  91. data/lib/geocoder/results/mapbox.rb +47 -0
  92. data/lib/geocoder/results/mapquest.rb +48 -0
  93. data/lib/geocoder/results/mapzen.rb +5 -0
  94. data/lib/geocoder/results/maxmind.rb +135 -0
  95. data/lib/geocoder/results/maxmind_geoip2.rb +9 -0
  96. data/lib/geocoder/results/maxmind_local.rb +49 -0
  97. data/lib/geocoder/results/nominatim.rb +99 -0
  98. data/lib/geocoder/results/okf.rb +106 -0
  99. data/lib/geocoder/results/opencagedata.rb +90 -0
  100. data/lib/geocoder/results/ovi.rb +71 -0
  101. data/lib/geocoder/results/pelias.rb +58 -0
  102. data/lib/geocoder/results/pointpin.rb +40 -0
  103. data/lib/geocoder/results/postcode_anywhere_uk.rb +42 -0
  104. data/lib/geocoder/results/smarty_streets.rb +106 -0
  105. data/lib/geocoder/results/telize.rb +45 -0
  106. data/lib/geocoder/results/test.rb +33 -0
  107. data/lib/geocoder/results/yandex.rb +92 -0
  108. data/lib/geocoder/sql.rb +107 -0
  109. data/lib/geocoder/stores/active_record.rb +305 -0
  110. data/lib/geocoder/stores/base.rb +116 -0
  111. data/lib/geocoder/stores/mongo_base.rb +58 -0
  112. data/lib/geocoder/stores/mongo_mapper.rb +13 -0
  113. data/lib/geocoder/stores/mongoid.rb +13 -0
  114. data/lib/geocoder/version.rb +3 -0
  115. data/lib/hash_recursive_merge.rb +74 -0
  116. data/lib/maxmind_database.rb +109 -0
  117. data/lib/tasks/geocoder.rake +38 -0
  118. data/lib/tasks/maxmind.rake +73 -0
  119. metadata +167 -0
@@ -0,0 +1,15 @@
1
+ require 'geocoder/lookups/pelias'
2
+ require 'geocoder/results/mapzen'
3
+
4
+ # https://mapzen.com/documentation/search/search/ for more information
5
+ module Geocoder::Lookup
6
+ class Mapzen < Pelias
7
+ def name
8
+ 'Mapzen'
9
+ end
10
+
11
+ def endpoint
12
+ configuration[:endpoint] || 'search.mapzen.com'
13
+ end
14
+ end
15
+ end
@@ -0,0 +1,90 @@
1
+ require 'geocoder/lookups/base'
2
+ require 'geocoder/results/maxmind'
3
+ require 'csv'
4
+
5
+ module Geocoder::Lookup
6
+ class Maxmind < Base
7
+
8
+ def name
9
+ "MaxMind"
10
+ end
11
+
12
+ def query_url(query)
13
+ "#{protocol}://geoip.maxmind.com/#{service_code}?" + url_query_string(query)
14
+ end
15
+
16
+ private # ---------------------------------------------------------------
17
+
18
+ ##
19
+ # Return the name of the configured service, or raise an exception.
20
+ #
21
+ def configured_service!
22
+ if s = configuration[:service] and services.keys.include?(s)
23
+ return s
24
+ else
25
+ raise(
26
+ Geocoder::ConfigurationError,
27
+ "When using MaxMind you MUST specify a service name: " +
28
+ "Geocoder.configure(:maxmind => {:service => ...}), " +
29
+ "where '...' is one of: #{services.keys.inspect}"
30
+ )
31
+ end
32
+ end
33
+
34
+ def service_code
35
+ services[configured_service!]
36
+ end
37
+
38
+ def service_response_fields_count
39
+ Geocoder::Result::Maxmind.field_names[configured_service!].size
40
+ end
41
+
42
+ def data_contains_error?(parsed_data)
43
+ # if all fields given then there is an error
44
+ parsed_data.size == service_response_fields_count and !parsed_data.last.nil?
45
+ end
46
+
47
+ ##
48
+ # Service names mapped to code used in URL.
49
+ #
50
+ def services
51
+ {
52
+ :country => "a",
53
+ :city => "b",
54
+ :city_isp_org => "f",
55
+ :omni => "e"
56
+ }
57
+ end
58
+
59
+ def results(query)
60
+ # don't look up a loopback address, just return the stored result
61
+ return [reserved_result] if query.loopback_ip_address?
62
+ doc = fetch_data(query)
63
+ if doc and doc.is_a?(Array)
64
+ if !data_contains_error?(doc)
65
+ return [doc]
66
+ elsif doc.last == "INVALID_LICENSE_KEY"
67
+ raise_error(Geocoder::InvalidApiKey) || Geocoder.log(:warn, "Invalid MaxMind API key.")
68
+ end
69
+ end
70
+ return []
71
+ end
72
+
73
+ def parse_raw_data(raw_data)
74
+ # Maxmind just returns text/plain as csv format but according to documentation,
75
+ # we get ISO-8859-1 encoded string. We need to convert it.
76
+ CSV.parse_line raw_data.force_encoding("ISO-8859-1").encode("UTF-8")
77
+ end
78
+
79
+ def reserved_result
80
+ ",,,,0,0,0,0,,,".split(",")
81
+ end
82
+
83
+ def query_url_params(query)
84
+ {
85
+ :l => configuration.api_key,
86
+ :i => query.sanitized_text
87
+ }.merge(super)
88
+ end
89
+ end
90
+ end
@@ -0,0 +1,69 @@
1
+ require 'geocoder/lookups/base'
2
+ require 'geocoder/results/maxmind_geoip2'
3
+
4
+ module Geocoder::Lookup
5
+ class MaxmindGeoip2 < Base
6
+
7
+ def name
8
+ "MaxMind GeoIP2"
9
+ end
10
+
11
+ # Maxmind's GeoIP2 Precision Services only supports HTTPS,
12
+ # otherwise a `404 Not Found` HTTP response will be returned
13
+ def supported_protocols
14
+ [:https]
15
+ end
16
+
17
+ def query_url(query)
18
+ "#{protocol}://geoip.maxmind.com/geoip/v2.1/#{configured_service!}/#{URI.escape(query.sanitized_text.strip)}"
19
+ end
20
+
21
+ private # ---------------------------------------------------------------
22
+
23
+ ##
24
+ # Return the name of the configured service, or raise an exception.
25
+ #
26
+ def configured_service!
27
+ if s = configuration[:service] and services.include?(s) and configuration[:basic_auth][:user] and configuration[:basic_auth][:password]
28
+ return s
29
+ else
30
+ raise(
31
+ Geocoder::ConfigurationError,
32
+ "When using MaxMind GeoIP2 you MUST specify a service name and basic_auth: " +
33
+ "Geocoder.configure(:maxmind_geoip2 => {:service => ...}, " +
34
+ ":basic_auth => {:user ..., :password => ...}), " +
35
+ "where service is one of: #{services.inspect}"
36
+ )
37
+ end
38
+ end
39
+
40
+ def data_contains_error?(doc)
41
+ (["code", "error"] - doc.keys).empty?
42
+ end
43
+
44
+ ##
45
+ # Service names used in URL.
46
+ #
47
+ def services
48
+ [
49
+ :country,
50
+ :city,
51
+ :insights,
52
+ ]
53
+ end
54
+
55
+ def results(query)
56
+ # don't look up a loopback address
57
+ return [] if query.loopback_ip_address?
58
+ doc = fetch_data(query)
59
+ if doc
60
+ if !data_contains_error?(doc)
61
+ return [doc]
62
+ else
63
+ Geocoder.log(:warn, "MaxMind GeoIP2 Geocoding API error: #{doc['code']} (#{doc['error']}).")
64
+ end
65
+ end
66
+ return []
67
+ end
68
+ end
69
+ end
@@ -0,0 +1,65 @@
1
+ require 'ipaddr'
2
+ require 'geocoder/lookups/base'
3
+ require 'geocoder/results/maxmind_local'
4
+
5
+ module Geocoder::Lookup
6
+ class MaxmindLocal < Base
7
+
8
+ def initialize
9
+ if !configuration[:file].nil?
10
+ begin
11
+ gem = RUBY_PLATFORM == 'java' ? 'jgeoip' : 'geoip'
12
+ require gem
13
+ rescue LoadError
14
+ raise "Could not load geoip dependency. To use MaxMind Local lookup you must add the #{gem} gem to your Gemfile or have it installed in your system."
15
+ end
16
+ end
17
+ super
18
+ end
19
+
20
+ def name
21
+ "MaxMind Local"
22
+ end
23
+
24
+ def required_api_key_parts
25
+ []
26
+ end
27
+
28
+ private
29
+
30
+ def results(query)
31
+ if configuration[:file]
32
+ geoip_class = RUBY_PLATFORM == "java" ? JGeoIP : GeoIP
33
+ result = geoip_class.new(configuration[:file]).city(query.to_s)
34
+ result.nil? ? [] : [encode_hash(result.to_hash)]
35
+ elsif configuration[:package] == :city
36
+ addr = IPAddr.new(query.text).to_i
37
+ q = "SELECT l.country, l.region, l.city, l.latitude, l.longitude
38
+ FROM maxmind_geolite_city_location l WHERE l.loc_id = (SELECT b.loc_id FROM maxmind_geolite_city_blocks b
39
+ WHERE b.start_ip_num <= #{addr} AND #{addr} <= b.end_ip_num)"
40
+ format_result(q, [:country_name, :region_name, :city_name, :latitude, :longitude])
41
+ elsif configuration[:package] == :country
42
+ addr = IPAddr.new(query.text).to_i
43
+ q = "SELECT country, country_code FROM maxmind_geolite_country
44
+ WHERE start_ip_num <= #{addr} AND #{addr} <= end_ip_num"
45
+ format_result(q, [:country_name, :country_code2])
46
+ end
47
+ end
48
+
49
+ def encode_hash(hash, encoding = "UTF-8")
50
+ hash.inject({}) do |h,i|
51
+ h[i[0]] = i[1].is_a?(String) ? i[1].encode(encoding) : i[1]
52
+ h
53
+ end
54
+ end
55
+
56
+ def format_result(query, attr_names)
57
+ if r = ActiveRecord::Base.connection.execute(query).first
58
+ r = r.values if r.is_a?(Hash) # some db adapters return Hash, some Array
59
+ [Hash[*attr_names.zip(r).flatten]]
60
+ else
61
+ []
62
+ end
63
+ end
64
+ end
65
+ end
@@ -0,0 +1,52 @@
1
+ require 'geocoder/lookups/base'
2
+ require "geocoder/results/nominatim"
3
+
4
+ module Geocoder::Lookup
5
+ class Nominatim < Base
6
+
7
+ def name
8
+ "Nominatim"
9
+ end
10
+
11
+ def map_link_url(coordinates)
12
+ "http://www.openstreetmap.org/?lat=#{coordinates[0]}&lon=#{coordinates[1]}&zoom=15&layers=M"
13
+ end
14
+
15
+ def query_url(query)
16
+ method = query.reverse_geocode? ? "reverse" : "search"
17
+ host = configuration[:host] || "nominatim.openstreetmap.org"
18
+ "#{protocol}://#{host}/#{method}?" + url_query_string(query)
19
+ end
20
+
21
+ private # ---------------------------------------------------------------
22
+
23
+ def results(query)
24
+ return [] unless doc = fetch_data(query)
25
+ doc.is_a?(Array) ? doc : [doc]
26
+ end
27
+
28
+ def parse_raw_data(raw_data)
29
+ if raw_data.include?("Bandwidth limit exceeded")
30
+ raise_error(Geocoder::OverQueryLimitError) || Geocoder.log(:warn, "Over API query limit.")
31
+ else
32
+ super(raw_data)
33
+ end
34
+ end
35
+
36
+ def query_url_params(query)
37
+ params = {
38
+ :format => "json",
39
+ :addressdetails => "1",
40
+ :"accept-language" => (query.language || configuration.language)
41
+ }.merge(super)
42
+ if query.reverse_geocode?
43
+ lat,lon = query.coordinates
44
+ params[:lat] = lat
45
+ params[:lon] = lon
46
+ else
47
+ params[:q] = query.sanitized_text
48
+ end
49
+ params
50
+ end
51
+ end
52
+ end
@@ -0,0 +1,44 @@
1
+ require 'geocoder/lookups/base'
2
+ require "geocoder/results/okf"
3
+
4
+ module Geocoder::Lookup
5
+ class Okf < Base
6
+
7
+ def name
8
+ "Okf"
9
+ end
10
+
11
+ def query_url(query)
12
+ "#{protocol}://data.okf.fi/gis/1/geocode/json?" + url_query_string(query)
13
+ end
14
+
15
+ private # ---------------------------------------------------------------
16
+
17
+ def valid_response?(response)
18
+ json = parse_json(response.body)
19
+ status = json["status"] if json
20
+ super(response) and ['OK', 'ZERO_RESULTS'].include?(status)
21
+ end
22
+
23
+ def results(query)
24
+ return [] unless doc = fetch_data(query)
25
+ case doc['status']; when "OK" # OK status implies >0 results
26
+ return doc['results']
27
+ end
28
+ return []
29
+ end
30
+
31
+ def query_url_okf_params(query)
32
+ params = {
33
+ (query.reverse_geocode? ? :latlng : :address) => query.sanitized_text,
34
+ :sensor => "false",
35
+ :language => (query.language || configuration.language)
36
+ }
37
+ params
38
+ end
39
+
40
+ def query_url_params(query)
41
+ query_url_okf_params(query).merge(super)
42
+ end
43
+ end
44
+ end
@@ -0,0 +1,58 @@
1
+ require 'geocoder/lookups/base'
2
+ require 'geocoder/results/opencagedata'
3
+
4
+ module Geocoder::Lookup
5
+ class Opencagedata < Base
6
+
7
+ def name
8
+ "OpenCageData"
9
+ end
10
+
11
+ def query_url(query)
12
+ "#{protocol}://api.opencagedata.com/geocode/v1/json?key=#{configuration.api_key}&#{url_query_string(query)}"
13
+ end
14
+
15
+ def required_api_key_parts
16
+ ["key"]
17
+ end
18
+
19
+ private
20
+
21
+ def results(query)
22
+ return [] unless doc = fetch_data(query)
23
+ # return doc["results"]
24
+
25
+ messages = doc['status']['message']
26
+ case doc['status']['code']
27
+ when 400 # Error with input
28
+ raise_error(Geocoder::InvalidRequest, messages) ||
29
+ Geocoder.log(:warn, "Opencagedata Geocoding API error: #{messages}")
30
+ when 403 # Key related error
31
+ raise_error(Geocoder::InvalidApiKey, messages) ||
32
+ Geocoder.log(:warn, "Opencagedata Geocoding API error: #{messages}")
33
+ when 402 # Quata Exceeded
34
+ raise_error(Geocoder::OverQueryLimitError, messages) ||
35
+ Geocoder.log(:warn, "Opencagedata Geocoding API error: #{messages}")
36
+ when 500 # Unknown error
37
+ raise_error(Geocoder::Error, messages) ||
38
+ Geocoder.log(:warn, "Opencagedata Geocoding API error: #{messages}")
39
+ end
40
+
41
+ return doc["results"]
42
+ end
43
+
44
+ def query_url_params(query)
45
+ params = {
46
+ :q => query.sanitized_text,
47
+ :language => (query.language || configuration.language)
48
+ }.merge(super)
49
+
50
+ unless (bounds = query.options[:bounds]).nil?
51
+ params[:bounds] = bounds.map{ |point| "%f,%f" % point }.join(',')
52
+ end
53
+
54
+ params
55
+ end
56
+
57
+ end
58
+ end
@@ -0,0 +1,62 @@
1
+ require 'geocoder/lookups/base'
2
+ require 'geocoder/results/ovi'
3
+
4
+ module Geocoder::Lookup
5
+ class Ovi < Base
6
+
7
+ def name
8
+ "Ovi"
9
+ end
10
+
11
+ def required_api_key_parts
12
+ []
13
+ end
14
+
15
+ def query_url(query)
16
+ "#{protocol}://lbs.ovi.com/search/6.2/#{if query.reverse_geocode? then 'reverse' end}geocode.json?" + url_query_string(query)
17
+ end
18
+
19
+ private # ---------------------------------------------------------------
20
+
21
+ def results(query)
22
+ return [] unless doc = fetch_data(query)
23
+ return [] unless doc['Response'] && doc['Response']['View']
24
+ if r=doc['Response']['View']
25
+ return [] if r.nil? || !r.is_a?(Array) || r.empty?
26
+ return r.first['Result']
27
+ end
28
+ []
29
+ end
30
+
31
+ def query_url_params(query)
32
+ options = {
33
+ :gen=>1,
34
+ :app_id=>api_key,
35
+ :app_code=>api_code
36
+ }
37
+
38
+ if query.reverse_geocode?
39
+ super.merge(options).merge(
40
+ :prox=>query.sanitized_text,
41
+ :mode=>:retrieveAddresses
42
+ )
43
+ else
44
+ super.merge(options).merge(
45
+ :searchtext=>query.sanitized_text
46
+ )
47
+ end
48
+ end
49
+
50
+ def api_key
51
+ if a=configuration.api_key
52
+ return a.first if a.is_a?(Array)
53
+ end
54
+ end
55
+
56
+ def api_code
57
+ if a=configuration.api_key
58
+ return a.last if a.is_a?(Array)
59
+ end
60
+ end
61
+ end
62
+ end