broken-geocoder 1.3.4

Sign up to get free protection for your applications and to get access to all the features.
Files changed (119) hide show
  1. checksums.yaml +7 -0
  2. data/CHANGELOG.md +467 -0
  3. data/LICENSE +20 -0
  4. data/README.md +1193 -0
  5. data/bin/geocode +5 -0
  6. data/examples/autoexpire_cache_dalli.rb +62 -0
  7. data/examples/autoexpire_cache_redis.rb +28 -0
  8. data/examples/cache_bypass.rb +48 -0
  9. data/examples/reverse_geocode_job.rb +40 -0
  10. data/lib/generators/geocoder/config/config_generator.rb +14 -0
  11. data/lib/generators/geocoder/config/templates/initializer.rb +21 -0
  12. data/lib/generators/geocoder/maxmind/geolite_city_generator.rb +28 -0
  13. data/lib/generators/geocoder/maxmind/geolite_country_generator.rb +28 -0
  14. data/lib/generators/geocoder/maxmind/templates/migration/geolite_city.rb +30 -0
  15. data/lib/generators/geocoder/maxmind/templates/migration/geolite_country.rb +17 -0
  16. data/lib/geocoder.rb +48 -0
  17. data/lib/geocoder/cache.rb +90 -0
  18. data/lib/geocoder/calculations.rb +431 -0
  19. data/lib/geocoder/cli.rb +121 -0
  20. data/lib/geocoder/configuration.rb +129 -0
  21. data/lib/geocoder/configuration_hash.rb +11 -0
  22. data/lib/geocoder/esri_token.rb +38 -0
  23. data/lib/geocoder/exceptions.rb +37 -0
  24. data/lib/geocoder/ip_address.rb +13 -0
  25. data/lib/geocoder/kernel_logger.rb +25 -0
  26. data/lib/geocoder/logger.rb +47 -0
  27. data/lib/geocoder/lookup.rb +110 -0
  28. data/lib/geocoder/lookups/baidu.rb +59 -0
  29. data/lib/geocoder/lookups/baidu_ip.rb +59 -0
  30. data/lib/geocoder/lookups/base.rb +325 -0
  31. data/lib/geocoder/lookups/bing.rb +80 -0
  32. data/lib/geocoder/lookups/dstk.rb +20 -0
  33. data/lib/geocoder/lookups/esri.rb +64 -0
  34. data/lib/geocoder/lookups/freegeoip.rb +51 -0
  35. data/lib/geocoder/lookups/geocoder_ca.rb +53 -0
  36. data/lib/geocoder/lookups/geocoder_us.rb +43 -0
  37. data/lib/geocoder/lookups/geocodio.rb +42 -0
  38. data/lib/geocoder/lookups/geoip2.rb +45 -0
  39. data/lib/geocoder/lookups/geoportail_lu.rb +65 -0
  40. data/lib/geocoder/lookups/google.rb +91 -0
  41. data/lib/geocoder/lookups/google_places_details.rb +50 -0
  42. data/lib/geocoder/lookups/google_premier.rb +47 -0
  43. data/lib/geocoder/lookups/here.rb +62 -0
  44. data/lib/geocoder/lookups/ipapi_com.rb +86 -0
  45. data/lib/geocoder/lookups/ipinfo_io.rb +55 -0
  46. data/lib/geocoder/lookups/latlon.rb +59 -0
  47. data/lib/geocoder/lookups/mapbox.rb +53 -0
  48. data/lib/geocoder/lookups/mapquest.rb +59 -0
  49. data/lib/geocoder/lookups/mapzen.rb +15 -0
  50. data/lib/geocoder/lookups/maxmind.rb +90 -0
  51. data/lib/geocoder/lookups/maxmind_geoip2.rb +69 -0
  52. data/lib/geocoder/lookups/maxmind_local.rb +65 -0
  53. data/lib/geocoder/lookups/nominatim.rb +52 -0
  54. data/lib/geocoder/lookups/okf.rb +44 -0
  55. data/lib/geocoder/lookups/opencagedata.rb +58 -0
  56. data/lib/geocoder/lookups/ovi.rb +62 -0
  57. data/lib/geocoder/lookups/pelias.rb +64 -0
  58. data/lib/geocoder/lookups/pointpin.rb +68 -0
  59. data/lib/geocoder/lookups/postcode_anywhere_uk.rb +51 -0
  60. data/lib/geocoder/lookups/smarty_streets.rb +50 -0
  61. data/lib/geocoder/lookups/telize.rb +55 -0
  62. data/lib/geocoder/lookups/test.rb +44 -0
  63. data/lib/geocoder/lookups/yandex.rb +58 -0
  64. data/lib/geocoder/models/active_record.rb +50 -0
  65. data/lib/geocoder/models/base.rb +39 -0
  66. data/lib/geocoder/models/mongo_base.rb +62 -0
  67. data/lib/geocoder/models/mongo_mapper.rb +26 -0
  68. data/lib/geocoder/models/mongoid.rb +32 -0
  69. data/lib/geocoder/query.rb +111 -0
  70. data/lib/geocoder/railtie.rb +26 -0
  71. data/lib/geocoder/request.rb +83 -0
  72. data/lib/geocoder/results/baidu.rb +79 -0
  73. data/lib/geocoder/results/baidu_ip.rb +62 -0
  74. data/lib/geocoder/results/base.rb +67 -0
  75. data/lib/geocoder/results/bing.rb +52 -0
  76. data/lib/geocoder/results/dstk.rb +6 -0
  77. data/lib/geocoder/results/esri.rb +75 -0
  78. data/lib/geocoder/results/freegeoip.rb +45 -0
  79. data/lib/geocoder/results/geocoder_ca.rb +60 -0
  80. data/lib/geocoder/results/geocoder_us.rb +39 -0
  81. data/lib/geocoder/results/geocodio.rb +70 -0
  82. data/lib/geocoder/results/geoip2.rb +62 -0
  83. data/lib/geocoder/results/geoportail_lu.rb +69 -0
  84. data/lib/geocoder/results/google.rb +139 -0
  85. data/lib/geocoder/results/google_places_details.rb +35 -0
  86. data/lib/geocoder/results/google_premier.rb +6 -0
  87. data/lib/geocoder/results/here.rb +71 -0
  88. data/lib/geocoder/results/ipapi_com.rb +45 -0
  89. data/lib/geocoder/results/ipinfo_io.rb +48 -0
  90. data/lib/geocoder/results/latlon.rb +71 -0
  91. data/lib/geocoder/results/mapbox.rb +47 -0
  92. data/lib/geocoder/results/mapquest.rb +48 -0
  93. data/lib/geocoder/results/mapzen.rb +5 -0
  94. data/lib/geocoder/results/maxmind.rb +135 -0
  95. data/lib/geocoder/results/maxmind_geoip2.rb +9 -0
  96. data/lib/geocoder/results/maxmind_local.rb +49 -0
  97. data/lib/geocoder/results/nominatim.rb +99 -0
  98. data/lib/geocoder/results/okf.rb +106 -0
  99. data/lib/geocoder/results/opencagedata.rb +90 -0
  100. data/lib/geocoder/results/ovi.rb +71 -0
  101. data/lib/geocoder/results/pelias.rb +58 -0
  102. data/lib/geocoder/results/pointpin.rb +40 -0
  103. data/lib/geocoder/results/postcode_anywhere_uk.rb +42 -0
  104. data/lib/geocoder/results/smarty_streets.rb +106 -0
  105. data/lib/geocoder/results/telize.rb +45 -0
  106. data/lib/geocoder/results/test.rb +33 -0
  107. data/lib/geocoder/results/yandex.rb +92 -0
  108. data/lib/geocoder/sql.rb +107 -0
  109. data/lib/geocoder/stores/active_record.rb +305 -0
  110. data/lib/geocoder/stores/base.rb +116 -0
  111. data/lib/geocoder/stores/mongo_base.rb +58 -0
  112. data/lib/geocoder/stores/mongo_mapper.rb +13 -0
  113. data/lib/geocoder/stores/mongoid.rb +13 -0
  114. data/lib/geocoder/version.rb +3 -0
  115. data/lib/hash_recursive_merge.rb +74 -0
  116. data/lib/maxmind_database.rb +109 -0
  117. data/lib/tasks/geocoder.rake +38 -0
  118. data/lib/tasks/maxmind.rake +73 -0
  119. metadata +167 -0
@@ -0,0 +1,59 @@
1
+ require 'geocoder/lookups/base'
2
+ require 'geocoder/results/baidu_ip'
3
+
4
+ module Geocoder::Lookup
5
+ class BaiduIp < Base
6
+
7
+ def name
8
+ "Baidu IP"
9
+ end
10
+
11
+ def required_api_key_parts
12
+ ["key"]
13
+ end
14
+
15
+ def query_url(query)
16
+ "#{protocol}://api.map.baidu.com/location/ip?" + url_query_string(query)
17
+ end
18
+
19
+ # HTTP only
20
+ def supported_protocols
21
+ [:http]
22
+ end
23
+
24
+ private # ---------------------------------------------------------------
25
+
26
+ def results(query, reverse = false)
27
+ return [] unless doc = fetch_data(query)
28
+ case doc['status']
29
+ when 0
30
+ return [doc['content']] unless doc['content'].blank?
31
+ when 1, 3, 4
32
+ raise_error(Geocoder::Error, "server error.") ||
33
+ Geocoder.log(:warn, "Baidu IP Geocoding API error: server error.")
34
+ when 2
35
+ raise_error(Geocoder::InvalidRequest, "invalid request.") ||
36
+ Geocoder.log(:warn, "Baidu IP Geocoding API error: invalid request.")
37
+ when 5
38
+ raise_error(Geocoder::InvalidApiKey, "invalid api key.") ||
39
+ Geocoder.log(:warn, "Baidu IP Geocoding API error: invalid api key.")
40
+ when 101, 102, 200..299
41
+ raise_error(Geocoder::RequestDenied, "request denied.") ||
42
+ Geocoder.log(:warn, "Baidu IP Geocoding API error: request denied.")
43
+ when 300..399
44
+ raise_error(Geocoder::OverQueryLimitError, "over query limit") ||
45
+ Geocoder.log(:warn, "Baidu IP Geocoding API error: over query limit.")
46
+ end
47
+ return []
48
+ end
49
+
50
+ def query_url_params(query)
51
+ {
52
+ :ip => query.sanitized_text,
53
+ :ak => configuration.api_key,
54
+ :coor => "bd09ll"
55
+ }.merge(super)
56
+ end
57
+
58
+ end
59
+ end
@@ -0,0 +1,325 @@
1
+ require 'net/http'
2
+ require 'net/https'
3
+ require 'uri'
4
+
5
+ unless defined?(ActiveSupport::JSON)
6
+ begin
7
+ require 'rubygems' # for Ruby 1.8
8
+ require 'json'
9
+ rescue LoadError
10
+ raise LoadError, "Please install the 'json' or 'json_pure' gem to parse geocoder results."
11
+ end
12
+ end
13
+
14
+ module Geocoder
15
+ module Lookup
16
+
17
+ class Base
18
+ def initialize
19
+ @cache = nil
20
+ end
21
+
22
+ ##
23
+ # Human-readable name of the geocoding API.
24
+ #
25
+ def name
26
+ fail
27
+ end
28
+
29
+ ##
30
+ # Symbol which is used in configuration to refer to this Lookup.
31
+ #
32
+ def handle
33
+ str = self.class.to_s
34
+ str[str.rindex(':')+1..-1].gsub(/([a-z\d]+)([A-Z])/,'\1_\2').downcase.to_sym
35
+ end
36
+
37
+ ##
38
+ # Query the geocoding API and return a Geocoder::Result object.
39
+ # Returns +nil+ on timeout or error.
40
+ #
41
+ # Takes a search string (eg: "Mississippi Coast Coliseumf, Biloxi, MS",
42
+ # "205.128.54.202") for geocoding, or coordinates (latitude, longitude)
43
+ # for reverse geocoding. Returns an array of <tt>Geocoder::Result</tt>s.
44
+ #
45
+ def search(query, options = {})
46
+ query = Geocoder::Query.new(query, options) unless query.is_a?(Geocoder::Query)
47
+ results(query).map{ |r|
48
+ result = result_class.new(r)
49
+ result.cache_hit = @cache_hit if cache
50
+ result
51
+ }
52
+ end
53
+
54
+ ##
55
+ # Return the URL for a map of the given coordinates.
56
+ #
57
+ # Not necessarily implemented by all subclasses as only some lookups
58
+ # also provide maps.
59
+ #
60
+ def map_link_url(coordinates)
61
+ nil
62
+ end
63
+
64
+ ##
65
+ # Array containing string descriptions of keys required by the API.
66
+ # Empty array if keys are optional or not required.
67
+ #
68
+ def required_api_key_parts
69
+ []
70
+ end
71
+
72
+ ##
73
+ # URL to use for querying the geocoding engine.
74
+ #
75
+ def query_url(query)
76
+ fail
77
+ end
78
+
79
+ ##
80
+ # The working Cache object.
81
+ #
82
+ def cache
83
+ if @cache.nil? and store = configuration.cache
84
+ @cache = Cache.new(store, configuration.cache_prefix)
85
+ end
86
+ @cache
87
+ end
88
+
89
+ ##
90
+ # Array containing the protocols supported by the api.
91
+ # Should be set to [:http] if only HTTP is supported
92
+ # or [:https] if only HTTPS is supported.
93
+ #
94
+ def supported_protocols
95
+ [:http, :https]
96
+ end
97
+
98
+ private # -------------------------------------------------------------
99
+
100
+ ##
101
+ # An object with configuration data for this particular lookup.
102
+ #
103
+ def configuration
104
+ Geocoder.config_for_lookup(handle)
105
+ end
106
+
107
+ ##
108
+ # Object used to make HTTP requests.
109
+ #
110
+ def http_client
111
+ proxy_name = "#{protocol}_proxy"
112
+ if proxy = configuration.send(proxy_name)
113
+ proxy_url = !!(proxy =~ /^#{protocol}/) ? proxy : protocol + '://' + proxy
114
+ begin
115
+ uri = URI.parse(proxy_url)
116
+ rescue URI::InvalidURIError
117
+ raise ConfigurationError,
118
+ "Error parsing #{protocol.upcase} proxy URL: '#{proxy_url}'"
119
+ end
120
+ Net::HTTP::Proxy(uri.host, uri.port, uri.user, uri.password)
121
+ else
122
+ Net::HTTP
123
+ end
124
+ end
125
+
126
+ ##
127
+ # Geocoder::Result object or nil on timeout or other error.
128
+ #
129
+ def results(query)
130
+ fail
131
+ end
132
+
133
+ def query_url_params(query)
134
+ query.options[:params] || {}
135
+ end
136
+
137
+ def url_query_string(query)
138
+ hash_to_query(
139
+ query_url_params(query).reject{ |key,value| value.nil? }
140
+ )
141
+ end
142
+
143
+ ##
144
+ # Key to use for caching a geocoding result. Usually this will be the
145
+ # request URL, but in cases where OAuth is used and the nonce,
146
+ # timestamp, etc varies from one request to another, we need to use
147
+ # something else (like the URL before OAuth encoding).
148
+ #
149
+ def cache_key(query)
150
+ query_url(query)
151
+ end
152
+
153
+ ##
154
+ # Class of the result objects
155
+ #
156
+ def result_class
157
+ Geocoder::Result.const_get(self.class.to_s.split(":").last)
158
+ end
159
+
160
+ ##
161
+ # Raise exception if configuration specifies it should be raised.
162
+ # Return false if exception not raised.
163
+ #
164
+ def raise_error(error, message = nil)
165
+ exceptions = configuration.always_raise
166
+ if exceptions == :all or exceptions.include?( error.is_a?(Class) ? error : error.class )
167
+ raise error, message
168
+ else
169
+ false
170
+ end
171
+ end
172
+
173
+ ##
174
+ # Returns a parsed search result (Ruby hash).
175
+ #
176
+ def fetch_data(query)
177
+ parse_raw_data fetch_raw_data(query)
178
+ rescue SocketError => err
179
+ raise_error(err) or Geocoder.log(:warn, "Geocoding API connection cannot be established.")
180
+ rescue Errno::ECONNREFUSED => err
181
+ raise_error(err) or Geocoder.log(:warn, "Geocoding API connection refused.")
182
+ rescue Timeout::Error => err
183
+ raise_error(err) or Geocoder.log(:warn, "Geocoding API not responding fast enough " +
184
+ "(use Geocoder.configure(:timeout => ...) to set limit).")
185
+ end
186
+
187
+ def parse_json(data)
188
+ if defined?(ActiveSupport::JSON)
189
+ ActiveSupport::JSON.decode(data)
190
+ else
191
+ JSON.parse(data)
192
+ end
193
+ rescue
194
+ raise_error(ResponseParseError.new(data)) or Geocoder.log(:warn, "Geocoding API's response was not valid JSON: #{data}")
195
+ end
196
+
197
+ ##
198
+ # Parses a raw search result (returns hash or array).
199
+ #
200
+ def parse_raw_data(raw_data)
201
+ parse_json(raw_data)
202
+ end
203
+
204
+ ##
205
+ # Protocol to use for communication with geocoding services.
206
+ # Set in configuration but not available for every service.
207
+ #
208
+ def protocol
209
+ "http" + (use_ssl? ? "s" : "")
210
+ end
211
+
212
+ def valid_response?(response)
213
+ (200..399).include?(response.code.to_i)
214
+ end
215
+
216
+ ##
217
+ # Fetch a raw geocoding result (JSON string).
218
+ # The result might or might not be cached.
219
+ #
220
+ def fetch_raw_data(query)
221
+ key = cache_key(query)
222
+ if cache and body = cache[key]
223
+ @cache_hit = true
224
+ else
225
+ check_api_key_configuration!(query)
226
+ response = make_api_request(query)
227
+ check_response_for_errors!(response)
228
+ body = response.body
229
+
230
+ # apply the charset from the Content-Type header, if possible
231
+ ct = response['content-type']
232
+
233
+ if ct && ct['charset']
234
+ charset = ct.split(';').select do |s|
235
+ s['charset']
236
+ end.first.to_s.split('=')
237
+ if charset.length == 2
238
+ body.force_encoding(charset.last) rescue ArgumentError
239
+ end
240
+ end
241
+
242
+ if cache and valid_response?(response)
243
+ cache[key] = body
244
+ end
245
+ @cache_hit = false
246
+ end
247
+ body
248
+ end
249
+
250
+ def check_response_for_errors!(response)
251
+ if response.code.to_i == 400
252
+ raise_error(Geocoder::InvalidRequest) ||
253
+ Geocoder.log(:warn, "Geocoding API error: 400 Bad Request")
254
+ elsif response.code.to_i == 401
255
+ raise_error(Geocoder::RequestDenied) ||
256
+ Geocoder.log(:warn, "Geocoding API error: 401 Unauthorized")
257
+ elsif response.code.to_i == 402
258
+ raise_error(Geocoder::OverQueryLimitError) ||
259
+ Geocoder.log(:warn, "Geocoding API error: 402 Payment Required")
260
+ elsif response.code.to_i == 429
261
+ raise_error(Geocoder::OverQueryLimitError) ||
262
+ Geocoder.log(:warn, "Geocoding API error: 429 Too Many Requests")
263
+ elsif response.code.to_i == 503
264
+ raise_error(Geocoder::ServiceUnavailable) ||
265
+ Geocoder.log(:warn, "Geocoding API error: 503 Service Unavailable")
266
+ end
267
+ end
268
+
269
+ ##
270
+ # Make an HTTP(S) request to a geocoding API and
271
+ # return the response object.
272
+ #
273
+ def make_api_request(query)
274
+ uri = URI.parse(query_url(query))
275
+ Geocoder.log(:debug, "Geocoder: HTTP request being made for #{uri.to_s}")
276
+ http_client.start(uri.host, uri.port, use_ssl: use_ssl?, open_timeout: configuration.timeout, read_timeout: configuration.timeout) do |client|
277
+ configure_ssl!(client) if use_ssl?
278
+ req = Net::HTTP::Get.new(uri.request_uri, configuration.http_headers)
279
+ if configuration.basic_auth[:user] and configuration.basic_auth[:password]
280
+ req.basic_auth(
281
+ configuration.basic_auth[:user],
282
+ configuration.basic_auth[:password]
283
+ )
284
+ end
285
+ client.request(req)
286
+ end
287
+ rescue Timeout::Error
288
+ raise Geocoder::LookupTimeout
289
+ end
290
+
291
+ def use_ssl?
292
+ if supported_protocols == [:https]
293
+ true
294
+ elsif supported_protocols == [:http]
295
+ false
296
+ else
297
+ configuration.use_https
298
+ end
299
+ end
300
+
301
+ def configure_ssl!(client); end
302
+
303
+ def check_api_key_configuration!(query)
304
+ key_parts = query.lookup.required_api_key_parts
305
+ if key_parts.size > Array(configuration.api_key).size
306
+ parts_string = key_parts.size == 1 ? key_parts.first : key_parts
307
+ raise Geocoder::ConfigurationError,
308
+ "The #{query.lookup.name} API requires a key to be configured: " +
309
+ parts_string.inspect
310
+ end
311
+ end
312
+
313
+ ##
314
+ # Simulate ActiveSupport's Object#to_query.
315
+ # Removes any keys with nil value.
316
+ #
317
+ def hash_to_query(hash)
318
+ require 'cgi' unless defined?(CGI) && defined?(CGI.escape)
319
+ hash.collect{ |p|
320
+ p[1].nil? ? nil : p.map{ |i| CGI.escape i.to_s } * '='
321
+ }.compact.sort * '&'
322
+ end
323
+ end
324
+ end
325
+ end
@@ -0,0 +1,80 @@
1
+ require 'geocoder/lookups/base'
2
+ require "geocoder/results/bing"
3
+
4
+ module Geocoder::Lookup
5
+ class Bing < Base
6
+
7
+ def name
8
+ "Bing"
9
+ end
10
+
11
+ def map_link_url(coordinates)
12
+ "http://www.bing.com/maps/default.aspx?cp=#{coordinates.join('~')}"
13
+ end
14
+
15
+ def required_api_key_parts
16
+ ["key"]
17
+ end
18
+
19
+ def query_url(query)
20
+ base_url(query) + url_query_string(query)
21
+ end
22
+
23
+ private # ---------------------------------------------------------------
24
+
25
+ def base_url(query)
26
+ url = "#{protocol}://dev.virtualearth.net/REST/v1/Locations"
27
+
28
+ if !query.reverse_geocode?
29
+ if r = query.options[:region]
30
+ url << "/#{r}"
31
+ end
32
+ # use the more forgiving 'unstructured' query format to allow special
33
+ # chars, newlines, brackets, typos.
34
+ url + "?q=" + URI.escape(query.sanitized_text.strip) + "&"
35
+ else
36
+ url + "/#{URI.escape(query.sanitized_text.strip)}?"
37
+ end
38
+ end
39
+
40
+ def results(query)
41
+ return [] unless doc = fetch_data(query)
42
+
43
+ if doc['statusCode'] == 200
44
+ return doc['resourceSets'].first['estimatedTotal'] > 0 ? doc['resourceSets'].first['resources'] : []
45
+ elsif doc['statusCode'] == 401 and doc["authenticationResultCode"] == "InvalidCredentials"
46
+ raise_error(Geocoder::InvalidApiKey) || Geocoder.log(:warn, "Invalid Bing API key.")
47
+ else
48
+ Geocoder.log(:warn, "Bing Geocoding API error: #{doc['statusCode']} (#{doc['statusDescription']}).")
49
+ end
50
+ return []
51
+ end
52
+
53
+ def query_url_params(query)
54
+ {
55
+ key: configuration.api_key
56
+ }.merge(super)
57
+ end
58
+
59
+ def check_response_for_errors!(response)
60
+ super
61
+ if server_overloaded?(response)
62
+ raise_error(Geocoder::ServiceUnavailable) ||
63
+ Geocoder.log(:warn, "Bing Geocoding API error: Service Unavailable")
64
+ end
65
+ end
66
+
67
+ def valid_response?(response)
68
+ super(response) and not server_overloaded?(response)
69
+ end
70
+
71
+ def server_overloaded?(response)
72
+ # Occasionally, the servers processing service requests can be overloaded,
73
+ # and you may receive some responses that contain no results for queries that
74
+ # you would normally receive a result. To identify this situation,
75
+ # check the HTTP headers of the response. If the HTTP header X-MS-BM-WS-INFO is set to 1,
76
+ # it is best to wait a few seconds and try again.
77
+ response['x-ms-bm-ws-info'].to_i == 1
78
+ end
79
+ end
80
+ end