broken-geocoder 1.3.4

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (119) hide show
  1. checksums.yaml +7 -0
  2. data/CHANGELOG.md +467 -0
  3. data/LICENSE +20 -0
  4. data/README.md +1193 -0
  5. data/bin/geocode +5 -0
  6. data/examples/autoexpire_cache_dalli.rb +62 -0
  7. data/examples/autoexpire_cache_redis.rb +28 -0
  8. data/examples/cache_bypass.rb +48 -0
  9. data/examples/reverse_geocode_job.rb +40 -0
  10. data/lib/generators/geocoder/config/config_generator.rb +14 -0
  11. data/lib/generators/geocoder/config/templates/initializer.rb +21 -0
  12. data/lib/generators/geocoder/maxmind/geolite_city_generator.rb +28 -0
  13. data/lib/generators/geocoder/maxmind/geolite_country_generator.rb +28 -0
  14. data/lib/generators/geocoder/maxmind/templates/migration/geolite_city.rb +30 -0
  15. data/lib/generators/geocoder/maxmind/templates/migration/geolite_country.rb +17 -0
  16. data/lib/geocoder.rb +48 -0
  17. data/lib/geocoder/cache.rb +90 -0
  18. data/lib/geocoder/calculations.rb +431 -0
  19. data/lib/geocoder/cli.rb +121 -0
  20. data/lib/geocoder/configuration.rb +129 -0
  21. data/lib/geocoder/configuration_hash.rb +11 -0
  22. data/lib/geocoder/esri_token.rb +38 -0
  23. data/lib/geocoder/exceptions.rb +37 -0
  24. data/lib/geocoder/ip_address.rb +13 -0
  25. data/lib/geocoder/kernel_logger.rb +25 -0
  26. data/lib/geocoder/logger.rb +47 -0
  27. data/lib/geocoder/lookup.rb +110 -0
  28. data/lib/geocoder/lookups/baidu.rb +59 -0
  29. data/lib/geocoder/lookups/baidu_ip.rb +59 -0
  30. data/lib/geocoder/lookups/base.rb +325 -0
  31. data/lib/geocoder/lookups/bing.rb +80 -0
  32. data/lib/geocoder/lookups/dstk.rb +20 -0
  33. data/lib/geocoder/lookups/esri.rb +64 -0
  34. data/lib/geocoder/lookups/freegeoip.rb +51 -0
  35. data/lib/geocoder/lookups/geocoder_ca.rb +53 -0
  36. data/lib/geocoder/lookups/geocoder_us.rb +43 -0
  37. data/lib/geocoder/lookups/geocodio.rb +42 -0
  38. data/lib/geocoder/lookups/geoip2.rb +45 -0
  39. data/lib/geocoder/lookups/geoportail_lu.rb +65 -0
  40. data/lib/geocoder/lookups/google.rb +91 -0
  41. data/lib/geocoder/lookups/google_places_details.rb +50 -0
  42. data/lib/geocoder/lookups/google_premier.rb +47 -0
  43. data/lib/geocoder/lookups/here.rb +62 -0
  44. data/lib/geocoder/lookups/ipapi_com.rb +86 -0
  45. data/lib/geocoder/lookups/ipinfo_io.rb +55 -0
  46. data/lib/geocoder/lookups/latlon.rb +59 -0
  47. data/lib/geocoder/lookups/mapbox.rb +53 -0
  48. data/lib/geocoder/lookups/mapquest.rb +59 -0
  49. data/lib/geocoder/lookups/mapzen.rb +15 -0
  50. data/lib/geocoder/lookups/maxmind.rb +90 -0
  51. data/lib/geocoder/lookups/maxmind_geoip2.rb +69 -0
  52. data/lib/geocoder/lookups/maxmind_local.rb +65 -0
  53. data/lib/geocoder/lookups/nominatim.rb +52 -0
  54. data/lib/geocoder/lookups/okf.rb +44 -0
  55. data/lib/geocoder/lookups/opencagedata.rb +58 -0
  56. data/lib/geocoder/lookups/ovi.rb +62 -0
  57. data/lib/geocoder/lookups/pelias.rb +64 -0
  58. data/lib/geocoder/lookups/pointpin.rb +68 -0
  59. data/lib/geocoder/lookups/postcode_anywhere_uk.rb +51 -0
  60. data/lib/geocoder/lookups/smarty_streets.rb +50 -0
  61. data/lib/geocoder/lookups/telize.rb +55 -0
  62. data/lib/geocoder/lookups/test.rb +44 -0
  63. data/lib/geocoder/lookups/yandex.rb +58 -0
  64. data/lib/geocoder/models/active_record.rb +50 -0
  65. data/lib/geocoder/models/base.rb +39 -0
  66. data/lib/geocoder/models/mongo_base.rb +62 -0
  67. data/lib/geocoder/models/mongo_mapper.rb +26 -0
  68. data/lib/geocoder/models/mongoid.rb +32 -0
  69. data/lib/geocoder/query.rb +111 -0
  70. data/lib/geocoder/railtie.rb +26 -0
  71. data/lib/geocoder/request.rb +83 -0
  72. data/lib/geocoder/results/baidu.rb +79 -0
  73. data/lib/geocoder/results/baidu_ip.rb +62 -0
  74. data/lib/geocoder/results/base.rb +67 -0
  75. data/lib/geocoder/results/bing.rb +52 -0
  76. data/lib/geocoder/results/dstk.rb +6 -0
  77. data/lib/geocoder/results/esri.rb +75 -0
  78. data/lib/geocoder/results/freegeoip.rb +45 -0
  79. data/lib/geocoder/results/geocoder_ca.rb +60 -0
  80. data/lib/geocoder/results/geocoder_us.rb +39 -0
  81. data/lib/geocoder/results/geocodio.rb +70 -0
  82. data/lib/geocoder/results/geoip2.rb +62 -0
  83. data/lib/geocoder/results/geoportail_lu.rb +69 -0
  84. data/lib/geocoder/results/google.rb +139 -0
  85. data/lib/geocoder/results/google_places_details.rb +35 -0
  86. data/lib/geocoder/results/google_premier.rb +6 -0
  87. data/lib/geocoder/results/here.rb +71 -0
  88. data/lib/geocoder/results/ipapi_com.rb +45 -0
  89. data/lib/geocoder/results/ipinfo_io.rb +48 -0
  90. data/lib/geocoder/results/latlon.rb +71 -0
  91. data/lib/geocoder/results/mapbox.rb +47 -0
  92. data/lib/geocoder/results/mapquest.rb +48 -0
  93. data/lib/geocoder/results/mapzen.rb +5 -0
  94. data/lib/geocoder/results/maxmind.rb +135 -0
  95. data/lib/geocoder/results/maxmind_geoip2.rb +9 -0
  96. data/lib/geocoder/results/maxmind_local.rb +49 -0
  97. data/lib/geocoder/results/nominatim.rb +99 -0
  98. data/lib/geocoder/results/okf.rb +106 -0
  99. data/lib/geocoder/results/opencagedata.rb +90 -0
  100. data/lib/geocoder/results/ovi.rb +71 -0
  101. data/lib/geocoder/results/pelias.rb +58 -0
  102. data/lib/geocoder/results/pointpin.rb +40 -0
  103. data/lib/geocoder/results/postcode_anywhere_uk.rb +42 -0
  104. data/lib/geocoder/results/smarty_streets.rb +106 -0
  105. data/lib/geocoder/results/telize.rb +45 -0
  106. data/lib/geocoder/results/test.rb +33 -0
  107. data/lib/geocoder/results/yandex.rb +92 -0
  108. data/lib/geocoder/sql.rb +107 -0
  109. data/lib/geocoder/stores/active_record.rb +305 -0
  110. data/lib/geocoder/stores/base.rb +116 -0
  111. data/lib/geocoder/stores/mongo_base.rb +58 -0
  112. data/lib/geocoder/stores/mongo_mapper.rb +13 -0
  113. data/lib/geocoder/stores/mongoid.rb +13 -0
  114. data/lib/geocoder/version.rb +3 -0
  115. data/lib/hash_recursive_merge.rb +74 -0
  116. data/lib/maxmind_database.rb +109 -0
  117. data/lib/tasks/geocoder.rake +38 -0
  118. data/lib/tasks/maxmind.rake +73 -0
  119. metadata +167 -0
@@ -0,0 +1,59 @@
1
+ require 'geocoder/lookups/base'
2
+ require 'geocoder/results/baidu_ip'
3
+
4
+ module Geocoder::Lookup
5
+ class BaiduIp < Base
6
+
7
+ def name
8
+ "Baidu IP"
9
+ end
10
+
11
+ def required_api_key_parts
12
+ ["key"]
13
+ end
14
+
15
+ def query_url(query)
16
+ "#{protocol}://api.map.baidu.com/location/ip?" + url_query_string(query)
17
+ end
18
+
19
+ # HTTP only
20
+ def supported_protocols
21
+ [:http]
22
+ end
23
+
24
+ private # ---------------------------------------------------------------
25
+
26
+ def results(query, reverse = false)
27
+ return [] unless doc = fetch_data(query)
28
+ case doc['status']
29
+ when 0
30
+ return [doc['content']] unless doc['content'].blank?
31
+ when 1, 3, 4
32
+ raise_error(Geocoder::Error, "server error.") ||
33
+ Geocoder.log(:warn, "Baidu IP Geocoding API error: server error.")
34
+ when 2
35
+ raise_error(Geocoder::InvalidRequest, "invalid request.") ||
36
+ Geocoder.log(:warn, "Baidu IP Geocoding API error: invalid request.")
37
+ when 5
38
+ raise_error(Geocoder::InvalidApiKey, "invalid api key.") ||
39
+ Geocoder.log(:warn, "Baidu IP Geocoding API error: invalid api key.")
40
+ when 101, 102, 200..299
41
+ raise_error(Geocoder::RequestDenied, "request denied.") ||
42
+ Geocoder.log(:warn, "Baidu IP Geocoding API error: request denied.")
43
+ when 300..399
44
+ raise_error(Geocoder::OverQueryLimitError, "over query limit") ||
45
+ Geocoder.log(:warn, "Baidu IP Geocoding API error: over query limit.")
46
+ end
47
+ return []
48
+ end
49
+
50
+ def query_url_params(query)
51
+ {
52
+ :ip => query.sanitized_text,
53
+ :ak => configuration.api_key,
54
+ :coor => "bd09ll"
55
+ }.merge(super)
56
+ end
57
+
58
+ end
59
+ end
@@ -0,0 +1,325 @@
1
+ require 'net/http'
2
+ require 'net/https'
3
+ require 'uri'
4
+
5
+ unless defined?(ActiveSupport::JSON)
6
+ begin
7
+ require 'rubygems' # for Ruby 1.8
8
+ require 'json'
9
+ rescue LoadError
10
+ raise LoadError, "Please install the 'json' or 'json_pure' gem to parse geocoder results."
11
+ end
12
+ end
13
+
14
+ module Geocoder
15
+ module Lookup
16
+
17
+ class Base
18
+ def initialize
19
+ @cache = nil
20
+ end
21
+
22
+ ##
23
+ # Human-readable name of the geocoding API.
24
+ #
25
+ def name
26
+ fail
27
+ end
28
+
29
+ ##
30
+ # Symbol which is used in configuration to refer to this Lookup.
31
+ #
32
+ def handle
33
+ str = self.class.to_s
34
+ str[str.rindex(':')+1..-1].gsub(/([a-z\d]+)([A-Z])/,'\1_\2').downcase.to_sym
35
+ end
36
+
37
+ ##
38
+ # Query the geocoding API and return a Geocoder::Result object.
39
+ # Returns +nil+ on timeout or error.
40
+ #
41
+ # Takes a search string (eg: "Mississippi Coast Coliseumf, Biloxi, MS",
42
+ # "205.128.54.202") for geocoding, or coordinates (latitude, longitude)
43
+ # for reverse geocoding. Returns an array of <tt>Geocoder::Result</tt>s.
44
+ #
45
+ def search(query, options = {})
46
+ query = Geocoder::Query.new(query, options) unless query.is_a?(Geocoder::Query)
47
+ results(query).map{ |r|
48
+ result = result_class.new(r)
49
+ result.cache_hit = @cache_hit if cache
50
+ result
51
+ }
52
+ end
53
+
54
+ ##
55
+ # Return the URL for a map of the given coordinates.
56
+ #
57
+ # Not necessarily implemented by all subclasses as only some lookups
58
+ # also provide maps.
59
+ #
60
+ def map_link_url(coordinates)
61
+ nil
62
+ end
63
+
64
+ ##
65
+ # Array containing string descriptions of keys required by the API.
66
+ # Empty array if keys are optional or not required.
67
+ #
68
+ def required_api_key_parts
69
+ []
70
+ end
71
+
72
+ ##
73
+ # URL to use for querying the geocoding engine.
74
+ #
75
+ def query_url(query)
76
+ fail
77
+ end
78
+
79
+ ##
80
+ # The working Cache object.
81
+ #
82
+ def cache
83
+ if @cache.nil? and store = configuration.cache
84
+ @cache = Cache.new(store, configuration.cache_prefix)
85
+ end
86
+ @cache
87
+ end
88
+
89
+ ##
90
+ # Array containing the protocols supported by the api.
91
+ # Should be set to [:http] if only HTTP is supported
92
+ # or [:https] if only HTTPS is supported.
93
+ #
94
+ def supported_protocols
95
+ [:http, :https]
96
+ end
97
+
98
+ private # -------------------------------------------------------------
99
+
100
+ ##
101
+ # An object with configuration data for this particular lookup.
102
+ #
103
+ def configuration
104
+ Geocoder.config_for_lookup(handle)
105
+ end
106
+
107
+ ##
108
+ # Object used to make HTTP requests.
109
+ #
110
+ def http_client
111
+ proxy_name = "#{protocol}_proxy"
112
+ if proxy = configuration.send(proxy_name)
113
+ proxy_url = !!(proxy =~ /^#{protocol}/) ? proxy : protocol + '://' + proxy
114
+ begin
115
+ uri = URI.parse(proxy_url)
116
+ rescue URI::InvalidURIError
117
+ raise ConfigurationError,
118
+ "Error parsing #{protocol.upcase} proxy URL: '#{proxy_url}'"
119
+ end
120
+ Net::HTTP::Proxy(uri.host, uri.port, uri.user, uri.password)
121
+ else
122
+ Net::HTTP
123
+ end
124
+ end
125
+
126
+ ##
127
+ # Geocoder::Result object or nil on timeout or other error.
128
+ #
129
+ def results(query)
130
+ fail
131
+ end
132
+
133
+ def query_url_params(query)
134
+ query.options[:params] || {}
135
+ end
136
+
137
+ def url_query_string(query)
138
+ hash_to_query(
139
+ query_url_params(query).reject{ |key,value| value.nil? }
140
+ )
141
+ end
142
+
143
+ ##
144
+ # Key to use for caching a geocoding result. Usually this will be the
145
+ # request URL, but in cases where OAuth is used and the nonce,
146
+ # timestamp, etc varies from one request to another, we need to use
147
+ # something else (like the URL before OAuth encoding).
148
+ #
149
+ def cache_key(query)
150
+ query_url(query)
151
+ end
152
+
153
+ ##
154
+ # Class of the result objects
155
+ #
156
+ def result_class
157
+ Geocoder::Result.const_get(self.class.to_s.split(":").last)
158
+ end
159
+
160
+ ##
161
+ # Raise exception if configuration specifies it should be raised.
162
+ # Return false if exception not raised.
163
+ #
164
+ def raise_error(error, message = nil)
165
+ exceptions = configuration.always_raise
166
+ if exceptions == :all or exceptions.include?( error.is_a?(Class) ? error : error.class )
167
+ raise error, message
168
+ else
169
+ false
170
+ end
171
+ end
172
+
173
+ ##
174
+ # Returns a parsed search result (Ruby hash).
175
+ #
176
+ def fetch_data(query)
177
+ parse_raw_data fetch_raw_data(query)
178
+ rescue SocketError => err
179
+ raise_error(err) or Geocoder.log(:warn, "Geocoding API connection cannot be established.")
180
+ rescue Errno::ECONNREFUSED => err
181
+ raise_error(err) or Geocoder.log(:warn, "Geocoding API connection refused.")
182
+ rescue Timeout::Error => err
183
+ raise_error(err) or Geocoder.log(:warn, "Geocoding API not responding fast enough " +
184
+ "(use Geocoder.configure(:timeout => ...) to set limit).")
185
+ end
186
+
187
+ def parse_json(data)
188
+ if defined?(ActiveSupport::JSON)
189
+ ActiveSupport::JSON.decode(data)
190
+ else
191
+ JSON.parse(data)
192
+ end
193
+ rescue
194
+ raise_error(ResponseParseError.new(data)) or Geocoder.log(:warn, "Geocoding API's response was not valid JSON: #{data}")
195
+ end
196
+
197
+ ##
198
+ # Parses a raw search result (returns hash or array).
199
+ #
200
+ def parse_raw_data(raw_data)
201
+ parse_json(raw_data)
202
+ end
203
+
204
+ ##
205
+ # Protocol to use for communication with geocoding services.
206
+ # Set in configuration but not available for every service.
207
+ #
208
+ def protocol
209
+ "http" + (use_ssl? ? "s" : "")
210
+ end
211
+
212
+ def valid_response?(response)
213
+ (200..399).include?(response.code.to_i)
214
+ end
215
+
216
+ ##
217
+ # Fetch a raw geocoding result (JSON string).
218
+ # The result might or might not be cached.
219
+ #
220
+ def fetch_raw_data(query)
221
+ key = cache_key(query)
222
+ if cache and body = cache[key]
223
+ @cache_hit = true
224
+ else
225
+ check_api_key_configuration!(query)
226
+ response = make_api_request(query)
227
+ check_response_for_errors!(response)
228
+ body = response.body
229
+
230
+ # apply the charset from the Content-Type header, if possible
231
+ ct = response['content-type']
232
+
233
+ if ct && ct['charset']
234
+ charset = ct.split(';').select do |s|
235
+ s['charset']
236
+ end.first.to_s.split('=')
237
+ if charset.length == 2
238
+ body.force_encoding(charset.last) rescue ArgumentError
239
+ end
240
+ end
241
+
242
+ if cache and valid_response?(response)
243
+ cache[key] = body
244
+ end
245
+ @cache_hit = false
246
+ end
247
+ body
248
+ end
249
+
250
+ def check_response_for_errors!(response)
251
+ if response.code.to_i == 400
252
+ raise_error(Geocoder::InvalidRequest) ||
253
+ Geocoder.log(:warn, "Geocoding API error: 400 Bad Request")
254
+ elsif response.code.to_i == 401
255
+ raise_error(Geocoder::RequestDenied) ||
256
+ Geocoder.log(:warn, "Geocoding API error: 401 Unauthorized")
257
+ elsif response.code.to_i == 402
258
+ raise_error(Geocoder::OverQueryLimitError) ||
259
+ Geocoder.log(:warn, "Geocoding API error: 402 Payment Required")
260
+ elsif response.code.to_i == 429
261
+ raise_error(Geocoder::OverQueryLimitError) ||
262
+ Geocoder.log(:warn, "Geocoding API error: 429 Too Many Requests")
263
+ elsif response.code.to_i == 503
264
+ raise_error(Geocoder::ServiceUnavailable) ||
265
+ Geocoder.log(:warn, "Geocoding API error: 503 Service Unavailable")
266
+ end
267
+ end
268
+
269
+ ##
270
+ # Make an HTTP(S) request to a geocoding API and
271
+ # return the response object.
272
+ #
273
+ def make_api_request(query)
274
+ uri = URI.parse(query_url(query))
275
+ Geocoder.log(:debug, "Geocoder: HTTP request being made for #{uri.to_s}")
276
+ http_client.start(uri.host, uri.port, use_ssl: use_ssl?, open_timeout: configuration.timeout, read_timeout: configuration.timeout) do |client|
277
+ configure_ssl!(client) if use_ssl?
278
+ req = Net::HTTP::Get.new(uri.request_uri, configuration.http_headers)
279
+ if configuration.basic_auth[:user] and configuration.basic_auth[:password]
280
+ req.basic_auth(
281
+ configuration.basic_auth[:user],
282
+ configuration.basic_auth[:password]
283
+ )
284
+ end
285
+ client.request(req)
286
+ end
287
+ rescue Timeout::Error
288
+ raise Geocoder::LookupTimeout
289
+ end
290
+
291
+ def use_ssl?
292
+ if supported_protocols == [:https]
293
+ true
294
+ elsif supported_protocols == [:http]
295
+ false
296
+ else
297
+ configuration.use_https
298
+ end
299
+ end
300
+
301
+ def configure_ssl!(client); end
302
+
303
+ def check_api_key_configuration!(query)
304
+ key_parts = query.lookup.required_api_key_parts
305
+ if key_parts.size > Array(configuration.api_key).size
306
+ parts_string = key_parts.size == 1 ? key_parts.first : key_parts
307
+ raise Geocoder::ConfigurationError,
308
+ "The #{query.lookup.name} API requires a key to be configured: " +
309
+ parts_string.inspect
310
+ end
311
+ end
312
+
313
+ ##
314
+ # Simulate ActiveSupport's Object#to_query.
315
+ # Removes any keys with nil value.
316
+ #
317
+ def hash_to_query(hash)
318
+ require 'cgi' unless defined?(CGI) && defined?(CGI.escape)
319
+ hash.collect{ |p|
320
+ p[1].nil? ? nil : p.map{ |i| CGI.escape i.to_s } * '='
321
+ }.compact.sort * '&'
322
+ end
323
+ end
324
+ end
325
+ end
@@ -0,0 +1,80 @@
1
+ require 'geocoder/lookups/base'
2
+ require "geocoder/results/bing"
3
+
4
+ module Geocoder::Lookup
5
+ class Bing < Base
6
+
7
+ def name
8
+ "Bing"
9
+ end
10
+
11
+ def map_link_url(coordinates)
12
+ "http://www.bing.com/maps/default.aspx?cp=#{coordinates.join('~')}"
13
+ end
14
+
15
+ def required_api_key_parts
16
+ ["key"]
17
+ end
18
+
19
+ def query_url(query)
20
+ base_url(query) + url_query_string(query)
21
+ end
22
+
23
+ private # ---------------------------------------------------------------
24
+
25
+ def base_url(query)
26
+ url = "#{protocol}://dev.virtualearth.net/REST/v1/Locations"
27
+
28
+ if !query.reverse_geocode?
29
+ if r = query.options[:region]
30
+ url << "/#{r}"
31
+ end
32
+ # use the more forgiving 'unstructured' query format to allow special
33
+ # chars, newlines, brackets, typos.
34
+ url + "?q=" + URI.escape(query.sanitized_text.strip) + "&"
35
+ else
36
+ url + "/#{URI.escape(query.sanitized_text.strip)}?"
37
+ end
38
+ end
39
+
40
+ def results(query)
41
+ return [] unless doc = fetch_data(query)
42
+
43
+ if doc['statusCode'] == 200
44
+ return doc['resourceSets'].first['estimatedTotal'] > 0 ? doc['resourceSets'].first['resources'] : []
45
+ elsif doc['statusCode'] == 401 and doc["authenticationResultCode"] == "InvalidCredentials"
46
+ raise_error(Geocoder::InvalidApiKey) || Geocoder.log(:warn, "Invalid Bing API key.")
47
+ else
48
+ Geocoder.log(:warn, "Bing Geocoding API error: #{doc['statusCode']} (#{doc['statusDescription']}).")
49
+ end
50
+ return []
51
+ end
52
+
53
+ def query_url_params(query)
54
+ {
55
+ key: configuration.api_key
56
+ }.merge(super)
57
+ end
58
+
59
+ def check_response_for_errors!(response)
60
+ super
61
+ if server_overloaded?(response)
62
+ raise_error(Geocoder::ServiceUnavailable) ||
63
+ Geocoder.log(:warn, "Bing Geocoding API error: Service Unavailable")
64
+ end
65
+ end
66
+
67
+ def valid_response?(response)
68
+ super(response) and not server_overloaded?(response)
69
+ end
70
+
71
+ def server_overloaded?(response)
72
+ # Occasionally, the servers processing service requests can be overloaded,
73
+ # and you may receive some responses that contain no results for queries that
74
+ # you would normally receive a result. To identify this situation,
75
+ # check the HTTP headers of the response. If the HTTP header X-MS-BM-WS-INFO is set to 1,
76
+ # it is best to wait a few seconds and try again.
77
+ response['x-ms-bm-ws-info'].to_i == 1
78
+ end
79
+ end
80
+ end