bw-geocoder 1.2.5

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (233) hide show
  1. data/.gitignore +6 -0
  2. data/.travis.yml +31 -0
  3. data/CHANGELOG.md +377 -0
  4. data/LICENSE +20 -0
  5. data/README.md +1041 -0
  6. data/Rakefile +25 -0
  7. data/bin/geocode +5 -0
  8. data/examples/autoexpire_cache_dalli.rb +62 -0
  9. data/examples/autoexpire_cache_redis.rb +28 -0
  10. data/examples/cache_bypass.rb +48 -0
  11. data/gemfiles/Gemfile.mongoid-2.4.x +16 -0
  12. data/lib/generators/geocoder/config/config_generator.rb +14 -0
  13. data/lib/generators/geocoder/config/templates/initializer.rb +21 -0
  14. data/lib/generators/geocoder/maxmind/geolite_city_generator.rb +28 -0
  15. data/lib/generators/geocoder/maxmind/geolite_country_generator.rb +28 -0
  16. data/lib/generators/geocoder/maxmind/templates/migration/geolite_city.rb +30 -0
  17. data/lib/generators/geocoder/maxmind/templates/migration/geolite_country.rb +17 -0
  18. data/lib/geocoder/cache.rb +90 -0
  19. data/lib/geocoder/calculations.rb +428 -0
  20. data/lib/geocoder/cli.rb +121 -0
  21. data/lib/geocoder/configuration.rb +124 -0
  22. data/lib/geocoder/configuration_hash.rb +11 -0
  23. data/lib/geocoder/exceptions.rb +21 -0
  24. data/lib/geocoder/ip_address.rb +21 -0
  25. data/lib/geocoder/lookup.rb +100 -0
  26. data/lib/geocoder/lookups/baidu.rb +55 -0
  27. data/lib/geocoder/lookups/baidu_ip.rb +54 -0
  28. data/lib/geocoder/lookups/base.rb +302 -0
  29. data/lib/geocoder/lookups/bing.rb +59 -0
  30. data/lib/geocoder/lookups/dstk.rb +20 -0
  31. data/lib/geocoder/lookups/esri.rb +48 -0
  32. data/lib/geocoder/lookups/freegeoip.rb +47 -0
  33. data/lib/geocoder/lookups/geocoder_ca.rb +54 -0
  34. data/lib/geocoder/lookups/geocoder_us.rb +39 -0
  35. data/lib/geocoder/lookups/geocodio.rb +42 -0
  36. data/lib/geocoder/lookups/google.rb +67 -0
  37. data/lib/geocoder/lookups/google_places_details.rb +50 -0
  38. data/lib/geocoder/lookups/google_premier.rb +47 -0
  39. data/lib/geocoder/lookups/here.rb +62 -0
  40. data/lib/geocoder/lookups/ip_address_labs.rb +43 -0
  41. data/lib/geocoder/lookups/mapquest.rb +60 -0
  42. data/lib/geocoder/lookups/maxmind.rb +90 -0
  43. data/lib/geocoder/lookups/maxmind_local.rb +58 -0
  44. data/lib/geocoder/lookups/nominatim.rb +52 -0
  45. data/lib/geocoder/lookups/okf.rb +43 -0
  46. data/lib/geocoder/lookups/opencagedata.rb +58 -0
  47. data/lib/geocoder/lookups/ovi.rb +62 -0
  48. data/lib/geocoder/lookups/pointpin.rb +68 -0
  49. data/lib/geocoder/lookups/smarty_streets.rb +45 -0
  50. data/lib/geocoder/lookups/telize.rb +40 -0
  51. data/lib/geocoder/lookups/test.rb +44 -0
  52. data/lib/geocoder/lookups/yahoo.rb +88 -0
  53. data/lib/geocoder/lookups/yandex.rb +54 -0
  54. data/lib/geocoder/models/active_record.rb +50 -0
  55. data/lib/geocoder/models/base.rb +39 -0
  56. data/lib/geocoder/models/mongo_base.rb +64 -0
  57. data/lib/geocoder/models/mongo_mapper.rb +26 -0
  58. data/lib/geocoder/models/mongoid.rb +32 -0
  59. data/lib/geocoder/query.rb +111 -0
  60. data/lib/geocoder/railtie.rb +26 -0
  61. data/lib/geocoder/request.rb +25 -0
  62. data/lib/geocoder/results/baidu.rb +79 -0
  63. data/lib/geocoder/results/baidu_ip.rb +62 -0
  64. data/lib/geocoder/results/base.rb +67 -0
  65. data/lib/geocoder/results/bing.rb +48 -0
  66. data/lib/geocoder/results/dstk.rb +6 -0
  67. data/lib/geocoder/results/esri.rb +51 -0
  68. data/lib/geocoder/results/freegeoip.rb +45 -0
  69. data/lib/geocoder/results/geocoder_ca.rb +60 -0
  70. data/lib/geocoder/results/geocoder_us.rb +39 -0
  71. data/lib/geocoder/results/geocodio.rb +66 -0
  72. data/lib/geocoder/results/google.rb +124 -0
  73. data/lib/geocoder/results/google_places_details.rb +35 -0
  74. data/lib/geocoder/results/google_premier.rb +6 -0
  75. data/lib/geocoder/results/here.rb +62 -0
  76. data/lib/geocoder/results/ip_address_labs.rb +78 -0
  77. data/lib/geocoder/results/mapquest.rb +51 -0
  78. data/lib/geocoder/results/maxmind.rb +135 -0
  79. data/lib/geocoder/results/maxmind_local.rb +49 -0
  80. data/lib/geocoder/results/nominatim.rb +94 -0
  81. data/lib/geocoder/results/okf.rb +106 -0
  82. data/lib/geocoder/results/opencagedata.rb +82 -0
  83. data/lib/geocoder/results/ovi.rb +62 -0
  84. data/lib/geocoder/results/pointpin.rb +44 -0
  85. data/lib/geocoder/results/smarty_streets.rb +106 -0
  86. data/lib/geocoder/results/telize.rb +45 -0
  87. data/lib/geocoder/results/test.rb +33 -0
  88. data/lib/geocoder/results/yahoo.rb +55 -0
  89. data/lib/geocoder/results/yandex.rb +84 -0
  90. data/lib/geocoder/sql.rb +107 -0
  91. data/lib/geocoder/stores/active_record.rb +278 -0
  92. data/lib/geocoder/stores/base.rb +127 -0
  93. data/lib/geocoder/stores/mongo_base.rb +89 -0
  94. data/lib/geocoder/stores/mongo_mapper.rb +13 -0
  95. data/lib/geocoder/stores/mongoid.rb +13 -0
  96. data/lib/geocoder/version.rb +3 -0
  97. data/lib/geocoder.rb +47 -0
  98. data/lib/hash_recursive_merge.rb +74 -0
  99. data/lib/maxmind_database.rb +109 -0
  100. data/lib/oauth_util.rb +112 -0
  101. data/lib/tasks/geocoder.rake +29 -0
  102. data/lib/tasks/maxmind.rake +73 -0
  103. data/test/fixtures/baidu_invalid_key +1 -0
  104. data/test/fixtures/baidu_ip_202_198_16_3 +19 -0
  105. data/test/fixtures/baidu_ip_invalid_key +1 -0
  106. data/test/fixtures/baidu_ip_no_results +1 -0
  107. data/test/fixtures/baidu_no_results +1 -0
  108. data/test/fixtures/baidu_reverse +1 -0
  109. data/test/fixtures/baidu_shanghai_pearl_tower +12 -0
  110. data/test/fixtures/bing_invalid_key +1 -0
  111. data/test/fixtures/bing_madison_square_garden +40 -0
  112. data/test/fixtures/bing_no_results +16 -0
  113. data/test/fixtures/bing_reverse +42 -0
  114. data/test/fixtures/cloudmade_invalid_key +1 -0
  115. data/test/fixtures/cloudmade_madison_square_garden +1 -0
  116. data/test/fixtures/cloudmade_no_results +1 -0
  117. data/test/fixtures/esri_madison_square_garden +59 -0
  118. data/test/fixtures/esri_no_results +8 -0
  119. data/test/fixtures/esri_reverse +21 -0
  120. data/test/fixtures/freegeoip_74_200_247_59 +12 -0
  121. data/test/fixtures/freegeoip_no_results +1 -0
  122. data/test/fixtures/geocoder_ca_madison_square_garden +1 -0
  123. data/test/fixtures/geocoder_ca_no_results +1 -0
  124. data/test/fixtures/geocoder_ca_reverse +34 -0
  125. data/test/fixtures/geocoder_us_madison_square_garden +1 -0
  126. data/test/fixtures/geocoder_us_no_results +1 -0
  127. data/test/fixtures/geocodio_1101_pennsylvania_ave +1 -0
  128. data/test/fixtures/geocodio_bad_api_key +3 -0
  129. data/test/fixtures/geocodio_invalid +4 -0
  130. data/test/fixtures/geocodio_no_results +1 -0
  131. data/test/fixtures/geocodio_over_query_limit +4 -0
  132. data/test/fixtures/google_garbage +456 -0
  133. data/test/fixtures/google_madison_square_garden +57 -0
  134. data/test/fixtures/google_no_city_data +44 -0
  135. data/test/fixtures/google_no_locality +51 -0
  136. data/test/fixtures/google_no_results +4 -0
  137. data/test/fixtures/google_over_limit +4 -0
  138. data/test/fixtures/google_places_details_invalid_request +4 -0
  139. data/test/fixtures/google_places_details_madison_square_garden +120 -0
  140. data/test/fixtures/google_places_details_no_results +4 -0
  141. data/test/fixtures/google_places_details_no_reviews +60 -0
  142. data/test/fixtures/google_places_details_no_types +66 -0
  143. data/test/fixtures/here_madison_square_garden +72 -0
  144. data/test/fixtures/here_no_results +8 -0
  145. data/test/fixtures/mapquest_error +16 -0
  146. data/test/fixtures/mapquest_invalid_api_key +16 -0
  147. data/test/fixtures/mapquest_invalid_request +16 -0
  148. data/test/fixtures/mapquest_madison_square_garden +52 -0
  149. data/test/fixtures/mapquest_no_results +16 -0
  150. data/test/fixtures/maxmind_24_24_24_21 +1 -0
  151. data/test/fixtures/maxmind_24_24_24_22 +1 -0
  152. data/test/fixtures/maxmind_24_24_24_23 +1 -0
  153. data/test/fixtures/maxmind_24_24_24_24 +1 -0
  154. data/test/fixtures/maxmind_74_200_247_59 +1 -0
  155. data/test/fixtures/maxmind_invalid_key +1 -0
  156. data/test/fixtures/maxmind_no_results +1 -0
  157. data/test/fixtures/nominatim_madison_square_garden +150 -0
  158. data/test/fixtures/nominatim_no_results +1 -0
  159. data/test/fixtures/nominatim_over_limit +1 -0
  160. data/test/fixtures/okf_kirstinmaki +67 -0
  161. data/test/fixtures/okf_no_results +4 -0
  162. data/test/fixtures/opencagedata_invalid_api_key +25 -0
  163. data/test/fixtures/opencagedata_invalid_request +26 -0
  164. data/test/fixtures/opencagedata_madison_square_garden +73 -0
  165. data/test/fixtures/opencagedata_no_results +29 -0
  166. data/test/fixtures/opencagedata_over_limit +31 -0
  167. data/test/fixtures/ovi_madison_square_garden +72 -0
  168. data/test/fixtures/ovi_no_results +8 -0
  169. data/test/fixtures/pointpin_10_10_10_10 +1 -0
  170. data/test/fixtures/pointpin_555_555_555_555 +1 -0
  171. data/test/fixtures/pointpin_80_111_555_555 +1 -0
  172. data/test/fixtures/pointpin_no_results +1 -0
  173. data/test/fixtures/smarty_streets_11211 +1 -0
  174. data/test/fixtures/smarty_streets_madison_square_garden +47 -0
  175. data/test/fixtures/smarty_streets_no_results +1 -0
  176. data/test/fixtures/telize_10_10_10_10 +1 -0
  177. data/test/fixtures/telize_555_555_555_555 +4 -0
  178. data/test/fixtures/telize_74_200_247_59 +1 -0
  179. data/test/fixtures/telize_no_results +1 -0
  180. data/test/fixtures/yahoo_error +1 -0
  181. data/test/fixtures/yahoo_invalid_key +2 -0
  182. data/test/fixtures/yahoo_madison_square_garden +52 -0
  183. data/test/fixtures/yahoo_no_results +10 -0
  184. data/test/fixtures/yahoo_over_limit +2 -0
  185. data/test/fixtures/yandex_canada_rue_dupuis_14 +446 -0
  186. data/test/fixtures/yandex_invalid_key +1 -0
  187. data/test/fixtures/yandex_kremlin +48 -0
  188. data/test/fixtures/yandex_new_york +1 -0
  189. data/test/fixtures/yandex_no_city_and_town +112 -0
  190. data/test/fixtures/yandex_no_results +16 -0
  191. data/test/integration/http_client_test.rb +31 -0
  192. data/test/mongoid_test_helper.rb +43 -0
  193. data/test/test_helper.rb +386 -0
  194. data/test/unit/active_record_test.rb +16 -0
  195. data/test/unit/cache_test.rb +37 -0
  196. data/test/unit/calculations_test.rb +220 -0
  197. data/test/unit/configuration_test.rb +55 -0
  198. data/test/unit/error_handling_test.rb +56 -0
  199. data/test/unit/geocoder_test.rb +78 -0
  200. data/test/unit/https_test.rb +17 -0
  201. data/test/unit/ip_address_test.rb +27 -0
  202. data/test/unit/lookup_test.rb +153 -0
  203. data/test/unit/lookups/bing_test.rb +68 -0
  204. data/test/unit/lookups/dstk_test.rb +26 -0
  205. data/test/unit/lookups/esri_test.rb +48 -0
  206. data/test/unit/lookups/freegeoip_test.rb +27 -0
  207. data/test/unit/lookups/geocoder_ca_test.rb +17 -0
  208. data/test/unit/lookups/geocodio_test.rb +55 -0
  209. data/test/unit/lookups/google_places_details_test.rb +122 -0
  210. data/test/unit/lookups/google_premier_test.rb +22 -0
  211. data/test/unit/lookups/google_test.rb +84 -0
  212. data/test/unit/lookups/mapquest_test.rb +60 -0
  213. data/test/unit/lookups/maxmind_local_test.rb +28 -0
  214. data/test/unit/lookups/maxmind_test.rb +63 -0
  215. data/test/unit/lookups/nominatim_test.rb +31 -0
  216. data/test/unit/lookups/okf_test.rb +38 -0
  217. data/test/unit/lookups/opencagedata_test.rb +64 -0
  218. data/test/unit/lookups/pointpin_test.rb +30 -0
  219. data/test/unit/lookups/smarty_streets_test.rb +71 -0
  220. data/test/unit/lookups/telize_test.rb +36 -0
  221. data/test/unit/lookups/yahoo_test.rb +35 -0
  222. data/test/unit/method_aliases_test.rb +26 -0
  223. data/test/unit/model_test.rb +38 -0
  224. data/test/unit/mongoid_test.rb +47 -0
  225. data/test/unit/near_test.rb +87 -0
  226. data/test/unit/oauth_util_test.rb +31 -0
  227. data/test/unit/proxy_test.rb +37 -0
  228. data/test/unit/query_test.rb +52 -0
  229. data/test/unit/rake_task_test.rb +21 -0
  230. data/test/unit/request_test.rb +35 -0
  231. data/test/unit/result_test.rb +72 -0
  232. data/test/unit/test_mode_test.rb +70 -0
  233. metadata +281 -0
@@ -0,0 +1,74 @@
1
+ #
2
+ # = Hash Recursive Merge
3
+ #
4
+ # Merges a Ruby Hash recursively, Also known as deep merge.
5
+ # Recursive version of Hash#merge and Hash#merge!.
6
+ #
7
+ # Category:: Ruby
8
+ # Package:: Hash
9
+ # Author:: Simone Carletti <weppos@weppos.net>
10
+ # Copyright:: 2007-2008 The Authors
11
+ # License:: MIT License
12
+ # Link:: http://www.simonecarletti.com/
13
+ # Source:: http://gist.github.com/gists/6391/
14
+ #
15
+ module HashRecursiveMerge
16
+
17
+ #
18
+ # Recursive version of Hash#merge!
19
+ #
20
+ # Adds the contents of +other_hash+ to +hsh+,
21
+ # merging entries in +hsh+ with duplicate keys with those from +other_hash+.
22
+ #
23
+ # Compared with Hash#merge!, this method supports nested hashes.
24
+ # When both +hsh+ and +other_hash+ contains an entry with the same key,
25
+ # it merges and returns the values from both arrays.
26
+ #
27
+ # h1 = {"a" => 100, "b" => 200, "c" => {"c1" => 12, "c2" => 14}}
28
+ # h2 = {"b" => 254, "c" => {"c1" => 16, "c3" => 94}}
29
+ # h1.rmerge!(h2) #=> {"a" => 100, "b" => 254, "c" => {"c1" => 16, "c2" => 14, "c3" => 94}}
30
+ #
31
+ # Simply using Hash#merge! would return
32
+ #
33
+ # h1.merge!(h2) #=> {"a" => 100, "b" = >254, "c" => {"c1" => 16, "c3" => 94}}
34
+ #
35
+ def rmerge!(other_hash)
36
+ merge!(other_hash) do |key, oldval, newval|
37
+ oldval.class == self.class ? oldval.rmerge!(newval) : newval
38
+ end
39
+ end
40
+
41
+ #
42
+ # Recursive version of Hash#merge
43
+ #
44
+ # Compared with Hash#merge!, this method supports nested hashes.
45
+ # When both +hsh+ and +other_hash+ contains an entry with the same key,
46
+ # it merges and returns the values from both arrays.
47
+ #
48
+ # Compared with Hash#merge, this method provides a different approch
49
+ # for merging nasted hashes.
50
+ # If the value of a given key is an Hash and both +other_hash+ abd +hsh
51
+ # includes the same key, the value is merged instead replaced with
52
+ # +other_hash+ value.
53
+ #
54
+ # h1 = {"a" => 100, "b" => 200, "c" => {"c1" => 12, "c2" => 14}}
55
+ # h2 = {"b" => 254, "c" => {"c1" => 16, "c3" => 94}}
56
+ # h1.rmerge(h2) #=> {"a" => 100, "b" => 254, "c" => {"c1" => 16, "c2" => 14, "c3" => 94}}
57
+ #
58
+ # Simply using Hash#merge would return
59
+ #
60
+ # h1.merge(h2) #=> {"a" => 100, "b" = >254, "c" => {"c1" => 16, "c3" => 94}}
61
+ #
62
+ def rmerge(other_hash)
63
+ r = {}
64
+ merge(other_hash) do |key, oldval, newval|
65
+ r[key] = oldval.class == self.class ? oldval.rmerge(newval) : newval
66
+ end
67
+ end
68
+
69
+ end
70
+
71
+
72
+ class Hash
73
+ include HashRecursiveMerge
74
+ end
@@ -0,0 +1,109 @@
1
+ require 'csv'
2
+ require 'net/http'
3
+
4
+ module Geocoder
5
+ module MaxmindDatabase
6
+ extend self
7
+
8
+ def download(package, dir = "tmp")
9
+ filepath = File.expand_path(File.join(dir, archive_filename(package)))
10
+ open(filepath, 'wb') do |file|
11
+ uri = URI.parse(archive_url(package))
12
+ Net::HTTP.start(uri.host, uri.port) do |http|
13
+ http.request_get(uri.path) do |resp|
14
+ # TODO: show progress
15
+ resp.read_body do |segment|
16
+ file.write(segment)
17
+ end
18
+ end
19
+ end
20
+ end
21
+ end
22
+
23
+ def insert(package, dir = "tmp")
24
+ data_files(package).each do |filepath,table|
25
+ print "Resetting table #{table}..."
26
+ ActiveRecord::Base.connection.execute("DELETE FROM #{table}")
27
+ puts "done"
28
+ insert_into_table(table, filepath)
29
+ end
30
+ end
31
+
32
+ def archive_filename(package)
33
+ p = archive_url_path(package)
34
+ s = !(pos = p.rindex('/')).nil? && pos + 1 || 0
35
+ p[s..-1]
36
+ end
37
+
38
+ private # -------------------------------------------------------------
39
+
40
+ def table_columns(table_name)
41
+ {
42
+ maxmind_geolite_city_blocks: %w[start_ip_num end_ip_num loc_id],
43
+ maxmind_geolite_city_location: %w[loc_id country region city postal_code latitude longitude metro_code area_code],
44
+ maxmind_geolite_country: %w[start_ip end_ip start_ip_num end_ip_num country_code country]
45
+ }[table_name.to_sym]
46
+ end
47
+
48
+ def insert_into_table(table, filepath)
49
+ start_time = Time.now
50
+ print "Loading data for table #{table}"
51
+ rows = []
52
+ columns = table_columns(table)
53
+ CSV.foreach(filepath, encoding: "ISO-8859-1") do |line|
54
+ # Some files have header rows.
55
+ # skip if starts with "Copyright" or "locId" or "startIpNum"
56
+ next if line.first.match(/[A-z]/)
57
+ rows << line.to_a
58
+ if rows.size == 10000
59
+ insert_rows(table, columns, rows)
60
+ rows = []
61
+ print "."
62
+ end
63
+ end
64
+ insert_rows(table, columns, rows) if rows.size > 0
65
+ puts "done (#{Time.now - start_time} seconds)"
66
+ end
67
+
68
+ def insert_rows(table, headers, rows)
69
+ value_strings = rows.map do |row|
70
+ "(" + row.map{ |col| sql_escaped_value(col) }.join(',') + ")"
71
+ end
72
+ q = "INSERT INTO #{table} (#{headers.join(',')}) " +
73
+ "VALUES #{value_strings.join(',')}"
74
+ ActiveRecord::Base.connection.execute(q)
75
+ end
76
+
77
+ def sql_escaped_value(value)
78
+ value.to_i.to_s == value ? value :
79
+ ActiveRecord::Base.connection.quote(value)
80
+ end
81
+
82
+ def data_files(package, dir = "tmp")
83
+ case package
84
+ when :geolite_city_csv
85
+ # use the last two in case multiple versions exist
86
+ files = Dir.glob(File.join(dir, "GeoLiteCity_*/*.csv"))[-2..-1].sort
87
+ Hash[*files.zip(["maxmind_geolite_city_blocks", "maxmind_geolite_city_location"]).flatten]
88
+ when :geolite_country_csv
89
+ {File.join(dir, "GeoIPCountryWhois.csv") => "maxmind_geolite_country"}
90
+ end
91
+ end
92
+
93
+ def archive_url(package)
94
+ base_url + archive_url_path(package)
95
+ end
96
+
97
+ def archive_url_path(package)
98
+ {
99
+ geolite_country_csv: "GeoIPCountryCSV.zip",
100
+ geolite_city_csv: "GeoLiteCity_CSV/GeoLiteCity-latest.zip",
101
+ geolite_asn_csv: "asnum/GeoIPASNum2.zip"
102
+ }[package]
103
+ end
104
+
105
+ def base_url
106
+ "http://geolite.maxmind.com/download/geoip/database/"
107
+ end
108
+ end
109
+ end
data/lib/oauth_util.rb ADDED
@@ -0,0 +1,112 @@
1
+ # A utility for signing an url using OAuth in a way that's convenient for debugging
2
+ # Note: the standard Ruby OAuth lib is here http://github.com/mojodna/oauth
3
+ # Source: http://gist.github.com/383159
4
+ # License: http://gist.github.com/375593
5
+ # Usage: see example.rb below
6
+ #
7
+ # NOTE: This file has been modified from the original Gist:
8
+ #
9
+ # 1. Fix to prevent param-array conversion, as mentioned in Gist comment.
10
+ # 2. Query string escaping has been changed. See:
11
+ # https://github.com/alexreisner/geocoder/pull/360
12
+ #
13
+
14
+ require 'uri'
15
+ require 'cgi'
16
+ require 'openssl'
17
+ require 'base64'
18
+
19
+ class OauthUtil
20
+
21
+ attr_accessor :consumer_key, :consumer_secret, :token, :token_secret, :req_method,
22
+ :sig_method, :oauth_version, :callback_url, :params, :req_url, :base_str
23
+
24
+ def initialize
25
+ @consumer_key = ''
26
+ @consumer_secret = ''
27
+ @token = ''
28
+ @token_secret = ''
29
+ @req_method = 'GET'
30
+ @sig_method = 'HMAC-SHA1'
31
+ @oauth_version = '1.0'
32
+ @callback_url = ''
33
+ end
34
+
35
+ # openssl::random_bytes returns non-word chars, which need to be removed. using alt method to get length
36
+ # ref http://snippets.dzone.com/posts/show/491
37
+ def nonce
38
+ Array.new( 5 ) { rand(256) }.pack('C*').unpack('H*').first
39
+ end
40
+
41
+ def percent_encode( string )
42
+
43
+ # ref http://snippets.dzone.com/posts/show/1260
44
+ return URI.escape( string, Regexp.new("[^#{URI::PATTERN::UNRESERVED}]") ).gsub('*', '%2A')
45
+ end
46
+
47
+ # @ref http://oauth.net/core/1.0/#rfc.section.9.2
48
+ def signature
49
+ key = percent_encode( @consumer_secret ) + '&' + percent_encode( @token_secret )
50
+
51
+ # ref: http://blog.nathanielbibler.com/post/63031273/openssl-hmac-vs-ruby-hmac-benchmarks
52
+ digest = OpenSSL::Digest.new( 'sha1' )
53
+ hmac = OpenSSL::HMAC.digest( digest, key, @base_str )
54
+
55
+ # ref http://groups.google.com/group/oauth-ruby/browse_thread/thread/9110ed8c8f3cae81
56
+ Base64.encode64( hmac ).chomp.gsub( /\n/, '' )
57
+ end
58
+
59
+ # sort (very important as it affects the signature), concat, and percent encode
60
+ # @ref http://oauth.net/core/1.0/#rfc.section.9.1.1
61
+ # @ref http://oauth.net/core/1.0/#9.2.1
62
+ # @ref http://oauth.net/core/1.0/#rfc.section.A.5.1
63
+ def query_string
64
+ pairs = []
65
+ @params.sort.each { | key, val |
66
+ pairs.push( "#{ CGI.escape(key.to_s).gsub(/%(5B|5D)/n) { [$1].pack('H*') } }=#{ CGI.escape(val.to_s) }" )
67
+ }
68
+ pairs.join '&'
69
+ end
70
+
71
+ # organize params & create signature
72
+ def sign( parsed_url )
73
+
74
+ @params = {
75
+ 'oauth_consumer_key' => @consumer_key,
76
+ 'oauth_nonce' => nonce,
77
+ 'oauth_signature_method' => @sig_method,
78
+ 'oauth_timestamp' => Time.now.to_i.to_s,
79
+ 'oauth_version' => @oauth_version
80
+ }
81
+
82
+ # if url has query, merge key/values into params obj overwriting defaults
83
+ if parsed_url.query
84
+ CGI.parse( parsed_url.query ).each do |k,v|
85
+ if v.is_a?(Array) && v.count == 1
86
+ @params[k] = v.first
87
+ else
88
+ @params[k] = v
89
+ end
90
+ end
91
+ end
92
+
93
+ # @ref http://oauth.net/core/1.0/#rfc.section.9.1.2
94
+ @req_url = parsed_url.scheme + '://' + parsed_url.host + parsed_url.path
95
+
96
+ # create base str. make it an object attr for ez debugging
97
+ # ref http://oauth.net/core/1.0/#anchor14
98
+ @base_str = [
99
+ @req_method,
100
+ percent_encode( req_url ),
101
+
102
+ # normalization is just x-www-form-urlencoded
103
+ percent_encode( query_string )
104
+
105
+ ].join( '&' )
106
+
107
+ # add signature
108
+ @params[ 'oauth_signature' ] = signature
109
+
110
+ return self
111
+ end
112
+ end
@@ -0,0 +1,29 @@
1
+ namespace :geocode do
2
+ desc "Geocode all objects without coordinates."
3
+ task :all => :environment do
4
+ class_name = ENV['CLASS'] || ENV['class']
5
+ sleep_timer = ENV['SLEEP'] || ENV['sleep']
6
+ batch = ENV['BATCH'] || ENV['batch']
7
+ raise "Please specify a CLASS (model)" unless class_name
8
+ klass = class_from_string(class_name)
9
+ batch = batch.to_i unless batch.nil?
10
+
11
+ klass.not_geocoded.find_each(batch_size: batch) do |obj|
12
+ obj.geocode; obj.save
13
+ sleep(sleep_timer.to_f) unless sleep_timer.nil?
14
+ end
15
+ end
16
+ end
17
+
18
+ ##
19
+ # Get a class object from the string given in the shell environment.
20
+ # Similar to ActiveSupport's +constantize+ method.
21
+ #
22
+ def class_from_string(class_name)
23
+ parts = class_name.split("::")
24
+ constant = Object
25
+ parts.each do |part|
26
+ constant = constant.const_get(part)
27
+ end
28
+ constant
29
+ end
@@ -0,0 +1,73 @@
1
+ require 'maxmind_database'
2
+
3
+ namespace :geocoder do
4
+ namespace :maxmind do
5
+ namespace :geolite do
6
+
7
+ desc "Download and load/refresh MaxMind GeoLite City data"
8
+ task load: [:download, :extract, :insert]
9
+
10
+ desc "Download MaxMind GeoLite City data"
11
+ task :download do
12
+ p = MaxmindTask.check_for_package!
13
+ MaxmindTask.download!(p, dir: ENV['DIR'] || "tmp/")
14
+ end
15
+
16
+ desc "Extract (unzip) MaxMind GeoLite City data"
17
+ task :extract do
18
+ p = MaxmindTask.check_for_package!
19
+ MaxmindTask.extract!(p, dir: ENV['DIR'] || "tmp/")
20
+ end
21
+
22
+ desc "Load/refresh MaxMind GeoLite City data"
23
+ task insert: [:environment] do
24
+ p = MaxmindTask.check_for_package!
25
+ MaxmindTask.insert!(p, dir: ENV['DIR'] || "tmp/")
26
+ end
27
+ end
28
+ end
29
+ end
30
+
31
+ module MaxmindTask
32
+ extend self
33
+
34
+ def check_for_package!
35
+ if %w[city country].include?(p = ENV['PACKAGE'])
36
+ return p
37
+ else
38
+ puts "Please specify PACKAGE=city or PACKAGE=country"
39
+ exit
40
+ end
41
+ end
42
+
43
+ def download!(package, options = {})
44
+ p = "geolite_#{package}_csv".intern
45
+ Geocoder::MaxmindDatabase.download(p, options[:dir])
46
+ end
47
+
48
+ def extract!(package, options = {})
49
+ begin
50
+ require 'zip'
51
+ rescue LoadError
52
+ puts "Please install gem: rubyzip (>= 1.0.0)"
53
+ exit
54
+ end
55
+ require 'fileutils'
56
+ p = "geolite_#{package}_csv".intern
57
+ archive_filename = Geocoder::MaxmindDatabase.archive_filename(p)
58
+ Zip::File.open(File.join(options[:dir], archive_filename)).each do |entry|
59
+ filepath = File.join(options[:dir], entry.name)
60
+ if File.exist? filepath
61
+ warn "File already exists (#{entry.name}), skipping"
62
+ else
63
+ FileUtils.mkdir_p(File.dirname(filepath))
64
+ entry.extract(filepath)
65
+ end
66
+ end
67
+ end
68
+
69
+ def insert!(package, options = {})
70
+ p = "geolite_#{package}_csv".intern
71
+ Geocoder::MaxmindDatabase.insert(p, options[:dir])
72
+ end
73
+ end
@@ -0,0 +1 @@
1
+ {"results":[],"status":5,"msg":"AK Illegal or Not Exist:"}
@@ -0,0 +1,19 @@
1
+ {
2
+ "address": "CN|北京|北京|None|CHINANET|1|None",
3
+ "content": {
4
+ "address": "北京市",
5
+ "address_detail": {
6
+ "city": "北京市",
7
+ "city_code": 131,
8
+ "district": "",
9
+ "province": "北京市",
10
+ "street": "",
11
+ "street_number": ""
12
+ },
13
+ "point": {
14
+ "x": "116.39564504",
15
+ "y": "39.92998578"
16
+ }
17
+ },
18
+ "status": 0
19
+ }
@@ -0,0 +1 @@
1
+ {"status":5,"uid":null,"sk":null,"logformat":null}
@@ -0,0 +1 @@
1
+ {"status":0, "content":{}}
@@ -0,0 +1 @@
1
+ {"status":0,"result":[]}
@@ -0,0 +1 @@
1
+ {"status":0,"result":{"location":{"lng":121.48789948569,"lat":31.249161555654},"formatted_address":"上海市闸北区天潼路619号","business":"七浦路,海宁路,北京东路","addressComponent":{"city":"上海市","district":"闸北区","province":"上海市","street":"天潼路","street_number":"619号"},"cityCode":289}}
@@ -0,0 +1,12 @@
1
+ {
2
+ "status":0,
3
+ "result":{
4
+ "location":{
5
+ "lng":116.30814954222,
6
+ "lat":40.056885091681
7
+ },
8
+ "precise":1,
9
+ "confidence":80,
10
+ "level":"\u5546\u52a1\u5927\u53a6"
11
+ }
12
+ }
@@ -0,0 +1 @@
1
+ {"authenticationResultCode":"InvalidCredentials","brandLogoUri":"http:\\/\\/dev.virtualearth.net\\/Branding\\/logo_powered_by.png","copyright":"Copyright \xC2\xA9 2012 Microsoft and its suppliers. All rights reserved. This API cannot be accessed and the content and any results may not be used, reproduced or transmitted in any manner without express written permission from Microsoft Corporation.","errorDetails":["Access was denied. You may have entered your credentials incorrectly, or you might not have access to the requested resource or operation."],"resourceSets":[],"statusCode":401,"statusDescription":"Unauthorized","traceId":"5c539f6e70c44b2e858741b6c932318e|EWRM001670|02.00.83.1900|"}
@@ -0,0 +1,40 @@
1
+ {
2
+ "authenticationResultCode":"ValidCredentials",
3
+ "brandLogoUri":"http:\/\/dev.virtualearth.net\/Branding\/logo_powered_by.png",
4
+ "copyright":"Copyright © 2011 Microsoft and its suppliers. All rights reserved. This API cannot be accessed and the content and any results may not be used, reproduced or transmitted in any manner without express written permission from Microsoft Corporation.",
5
+ "resourceSets":[
6
+ {
7
+ "estimatedTotal":1,
8
+ "resources":[
9
+ {
10
+ "__type":"Location:http:\/\/schemas.microsoft.com\/search\/local\/ws\/rest\/v1",
11
+ "bbox":[
12
+ 40.744944289326668,
13
+ -74.002353921532631,
14
+ 40.755675807595253,
15
+ -73.983625397086143
16
+ ],
17
+ "name":"Madison Square Garden, NY",
18
+ "point":{
19
+ "type":"Point",
20
+ "coordinates":[
21
+ 40.75031,
22
+ -73.99299
23
+ ]
24
+ },
25
+ "address":{
26
+ "adminDistrict":"NY",
27
+ "countryRegion":"United States",
28
+ "formattedAddress":"Madison Square Garden, NY",
29
+ "locality":"New York"
30
+ },
31
+ "confidence":"High",
32
+ "entityType":"Stadium"
33
+ }
34
+ ]
35
+ }
36
+ ],
37
+ "statusCode":200,
38
+ "statusDescription":"OK",
39
+ "traceId":"55094ee53c8d45e789794014666328cd|CH1M001466|02.00.82.2800|CH1MSNVM001396, CH1MSNVM001370, CH1MSNVM001397"
40
+ }
@@ -0,0 +1,16 @@
1
+ {
2
+ "authenticationResultCode":"ValidCredentials",
3
+ "brandLogoUri":"http:\/\/dev.virtualearth.net\/Branding\/logo_powered_by.png",
4
+ "copyright":"Copyright © 2011 Microsoft and its suppliers. All rights reserved. This API cannot be accessed and the content and any results may not be used, reproduced or transmitted in any manner without express written permission from Microsoft Corporation.",
5
+ "resourceSets":[
6
+ {
7
+ "estimatedTotal":0,
8
+ "resources":[
9
+
10
+ ]
11
+ }
12
+ ],
13
+ "statusCode":200,
14
+ "statusDescription":"OK",
15
+ "traceId":"907b76a307bc49129a489de3d4c992ea|CH1M001463|02.00.82.2800|CH1MSNVM001383, CH1MSNVM001358, CH1MSNVM001397"
16
+ }
@@ -0,0 +1,42 @@
1
+ {
2
+ "authenticationResultCode":"ValidCredentials",
3
+ "brandLogoUri":"http:\/\/dev.virtualearth.net\/Branding\/logo_powered_by.png",
4
+ "copyright":"Copyright © 2011 Microsoft and its suppliers. All rights reserved. This API cannot be accessed and the content and any results may not be used, reproduced or transmitted in any manner without express written permission from Microsoft Corporation.",
5
+ "resourceSets":[
6
+ {
7
+ "estimatedTotal":1,
8
+ "resources":[
9
+ {
10
+ "__type":"Location:http:\/\/schemas.microsoft.com\/search\/local\/ws\/rest\/v1",
11
+ "bbox":[
12
+ 45.419835111675845,
13
+ -75.683656128790716,
14
+ 45.4275605468172,
15
+ -75.66898098334994
16
+ ],
17
+ "name":"291 Rue Somerset E, Ottawa, ON, K1N",
18
+ "point":{
19
+ "type":"Point",
20
+ "coordinates":[
21
+ 45.423697829246521,
22
+ -75.676318556070328
23
+ ]
24
+ },
25
+ "address":{
26
+ "addressLine":"291 Rue Somerset E",
27
+ "adminDistrict":"ON",
28
+ "countryRegion":"Canada",
29
+ "formattedAddress":"291 Rue Somerset E, Ottawa, ON, K1N",
30
+ "locality":"Ottawa",
31
+ "postalCode":"K1N"
32
+ },
33
+ "confidence":"Medium",
34
+ "entityType":"Address"
35
+ }
36
+ ]
37
+ }
38
+ ],
39
+ "statusCode":200,
40
+ "statusDescription":"OK",
41
+ "traceId":"27bd5ed659e64ba6970c4144f1d4ea94|CH1M001470|02.00.82.2800|CH1MSNVM001396, CH1MSNVM001374"
42
+ }
@@ -0,0 +1 @@
1
+ Forbidden request
@@ -0,0 +1 @@
1
+ {"found": 2, "bounds": [[40.74983, -73.99433], [40.75116, -73.99266]], "features": [{"id": 32891803,"centroid": {"type":"POINT","coordinates":[40.75111, -73.99345]},"bounds": [[40.74983, -73.99433], [40.75116, -73.99266]],"properties": {"osm_element": "way", "sport": "hockey;basketball;lacrosse", "name": "Madison Square Garden", "leisure": "stadium", "osm_id": "24801588"},"location": {"county": "New York", "country": "United States of America", "postcode": "10119", "road": "West 31st Street", "city": "New York"},"type": "Feature"},{"id": 12977552,"centroid": {"type":"POINT","coordinates":[40.75066, -73.99347]},"bounds": [[40.75066, -73.99347], [40.75066, -73.99347]],"properties": {"building": "yes", "osm_element": "node", "name": "Madison Square Garden Center", "addr:state": "NY", "osm_id": "368045579"},"location": {"county": "New York", "country": "United States of America", "postcode": "10119", "road": "West 33rd Street", "city": "New York"},"type": "Feature"}], "type": "FeatureCollection", "crs": {"type": "EPSG", "properties": {"code": 4326, "coordinate_order": [0, 1]}}}
@@ -0,0 +1 @@
1
+ {}
@@ -0,0 +1,59 @@
1
+ {
2
+ "spatialReference": {
3
+ "wkid": 4326,
4
+ "latestWkid": 4326
5
+ },
6
+ "locations": [
7
+ {
8
+ "name": "Madison Square Garden",
9
+ "extent": {
10
+ "xmin": -74.000241000000003,
11
+ "ymin": 40.744050000000001,
12
+ "xmax": -73.988241000000002,
13
+ "ymax": 40.756050000000002
14
+ },
15
+ "feature": {
16
+ "geometry": {
17
+ "x": -73.994238897999651,
18
+ "y": 40.750049813000487
19
+ },
20
+ "attributes": {
21
+ "Loc_name": "Gaz.WorldGazetteer.POI2",
22
+ "Score": 100,
23
+ "Match_addr": "Madison Square Garden",
24
+ "Addr_type": "POI",
25
+ "Type": "Sports Complex",
26
+ "PlaceName": "Madison Square Garden",
27
+ "Rank": "18",
28
+ "AddBldg": "",
29
+ "AddNum": "",
30
+ "AddNumFrom": "",
31
+ "AddNumTo": "",
32
+ "Side": "",
33
+ "StPreDir": "",
34
+ "StPreType": "",
35
+ "StName": "",
36
+ "StType": "",
37
+ "StDir": "",
38
+ "Nbrhd": "",
39
+ "City": "New York",
40
+ "Subregion": "New York",
41
+ "Region": "New York",
42
+ "Postal": "10001",
43
+ "PostalExt": "",
44
+ "Country": "USA",
45
+ "LangCode": "",
46
+ "Distance": 0,
47
+ "X": -73.994240000000005,
48
+ "Y": 40.750050000000002,
49
+ "DisplayX": -73.994240000000005,
50
+ "DisplayY": 40.750050000000002,
51
+ "Xmin": -74.000241000000003,
52
+ "Xmax": -73.988241000000002,
53
+ "Ymin": 40.744050000000001,
54
+ "Ymax": 40.756050000000002
55
+ }
56
+ }
57
+ }
58
+ ]
59
+ }
@@ -0,0 +1,8 @@
1
+ {
2
+ "spatialReference": {
3
+ "wkid": 4326,
4
+ "latestWkid": 4326
5
+ },
6
+ "locations": [
7
+ ]
8
+ }
@@ -0,0 +1,21 @@
1
+ {
2
+ "address": {
3
+ "Address": "4 Avenue Gustave Eiffel",
4
+ "Neighborhood": "7e Arrondissement",
5
+ "City": "Paris",
6
+ "Subregion": "Paris",
7
+ "Region": "Île-de-France",
8
+ "Postal": "75007",
9
+ "PostalExt": null,
10
+ "CountryCode": "FRA",
11
+ "Loc_name": "FRA.PointAddress"
12
+ },
13
+ "location": {
14
+ "x": 2.2956200048981574,
15
+ "y": 48.858129997357558,
16
+ "spatialReference": {
17
+ "wkid": 4326,
18
+ "latestWkid": 4326
19
+ }
20
+ }
21
+ }
@@ -0,0 +1,12 @@
1
+ {
2
+ "city": "Plano",
3
+ "region_code": "TX",
4
+ "region_name": "Texas",
5
+ "metrocode": "623",
6
+ "zipcode": "75093",
7
+ "longitude": "-96.8134",
8
+ "country_name": "United States",
9
+ "country_code": "US",
10
+ "ip": "74.200.247.59",
11
+ "latitude": "33.0347"
12
+ }
@@ -0,0 +1 @@
1
+ <html><title>404: Not Found</title><body>404: Not Found</body></html>