bw-geocoder 1.2.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- data/.gitignore +6 -0
- data/.travis.yml +31 -0
- data/CHANGELOG.md +377 -0
- data/LICENSE +20 -0
- data/README.md +1041 -0
- data/Rakefile +25 -0
- data/bin/geocode +5 -0
- data/examples/autoexpire_cache_dalli.rb +62 -0
- data/examples/autoexpire_cache_redis.rb +28 -0
- data/examples/cache_bypass.rb +48 -0
- data/gemfiles/Gemfile.mongoid-2.4.x +16 -0
- data/lib/generators/geocoder/config/config_generator.rb +14 -0
- data/lib/generators/geocoder/config/templates/initializer.rb +21 -0
- data/lib/generators/geocoder/maxmind/geolite_city_generator.rb +28 -0
- data/lib/generators/geocoder/maxmind/geolite_country_generator.rb +28 -0
- data/lib/generators/geocoder/maxmind/templates/migration/geolite_city.rb +30 -0
- data/lib/generators/geocoder/maxmind/templates/migration/geolite_country.rb +17 -0
- data/lib/geocoder/cache.rb +90 -0
- data/lib/geocoder/calculations.rb +428 -0
- data/lib/geocoder/cli.rb +121 -0
- data/lib/geocoder/configuration.rb +124 -0
- data/lib/geocoder/configuration_hash.rb +11 -0
- data/lib/geocoder/exceptions.rb +21 -0
- data/lib/geocoder/ip_address.rb +21 -0
- data/lib/geocoder/lookup.rb +100 -0
- data/lib/geocoder/lookups/baidu.rb +55 -0
- data/lib/geocoder/lookups/baidu_ip.rb +54 -0
- data/lib/geocoder/lookups/base.rb +302 -0
- data/lib/geocoder/lookups/bing.rb +59 -0
- data/lib/geocoder/lookups/dstk.rb +20 -0
- data/lib/geocoder/lookups/esri.rb +48 -0
- data/lib/geocoder/lookups/freegeoip.rb +47 -0
- data/lib/geocoder/lookups/geocoder_ca.rb +54 -0
- data/lib/geocoder/lookups/geocoder_us.rb +39 -0
- data/lib/geocoder/lookups/geocodio.rb +42 -0
- data/lib/geocoder/lookups/google.rb +67 -0
- data/lib/geocoder/lookups/google_places_details.rb +50 -0
- data/lib/geocoder/lookups/google_premier.rb +47 -0
- data/lib/geocoder/lookups/here.rb +62 -0
- data/lib/geocoder/lookups/ip_address_labs.rb +43 -0
- data/lib/geocoder/lookups/mapquest.rb +60 -0
- data/lib/geocoder/lookups/maxmind.rb +90 -0
- data/lib/geocoder/lookups/maxmind_local.rb +58 -0
- data/lib/geocoder/lookups/nominatim.rb +52 -0
- data/lib/geocoder/lookups/okf.rb +43 -0
- data/lib/geocoder/lookups/opencagedata.rb +58 -0
- data/lib/geocoder/lookups/ovi.rb +62 -0
- data/lib/geocoder/lookups/pointpin.rb +68 -0
- data/lib/geocoder/lookups/smarty_streets.rb +45 -0
- data/lib/geocoder/lookups/telize.rb +40 -0
- data/lib/geocoder/lookups/test.rb +44 -0
- data/lib/geocoder/lookups/yahoo.rb +88 -0
- data/lib/geocoder/lookups/yandex.rb +54 -0
- data/lib/geocoder/models/active_record.rb +50 -0
- data/lib/geocoder/models/base.rb +39 -0
- data/lib/geocoder/models/mongo_base.rb +64 -0
- data/lib/geocoder/models/mongo_mapper.rb +26 -0
- data/lib/geocoder/models/mongoid.rb +32 -0
- data/lib/geocoder/query.rb +111 -0
- data/lib/geocoder/railtie.rb +26 -0
- data/lib/geocoder/request.rb +25 -0
- data/lib/geocoder/results/baidu.rb +79 -0
- data/lib/geocoder/results/baidu_ip.rb +62 -0
- data/lib/geocoder/results/base.rb +67 -0
- data/lib/geocoder/results/bing.rb +48 -0
- data/lib/geocoder/results/dstk.rb +6 -0
- data/lib/geocoder/results/esri.rb +51 -0
- data/lib/geocoder/results/freegeoip.rb +45 -0
- data/lib/geocoder/results/geocoder_ca.rb +60 -0
- data/lib/geocoder/results/geocoder_us.rb +39 -0
- data/lib/geocoder/results/geocodio.rb +66 -0
- data/lib/geocoder/results/google.rb +124 -0
- data/lib/geocoder/results/google_places_details.rb +35 -0
- data/lib/geocoder/results/google_premier.rb +6 -0
- data/lib/geocoder/results/here.rb +62 -0
- data/lib/geocoder/results/ip_address_labs.rb +78 -0
- data/lib/geocoder/results/mapquest.rb +51 -0
- data/lib/geocoder/results/maxmind.rb +135 -0
- data/lib/geocoder/results/maxmind_local.rb +49 -0
- data/lib/geocoder/results/nominatim.rb +94 -0
- data/lib/geocoder/results/okf.rb +106 -0
- data/lib/geocoder/results/opencagedata.rb +82 -0
- data/lib/geocoder/results/ovi.rb +62 -0
- data/lib/geocoder/results/pointpin.rb +44 -0
- data/lib/geocoder/results/smarty_streets.rb +106 -0
- data/lib/geocoder/results/telize.rb +45 -0
- data/lib/geocoder/results/test.rb +33 -0
- data/lib/geocoder/results/yahoo.rb +55 -0
- data/lib/geocoder/results/yandex.rb +84 -0
- data/lib/geocoder/sql.rb +107 -0
- data/lib/geocoder/stores/active_record.rb +278 -0
- data/lib/geocoder/stores/base.rb +127 -0
- data/lib/geocoder/stores/mongo_base.rb +89 -0
- data/lib/geocoder/stores/mongo_mapper.rb +13 -0
- data/lib/geocoder/stores/mongoid.rb +13 -0
- data/lib/geocoder/version.rb +3 -0
- data/lib/geocoder.rb +47 -0
- data/lib/hash_recursive_merge.rb +74 -0
- data/lib/maxmind_database.rb +109 -0
- data/lib/oauth_util.rb +112 -0
- data/lib/tasks/geocoder.rake +29 -0
- data/lib/tasks/maxmind.rake +73 -0
- data/test/fixtures/baidu_invalid_key +1 -0
- data/test/fixtures/baidu_ip_202_198_16_3 +19 -0
- data/test/fixtures/baidu_ip_invalid_key +1 -0
- data/test/fixtures/baidu_ip_no_results +1 -0
- data/test/fixtures/baidu_no_results +1 -0
- data/test/fixtures/baidu_reverse +1 -0
- data/test/fixtures/baidu_shanghai_pearl_tower +12 -0
- data/test/fixtures/bing_invalid_key +1 -0
- data/test/fixtures/bing_madison_square_garden +40 -0
- data/test/fixtures/bing_no_results +16 -0
- data/test/fixtures/bing_reverse +42 -0
- data/test/fixtures/cloudmade_invalid_key +1 -0
- data/test/fixtures/cloudmade_madison_square_garden +1 -0
- data/test/fixtures/cloudmade_no_results +1 -0
- data/test/fixtures/esri_madison_square_garden +59 -0
- data/test/fixtures/esri_no_results +8 -0
- data/test/fixtures/esri_reverse +21 -0
- data/test/fixtures/freegeoip_74_200_247_59 +12 -0
- data/test/fixtures/freegeoip_no_results +1 -0
- data/test/fixtures/geocoder_ca_madison_square_garden +1 -0
- data/test/fixtures/geocoder_ca_no_results +1 -0
- data/test/fixtures/geocoder_ca_reverse +34 -0
- data/test/fixtures/geocoder_us_madison_square_garden +1 -0
- data/test/fixtures/geocoder_us_no_results +1 -0
- data/test/fixtures/geocodio_1101_pennsylvania_ave +1 -0
- data/test/fixtures/geocodio_bad_api_key +3 -0
- data/test/fixtures/geocodio_invalid +4 -0
- data/test/fixtures/geocodio_no_results +1 -0
- data/test/fixtures/geocodio_over_query_limit +4 -0
- data/test/fixtures/google_garbage +456 -0
- data/test/fixtures/google_madison_square_garden +57 -0
- data/test/fixtures/google_no_city_data +44 -0
- data/test/fixtures/google_no_locality +51 -0
- data/test/fixtures/google_no_results +4 -0
- data/test/fixtures/google_over_limit +4 -0
- data/test/fixtures/google_places_details_invalid_request +4 -0
- data/test/fixtures/google_places_details_madison_square_garden +120 -0
- data/test/fixtures/google_places_details_no_results +4 -0
- data/test/fixtures/google_places_details_no_reviews +60 -0
- data/test/fixtures/google_places_details_no_types +66 -0
- data/test/fixtures/here_madison_square_garden +72 -0
- data/test/fixtures/here_no_results +8 -0
- data/test/fixtures/mapquest_error +16 -0
- data/test/fixtures/mapquest_invalid_api_key +16 -0
- data/test/fixtures/mapquest_invalid_request +16 -0
- data/test/fixtures/mapquest_madison_square_garden +52 -0
- data/test/fixtures/mapquest_no_results +16 -0
- data/test/fixtures/maxmind_24_24_24_21 +1 -0
- data/test/fixtures/maxmind_24_24_24_22 +1 -0
- data/test/fixtures/maxmind_24_24_24_23 +1 -0
- data/test/fixtures/maxmind_24_24_24_24 +1 -0
- data/test/fixtures/maxmind_74_200_247_59 +1 -0
- data/test/fixtures/maxmind_invalid_key +1 -0
- data/test/fixtures/maxmind_no_results +1 -0
- data/test/fixtures/nominatim_madison_square_garden +150 -0
- data/test/fixtures/nominatim_no_results +1 -0
- data/test/fixtures/nominatim_over_limit +1 -0
- data/test/fixtures/okf_kirstinmaki +67 -0
- data/test/fixtures/okf_no_results +4 -0
- data/test/fixtures/opencagedata_invalid_api_key +25 -0
- data/test/fixtures/opencagedata_invalid_request +26 -0
- data/test/fixtures/opencagedata_madison_square_garden +73 -0
- data/test/fixtures/opencagedata_no_results +29 -0
- data/test/fixtures/opencagedata_over_limit +31 -0
- data/test/fixtures/ovi_madison_square_garden +72 -0
- data/test/fixtures/ovi_no_results +8 -0
- data/test/fixtures/pointpin_10_10_10_10 +1 -0
- data/test/fixtures/pointpin_555_555_555_555 +1 -0
- data/test/fixtures/pointpin_80_111_555_555 +1 -0
- data/test/fixtures/pointpin_no_results +1 -0
- data/test/fixtures/smarty_streets_11211 +1 -0
- data/test/fixtures/smarty_streets_madison_square_garden +47 -0
- data/test/fixtures/smarty_streets_no_results +1 -0
- data/test/fixtures/telize_10_10_10_10 +1 -0
- data/test/fixtures/telize_555_555_555_555 +4 -0
- data/test/fixtures/telize_74_200_247_59 +1 -0
- data/test/fixtures/telize_no_results +1 -0
- data/test/fixtures/yahoo_error +1 -0
- data/test/fixtures/yahoo_invalid_key +2 -0
- data/test/fixtures/yahoo_madison_square_garden +52 -0
- data/test/fixtures/yahoo_no_results +10 -0
- data/test/fixtures/yahoo_over_limit +2 -0
- data/test/fixtures/yandex_canada_rue_dupuis_14 +446 -0
- data/test/fixtures/yandex_invalid_key +1 -0
- data/test/fixtures/yandex_kremlin +48 -0
- data/test/fixtures/yandex_new_york +1 -0
- data/test/fixtures/yandex_no_city_and_town +112 -0
- data/test/fixtures/yandex_no_results +16 -0
- data/test/integration/http_client_test.rb +31 -0
- data/test/mongoid_test_helper.rb +43 -0
- data/test/test_helper.rb +386 -0
- data/test/unit/active_record_test.rb +16 -0
- data/test/unit/cache_test.rb +37 -0
- data/test/unit/calculations_test.rb +220 -0
- data/test/unit/configuration_test.rb +55 -0
- data/test/unit/error_handling_test.rb +56 -0
- data/test/unit/geocoder_test.rb +78 -0
- data/test/unit/https_test.rb +17 -0
- data/test/unit/ip_address_test.rb +27 -0
- data/test/unit/lookup_test.rb +153 -0
- data/test/unit/lookups/bing_test.rb +68 -0
- data/test/unit/lookups/dstk_test.rb +26 -0
- data/test/unit/lookups/esri_test.rb +48 -0
- data/test/unit/lookups/freegeoip_test.rb +27 -0
- data/test/unit/lookups/geocoder_ca_test.rb +17 -0
- data/test/unit/lookups/geocodio_test.rb +55 -0
- data/test/unit/lookups/google_places_details_test.rb +122 -0
- data/test/unit/lookups/google_premier_test.rb +22 -0
- data/test/unit/lookups/google_test.rb +84 -0
- data/test/unit/lookups/mapquest_test.rb +60 -0
- data/test/unit/lookups/maxmind_local_test.rb +28 -0
- data/test/unit/lookups/maxmind_test.rb +63 -0
- data/test/unit/lookups/nominatim_test.rb +31 -0
- data/test/unit/lookups/okf_test.rb +38 -0
- data/test/unit/lookups/opencagedata_test.rb +64 -0
- data/test/unit/lookups/pointpin_test.rb +30 -0
- data/test/unit/lookups/smarty_streets_test.rb +71 -0
- data/test/unit/lookups/telize_test.rb +36 -0
- data/test/unit/lookups/yahoo_test.rb +35 -0
- data/test/unit/method_aliases_test.rb +26 -0
- data/test/unit/model_test.rb +38 -0
- data/test/unit/mongoid_test.rb +47 -0
- data/test/unit/near_test.rb +87 -0
- data/test/unit/oauth_util_test.rb +31 -0
- data/test/unit/proxy_test.rb +37 -0
- data/test/unit/query_test.rb +52 -0
- data/test/unit/rake_task_test.rb +21 -0
- data/test/unit/request_test.rb +35 -0
- data/test/unit/result_test.rb +72 -0
- data/test/unit/test_mode_test.rb +70 -0
- metadata +281 -0
@@ -0,0 +1,74 @@
|
|
1
|
+
#
|
2
|
+
# = Hash Recursive Merge
|
3
|
+
#
|
4
|
+
# Merges a Ruby Hash recursively, Also known as deep merge.
|
5
|
+
# Recursive version of Hash#merge and Hash#merge!.
|
6
|
+
#
|
7
|
+
# Category:: Ruby
|
8
|
+
# Package:: Hash
|
9
|
+
# Author:: Simone Carletti <weppos@weppos.net>
|
10
|
+
# Copyright:: 2007-2008 The Authors
|
11
|
+
# License:: MIT License
|
12
|
+
# Link:: http://www.simonecarletti.com/
|
13
|
+
# Source:: http://gist.github.com/gists/6391/
|
14
|
+
#
|
15
|
+
module HashRecursiveMerge
|
16
|
+
|
17
|
+
#
|
18
|
+
# Recursive version of Hash#merge!
|
19
|
+
#
|
20
|
+
# Adds the contents of +other_hash+ to +hsh+,
|
21
|
+
# merging entries in +hsh+ with duplicate keys with those from +other_hash+.
|
22
|
+
#
|
23
|
+
# Compared with Hash#merge!, this method supports nested hashes.
|
24
|
+
# When both +hsh+ and +other_hash+ contains an entry with the same key,
|
25
|
+
# it merges and returns the values from both arrays.
|
26
|
+
#
|
27
|
+
# h1 = {"a" => 100, "b" => 200, "c" => {"c1" => 12, "c2" => 14}}
|
28
|
+
# h2 = {"b" => 254, "c" => {"c1" => 16, "c3" => 94}}
|
29
|
+
# h1.rmerge!(h2) #=> {"a" => 100, "b" => 254, "c" => {"c1" => 16, "c2" => 14, "c3" => 94}}
|
30
|
+
#
|
31
|
+
# Simply using Hash#merge! would return
|
32
|
+
#
|
33
|
+
# h1.merge!(h2) #=> {"a" => 100, "b" = >254, "c" => {"c1" => 16, "c3" => 94}}
|
34
|
+
#
|
35
|
+
def rmerge!(other_hash)
|
36
|
+
merge!(other_hash) do |key, oldval, newval|
|
37
|
+
oldval.class == self.class ? oldval.rmerge!(newval) : newval
|
38
|
+
end
|
39
|
+
end
|
40
|
+
|
41
|
+
#
|
42
|
+
# Recursive version of Hash#merge
|
43
|
+
#
|
44
|
+
# Compared with Hash#merge!, this method supports nested hashes.
|
45
|
+
# When both +hsh+ and +other_hash+ contains an entry with the same key,
|
46
|
+
# it merges and returns the values from both arrays.
|
47
|
+
#
|
48
|
+
# Compared with Hash#merge, this method provides a different approch
|
49
|
+
# for merging nasted hashes.
|
50
|
+
# If the value of a given key is an Hash and both +other_hash+ abd +hsh
|
51
|
+
# includes the same key, the value is merged instead replaced with
|
52
|
+
# +other_hash+ value.
|
53
|
+
#
|
54
|
+
# h1 = {"a" => 100, "b" => 200, "c" => {"c1" => 12, "c2" => 14}}
|
55
|
+
# h2 = {"b" => 254, "c" => {"c1" => 16, "c3" => 94}}
|
56
|
+
# h1.rmerge(h2) #=> {"a" => 100, "b" => 254, "c" => {"c1" => 16, "c2" => 14, "c3" => 94}}
|
57
|
+
#
|
58
|
+
# Simply using Hash#merge would return
|
59
|
+
#
|
60
|
+
# h1.merge(h2) #=> {"a" => 100, "b" = >254, "c" => {"c1" => 16, "c3" => 94}}
|
61
|
+
#
|
62
|
+
def rmerge(other_hash)
|
63
|
+
r = {}
|
64
|
+
merge(other_hash) do |key, oldval, newval|
|
65
|
+
r[key] = oldval.class == self.class ? oldval.rmerge(newval) : newval
|
66
|
+
end
|
67
|
+
end
|
68
|
+
|
69
|
+
end
|
70
|
+
|
71
|
+
|
72
|
+
class Hash
|
73
|
+
include HashRecursiveMerge
|
74
|
+
end
|
@@ -0,0 +1,109 @@
|
|
1
|
+
require 'csv'
|
2
|
+
require 'net/http'
|
3
|
+
|
4
|
+
module Geocoder
|
5
|
+
module MaxmindDatabase
|
6
|
+
extend self
|
7
|
+
|
8
|
+
def download(package, dir = "tmp")
|
9
|
+
filepath = File.expand_path(File.join(dir, archive_filename(package)))
|
10
|
+
open(filepath, 'wb') do |file|
|
11
|
+
uri = URI.parse(archive_url(package))
|
12
|
+
Net::HTTP.start(uri.host, uri.port) do |http|
|
13
|
+
http.request_get(uri.path) do |resp|
|
14
|
+
# TODO: show progress
|
15
|
+
resp.read_body do |segment|
|
16
|
+
file.write(segment)
|
17
|
+
end
|
18
|
+
end
|
19
|
+
end
|
20
|
+
end
|
21
|
+
end
|
22
|
+
|
23
|
+
def insert(package, dir = "tmp")
|
24
|
+
data_files(package).each do |filepath,table|
|
25
|
+
print "Resetting table #{table}..."
|
26
|
+
ActiveRecord::Base.connection.execute("DELETE FROM #{table}")
|
27
|
+
puts "done"
|
28
|
+
insert_into_table(table, filepath)
|
29
|
+
end
|
30
|
+
end
|
31
|
+
|
32
|
+
def archive_filename(package)
|
33
|
+
p = archive_url_path(package)
|
34
|
+
s = !(pos = p.rindex('/')).nil? && pos + 1 || 0
|
35
|
+
p[s..-1]
|
36
|
+
end
|
37
|
+
|
38
|
+
private # -------------------------------------------------------------
|
39
|
+
|
40
|
+
def table_columns(table_name)
|
41
|
+
{
|
42
|
+
maxmind_geolite_city_blocks: %w[start_ip_num end_ip_num loc_id],
|
43
|
+
maxmind_geolite_city_location: %w[loc_id country region city postal_code latitude longitude metro_code area_code],
|
44
|
+
maxmind_geolite_country: %w[start_ip end_ip start_ip_num end_ip_num country_code country]
|
45
|
+
}[table_name.to_sym]
|
46
|
+
end
|
47
|
+
|
48
|
+
def insert_into_table(table, filepath)
|
49
|
+
start_time = Time.now
|
50
|
+
print "Loading data for table #{table}"
|
51
|
+
rows = []
|
52
|
+
columns = table_columns(table)
|
53
|
+
CSV.foreach(filepath, encoding: "ISO-8859-1") do |line|
|
54
|
+
# Some files have header rows.
|
55
|
+
# skip if starts with "Copyright" or "locId" or "startIpNum"
|
56
|
+
next if line.first.match(/[A-z]/)
|
57
|
+
rows << line.to_a
|
58
|
+
if rows.size == 10000
|
59
|
+
insert_rows(table, columns, rows)
|
60
|
+
rows = []
|
61
|
+
print "."
|
62
|
+
end
|
63
|
+
end
|
64
|
+
insert_rows(table, columns, rows) if rows.size > 0
|
65
|
+
puts "done (#{Time.now - start_time} seconds)"
|
66
|
+
end
|
67
|
+
|
68
|
+
def insert_rows(table, headers, rows)
|
69
|
+
value_strings = rows.map do |row|
|
70
|
+
"(" + row.map{ |col| sql_escaped_value(col) }.join(',') + ")"
|
71
|
+
end
|
72
|
+
q = "INSERT INTO #{table} (#{headers.join(',')}) " +
|
73
|
+
"VALUES #{value_strings.join(',')}"
|
74
|
+
ActiveRecord::Base.connection.execute(q)
|
75
|
+
end
|
76
|
+
|
77
|
+
def sql_escaped_value(value)
|
78
|
+
value.to_i.to_s == value ? value :
|
79
|
+
ActiveRecord::Base.connection.quote(value)
|
80
|
+
end
|
81
|
+
|
82
|
+
def data_files(package, dir = "tmp")
|
83
|
+
case package
|
84
|
+
when :geolite_city_csv
|
85
|
+
# use the last two in case multiple versions exist
|
86
|
+
files = Dir.glob(File.join(dir, "GeoLiteCity_*/*.csv"))[-2..-1].sort
|
87
|
+
Hash[*files.zip(["maxmind_geolite_city_blocks", "maxmind_geolite_city_location"]).flatten]
|
88
|
+
when :geolite_country_csv
|
89
|
+
{File.join(dir, "GeoIPCountryWhois.csv") => "maxmind_geolite_country"}
|
90
|
+
end
|
91
|
+
end
|
92
|
+
|
93
|
+
def archive_url(package)
|
94
|
+
base_url + archive_url_path(package)
|
95
|
+
end
|
96
|
+
|
97
|
+
def archive_url_path(package)
|
98
|
+
{
|
99
|
+
geolite_country_csv: "GeoIPCountryCSV.zip",
|
100
|
+
geolite_city_csv: "GeoLiteCity_CSV/GeoLiteCity-latest.zip",
|
101
|
+
geolite_asn_csv: "asnum/GeoIPASNum2.zip"
|
102
|
+
}[package]
|
103
|
+
end
|
104
|
+
|
105
|
+
def base_url
|
106
|
+
"http://geolite.maxmind.com/download/geoip/database/"
|
107
|
+
end
|
108
|
+
end
|
109
|
+
end
|
data/lib/oauth_util.rb
ADDED
@@ -0,0 +1,112 @@
|
|
1
|
+
# A utility for signing an url using OAuth in a way that's convenient for debugging
|
2
|
+
# Note: the standard Ruby OAuth lib is here http://github.com/mojodna/oauth
|
3
|
+
# Source: http://gist.github.com/383159
|
4
|
+
# License: http://gist.github.com/375593
|
5
|
+
# Usage: see example.rb below
|
6
|
+
#
|
7
|
+
# NOTE: This file has been modified from the original Gist:
|
8
|
+
#
|
9
|
+
# 1. Fix to prevent param-array conversion, as mentioned in Gist comment.
|
10
|
+
# 2. Query string escaping has been changed. See:
|
11
|
+
# https://github.com/alexreisner/geocoder/pull/360
|
12
|
+
#
|
13
|
+
|
14
|
+
require 'uri'
|
15
|
+
require 'cgi'
|
16
|
+
require 'openssl'
|
17
|
+
require 'base64'
|
18
|
+
|
19
|
+
class OauthUtil
|
20
|
+
|
21
|
+
attr_accessor :consumer_key, :consumer_secret, :token, :token_secret, :req_method,
|
22
|
+
:sig_method, :oauth_version, :callback_url, :params, :req_url, :base_str
|
23
|
+
|
24
|
+
def initialize
|
25
|
+
@consumer_key = ''
|
26
|
+
@consumer_secret = ''
|
27
|
+
@token = ''
|
28
|
+
@token_secret = ''
|
29
|
+
@req_method = 'GET'
|
30
|
+
@sig_method = 'HMAC-SHA1'
|
31
|
+
@oauth_version = '1.0'
|
32
|
+
@callback_url = ''
|
33
|
+
end
|
34
|
+
|
35
|
+
# openssl::random_bytes returns non-word chars, which need to be removed. using alt method to get length
|
36
|
+
# ref http://snippets.dzone.com/posts/show/491
|
37
|
+
def nonce
|
38
|
+
Array.new( 5 ) { rand(256) }.pack('C*').unpack('H*').first
|
39
|
+
end
|
40
|
+
|
41
|
+
def percent_encode( string )
|
42
|
+
|
43
|
+
# ref http://snippets.dzone.com/posts/show/1260
|
44
|
+
return URI.escape( string, Regexp.new("[^#{URI::PATTERN::UNRESERVED}]") ).gsub('*', '%2A')
|
45
|
+
end
|
46
|
+
|
47
|
+
# @ref http://oauth.net/core/1.0/#rfc.section.9.2
|
48
|
+
def signature
|
49
|
+
key = percent_encode( @consumer_secret ) + '&' + percent_encode( @token_secret )
|
50
|
+
|
51
|
+
# ref: http://blog.nathanielbibler.com/post/63031273/openssl-hmac-vs-ruby-hmac-benchmarks
|
52
|
+
digest = OpenSSL::Digest.new( 'sha1' )
|
53
|
+
hmac = OpenSSL::HMAC.digest( digest, key, @base_str )
|
54
|
+
|
55
|
+
# ref http://groups.google.com/group/oauth-ruby/browse_thread/thread/9110ed8c8f3cae81
|
56
|
+
Base64.encode64( hmac ).chomp.gsub( /\n/, '' )
|
57
|
+
end
|
58
|
+
|
59
|
+
# sort (very important as it affects the signature), concat, and percent encode
|
60
|
+
# @ref http://oauth.net/core/1.0/#rfc.section.9.1.1
|
61
|
+
# @ref http://oauth.net/core/1.0/#9.2.1
|
62
|
+
# @ref http://oauth.net/core/1.0/#rfc.section.A.5.1
|
63
|
+
def query_string
|
64
|
+
pairs = []
|
65
|
+
@params.sort.each { | key, val |
|
66
|
+
pairs.push( "#{ CGI.escape(key.to_s).gsub(/%(5B|5D)/n) { [$1].pack('H*') } }=#{ CGI.escape(val.to_s) }" )
|
67
|
+
}
|
68
|
+
pairs.join '&'
|
69
|
+
end
|
70
|
+
|
71
|
+
# organize params & create signature
|
72
|
+
def sign( parsed_url )
|
73
|
+
|
74
|
+
@params = {
|
75
|
+
'oauth_consumer_key' => @consumer_key,
|
76
|
+
'oauth_nonce' => nonce,
|
77
|
+
'oauth_signature_method' => @sig_method,
|
78
|
+
'oauth_timestamp' => Time.now.to_i.to_s,
|
79
|
+
'oauth_version' => @oauth_version
|
80
|
+
}
|
81
|
+
|
82
|
+
# if url has query, merge key/values into params obj overwriting defaults
|
83
|
+
if parsed_url.query
|
84
|
+
CGI.parse( parsed_url.query ).each do |k,v|
|
85
|
+
if v.is_a?(Array) && v.count == 1
|
86
|
+
@params[k] = v.first
|
87
|
+
else
|
88
|
+
@params[k] = v
|
89
|
+
end
|
90
|
+
end
|
91
|
+
end
|
92
|
+
|
93
|
+
# @ref http://oauth.net/core/1.0/#rfc.section.9.1.2
|
94
|
+
@req_url = parsed_url.scheme + '://' + parsed_url.host + parsed_url.path
|
95
|
+
|
96
|
+
# create base str. make it an object attr for ez debugging
|
97
|
+
# ref http://oauth.net/core/1.0/#anchor14
|
98
|
+
@base_str = [
|
99
|
+
@req_method,
|
100
|
+
percent_encode( req_url ),
|
101
|
+
|
102
|
+
# normalization is just x-www-form-urlencoded
|
103
|
+
percent_encode( query_string )
|
104
|
+
|
105
|
+
].join( '&' )
|
106
|
+
|
107
|
+
# add signature
|
108
|
+
@params[ 'oauth_signature' ] = signature
|
109
|
+
|
110
|
+
return self
|
111
|
+
end
|
112
|
+
end
|
@@ -0,0 +1,29 @@
|
|
1
|
+
namespace :geocode do
|
2
|
+
desc "Geocode all objects without coordinates."
|
3
|
+
task :all => :environment do
|
4
|
+
class_name = ENV['CLASS'] || ENV['class']
|
5
|
+
sleep_timer = ENV['SLEEP'] || ENV['sleep']
|
6
|
+
batch = ENV['BATCH'] || ENV['batch']
|
7
|
+
raise "Please specify a CLASS (model)" unless class_name
|
8
|
+
klass = class_from_string(class_name)
|
9
|
+
batch = batch.to_i unless batch.nil?
|
10
|
+
|
11
|
+
klass.not_geocoded.find_each(batch_size: batch) do |obj|
|
12
|
+
obj.geocode; obj.save
|
13
|
+
sleep(sleep_timer.to_f) unless sleep_timer.nil?
|
14
|
+
end
|
15
|
+
end
|
16
|
+
end
|
17
|
+
|
18
|
+
##
|
19
|
+
# Get a class object from the string given in the shell environment.
|
20
|
+
# Similar to ActiveSupport's +constantize+ method.
|
21
|
+
#
|
22
|
+
def class_from_string(class_name)
|
23
|
+
parts = class_name.split("::")
|
24
|
+
constant = Object
|
25
|
+
parts.each do |part|
|
26
|
+
constant = constant.const_get(part)
|
27
|
+
end
|
28
|
+
constant
|
29
|
+
end
|
@@ -0,0 +1,73 @@
|
|
1
|
+
require 'maxmind_database'
|
2
|
+
|
3
|
+
namespace :geocoder do
|
4
|
+
namespace :maxmind do
|
5
|
+
namespace :geolite do
|
6
|
+
|
7
|
+
desc "Download and load/refresh MaxMind GeoLite City data"
|
8
|
+
task load: [:download, :extract, :insert]
|
9
|
+
|
10
|
+
desc "Download MaxMind GeoLite City data"
|
11
|
+
task :download do
|
12
|
+
p = MaxmindTask.check_for_package!
|
13
|
+
MaxmindTask.download!(p, dir: ENV['DIR'] || "tmp/")
|
14
|
+
end
|
15
|
+
|
16
|
+
desc "Extract (unzip) MaxMind GeoLite City data"
|
17
|
+
task :extract do
|
18
|
+
p = MaxmindTask.check_for_package!
|
19
|
+
MaxmindTask.extract!(p, dir: ENV['DIR'] || "tmp/")
|
20
|
+
end
|
21
|
+
|
22
|
+
desc "Load/refresh MaxMind GeoLite City data"
|
23
|
+
task insert: [:environment] do
|
24
|
+
p = MaxmindTask.check_for_package!
|
25
|
+
MaxmindTask.insert!(p, dir: ENV['DIR'] || "tmp/")
|
26
|
+
end
|
27
|
+
end
|
28
|
+
end
|
29
|
+
end
|
30
|
+
|
31
|
+
module MaxmindTask
|
32
|
+
extend self
|
33
|
+
|
34
|
+
def check_for_package!
|
35
|
+
if %w[city country].include?(p = ENV['PACKAGE'])
|
36
|
+
return p
|
37
|
+
else
|
38
|
+
puts "Please specify PACKAGE=city or PACKAGE=country"
|
39
|
+
exit
|
40
|
+
end
|
41
|
+
end
|
42
|
+
|
43
|
+
def download!(package, options = {})
|
44
|
+
p = "geolite_#{package}_csv".intern
|
45
|
+
Geocoder::MaxmindDatabase.download(p, options[:dir])
|
46
|
+
end
|
47
|
+
|
48
|
+
def extract!(package, options = {})
|
49
|
+
begin
|
50
|
+
require 'zip'
|
51
|
+
rescue LoadError
|
52
|
+
puts "Please install gem: rubyzip (>= 1.0.0)"
|
53
|
+
exit
|
54
|
+
end
|
55
|
+
require 'fileutils'
|
56
|
+
p = "geolite_#{package}_csv".intern
|
57
|
+
archive_filename = Geocoder::MaxmindDatabase.archive_filename(p)
|
58
|
+
Zip::File.open(File.join(options[:dir], archive_filename)).each do |entry|
|
59
|
+
filepath = File.join(options[:dir], entry.name)
|
60
|
+
if File.exist? filepath
|
61
|
+
warn "File already exists (#{entry.name}), skipping"
|
62
|
+
else
|
63
|
+
FileUtils.mkdir_p(File.dirname(filepath))
|
64
|
+
entry.extract(filepath)
|
65
|
+
end
|
66
|
+
end
|
67
|
+
end
|
68
|
+
|
69
|
+
def insert!(package, options = {})
|
70
|
+
p = "geolite_#{package}_csv".intern
|
71
|
+
Geocoder::MaxmindDatabase.insert(p, options[:dir])
|
72
|
+
end
|
73
|
+
end
|
@@ -0,0 +1 @@
|
|
1
|
+
{"results":[],"status":5,"msg":"AK Illegal or Not Exist:"}
|
@@ -0,0 +1,19 @@
|
|
1
|
+
{
|
2
|
+
"address": "CN|北京|北京|None|CHINANET|1|None",
|
3
|
+
"content": {
|
4
|
+
"address": "北京市",
|
5
|
+
"address_detail": {
|
6
|
+
"city": "北京市",
|
7
|
+
"city_code": 131,
|
8
|
+
"district": "",
|
9
|
+
"province": "北京市",
|
10
|
+
"street": "",
|
11
|
+
"street_number": ""
|
12
|
+
},
|
13
|
+
"point": {
|
14
|
+
"x": "116.39564504",
|
15
|
+
"y": "39.92998578"
|
16
|
+
}
|
17
|
+
},
|
18
|
+
"status": 0
|
19
|
+
}
|
@@ -0,0 +1 @@
|
|
1
|
+
{"status":5,"uid":null,"sk":null,"logformat":null}
|
@@ -0,0 +1 @@
|
|
1
|
+
{"status":0, "content":{}}
|
@@ -0,0 +1 @@
|
|
1
|
+
{"status":0,"result":[]}
|
@@ -0,0 +1 @@
|
|
1
|
+
{"status":0,"result":{"location":{"lng":121.48789948569,"lat":31.249161555654},"formatted_address":"上海市闸北区天潼路619号","business":"七浦路,海宁路,北京东路","addressComponent":{"city":"上海市","district":"闸北区","province":"上海市","street":"天潼路","street_number":"619号"},"cityCode":289}}
|
@@ -0,0 +1 @@
|
|
1
|
+
{"authenticationResultCode":"InvalidCredentials","brandLogoUri":"http:\\/\\/dev.virtualearth.net\\/Branding\\/logo_powered_by.png","copyright":"Copyright \xC2\xA9 2012 Microsoft and its suppliers. All rights reserved. This API cannot be accessed and the content and any results may not be used, reproduced or transmitted in any manner without express written permission from Microsoft Corporation.","errorDetails":["Access was denied. You may have entered your credentials incorrectly, or you might not have access to the requested resource or operation."],"resourceSets":[],"statusCode":401,"statusDescription":"Unauthorized","traceId":"5c539f6e70c44b2e858741b6c932318e|EWRM001670|02.00.83.1900|"}
|
@@ -0,0 +1,40 @@
|
|
1
|
+
{
|
2
|
+
"authenticationResultCode":"ValidCredentials",
|
3
|
+
"brandLogoUri":"http:\/\/dev.virtualearth.net\/Branding\/logo_powered_by.png",
|
4
|
+
"copyright":"Copyright © 2011 Microsoft and its suppliers. All rights reserved. This API cannot be accessed and the content and any results may not be used, reproduced or transmitted in any manner without express written permission from Microsoft Corporation.",
|
5
|
+
"resourceSets":[
|
6
|
+
{
|
7
|
+
"estimatedTotal":1,
|
8
|
+
"resources":[
|
9
|
+
{
|
10
|
+
"__type":"Location:http:\/\/schemas.microsoft.com\/search\/local\/ws\/rest\/v1",
|
11
|
+
"bbox":[
|
12
|
+
40.744944289326668,
|
13
|
+
-74.002353921532631,
|
14
|
+
40.755675807595253,
|
15
|
+
-73.983625397086143
|
16
|
+
],
|
17
|
+
"name":"Madison Square Garden, NY",
|
18
|
+
"point":{
|
19
|
+
"type":"Point",
|
20
|
+
"coordinates":[
|
21
|
+
40.75031,
|
22
|
+
-73.99299
|
23
|
+
]
|
24
|
+
},
|
25
|
+
"address":{
|
26
|
+
"adminDistrict":"NY",
|
27
|
+
"countryRegion":"United States",
|
28
|
+
"formattedAddress":"Madison Square Garden, NY",
|
29
|
+
"locality":"New York"
|
30
|
+
},
|
31
|
+
"confidence":"High",
|
32
|
+
"entityType":"Stadium"
|
33
|
+
}
|
34
|
+
]
|
35
|
+
}
|
36
|
+
],
|
37
|
+
"statusCode":200,
|
38
|
+
"statusDescription":"OK",
|
39
|
+
"traceId":"55094ee53c8d45e789794014666328cd|CH1M001466|02.00.82.2800|CH1MSNVM001396, CH1MSNVM001370, CH1MSNVM001397"
|
40
|
+
}
|
@@ -0,0 +1,16 @@
|
|
1
|
+
{
|
2
|
+
"authenticationResultCode":"ValidCredentials",
|
3
|
+
"brandLogoUri":"http:\/\/dev.virtualearth.net\/Branding\/logo_powered_by.png",
|
4
|
+
"copyright":"Copyright © 2011 Microsoft and its suppliers. All rights reserved. This API cannot be accessed and the content and any results may not be used, reproduced or transmitted in any manner without express written permission from Microsoft Corporation.",
|
5
|
+
"resourceSets":[
|
6
|
+
{
|
7
|
+
"estimatedTotal":0,
|
8
|
+
"resources":[
|
9
|
+
|
10
|
+
]
|
11
|
+
}
|
12
|
+
],
|
13
|
+
"statusCode":200,
|
14
|
+
"statusDescription":"OK",
|
15
|
+
"traceId":"907b76a307bc49129a489de3d4c992ea|CH1M001463|02.00.82.2800|CH1MSNVM001383, CH1MSNVM001358, CH1MSNVM001397"
|
16
|
+
}
|
@@ -0,0 +1,42 @@
|
|
1
|
+
{
|
2
|
+
"authenticationResultCode":"ValidCredentials",
|
3
|
+
"brandLogoUri":"http:\/\/dev.virtualearth.net\/Branding\/logo_powered_by.png",
|
4
|
+
"copyright":"Copyright © 2011 Microsoft and its suppliers. All rights reserved. This API cannot be accessed and the content and any results may not be used, reproduced or transmitted in any manner without express written permission from Microsoft Corporation.",
|
5
|
+
"resourceSets":[
|
6
|
+
{
|
7
|
+
"estimatedTotal":1,
|
8
|
+
"resources":[
|
9
|
+
{
|
10
|
+
"__type":"Location:http:\/\/schemas.microsoft.com\/search\/local\/ws\/rest\/v1",
|
11
|
+
"bbox":[
|
12
|
+
45.419835111675845,
|
13
|
+
-75.683656128790716,
|
14
|
+
45.4275605468172,
|
15
|
+
-75.66898098334994
|
16
|
+
],
|
17
|
+
"name":"291 Rue Somerset E, Ottawa, ON, K1N",
|
18
|
+
"point":{
|
19
|
+
"type":"Point",
|
20
|
+
"coordinates":[
|
21
|
+
45.423697829246521,
|
22
|
+
-75.676318556070328
|
23
|
+
]
|
24
|
+
},
|
25
|
+
"address":{
|
26
|
+
"addressLine":"291 Rue Somerset E",
|
27
|
+
"adminDistrict":"ON",
|
28
|
+
"countryRegion":"Canada",
|
29
|
+
"formattedAddress":"291 Rue Somerset E, Ottawa, ON, K1N",
|
30
|
+
"locality":"Ottawa",
|
31
|
+
"postalCode":"K1N"
|
32
|
+
},
|
33
|
+
"confidence":"Medium",
|
34
|
+
"entityType":"Address"
|
35
|
+
}
|
36
|
+
]
|
37
|
+
}
|
38
|
+
],
|
39
|
+
"statusCode":200,
|
40
|
+
"statusDescription":"OK",
|
41
|
+
"traceId":"27bd5ed659e64ba6970c4144f1d4ea94|CH1M001470|02.00.82.2800|CH1MSNVM001396, CH1MSNVM001374"
|
42
|
+
}
|
@@ -0,0 +1 @@
|
|
1
|
+
Forbidden request
|
@@ -0,0 +1 @@
|
|
1
|
+
{"found": 2, "bounds": [[40.74983, -73.99433], [40.75116, -73.99266]], "features": [{"id": 32891803,"centroid": {"type":"POINT","coordinates":[40.75111, -73.99345]},"bounds": [[40.74983, -73.99433], [40.75116, -73.99266]],"properties": {"osm_element": "way", "sport": "hockey;basketball;lacrosse", "name": "Madison Square Garden", "leisure": "stadium", "osm_id": "24801588"},"location": {"county": "New York", "country": "United States of America", "postcode": "10119", "road": "West 31st Street", "city": "New York"},"type": "Feature"},{"id": 12977552,"centroid": {"type":"POINT","coordinates":[40.75066, -73.99347]},"bounds": [[40.75066, -73.99347], [40.75066, -73.99347]],"properties": {"building": "yes", "osm_element": "node", "name": "Madison Square Garden Center", "addr:state": "NY", "osm_id": "368045579"},"location": {"county": "New York", "country": "United States of America", "postcode": "10119", "road": "West 33rd Street", "city": "New York"},"type": "Feature"}], "type": "FeatureCollection", "crs": {"type": "EPSG", "properties": {"code": 4326, "coordinate_order": [0, 1]}}}
|
@@ -0,0 +1 @@
|
|
1
|
+
{}
|
@@ -0,0 +1,59 @@
|
|
1
|
+
{
|
2
|
+
"spatialReference": {
|
3
|
+
"wkid": 4326,
|
4
|
+
"latestWkid": 4326
|
5
|
+
},
|
6
|
+
"locations": [
|
7
|
+
{
|
8
|
+
"name": "Madison Square Garden",
|
9
|
+
"extent": {
|
10
|
+
"xmin": -74.000241000000003,
|
11
|
+
"ymin": 40.744050000000001,
|
12
|
+
"xmax": -73.988241000000002,
|
13
|
+
"ymax": 40.756050000000002
|
14
|
+
},
|
15
|
+
"feature": {
|
16
|
+
"geometry": {
|
17
|
+
"x": -73.994238897999651,
|
18
|
+
"y": 40.750049813000487
|
19
|
+
},
|
20
|
+
"attributes": {
|
21
|
+
"Loc_name": "Gaz.WorldGazetteer.POI2",
|
22
|
+
"Score": 100,
|
23
|
+
"Match_addr": "Madison Square Garden",
|
24
|
+
"Addr_type": "POI",
|
25
|
+
"Type": "Sports Complex",
|
26
|
+
"PlaceName": "Madison Square Garden",
|
27
|
+
"Rank": "18",
|
28
|
+
"AddBldg": "",
|
29
|
+
"AddNum": "",
|
30
|
+
"AddNumFrom": "",
|
31
|
+
"AddNumTo": "",
|
32
|
+
"Side": "",
|
33
|
+
"StPreDir": "",
|
34
|
+
"StPreType": "",
|
35
|
+
"StName": "",
|
36
|
+
"StType": "",
|
37
|
+
"StDir": "",
|
38
|
+
"Nbrhd": "",
|
39
|
+
"City": "New York",
|
40
|
+
"Subregion": "New York",
|
41
|
+
"Region": "New York",
|
42
|
+
"Postal": "10001",
|
43
|
+
"PostalExt": "",
|
44
|
+
"Country": "USA",
|
45
|
+
"LangCode": "",
|
46
|
+
"Distance": 0,
|
47
|
+
"X": -73.994240000000005,
|
48
|
+
"Y": 40.750050000000002,
|
49
|
+
"DisplayX": -73.994240000000005,
|
50
|
+
"DisplayY": 40.750050000000002,
|
51
|
+
"Xmin": -74.000241000000003,
|
52
|
+
"Xmax": -73.988241000000002,
|
53
|
+
"Ymin": 40.744050000000001,
|
54
|
+
"Ymax": 40.756050000000002
|
55
|
+
}
|
56
|
+
}
|
57
|
+
}
|
58
|
+
]
|
59
|
+
}
|
@@ -0,0 +1,21 @@
|
|
1
|
+
{
|
2
|
+
"address": {
|
3
|
+
"Address": "4 Avenue Gustave Eiffel",
|
4
|
+
"Neighborhood": "7e Arrondissement",
|
5
|
+
"City": "Paris",
|
6
|
+
"Subregion": "Paris",
|
7
|
+
"Region": "Île-de-France",
|
8
|
+
"Postal": "75007",
|
9
|
+
"PostalExt": null,
|
10
|
+
"CountryCode": "FRA",
|
11
|
+
"Loc_name": "FRA.PointAddress"
|
12
|
+
},
|
13
|
+
"location": {
|
14
|
+
"x": 2.2956200048981574,
|
15
|
+
"y": 48.858129997357558,
|
16
|
+
"spatialReference": {
|
17
|
+
"wkid": 4326,
|
18
|
+
"latestWkid": 4326
|
19
|
+
}
|
20
|
+
}
|
21
|
+
}
|
@@ -0,0 +1 @@
|
|
1
|
+
<html><title>404: Not Found</title><body>404: Not Found</body></html>
|