publicstorage 0.1.1 → 0.2.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/README.md +11 -15
- data/exe/publicstorage +7 -0
- data/lib/publicstorage/address.rb +6 -1
- data/lib/publicstorage/cli.rb +58 -0
- data/lib/publicstorage/crawler.rb +1 -1
- data/lib/publicstorage/dimensions.rb +6 -1
- data/lib/publicstorage/facility.rb +67 -24
- data/lib/publicstorage/geocode.rb +6 -1
- data/lib/publicstorage/price.rb +6 -1
- data/lib/publicstorage/rates.rb +6 -1
- data/lib/publicstorage/sitemap.rb +3 -1
- data/lib/publicstorage/version.rb +1 -1
- data/lib/publicstorage.rb +1 -0
- metadata +9 -6
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 5f5319690995d8d7f216e02fa0ead21f9c3bfc136e2e9ed8d46333f840909474
|
4
|
+
data.tar.gz: f0da92039ca2c9dd5dbf1256a5e3df2cffa304763ee766229e0175bfcf817047
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 723e933f0da23b57b731e9123a7a412eb9984e7f54897a77304d1f0753cc274f1ffbf68711b59d25e4f9283f38ca4e3ee0aa8ffc9e606cdf26f0a92d4581e634
|
7
|
+
data.tar.gz: 5149248cc1fa272c78cc9ac8666bfe52501b23228a2dba66533c94c25ca5294fc9e31717fae635ea46361253428197c0007f1f577fca0e34cde15f49021bb4bf
|
data/README.md
CHANGED
@@ -20,24 +20,20 @@ require 'publicstorage'
|
|
20
20
|
sitemap = PublicStorage::Facility.sitemap
|
21
21
|
sitemap.links.each do |link|
|
22
22
|
url = link.loc
|
23
|
+
facility = ExtraSpace::Facility.fetch(url:)
|
23
24
|
|
24
|
-
facility
|
25
|
-
|
26
|
-
puts "Street: #{facility.address.street}"
|
27
|
-
puts "City: #{facility.address.city}"
|
28
|
-
puts "State: #{facility.address.state}"
|
29
|
-
puts "ZIP: #{facility.address.zip}"
|
30
|
-
puts "Latitude: #{facility.geocode.latitude}"
|
31
|
-
puts "Longitude: #{facility.geocode.longitude}"
|
32
|
-
puts
|
25
|
+
puts facility.text
|
33
26
|
|
34
27
|
facility.prices.each do |price|
|
35
|
-
puts
|
36
|
-
puts "Width: #{price.dimensions.width}"
|
37
|
-
puts "Depth: #{price.dimensions.depth}"
|
38
|
-
puts "SQFT: #{price.dimensions.sqft}"
|
39
|
-
puts "Rates: $#{price.rates.street} (street) / $#{price.rates.web} (web)"
|
40
|
-
puts
|
28
|
+
puts price.text
|
41
29
|
end
|
30
|
+
|
31
|
+
puts
|
42
32
|
end
|
43
33
|
```
|
34
|
+
|
35
|
+
## CLI
|
36
|
+
|
37
|
+
```bash
|
38
|
+
publicstorage crawl
|
39
|
+
```
|
data/exe/publicstorage
ADDED
@@ -1,7 +1,7 @@
|
|
1
1
|
# frozen_string_literal: true
|
2
2
|
|
3
3
|
module PublicStorage
|
4
|
-
#
|
4
|
+
# The address (street + city + state + zip) of a facility.
|
5
5
|
class Address
|
6
6
|
# @attribute [rw] street
|
7
7
|
# @return [String]
|
@@ -41,6 +41,11 @@ module PublicStorage
|
|
41
41
|
"#<#{self.class.name} #{props.join(' ')}>"
|
42
42
|
end
|
43
43
|
|
44
|
+
# @return [String]
|
45
|
+
def text
|
46
|
+
"#{street}, #{city}, #{state} #{zip}"
|
47
|
+
end
|
48
|
+
|
44
49
|
# @param data [Hash]
|
45
50
|
#
|
46
51
|
# @return [Address]
|
@@ -0,0 +1,58 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
require 'optparse'
|
4
|
+
|
5
|
+
module PublicStorage
|
6
|
+
# Used when interacting with the library from the command line interface (CLI).
|
7
|
+
#
|
8
|
+
# Usage:
|
9
|
+
#
|
10
|
+
# cli = PublicStorage::CLI.new
|
11
|
+
# cli.parse
|
12
|
+
class CLI
|
13
|
+
module Code
|
14
|
+
OK = 0
|
15
|
+
ERROR = 1
|
16
|
+
end
|
17
|
+
|
18
|
+
# @param argv [Array<String>]
|
19
|
+
def parse(argv = ARGV)
|
20
|
+
parser.parse!(argv)
|
21
|
+
command = argv.shift
|
22
|
+
|
23
|
+
case command
|
24
|
+
when 'crawl' then crawl
|
25
|
+
else
|
26
|
+
warn("unsupported command=#{command.inspect}")
|
27
|
+
exit(Code::ERROR)
|
28
|
+
end
|
29
|
+
end
|
30
|
+
|
31
|
+
private
|
32
|
+
|
33
|
+
def crawl
|
34
|
+
PublicStorage::Facility.crawl
|
35
|
+
exit(Code::OK)
|
36
|
+
end
|
37
|
+
|
38
|
+
def help(options)
|
39
|
+
puts(options)
|
40
|
+
exit(Code::OK)
|
41
|
+
end
|
42
|
+
|
43
|
+
def version
|
44
|
+
puts(VERSION)
|
45
|
+
exit(Code::OK)
|
46
|
+
end
|
47
|
+
|
48
|
+
# @return [OptionParser]
|
49
|
+
def parser
|
50
|
+
OptionParser.new do |options|
|
51
|
+
options.banner = 'usage: PublicStorage [options] <command> [<args>]'
|
52
|
+
|
53
|
+
options.on('-h', '--help', 'help') { help(options) }
|
54
|
+
options.on('-v', '--version', 'version') { version }
|
55
|
+
end
|
56
|
+
end
|
57
|
+
end
|
58
|
+
end
|
@@ -30,7 +30,7 @@ module PublicStorage
|
|
30
30
|
# @return [HTTP::Response]
|
31
31
|
def fetch(url:)
|
32
32
|
response = HTTP.get(url)
|
33
|
-
raise FetchError(url:, response: response.flush) unless response.status.ok?
|
33
|
+
raise FetchError.new(url:, response: response.flush) unless response.status.ok?
|
34
34
|
|
35
35
|
response
|
36
36
|
end
|
@@ -1,7 +1,7 @@
|
|
1
1
|
# frozen_string_literal: true
|
2
2
|
|
3
3
|
module PublicStorage
|
4
|
-
# The dimensions
|
4
|
+
# The dimensions (width + depth + sqft) of a price.
|
5
5
|
class Dimensions
|
6
6
|
SELECTOR = '.unit-size'
|
7
7
|
|
@@ -36,6 +36,11 @@ module PublicStorage
|
|
36
36
|
"#<#{self.class.name} #{props.join(' ')}>"
|
37
37
|
end
|
38
38
|
|
39
|
+
# @return [String] e.g. "10' × 10' (100 sqft)"
|
40
|
+
def text
|
41
|
+
"#{format('%g', @width)}' × #{format('%g', @depth)}' (#{@sqft} sqft)"
|
42
|
+
end
|
43
|
+
|
39
44
|
# @param element [Nokogiri::XML::Element]
|
40
45
|
#
|
41
46
|
# @return [Dimensions]
|
@@ -6,6 +6,17 @@ module PublicStorage
|
|
6
6
|
SITEMAP_URL = 'https://www.publicstorage.com/sitemap_0-product.xml'
|
7
7
|
|
8
8
|
PRICE_SELECTOR = '.units-results-section .unit-list-group .unit-list-item'
|
9
|
+
LD_SELECTOR = 'script[type="application/ld+json"]'
|
10
|
+
|
11
|
+
ID_REGEX = /(?<id>\d+)\.html/
|
12
|
+
|
13
|
+
# @attribute [rw] id
|
14
|
+
# @return [Integer]
|
15
|
+
attr_accessor :id
|
16
|
+
|
17
|
+
# @attribute [rw] name
|
18
|
+
# @return [String]
|
19
|
+
attr_accessor :name
|
9
20
|
|
10
21
|
# @attribute [rw] address
|
11
22
|
# @return [Address]
|
@@ -19,25 +30,6 @@ module PublicStorage
|
|
19
30
|
# @return [Array<Price>]
|
20
31
|
attr_accessor :prices
|
21
32
|
|
22
|
-
# @param address [Address]
|
23
|
-
# @param geocode [Geocode]
|
24
|
-
# @param prices [Array<Price>]
|
25
|
-
def initialize(address:, geocode:, prices:)
|
26
|
-
@address = address
|
27
|
-
@geocode = geocode
|
28
|
-
@prices = prices
|
29
|
-
end
|
30
|
-
|
31
|
-
# @return [String]
|
32
|
-
def inspect
|
33
|
-
props = [
|
34
|
-
"address=#{@address.inspect}",
|
35
|
-
"geocode=#{@geocode.inspect}",
|
36
|
-
"prices=#{@prices.inspect}"
|
37
|
-
]
|
38
|
-
"#<#{self.class.name} #{props.join(' ')}>"
|
39
|
-
end
|
40
|
-
|
41
33
|
# @return [Sitemap]
|
42
34
|
def self.sitemap
|
43
35
|
Sitemap.fetch(url: SITEMAP_URL)
|
@@ -55,14 +47,65 @@ module PublicStorage
|
|
55
47
|
#
|
56
48
|
# @return [Facility]
|
57
49
|
def self.parse(document:)
|
58
|
-
data =
|
59
|
-
|
60
|
-
|
61
|
-
|
50
|
+
data = parse_ld(document:)
|
51
|
+
id = Integer(data['url'].match(ID_REGEX)[:id])
|
52
|
+
name = data['name']
|
53
|
+
address = Address.parse(data: data['address'])
|
54
|
+
geocode = Geocode.parse(data: data['geo'])
|
62
55
|
|
63
56
|
prices = document.css(PRICE_SELECTOR).map { |element| Price.parse(element:) }
|
64
57
|
|
65
|
-
new(address:, geocode:, prices:)
|
58
|
+
new(id:, name:, address:, geocode:, prices:)
|
59
|
+
end
|
60
|
+
|
61
|
+
# @param document [NokoGiri::XML::Document]
|
62
|
+
#
|
63
|
+
# @return [Hash]
|
64
|
+
def self.parse_ld(document:)
|
65
|
+
JSON.parse(document.at_css(LD_SELECTOR).text).find { |entry| entry['@type'] == 'SelfStorage' }
|
66
|
+
end
|
67
|
+
|
68
|
+
def self.crawl
|
69
|
+
sitemap.links.each do |link|
|
70
|
+
url = link.loc
|
71
|
+
|
72
|
+
facility = fetch(url:)
|
73
|
+
puts facility.text
|
74
|
+
|
75
|
+
facility.prices.each do |price|
|
76
|
+
puts price.text
|
77
|
+
end
|
78
|
+
|
79
|
+
puts
|
80
|
+
end
|
81
|
+
end
|
82
|
+
|
83
|
+
# @param id [String]
|
84
|
+
# @param name [String]
|
85
|
+
# @param address [Address]
|
86
|
+
# @param geocode [Geocode]
|
87
|
+
# @param prices [Array<Price>]
|
88
|
+
def initialize(id:, name:, address:, geocode:, prices:)
|
89
|
+
@id = id
|
90
|
+
@name = name
|
91
|
+
@address = address
|
92
|
+
@geocode = geocode
|
93
|
+
@prices = prices
|
94
|
+
end
|
95
|
+
|
96
|
+
# @return [String]
|
97
|
+
def inspect
|
98
|
+
props = [
|
99
|
+
"address=#{@address.inspect}",
|
100
|
+
"geocode=#{@geocode.inspect}",
|
101
|
+
"prices=#{@prices.inspect}"
|
102
|
+
]
|
103
|
+
"#<#{self.class.name} #{props.join(' ')}>"
|
104
|
+
end
|
105
|
+
|
106
|
+
# @return [String]
|
107
|
+
def text
|
108
|
+
"#{@id} | #{@name} | #{@address.text} | #{@geocode.text}"
|
66
109
|
end
|
67
110
|
end
|
68
111
|
end
|
@@ -1,7 +1,7 @@
|
|
1
1
|
# frozen_string_literal: true
|
2
2
|
|
3
3
|
module PublicStorage
|
4
|
-
#
|
4
|
+
# The geocode (latitude + longitude) of a facility.
|
5
5
|
class Geocode
|
6
6
|
# @attribute [rw] latitude
|
7
7
|
# @return [Float]
|
@@ -27,6 +27,11 @@ module PublicStorage
|
|
27
27
|
"#<#{self.class.name} #{props.join(' ')}>"
|
28
28
|
end
|
29
29
|
|
30
|
+
# @return [String]
|
31
|
+
def text
|
32
|
+
"#{@latitude},#{@longitude}"
|
33
|
+
end
|
34
|
+
|
30
35
|
# @param data [Hash]
|
31
36
|
#
|
32
37
|
# @return [Geocode]
|
data/lib/publicstorage/price.rb
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
# frozen_string_literal: true
|
2
2
|
|
3
3
|
module PublicStorage
|
4
|
-
#
|
4
|
+
# The price (id + dimensions + rate) for a facility
|
5
5
|
class Price
|
6
6
|
# @attribute [rw] id
|
7
7
|
# @return [String]
|
@@ -34,6 +34,11 @@ module PublicStorage
|
|
34
34
|
"#<#{self.class.name} #{props.join(' ')}>"
|
35
35
|
end
|
36
36
|
|
37
|
+
# @return [String] e.g. "123 | 5' × 5' (25 sqft) | $100 (street) / $90 (web)"
|
38
|
+
def text
|
39
|
+
"#{@id} | #{@dimensions.text} | #{@rates.text}"
|
40
|
+
end
|
41
|
+
|
37
42
|
# @param element [Nokogiri::XML::Element]
|
38
43
|
#
|
39
44
|
# @return [Price]
|
data/lib/publicstorage/rates.rb
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
# frozen_string_literal: true
|
2
2
|
|
3
3
|
module PublicStorage
|
4
|
-
# The rates
|
4
|
+
# The rates (street + web) for a facility
|
5
5
|
class Rates
|
6
6
|
STREET_SELECTOR = '.unit-prices .unit-pricing .unit-strike-through-price'
|
7
7
|
WEB_SELECTOR = '.unit-prices .unit-pricing .unit-price'
|
@@ -30,6 +30,11 @@ module PublicStorage
|
|
30
30
|
"#<#{self.class.name} #{props.join(' ')}>"
|
31
31
|
end
|
32
32
|
|
33
|
+
# @return [String] e.g. "$80 (street) | $60 (web)"
|
34
|
+
def text
|
35
|
+
"$#{@street} (street) | $#{@web} (web)"
|
36
|
+
end
|
37
|
+
|
33
38
|
# @param element [Nokogiri::XML::Element]
|
34
39
|
#
|
35
40
|
# @return [Rates]
|
data/lib/publicstorage.rb
CHANGED
metadata
CHANGED
@@ -1,11 +1,11 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: publicstorage
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.
|
4
|
+
version: 0.2.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Kevin Sylvestre
|
8
|
-
autorequire:
|
8
|
+
autorequire:
|
9
9
|
bindir: exe
|
10
10
|
cert_chain: []
|
11
11
|
date: 2024-11-27 00:00:00.000000000 Z
|
@@ -69,7 +69,8 @@ dependencies:
|
|
69
69
|
description: Uses HTTP.rb to scrape publicstorage.com.
|
70
70
|
email:
|
71
71
|
- kevin@ksylvest.com
|
72
|
-
executables:
|
72
|
+
executables:
|
73
|
+
- publicstorage
|
73
74
|
extensions: []
|
74
75
|
extra_rdoc_files: []
|
75
76
|
files:
|
@@ -77,8 +78,10 @@ files:
|
|
77
78
|
- README.md
|
78
79
|
- bin/console
|
79
80
|
- bin/setup
|
81
|
+
- exe/publicstorage
|
80
82
|
- lib/publicstorage.rb
|
81
83
|
- lib/publicstorage/address.rb
|
84
|
+
- lib/publicstorage/cli.rb
|
82
85
|
- lib/publicstorage/crawler.rb
|
83
86
|
- lib/publicstorage/dimensions.rb
|
84
87
|
- lib/publicstorage/facility.rb
|
@@ -96,7 +99,7 @@ metadata:
|
|
96
99
|
homepage_uri: https://github.com/ksylvest/publicstorage
|
97
100
|
source_code_uri: https://github.com/ksylvest/publicstorage
|
98
101
|
changelog_uri: https://github.com/ksylvest/publicstorage
|
99
|
-
post_install_message:
|
102
|
+
post_install_message:
|
100
103
|
rdoc_options: []
|
101
104
|
require_paths:
|
102
105
|
- lib
|
@@ -111,8 +114,8 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
111
114
|
- !ruby/object:Gem::Version
|
112
115
|
version: '0'
|
113
116
|
requirements: []
|
114
|
-
rubygems_version: 3.5.
|
115
|
-
signing_key:
|
117
|
+
rubygems_version: 3.5.23
|
118
|
+
signing_key:
|
116
119
|
specification_version: 4
|
117
120
|
summary: A crawler for PublicStorage.
|
118
121
|
test_files: []
|