publicstorage 0.1.2 → 0.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/README.md +22 -15
- data/exe/publicstorage +7 -0
- data/lib/publicstorage/address.rb +6 -1
- data/lib/publicstorage/cli.rb +58 -0
- data/lib/publicstorage/config.rb +19 -0
- data/lib/publicstorage/crawler.rb +16 -2
- data/lib/publicstorage/dimensions.rb +6 -1
- data/lib/publicstorage/facility.rb +67 -24
- data/lib/publicstorage/geocode.rb +6 -1
- data/lib/publicstorage/price.rb +6 -1
- data/lib/publicstorage/rates.rb +6 -1
- data/lib/publicstorage/sitemap.rb +3 -1
- data/lib/publicstorage/version.rb +1 -1
- data/lib/publicstorage.rb +13 -0
- metadata +6 -2
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 34a6d2f42dac72b5aac92461444a57e40f3dd42ace468fc24f61ac62c81d09e1
|
4
|
+
data.tar.gz: bbb5c737bcf5236e4236fe6d0b336497557f1e300938924b1eb9727c204a9ff1
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: b1bed2e588707393e89bf67a416bc8f3622dda29e3c8b4754cfd015f224555530a628cce7cbb68e5c4787c745905493ae8c6e6efe14a8c6d84fae852f213474b
|
7
|
+
data.tar.gz: e46336139bbf2f3054cff093e650b88c5e84b7bb0beaa876b5f833d192c641fa3e48449ca27a3af1215a2fdd9dafd284c937348951248e5704ca62668b43f11d
|
data/README.md
CHANGED
@@ -12,6 +12,17 @@
|
|
12
12
|
gem install publicstorage
|
13
13
|
```
|
14
14
|
|
15
|
+
## Configuration
|
16
|
+
|
17
|
+
```ruby
|
18
|
+
require 'publicstorage'
|
19
|
+
|
20
|
+
PublicStorage.configure do |config|
|
21
|
+
config.user_agent = '../..' # ENV['PUBLICSTORAGE_USER_AGENT']
|
22
|
+
config.timeout = 30 # ENV['PUBLICSTORAGE_TIMEOUT']
|
23
|
+
end
|
24
|
+
```
|
25
|
+
|
15
26
|
## Usage
|
16
27
|
|
17
28
|
```ruby
|
@@ -20,24 +31,20 @@ require 'publicstorage'
|
|
20
31
|
sitemap = PublicStorage::Facility.sitemap
|
21
32
|
sitemap.links.each do |link|
|
22
33
|
url = link.loc
|
34
|
+
facility = ExtraSpace::Facility.fetch(url:)
|
23
35
|
|
24
|
-
facility
|
25
|
-
|
26
|
-
puts "Street: #{facility.address.street}"
|
27
|
-
puts "City: #{facility.address.city}"
|
28
|
-
puts "State: #{facility.address.state}"
|
29
|
-
puts "ZIP: #{facility.address.zip}"
|
30
|
-
puts "Latitude: #{facility.geocode.latitude}"
|
31
|
-
puts "Longitude: #{facility.geocode.longitude}"
|
32
|
-
puts
|
36
|
+
puts facility.text
|
33
37
|
|
34
38
|
facility.prices.each do |price|
|
35
|
-
puts
|
36
|
-
puts "Width: #{price.dimensions.width}"
|
37
|
-
puts "Depth: #{price.dimensions.depth}"
|
38
|
-
puts "SQFT: #{price.dimensions.sqft}"
|
39
|
-
puts "Rates: $#{price.rates.street} (street) / $#{price.rates.web} (web)"
|
40
|
-
puts
|
39
|
+
puts price.text
|
41
40
|
end
|
41
|
+
|
42
|
+
puts
|
42
43
|
end
|
43
44
|
```
|
45
|
+
|
46
|
+
## CLI
|
47
|
+
|
48
|
+
```bash
|
49
|
+
publicstorage crawl
|
50
|
+
```
|
data/exe/publicstorage
ADDED
@@ -1,7 +1,7 @@
|
|
1
1
|
# frozen_string_literal: true
|
2
2
|
|
3
3
|
module PublicStorage
|
4
|
-
#
|
4
|
+
# The address (street + city + state + zip) of a facility.
|
5
5
|
class Address
|
6
6
|
# @attribute [rw] street
|
7
7
|
# @return [String]
|
@@ -41,6 +41,11 @@ module PublicStorage
|
|
41
41
|
"#<#{self.class.name} #{props.join(' ')}>"
|
42
42
|
end
|
43
43
|
|
44
|
+
# @return [String]
|
45
|
+
def text
|
46
|
+
"#{street}, #{city}, #{state} #{zip}"
|
47
|
+
end
|
48
|
+
|
44
49
|
# @param data [Hash]
|
45
50
|
#
|
46
51
|
# @return [Address]
|
@@ -0,0 +1,58 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
require 'optparse'
|
4
|
+
|
5
|
+
module PublicStorage
|
6
|
+
# Used when interacting with the library from the command line interface (CLI).
|
7
|
+
#
|
8
|
+
# Usage:
|
9
|
+
#
|
10
|
+
# cli = PublicStorage::CLI.new
|
11
|
+
# cli.parse
|
12
|
+
class CLI
|
13
|
+
module Code
|
14
|
+
OK = 0
|
15
|
+
ERROR = 1
|
16
|
+
end
|
17
|
+
|
18
|
+
# @param argv [Array<String>]
|
19
|
+
def parse(argv = ARGV)
|
20
|
+
parser.parse!(argv)
|
21
|
+
command = argv.shift
|
22
|
+
|
23
|
+
case command
|
24
|
+
when 'crawl' then crawl
|
25
|
+
else
|
26
|
+
warn("unsupported command=#{command.inspect}")
|
27
|
+
exit(Code::ERROR)
|
28
|
+
end
|
29
|
+
end
|
30
|
+
|
31
|
+
private
|
32
|
+
|
33
|
+
def crawl
|
34
|
+
PublicStorage::Facility.crawl
|
35
|
+
exit(Code::OK)
|
36
|
+
end
|
37
|
+
|
38
|
+
def help(options)
|
39
|
+
puts(options)
|
40
|
+
exit(Code::OK)
|
41
|
+
end
|
42
|
+
|
43
|
+
def version
|
44
|
+
puts(VERSION)
|
45
|
+
exit(Code::OK)
|
46
|
+
end
|
47
|
+
|
48
|
+
# @return [OptionParser]
|
49
|
+
def parser
|
50
|
+
OptionParser.new do |options|
|
51
|
+
options.banner = 'usage: PublicStorage [options] <command> [<args>]'
|
52
|
+
|
53
|
+
options.on('-h', '--help', 'help') { help(options) }
|
54
|
+
options.on('-v', '--version', 'version') { version }
|
55
|
+
end
|
56
|
+
end
|
57
|
+
end
|
58
|
+
end
|
@@ -0,0 +1,19 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
module PublicStorage
|
4
|
+
# The core configuration.
|
5
|
+
class Config
|
6
|
+
# @attribute [rw] user_agent
|
7
|
+
# @return [String]
|
8
|
+
attr_accessor :user_agent
|
9
|
+
|
10
|
+
# @attribute [rw] timeout
|
11
|
+
# @return [Integer]
|
12
|
+
attr_accessor :timeout
|
13
|
+
|
14
|
+
def initialize
|
15
|
+
@user_agent = ENV.fetch('PUBLIC_STORAGE_USER_AGENT', "publicstorage.rb/#{VERSION}")
|
16
|
+
@timeout = Integer(ENV.fetch('PUBLICSTORAGE_TIMEOUT', 60))
|
17
|
+
end
|
18
|
+
end
|
19
|
+
end
|
@@ -3,6 +3,8 @@
|
|
3
3
|
module PublicStorage
|
4
4
|
# Used to fetch and parse either HTML or XML via a URL.
|
5
5
|
class Crawler
|
6
|
+
HOST = 'https://www.publicstorage.com'
|
7
|
+
|
6
8
|
# Raised for unexpected HTTP responses.
|
7
9
|
class FetchError < StandardError
|
8
10
|
# @param url [String]
|
@@ -26,11 +28,23 @@ module PublicStorage
|
|
26
28
|
new.xml(url:)
|
27
29
|
end
|
28
30
|
|
31
|
+
# @return [HTTP::Client]
|
32
|
+
def connection
|
33
|
+
@connection ||= begin
|
34
|
+
config = PublicStorage.config
|
35
|
+
|
36
|
+
connection = HTTP.persistent(HOST)
|
37
|
+
connection = connection.headers('User-Agent' => config.user_agent) if config.user_agent
|
38
|
+
connection = connection.timeout(config.timeout) if config.timeout
|
39
|
+
connection
|
40
|
+
end
|
41
|
+
end
|
42
|
+
|
29
43
|
# @param url [String]
|
30
44
|
# @return [HTTP::Response]
|
31
45
|
def fetch(url:)
|
32
|
-
response =
|
33
|
-
raise FetchError(url:, response: response.flush) unless response.status.ok?
|
46
|
+
response = connection.get(url)
|
47
|
+
raise FetchError.new(url:, response: response.flush) unless response.status.ok?
|
34
48
|
|
35
49
|
response
|
36
50
|
end
|
@@ -1,7 +1,7 @@
|
|
1
1
|
# frozen_string_literal: true
|
2
2
|
|
3
3
|
module PublicStorage
|
4
|
-
# The dimensions
|
4
|
+
# The dimensions (width + depth + sqft) of a price.
|
5
5
|
class Dimensions
|
6
6
|
SELECTOR = '.unit-size'
|
7
7
|
|
@@ -36,6 +36,11 @@ module PublicStorage
|
|
36
36
|
"#<#{self.class.name} #{props.join(' ')}>"
|
37
37
|
end
|
38
38
|
|
39
|
+
# @return [String] e.g. "10' × 10' (100 sqft)"
|
40
|
+
def text
|
41
|
+
"#{format('%g', @width)}' × #{format('%g', @depth)}' (#{@sqft} sqft)"
|
42
|
+
end
|
43
|
+
|
39
44
|
# @param element [Nokogiri::XML::Element]
|
40
45
|
#
|
41
46
|
# @return [Dimensions]
|
@@ -6,6 +6,17 @@ module PublicStorage
|
|
6
6
|
SITEMAP_URL = 'https://www.publicstorage.com/sitemap_0-product.xml'
|
7
7
|
|
8
8
|
PRICE_SELECTOR = '.units-results-section .unit-list-group .unit-list-item'
|
9
|
+
LD_SELECTOR = 'script[type="application/ld+json"]'
|
10
|
+
|
11
|
+
ID_REGEX = /(?<id>\d+)\.html/
|
12
|
+
|
13
|
+
# @attribute [rw] id
|
14
|
+
# @return [Integer]
|
15
|
+
attr_accessor :id
|
16
|
+
|
17
|
+
# @attribute [rw] name
|
18
|
+
# @return [String]
|
19
|
+
attr_accessor :name
|
9
20
|
|
10
21
|
# @attribute [rw] address
|
11
22
|
# @return [Address]
|
@@ -19,25 +30,6 @@ module PublicStorage
|
|
19
30
|
# @return [Array<Price>]
|
20
31
|
attr_accessor :prices
|
21
32
|
|
22
|
-
# @param address [Address]
|
23
|
-
# @param geocode [Geocode]
|
24
|
-
# @param prices [Array<Price>]
|
25
|
-
def initialize(address:, geocode:, prices:)
|
26
|
-
@address = address
|
27
|
-
@geocode = geocode
|
28
|
-
@prices = prices
|
29
|
-
end
|
30
|
-
|
31
|
-
# @return [String]
|
32
|
-
def inspect
|
33
|
-
props = [
|
34
|
-
"address=#{@address.inspect}",
|
35
|
-
"geocode=#{@geocode.inspect}",
|
36
|
-
"prices=#{@prices.inspect}"
|
37
|
-
]
|
38
|
-
"#<#{self.class.name} #{props.join(' ')}>"
|
39
|
-
end
|
40
|
-
|
41
33
|
# @return [Sitemap]
|
42
34
|
def self.sitemap
|
43
35
|
Sitemap.fetch(url: SITEMAP_URL)
|
@@ -55,14 +47,65 @@ module PublicStorage
|
|
55
47
|
#
|
56
48
|
# @return [Facility]
|
57
49
|
def self.parse(document:)
|
58
|
-
data =
|
59
|
-
|
60
|
-
|
61
|
-
|
50
|
+
data = parse_ld(document:)
|
51
|
+
id = Integer(data['url'].match(ID_REGEX)[:id])
|
52
|
+
name = data['name']
|
53
|
+
address = Address.parse(data: data['address'])
|
54
|
+
geocode = Geocode.parse(data: data['geo'])
|
62
55
|
|
63
56
|
prices = document.css(PRICE_SELECTOR).map { |element| Price.parse(element:) }
|
64
57
|
|
65
|
-
new(address:, geocode:, prices:)
|
58
|
+
new(id:, name:, address:, geocode:, prices:)
|
59
|
+
end
|
60
|
+
|
61
|
+
# @param document [NokoGiri::XML::Document]
|
62
|
+
#
|
63
|
+
# @return [Hash]
|
64
|
+
def self.parse_ld(document:)
|
65
|
+
JSON.parse(document.at_css(LD_SELECTOR).text).find { |entry| entry['@type'] == 'SelfStorage' }
|
66
|
+
end
|
67
|
+
|
68
|
+
def self.crawl
|
69
|
+
sitemap.links.each do |link|
|
70
|
+
url = link.loc
|
71
|
+
|
72
|
+
facility = fetch(url:)
|
73
|
+
puts facility.text
|
74
|
+
|
75
|
+
facility.prices.each do |price|
|
76
|
+
puts price.text
|
77
|
+
end
|
78
|
+
|
79
|
+
puts
|
80
|
+
end
|
81
|
+
end
|
82
|
+
|
83
|
+
# @param id [String]
|
84
|
+
# @param name [String]
|
85
|
+
# @param address [Address]
|
86
|
+
# @param geocode [Geocode]
|
87
|
+
# @param prices [Array<Price>]
|
88
|
+
def initialize(id:, name:, address:, geocode:, prices:)
|
89
|
+
@id = id
|
90
|
+
@name = name
|
91
|
+
@address = address
|
92
|
+
@geocode = geocode
|
93
|
+
@prices = prices
|
94
|
+
end
|
95
|
+
|
96
|
+
# @return [String]
|
97
|
+
def inspect
|
98
|
+
props = [
|
99
|
+
"address=#{@address.inspect}",
|
100
|
+
"geocode=#{@geocode.inspect}",
|
101
|
+
"prices=#{@prices.inspect}"
|
102
|
+
]
|
103
|
+
"#<#{self.class.name} #{props.join(' ')}>"
|
104
|
+
end
|
105
|
+
|
106
|
+
# @return [String]
|
107
|
+
def text
|
108
|
+
"#{@id} | #{@name} | #{@address.text} | #{@geocode.text}"
|
66
109
|
end
|
67
110
|
end
|
68
111
|
end
|
@@ -1,7 +1,7 @@
|
|
1
1
|
# frozen_string_literal: true
|
2
2
|
|
3
3
|
module PublicStorage
|
4
|
-
#
|
4
|
+
# The geocode (latitude + longitude) of a facility.
|
5
5
|
class Geocode
|
6
6
|
# @attribute [rw] latitude
|
7
7
|
# @return [Float]
|
@@ -27,6 +27,11 @@ module PublicStorage
|
|
27
27
|
"#<#{self.class.name} #{props.join(' ')}>"
|
28
28
|
end
|
29
29
|
|
30
|
+
# @return [String]
|
31
|
+
def text
|
32
|
+
"#{@latitude},#{@longitude}"
|
33
|
+
end
|
34
|
+
|
30
35
|
# @param data [Hash]
|
31
36
|
#
|
32
37
|
# @return [Geocode]
|
data/lib/publicstorage/price.rb
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
# frozen_string_literal: true
|
2
2
|
|
3
3
|
module PublicStorage
|
4
|
-
#
|
4
|
+
# The price (id + dimensions + rate) for a facility
|
5
5
|
class Price
|
6
6
|
# @attribute [rw] id
|
7
7
|
# @return [String]
|
@@ -34,6 +34,11 @@ module PublicStorage
|
|
34
34
|
"#<#{self.class.name} #{props.join(' ')}>"
|
35
35
|
end
|
36
36
|
|
37
|
+
# @return [String] e.g. "123 | 5' × 5' (25 sqft) | $100 (street) / $90 (web)"
|
38
|
+
def text
|
39
|
+
"#{@id} | #{@dimensions.text} | #{@rates.text}"
|
40
|
+
end
|
41
|
+
|
37
42
|
# @param element [Nokogiri::XML::Element]
|
38
43
|
#
|
39
44
|
# @return [Price]
|
data/lib/publicstorage/rates.rb
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
# frozen_string_literal: true
|
2
2
|
|
3
3
|
module PublicStorage
|
4
|
-
# The rates
|
4
|
+
# The rates (street + web) for a facility
|
5
5
|
class Rates
|
6
6
|
STREET_SELECTOR = '.unit-prices .unit-pricing .unit-strike-through-price'
|
7
7
|
WEB_SELECTOR = '.unit-prices .unit-pricing .unit-price'
|
@@ -30,6 +30,11 @@ module PublicStorage
|
|
30
30
|
"#<#{self.class.name} #{props.join(' ')}>"
|
31
31
|
end
|
32
32
|
|
33
|
+
# @return [String] e.g. "$80 (street) | $60 (web)"
|
34
|
+
def text
|
35
|
+
"$#{@street} (street) | $#{@web} (web)"
|
36
|
+
end
|
37
|
+
|
33
38
|
# @param element [Nokogiri::XML::Element]
|
34
39
|
#
|
35
40
|
# @return [Rates]
|
data/lib/publicstorage.rb
CHANGED
@@ -6,8 +6,21 @@ require 'zeitwerk'
|
|
6
6
|
|
7
7
|
loader = Zeitwerk::Loader.for_gem
|
8
8
|
loader.inflector.inflect 'publicstorage' => 'PublicStorage'
|
9
|
+
loader.inflector.inflect 'cli' => 'CLI'
|
9
10
|
loader.setup
|
10
11
|
|
12
|
+
# An interface for PublicStorage.
|
11
13
|
module PublicStorage
|
12
14
|
class Error < StandardError; end
|
15
|
+
|
16
|
+
# @return [Config]
|
17
|
+
def self.config
|
18
|
+
@config ||= Config.new
|
19
|
+
end
|
20
|
+
|
21
|
+
# @yield [config]
|
22
|
+
# @yieldparam config [Config]
|
23
|
+
def self.configure
|
24
|
+
yield config
|
25
|
+
end
|
13
26
|
end
|
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: publicstorage
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.
|
4
|
+
version: 0.3.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Kevin Sylvestre
|
@@ -69,7 +69,8 @@ dependencies:
|
|
69
69
|
description: Uses HTTP.rb to scrape publicstorage.com.
|
70
70
|
email:
|
71
71
|
- kevin@ksylvest.com
|
72
|
-
executables:
|
72
|
+
executables:
|
73
|
+
- publicstorage
|
73
74
|
extensions: []
|
74
75
|
extra_rdoc_files: []
|
75
76
|
files:
|
@@ -77,8 +78,11 @@ files:
|
|
77
78
|
- README.md
|
78
79
|
- bin/console
|
79
80
|
- bin/setup
|
81
|
+
- exe/publicstorage
|
80
82
|
- lib/publicstorage.rb
|
81
83
|
- lib/publicstorage/address.rb
|
84
|
+
- lib/publicstorage/cli.rb
|
85
|
+
- lib/publicstorage/config.rb
|
82
86
|
- lib/publicstorage/crawler.rb
|
83
87
|
- lib/publicstorage/dimensions.rb
|
84
88
|
- lib/publicstorage/facility.rb
|