publicstorage 0.1.2 → 0.3.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: fc55c7ceabfcca39423c75b3315ded193267c2d02da648056b472c4f6e5ca64a
4
- data.tar.gz: ed7aaea3035fe7aaa876febfa809a90cf9559080e61aeca1fdffb8ed338bba61
3
+ metadata.gz: 34a6d2f42dac72b5aac92461444a57e40f3dd42ace468fc24f61ac62c81d09e1
4
+ data.tar.gz: bbb5c737bcf5236e4236fe6d0b336497557f1e300938924b1eb9727c204a9ff1
5
5
  SHA512:
6
- metadata.gz: b4d628a25a0af8d0754c98868d3a7cd18c2a8b7d6d9dc6f66fbd1a1a83454aea72c108806f3e25a4311d5aa80862cb4a7798cd5645e08484b4f44179139f1cd3
7
- data.tar.gz: 46684da48469510751828b452b5f00381caf94b460da89383675429ed8974878355e03cb5e2fdc879db59b5a8e591a83a557b77f131558a4ad91fafb84dc05d9
6
+ metadata.gz: b1bed2e588707393e89bf67a416bc8f3622dda29e3c8b4754cfd015f224555530a628cce7cbb68e5c4787c745905493ae8c6e6efe14a8c6d84fae852f213474b
7
+ data.tar.gz: e46336139bbf2f3054cff093e650b88c5e84b7bb0beaa876b5f833d192c641fa3e48449ca27a3af1215a2fdd9dafd284c937348951248e5704ca62668b43f11d
data/README.md CHANGED
@@ -12,6 +12,17 @@
12
12
  gem install publicstorage
13
13
  ```
14
14
 
15
+ ## Configuration
16
+
17
+ ```ruby
18
+ require 'publicstorage'
19
+
20
+ PublicStorage.configure do |config|
21
+ config.user_agent = '../..' # ENV['PUBLICSTORAGE_USER_AGENT']
22
+ config.timeout = 30 # ENV['PUBLICSTORAGE_TIMEOUT']
23
+ end
24
+ ```
25
+
15
26
  ## Usage
16
27
 
17
28
  ```ruby
@@ -20,24 +31,20 @@ require 'publicstorage'
20
31
  sitemap = PublicStorage::Facility.sitemap
21
32
  sitemap.links.each do |link|
22
33
  url = link.loc
34
+ facility = ExtraSpace::Facility.fetch(url:)
23
35
 
24
- facility = PublicStorage::Facility.fetch(url:)
25
-
26
- puts "Street: #{facility.address.street}"
27
- puts "City: #{facility.address.city}"
28
- puts "State: #{facility.address.state}"
29
- puts "ZIP: #{facility.address.zip}"
30
- puts "Latitude: #{facility.geocode.latitude}"
31
- puts "Longitude: #{facility.geocode.longitude}"
32
- puts
36
+ puts facility.text
33
37
 
34
38
  facility.prices.each do |price|
35
- puts "ID: #{price.id}"
36
- puts "Width: #{price.dimensions.width}"
37
- puts "Depth: #{price.dimensions.depth}"
38
- puts "SQFT: #{price.dimensions.sqft}"
39
- puts "Rates: $#{price.rates.street} (street) / $#{price.rates.web} (web)"
40
- puts
39
+ puts price.text
41
40
  end
41
+
42
+ puts
42
43
  end
43
44
  ```
45
+
46
+ ## CLI
47
+
48
+ ```bash
49
+ publicstorage crawl
50
+ ```
data/exe/publicstorage ADDED
@@ -0,0 +1,7 @@
1
+ #!/usr/bin/env ruby
2
+ # frozen_string_literal: true
3
+
4
+ require 'publicstorage'
5
+
6
+ cli = PublicStorage::CLI.new
7
+ cli.parse
@@ -1,7 +1,7 @@
1
1
  # frozen_string_literal: true
2
2
 
3
3
  module PublicStorage
4
- # Represents an address associated with a facility.
4
+ # The address (street + city + state + zip) of a facility.
5
5
  class Address
6
6
  # @attribute [rw] street
7
7
  # @return [String]
@@ -41,6 +41,11 @@ module PublicStorage
41
41
  "#<#{self.class.name} #{props.join(' ')}>"
42
42
  end
43
43
 
44
+ # @return [String]
45
+ def text
46
+ "#{street}, #{city}, #{state} #{zip}"
47
+ end
48
+
44
49
  # @param data [Hash]
45
50
  #
46
51
  # @return [Address]
@@ -0,0 +1,58 @@
1
+ # frozen_string_literal: true
2
+
3
+ require 'optparse'
4
+
5
+ module PublicStorage
6
+ # Used when interacting with the library from the command line interface (CLI).
7
+ #
8
+ # Usage:
9
+ #
10
+ # cli = PublicStorage::CLI.new
11
+ # cli.parse
12
+ class CLI
13
+ module Code
14
+ OK = 0
15
+ ERROR = 1
16
+ end
17
+
18
+ # @param argv [Array<String>]
19
+ def parse(argv = ARGV)
20
+ parser.parse!(argv)
21
+ command = argv.shift
22
+
23
+ case command
24
+ when 'crawl' then crawl
25
+ else
26
+ warn("unsupported command=#{command.inspect}")
27
+ exit(Code::ERROR)
28
+ end
29
+ end
30
+
31
+ private
32
+
33
+ def crawl
34
+ PublicStorage::Facility.crawl
35
+ exit(Code::OK)
36
+ end
37
+
38
+ def help(options)
39
+ puts(options)
40
+ exit(Code::OK)
41
+ end
42
+
43
+ def version
44
+ puts(VERSION)
45
+ exit(Code::OK)
46
+ end
47
+
48
+ # @return [OptionParser]
49
+ def parser
50
+ OptionParser.new do |options|
51
+ options.banner = 'usage: PublicStorage [options] <command> [<args>]'
52
+
53
+ options.on('-h', '--help', 'help') { help(options) }
54
+ options.on('-v', '--version', 'version') { version }
55
+ end
56
+ end
57
+ end
58
+ end
@@ -0,0 +1,19 @@
1
+ # frozen_string_literal: true
2
+
3
+ module PublicStorage
4
+ # The core configuration.
5
+ class Config
6
+ # @attribute [rw] user_agent
7
+ # @return [String]
8
+ attr_accessor :user_agent
9
+
10
+ # @attribute [rw] timeout
11
+ # @return [Integer]
12
+ attr_accessor :timeout
13
+
14
+ def initialize
15
+ @user_agent = ENV.fetch('PUBLIC_STORAGE_USER_AGENT', "publicstorage.rb/#{VERSION}")
16
+ @timeout = Integer(ENV.fetch('PUBLICSTORAGE_TIMEOUT', 60))
17
+ end
18
+ end
19
+ end
@@ -3,6 +3,8 @@
3
3
  module PublicStorage
4
4
  # Used to fetch and parse either HTML or XML via a URL.
5
5
  class Crawler
6
+ HOST = 'https://www.publicstorage.com'
7
+
6
8
  # Raised for unexpected HTTP responses.
7
9
  class FetchError < StandardError
8
10
  # @param url [String]
@@ -26,11 +28,23 @@ module PublicStorage
26
28
  new.xml(url:)
27
29
  end
28
30
 
31
+ # @return [HTTP::Client]
32
+ def connection
33
+ @connection ||= begin
34
+ config = PublicStorage.config
35
+
36
+ connection = HTTP.persistent(HOST)
37
+ connection = connection.headers('User-Agent' => config.user_agent) if config.user_agent
38
+ connection = connection.timeout(config.timeout) if config.timeout
39
+ connection
40
+ end
41
+ end
42
+
29
43
  # @param url [String]
30
44
  # @return [HTTP::Response]
31
45
  def fetch(url:)
32
- response = HTTP.get(url)
33
- raise FetchError(url:, response: response.flush) unless response.status.ok?
46
+ response = connection.get(url)
47
+ raise FetchError.new(url:, response: response.flush) unless response.status.ok?
34
48
 
35
49
  response
36
50
  end
@@ -1,7 +1,7 @@
1
1
  # frozen_string_literal: true
2
2
 
3
3
  module PublicStorage
4
- # The dimensions associated with a price.
4
+ # The dimensions (width + depth + sqft) of a price.
5
5
  class Dimensions
6
6
  SELECTOR = '.unit-size'
7
7
 
@@ -36,6 +36,11 @@ module PublicStorage
36
36
  "#<#{self.class.name} #{props.join(' ')}>"
37
37
  end
38
38
 
39
+ # @return [String] e.g. "10' × 10' (100 sqft)"
40
+ def text
41
+ "#{format('%g', @width)}' × #{format('%g', @depth)}' (#{@sqft} sqft)"
42
+ end
43
+
39
44
  # @param element [Nokogiri::XML::Element]
40
45
  #
41
46
  # @return [Dimensions]
@@ -6,6 +6,17 @@ module PublicStorage
6
6
  SITEMAP_URL = 'https://www.publicstorage.com/sitemap_0-product.xml'
7
7
 
8
8
  PRICE_SELECTOR = '.units-results-section .unit-list-group .unit-list-item'
9
+ LD_SELECTOR = 'script[type="application/ld+json"]'
10
+
11
+ ID_REGEX = /(?<id>\d+)\.html/
12
+
13
+ # @attribute [rw] id
14
+ # @return [Integer]
15
+ attr_accessor :id
16
+
17
+ # @attribute [rw] name
18
+ # @return [String]
19
+ attr_accessor :name
9
20
 
10
21
  # @attribute [rw] address
11
22
  # @return [Address]
@@ -19,25 +30,6 @@ module PublicStorage
19
30
  # @return [Array<Price>]
20
31
  attr_accessor :prices
21
32
 
22
- # @param address [Address]
23
- # @param geocode [Geocode]
24
- # @param prices [Array<Price>]
25
- def initialize(address:, geocode:, prices:)
26
- @address = address
27
- @geocode = geocode
28
- @prices = prices
29
- end
30
-
31
- # @return [String]
32
- def inspect
33
- props = [
34
- "address=#{@address.inspect}",
35
- "geocode=#{@geocode.inspect}",
36
- "prices=#{@prices.inspect}"
37
- ]
38
- "#<#{self.class.name} #{props.join(' ')}>"
39
- end
40
-
41
33
  # @return [Sitemap]
42
34
  def self.sitemap
43
35
  Sitemap.fetch(url: SITEMAP_URL)
@@ -55,14 +47,65 @@ module PublicStorage
55
47
  #
56
48
  # @return [Facility]
57
49
  def self.parse(document:)
58
- data = JSON.parse(document.at_css('script[type="application/ld+json"]').text)
59
- item = data.find { |entry| entry['@type'] == 'SelfStorage' }
60
- address = Address.parse(data: item['address'])
61
- geocode = Geocode.parse(data: item['geo'])
50
+ data = parse_ld(document:)
51
+ id = Integer(data['url'].match(ID_REGEX)[:id])
52
+ name = data['name']
53
+ address = Address.parse(data: data['address'])
54
+ geocode = Geocode.parse(data: data['geo'])
62
55
 
63
56
  prices = document.css(PRICE_SELECTOR).map { |element| Price.parse(element:) }
64
57
 
65
- new(address:, geocode:, prices:)
58
+ new(id:, name:, address:, geocode:, prices:)
59
+ end
60
+
61
+ # @param document [NokoGiri::XML::Document]
62
+ #
63
+ # @return [Hash]
64
+ def self.parse_ld(document:)
65
+ JSON.parse(document.at_css(LD_SELECTOR).text).find { |entry| entry['@type'] == 'SelfStorage' }
66
+ end
67
+
68
+ def self.crawl
69
+ sitemap.links.each do |link|
70
+ url = link.loc
71
+
72
+ facility = fetch(url:)
73
+ puts facility.text
74
+
75
+ facility.prices.each do |price|
76
+ puts price.text
77
+ end
78
+
79
+ puts
80
+ end
81
+ end
82
+
83
+ # @param id [String]
84
+ # @param name [String]
85
+ # @param address [Address]
86
+ # @param geocode [Geocode]
87
+ # @param prices [Array<Price>]
88
+ def initialize(id:, name:, address:, geocode:, prices:)
89
+ @id = id
90
+ @name = name
91
+ @address = address
92
+ @geocode = geocode
93
+ @prices = prices
94
+ end
95
+
96
+ # @return [String]
97
+ def inspect
98
+ props = [
99
+ "address=#{@address.inspect}",
100
+ "geocode=#{@geocode.inspect}",
101
+ "prices=#{@prices.inspect}"
102
+ ]
103
+ "#<#{self.class.name} #{props.join(' ')}>"
104
+ end
105
+
106
+ # @return [String]
107
+ def text
108
+ "#{@id} | #{@name} | #{@address.text} | #{@geocode.text}"
66
109
  end
67
110
  end
68
111
  end
@@ -1,7 +1,7 @@
1
1
  # frozen_string_literal: true
2
2
 
3
3
  module PublicStorage
4
- # A geocode associated with a facility.
4
+ # The geocode (latitude + longitude) of a facility.
5
5
  class Geocode
6
6
  # @attribute [rw] latitude
7
7
  # @return [Float]
@@ -27,6 +27,11 @@ module PublicStorage
27
27
  "#<#{self.class.name} #{props.join(' ')}>"
28
28
  end
29
29
 
30
+ # @return [String]
31
+ def text
32
+ "#{@latitude},#{@longitude}"
33
+ end
34
+
30
35
  # @param data [Hash]
31
36
  #
32
37
  # @return [Geocode]
@@ -1,7 +1,7 @@
1
1
  # frozen_string_literal: true
2
2
 
3
3
  module PublicStorage
4
- # A price associated with a unit.
4
+ # The price (id + dimensions + rate) for a facility
5
5
  class Price
6
6
  # @attribute [rw] id
7
7
  # @return [String]
@@ -34,6 +34,11 @@ module PublicStorage
34
34
  "#<#{self.class.name} #{props.join(' ')}>"
35
35
  end
36
36
 
37
+ # @return [String] e.g. "123 | 5' × 5' (25 sqft) | $100 (street) / $90 (web)"
38
+ def text
39
+ "#{@id} | #{@dimensions.text} | #{@rates.text}"
40
+ end
41
+
37
42
  # @param element [Nokogiri::XML::Element]
38
43
  #
39
44
  # @return [Price]
@@ -1,7 +1,7 @@
1
1
  # frozen_string_literal: true
2
2
 
3
3
  module PublicStorage
4
- # The rates associated with a price
4
+ # The rates (street + web) for a facility
5
5
  class Rates
6
6
  STREET_SELECTOR = '.unit-prices .unit-pricing .unit-strike-through-price'
7
7
  WEB_SELECTOR = '.unit-prices .unit-pricing .unit-price'
@@ -30,6 +30,11 @@ module PublicStorage
30
30
  "#<#{self.class.name} #{props.join(' ')}>"
31
31
  end
32
32
 
33
+ # @return [String] e.g. "$80 (street) | $60 (web)"
34
+ def text
35
+ "$#{@street} (street) | $#{@web} (web)"
36
+ end
37
+
33
38
  # @param element [Nokogiri::XML::Element]
34
39
  #
35
40
  # @return [Rates]
@@ -1,7 +1,9 @@
1
1
  # frozen_string_literal: true
2
2
 
3
3
  module PublicStorage
4
- # A sitemap.
4
+ # A sitemap on publicstorage.com.
5
+ #
6
+ # e.g. https://www.publicstorage.com/sitemap_0-product.xml
5
7
  class Sitemap
6
8
  # @attribute [rw] links
7
9
  # @return [Array<Link>]
@@ -1,5 +1,5 @@
1
1
  # frozen_string_literal: true
2
2
 
3
3
  module PublicStorage
4
- VERSION = '0.1.2'
4
+ VERSION = '0.3.0'
5
5
  end
data/lib/publicstorage.rb CHANGED
@@ -6,8 +6,21 @@ require 'zeitwerk'
6
6
 
7
7
  loader = Zeitwerk::Loader.for_gem
8
8
  loader.inflector.inflect 'publicstorage' => 'PublicStorage'
9
+ loader.inflector.inflect 'cli' => 'CLI'
9
10
  loader.setup
10
11
 
12
+ # An interface for PublicStorage.
11
13
  module PublicStorage
12
14
  class Error < StandardError; end
15
+
16
+ # @return [Config]
17
+ def self.config
18
+ @config ||= Config.new
19
+ end
20
+
21
+ # @yield [config]
22
+ # @yieldparam config [Config]
23
+ def self.configure
24
+ yield config
25
+ end
13
26
  end
metadata CHANGED
@@ -1,7 +1,7 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: publicstorage
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.1.2
4
+ version: 0.3.0
5
5
  platform: ruby
6
6
  authors:
7
7
  - Kevin Sylvestre
@@ -69,7 +69,8 @@ dependencies:
69
69
  description: Uses HTTP.rb to scrape publicstorage.com.
70
70
  email:
71
71
  - kevin@ksylvest.com
72
- executables: []
72
+ executables:
73
+ - publicstorage
73
74
  extensions: []
74
75
  extra_rdoc_files: []
75
76
  files:
@@ -77,8 +78,11 @@ files:
77
78
  - README.md
78
79
  - bin/console
79
80
  - bin/setup
81
+ - exe/publicstorage
80
82
  - lib/publicstorage.rb
81
83
  - lib/publicstorage/address.rb
84
+ - lib/publicstorage/cli.rb
85
+ - lib/publicstorage/config.rb
82
86
  - lib/publicstorage/crawler.rb
83
87
  - lib/publicstorage/dimensions.rb
84
88
  - lib/publicstorage/facility.rb