cubesmart 0.1.0 → 0.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/lib/cubesmart/address.rb +3 -5
- data/lib/cubesmart/cli.rb +1 -1
- data/lib/cubesmart/crawl.rb +37 -0
- data/lib/cubesmart/crawler.rb +0 -9
- data/lib/cubesmart/dimensions.rb +10 -3
- data/lib/cubesmart/facility.rb +24 -18
- data/lib/cubesmart/fetch_error.rb +12 -0
- data/lib/cubesmart/price.rb +6 -5
- data/lib/cubesmart/rates.rb +24 -10
- data/lib/cubesmart/version.rb +1 -1
- metadata +8 -6
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 3e26402813c2f507a7a4902766a26821194de2f1c9c2d31ef633b2138c45e2f5
|
4
|
+
data.tar.gz: 91628c2ca652fc5be1788d4616652b82483863ab3b0bc3107cf7f3b9861cac51
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: e1f88ebc04f150dc9cbf0d590e01f05245be3a3a323d31526b6f5cdd6f445b38ad66c2cf7a0f77847ba20c62c364b58a1fd9db1bb87ee2fd8d610f0fc30f53cc
|
7
|
+
data.tar.gz: 04f620cbfe554d8ecaeb77e363ad7a31c506a954c86deec4f6ad597af182c39c42340667f202fe9a7880740d4c4525c657368813d8a9bd6d6751aea8398db3ca
|
data/lib/cubesmart/address.rb
CHANGED
@@ -50,12 +50,10 @@ module CubeSmart
|
|
50
50
|
#
|
51
51
|
# @return [Address]
|
52
52
|
def self.parse(data:)
|
53
|
-
lines = %w[line1 line2 line3 line4].map { |key| data[key] }
|
54
|
-
|
55
53
|
new(
|
56
|
-
street:
|
57
|
-
city: data['
|
58
|
-
state: data['
|
54
|
+
street: data['streetAddress'],
|
55
|
+
city: data['addressLocality'],
|
56
|
+
state: data['addressRegion'],
|
59
57
|
zip: data['postalCode']
|
60
58
|
)
|
61
59
|
end
|
data/lib/cubesmart/cli.rb
CHANGED
@@ -0,0 +1,37 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
module CubeSmart
|
4
|
+
# Handles the crawl command via CLI.
|
5
|
+
class Crawl
|
6
|
+
def self.run(...)
|
7
|
+
new(...).run
|
8
|
+
end
|
9
|
+
|
10
|
+
# @param stdout [IO] optional
|
11
|
+
# @param stderr [IO] optional
|
12
|
+
# @param options [Hash] optional
|
13
|
+
def initialize(stdout: $stdout, stderr: $stderr, options: {})
|
14
|
+
@stdout = stdout
|
15
|
+
@stderr = stderr
|
16
|
+
@options = options
|
17
|
+
end
|
18
|
+
|
19
|
+
def run
|
20
|
+
sitemap = Facility.sitemap
|
21
|
+
@stdout.puts("count=#{sitemap.links.count}")
|
22
|
+
@stdout.puts
|
23
|
+
|
24
|
+
sitemap.links.each { |link| process(url: link.loc) }
|
25
|
+
end
|
26
|
+
|
27
|
+
def process(url:)
|
28
|
+
@stdout.puts(url)
|
29
|
+
facility = Facility.fetch(url: url)
|
30
|
+
@stdout.puts(facility.text)
|
31
|
+
facility.prices.each { |price| @stdout.puts(price.text) }
|
32
|
+
@stdout.puts
|
33
|
+
rescue FetchError => e
|
34
|
+
@stderr.puts("url=#{url} error=#{e.message}")
|
35
|
+
end
|
36
|
+
end
|
37
|
+
end
|
data/lib/cubesmart/crawler.rb
CHANGED
@@ -5,15 +5,6 @@ module CubeSmart
|
|
5
5
|
class Crawler
|
6
6
|
HOST = 'https://www.cubesmart.com'
|
7
7
|
|
8
|
-
# Raised for unexpected HTTP responses.
|
9
|
-
class FetchError < StandardError
|
10
|
-
# @param url [String]
|
11
|
-
# @param response [HTTP::Response]
|
12
|
-
def initialize(url:, response:)
|
13
|
-
super("url=#{url} status=#{response.status.inspect} body=#{String(response.body).inspect}")
|
14
|
-
end
|
15
|
-
end
|
16
|
-
|
17
8
|
# @param url [String]
|
18
9
|
# @raise [FetchError]
|
19
10
|
# @return [Nokogiri::HTML::Document]
|
data/lib/cubesmart/dimensions.rb
CHANGED
@@ -39,11 +39,18 @@ module CubeSmart
|
|
39
39
|
"#{format('%g', @width)}' × #{format('%g', @depth)}' (#{@sqft} sqft)"
|
40
40
|
end
|
41
41
|
|
42
|
-
# @param
|
42
|
+
# @param element [Nokogiri::XML::Element]
|
43
43
|
#
|
44
44
|
# @return [Dimensions]
|
45
|
-
def self.parse(
|
46
|
-
|
45
|
+
def self.parse(element:)
|
46
|
+
text = element.text
|
47
|
+
match = text.match(/(?<width>[\d\.]+)'x(?<depth>[\d\.]+)'/)
|
48
|
+
raise text.inspect if match.nil?
|
49
|
+
|
50
|
+
width = Float(match[:width])
|
51
|
+
depth = Float(match[:depth])
|
52
|
+
sqft = Integer(width * depth)
|
53
|
+
new(depth:, width:, sqft:)
|
47
54
|
end
|
48
55
|
end
|
49
56
|
end
|
data/lib/cubesmart/facility.rb
CHANGED
@@ -7,8 +7,15 @@ module CubeSmart
|
|
7
7
|
class Facility
|
8
8
|
class ParseError < StandardError; end
|
9
9
|
|
10
|
+
DEFAULT_EMAIL = 'webleads@cubesmart.com'
|
11
|
+
DEFAULT_PHONE = '1-877-279-7585'
|
12
|
+
|
10
13
|
SITEMAP_URL = 'https://www.cubesmart.com/sitemap-facility.xml'
|
11
14
|
|
15
|
+
PRICE_SELECTOR = %w[small medium large].map do |group|
|
16
|
+
".#{group}-group ul.csStorageSizeDimension li.csStorageSizeDimension"
|
17
|
+
end.join(', ')
|
18
|
+
|
12
19
|
ID_REGEX = /(?<id>\d+)\.html/
|
13
20
|
|
14
21
|
# @attribute [rw] id
|
@@ -19,6 +26,14 @@ module CubeSmart
|
|
19
26
|
# @return [String]
|
20
27
|
attr_accessor :name
|
21
28
|
|
29
|
+
# @attribute [rw] phone
|
30
|
+
# @return [String]
|
31
|
+
attr_accessor :phone
|
32
|
+
|
33
|
+
# @attribute [rw] email
|
34
|
+
# @return [String]
|
35
|
+
attr_accessor :email
|
36
|
+
|
22
37
|
# @attribute [rw] address
|
23
38
|
# @return [Address]
|
24
39
|
attr_accessor :address
|
@@ -54,7 +69,7 @@ module CubeSmart
|
|
54
69
|
name = data['name']
|
55
70
|
address = Address.parse(data: data['address'])
|
56
71
|
geocode = Geocode.parse(data: data['geo'])
|
57
|
-
prices =
|
72
|
+
prices = document.css(PRICE_SELECTOR).map { |element| Price.parse(element: element) }
|
58
73
|
|
59
74
|
new(id:, name:, address:, geocode:, prices:)
|
60
75
|
end
|
@@ -69,31 +84,20 @@ module CubeSmart
|
|
69
84
|
graph.find { |entry| entry['@type'] == 'SelfStorage' } || raise(ParseError, 'missing @graph')
|
70
85
|
end
|
71
86
|
|
72
|
-
def self.crawl
|
73
|
-
sitemap.links.each do |link|
|
74
|
-
url = link.loc
|
75
|
-
|
76
|
-
facility = fetch(url:)
|
77
|
-
puts facility.text
|
78
|
-
|
79
|
-
facility.prices.each do |price|
|
80
|
-
puts price.text
|
81
|
-
end
|
82
|
-
|
83
|
-
puts
|
84
|
-
end
|
85
|
-
end
|
86
|
-
|
87
87
|
# @param id [String]
|
88
88
|
# @param name [String]
|
89
89
|
# @param address [Address]
|
90
90
|
# @param geocode [Geocode]
|
91
|
+
# @param phone [String]
|
92
|
+
# @param email [String]
|
91
93
|
# @param prices [Array<Price>]
|
92
|
-
def initialize(id:, name:, address:, geocode:, prices:)
|
94
|
+
def initialize(id:, name:, address:, geocode:, phone: DEFAULT_PHONE, email: DEFAULT_EMAIL, prices: [])
|
93
95
|
@id = id
|
94
96
|
@name = name
|
95
97
|
@address = address
|
96
98
|
@geocode = geocode
|
99
|
+
@phone = phone
|
100
|
+
@email = email
|
97
101
|
@prices = prices
|
98
102
|
end
|
99
103
|
|
@@ -103,6 +107,8 @@ module CubeSmart
|
|
103
107
|
"id=#{@id.inspect}",
|
104
108
|
"address=#{@address.inspect}",
|
105
109
|
"geocode=#{@geocode.inspect}",
|
110
|
+
"phone=#{@phone.inspect}",
|
111
|
+
"email=#{@email.inspect}",
|
106
112
|
"prices=#{@prices.inspect}"
|
107
113
|
]
|
108
114
|
"#<#{self.class.name} #{props.join(' ')}>"
|
@@ -110,7 +116,7 @@ module CubeSmart
|
|
110
116
|
|
111
117
|
# @return [String]
|
112
118
|
def text
|
113
|
-
"#{@id} | #{@name} | #{@address.text} | #{@geocode.text}"
|
119
|
+
"#{@id} | #{@name} | #{@phone} | #{@email} | #{@address.text} | #{@geocode.text}"
|
114
120
|
end
|
115
121
|
end
|
116
122
|
end
|
@@ -0,0 +1,12 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
module CubeSmart
|
4
|
+
# Raised for unexpected HTTP responses.
|
5
|
+
class FetchError < Error
|
6
|
+
# @param url [String]
|
7
|
+
# @param response [HTTP::Response]
|
8
|
+
def initialize(url:, response:)
|
9
|
+
super("url=#{url} status=#{response.status.inspect} body=#{String(response.body).inspect}")
|
10
|
+
end
|
11
|
+
end
|
12
|
+
end
|
data/lib/cubesmart/price.rb
CHANGED
@@ -39,14 +39,15 @@ module CubeSmart
|
|
39
39
|
"#{@id} | #{@dimensions.text} | #{@rates.text}"
|
40
40
|
end
|
41
41
|
|
42
|
-
# @param
|
42
|
+
# @param element [Nokogiri::XML::Element]
|
43
43
|
#
|
44
44
|
# @return [Price]
|
45
|
-
def self.parse(
|
46
|
-
|
47
|
-
|
45
|
+
def self.parse(element:)
|
46
|
+
id = element.attr('id')
|
47
|
+
dimensions = Dimensions.parse(element:)
|
48
|
+
rates = Rates.parse(element:)
|
48
49
|
new(
|
49
|
-
id
|
50
|
+
id:,
|
50
51
|
dimensions: dimensions,
|
51
52
|
rates: rates
|
52
53
|
)
|
data/lib/cubesmart/rates.rb
CHANGED
@@ -3,6 +3,10 @@
|
|
3
3
|
module CubeSmart
|
4
4
|
# The rates (street + web) for a facility
|
5
5
|
class Rates
|
6
|
+
STREET_SELECTOR = '.ptOriginalPriceSpan'
|
7
|
+
WEB_SELECTOR = '.ptDiscountPriceSpan'
|
8
|
+
VALUE_REGEX = /(?<value>[\d\.]+)/
|
9
|
+
|
6
10
|
# @attribute [rw] street
|
7
11
|
# @return [Integer]
|
8
12
|
attr_accessor :street
|
@@ -11,6 +15,26 @@ module CubeSmart
|
|
11
15
|
# @return [Integer]
|
12
16
|
attr_accessor :web
|
13
17
|
|
18
|
+
# @param element [Nokogiri::XML::Element]
|
19
|
+
#
|
20
|
+
# @return [Rates]
|
21
|
+
def self.parse(element:)
|
22
|
+
street = parse_value(element: element.at_css(STREET_SELECTOR))
|
23
|
+
web = parse_value(element: element.at_css(WEB_SELECTOR))
|
24
|
+
|
25
|
+
new(street: street || web, web: web || street)
|
26
|
+
end
|
27
|
+
|
28
|
+
# @param element [Nokogiri::XML::Element]
|
29
|
+
#
|
30
|
+
# @return [Float, nil]
|
31
|
+
def self.parse_value(element:)
|
32
|
+
return if element.nil?
|
33
|
+
|
34
|
+
match = VALUE_REGEX.match(element.text)
|
35
|
+
Float(match[:value]) if match
|
36
|
+
end
|
37
|
+
|
14
38
|
# @param street [Integer]
|
15
39
|
# @param web [Integer]
|
16
40
|
def initialize(street:, web:)
|
@@ -31,15 +55,5 @@ module CubeSmart
|
|
31
55
|
def text
|
32
56
|
"$#{@street} (street) | $#{@web} (web)"
|
33
57
|
end
|
34
|
-
|
35
|
-
# @param data [Hash]
|
36
|
-
#
|
37
|
-
# @return [Rates]
|
38
|
-
def self.parse(data:)
|
39
|
-
new(
|
40
|
-
street: data['street'],
|
41
|
-
web: data['web']
|
42
|
-
)
|
43
|
-
end
|
44
58
|
end
|
45
59
|
end
|
data/lib/cubesmart/version.rb
CHANGED
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: cubesmart
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.
|
4
|
+
version: 0.3.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Kevin Sylvestre
|
8
|
-
autorequire:
|
8
|
+
autorequire:
|
9
9
|
bindir: exe
|
10
10
|
cert_chain: []
|
11
|
-
date: 2024-12-
|
11
|
+
date: 2024-12-04 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: http
|
@@ -97,9 +97,11 @@ files:
|
|
97
97
|
- lib/cubesmart/address.rb
|
98
98
|
- lib/cubesmart/cli.rb
|
99
99
|
- lib/cubesmart/config.rb
|
100
|
+
- lib/cubesmart/crawl.rb
|
100
101
|
- lib/cubesmart/crawler.rb
|
101
102
|
- lib/cubesmart/dimensions.rb
|
102
103
|
- lib/cubesmart/facility.rb
|
104
|
+
- lib/cubesmart/fetch_error.rb
|
103
105
|
- lib/cubesmart/geocode.rb
|
104
106
|
- lib/cubesmart/link.rb
|
105
107
|
- lib/cubesmart/price.rb
|
@@ -114,7 +116,7 @@ metadata:
|
|
114
116
|
homepage_uri: https://github.com/ksylvest/cubesmart
|
115
117
|
source_code_uri: https://github.com/ksylvest/cubesmart
|
116
118
|
changelog_uri: https://github.com/ksylvest/cubesmart
|
117
|
-
post_install_message:
|
119
|
+
post_install_message:
|
118
120
|
rdoc_options: []
|
119
121
|
require_paths:
|
120
122
|
- lib
|
@@ -129,8 +131,8 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
129
131
|
- !ruby/object:Gem::Version
|
130
132
|
version: '0'
|
131
133
|
requirements: []
|
132
|
-
rubygems_version: 3.5.
|
133
|
-
signing_key:
|
134
|
+
rubygems_version: 3.5.23
|
135
|
+
signing_key:
|
134
136
|
specification_version: 4
|
135
137
|
summary: A crawler for CubeSmart.
|
136
138
|
test_files: []
|