cubesmart 0.6.0 → 1.0.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/README.md +6 -0
- data/lib/cubesmart/cli.rb +10 -4
- data/lib/cubesmart/config.rb +23 -28
- data/lib/cubesmart/crawl.rb +11 -8
- data/lib/cubesmart/crawler.rb +4 -4
- data/lib/cubesmart/dimensions.rb +8 -8
- data/lib/cubesmart/version.rb +1 -1
- metadata +5 -4
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 9f6ca103f9053e6250a85a42aa2a374edefdd702195777acbc7b3434e86b3775
|
4
|
+
data.tar.gz: b5852885f4fba9a2e9976c675a43d655afaf3cc1b090930b4d80a790c70b097b
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 3ac76e5460e15ec7f7f26acce3aca8682e01f63d81e41ba408588089da2458c7514e12990e6d6e1f27544a26e59055a3ccd1a6456913e61446ea7a0a55fdee5b
|
7
|
+
data.tar.gz: 3a3ed1f456a2740714cd240058ced94d2e7dc9b7cbef5a8c49742d3f395f7da27b93bd9aebd2242db032a3e682fe93d1debb696976b3be776b7241b0a7012ba8
|
data/README.md
CHANGED
@@ -6,6 +6,8 @@
|
|
6
6
|
[![Yard](https://img.shields.io/badge/docs-site-blue.svg)](https://cubesmart.ksylvest.com)
|
7
7
|
[![CircleCI](https://img.shields.io/circleci/build/github/ksylvest/cubesmart)](https://circleci.com/gh/ksylvest/cubesmart)
|
8
8
|
|
9
|
+
A Ruby library offering both a CLI and API for scraping [CubeSmart](hhttps://www.cubesmart.com/) self-storage facilities and prices.
|
10
|
+
|
9
11
|
## Installation
|
10
12
|
|
11
13
|
```bash
|
@@ -49,3 +51,7 @@ end
|
|
49
51
|
```bash
|
50
52
|
cubesmart crawl
|
51
53
|
```
|
54
|
+
|
55
|
+
```bash
|
56
|
+
cubesmart crawl "https://www.cubesmart.com/california-self-storage/los-angeles-self-storage/3844.html"
|
57
|
+
```
|
data/lib/cubesmart/cli.rb
CHANGED
@@ -21,7 +21,7 @@ module CubeSmart
|
|
21
21
|
command = argv.shift
|
22
22
|
|
23
23
|
case command
|
24
|
-
when 'crawl' then crawl
|
24
|
+
when 'crawl' then crawl(*argv)
|
25
25
|
else
|
26
26
|
warn("unsupported command=#{command.inspect}")
|
27
27
|
exit(Code::ERROR)
|
@@ -30,8 +30,9 @@ module CubeSmart
|
|
30
30
|
|
31
31
|
private
|
32
32
|
|
33
|
-
|
34
|
-
|
33
|
+
# @param url [String] optional
|
34
|
+
def crawl(url = nil)
|
35
|
+
Crawl.run(url: url)
|
35
36
|
exit(Code::OK)
|
36
37
|
end
|
37
38
|
|
@@ -48,10 +49,15 @@ module CubeSmart
|
|
48
49
|
# @return [OptionParser]
|
49
50
|
def parser
|
50
51
|
OptionParser.new do |options|
|
51
|
-
options.banner = 'usage:
|
52
|
+
options.banner = 'usage: extraspace [options] <command> [<args>]'
|
52
53
|
|
53
54
|
options.on('-h', '--help', 'help') { help(options) }
|
54
55
|
options.on('-v', '--version', 'version') { version }
|
56
|
+
|
57
|
+
options.separator <<~COMMANDS
|
58
|
+
commands:
|
59
|
+
crawl [url]
|
60
|
+
COMMANDS
|
55
61
|
end
|
56
62
|
end
|
57
63
|
end
|
data/lib/cubesmart/config.rb
CHANGED
@@ -3,6 +3,10 @@
|
|
3
3
|
module CubeSmart
|
4
4
|
# The core configuration.
|
5
5
|
class Config
|
6
|
+
# @attribute [rw] accept_language
|
7
|
+
# @return [String]
|
8
|
+
attr_accessor :accept_language
|
9
|
+
|
6
10
|
# @attribute [rw] user_agent
|
7
11
|
# @return [String]
|
8
12
|
attr_accessor :user_agent
|
@@ -16,48 +20,39 @@ module CubeSmart
|
|
16
20
|
attr_accessor :proxy_url
|
17
21
|
|
18
22
|
def initialize
|
23
|
+
@accept_language = ENV.fetch('CUBESMART_ACCEPT_LANGUAGE', 'en-US,en;q=0.9')
|
19
24
|
@user_agent = ENV.fetch('CUBESMART_USER_AGENT', "cubesmart.rb/#{VERSION}")
|
20
25
|
@timeout = Integer(ENV.fetch('CUBESMART_TIMEOUT', 60))
|
21
26
|
@proxy_url = ENV.fetch('CUBESMART_PROXY_URL', nil)
|
22
27
|
end
|
23
28
|
|
24
29
|
# @return [Boolean]
|
25
|
-
def
|
26
|
-
!@
|
27
|
-
end
|
28
|
-
|
29
|
-
# @return [URI]
|
30
|
-
def proxy_uri
|
31
|
-
@proxy_uri ||= URI.parse(proxy_url) if proxy?
|
32
|
-
end
|
33
|
-
|
34
|
-
# @return [Integer]
|
35
|
-
def proxy_port
|
36
|
-
proxy_uri&.port
|
37
|
-
end
|
38
|
-
|
39
|
-
def proxy_host
|
40
|
-
proxy_uri&.host
|
30
|
+
def headers?
|
31
|
+
!@user_agent.nil?
|
41
32
|
end
|
42
33
|
|
43
|
-
# @return [
|
44
|
-
def
|
45
|
-
|
34
|
+
# @return [Boolean]
|
35
|
+
def timeout?
|
36
|
+
!@timeout.zero?
|
46
37
|
end
|
47
38
|
|
48
|
-
# @return [
|
49
|
-
def
|
50
|
-
|
39
|
+
# @return [Boolean]
|
40
|
+
def proxy?
|
41
|
+
!@proxy_url.nil?
|
51
42
|
end
|
52
43
|
|
53
|
-
# @return [String]
|
54
|
-
def
|
55
|
-
|
44
|
+
# @return [Hash<String, String>] e.g { 'User-Agent' => 'cubesmart.rb/1.0.0' }
|
45
|
+
def headers
|
46
|
+
{
|
47
|
+
'Accept-Language' => @accept_language,
|
48
|
+
'User-Agent' => @user_agent
|
49
|
+
}
|
56
50
|
end
|
57
51
|
|
58
|
-
# @return [Array]
|
59
|
-
def
|
60
|
-
|
52
|
+
# @return [Array] e.g. ['proxy.example.com', 8080, 'user', 'pass']
|
53
|
+
def via
|
54
|
+
proxy_uri = URI.parse(@proxy_url)
|
55
|
+
[proxy_uri.host, proxy_uri.port, proxy_uri.user, proxy_uri.password]
|
61
56
|
end
|
62
57
|
end
|
63
58
|
end
|
data/lib/cubesmart/crawl.rb
CHANGED
@@ -9,19 +9,22 @@ module CubeSmart
|
|
9
9
|
|
10
10
|
# @param stdout [IO] optional
|
11
11
|
# @param stderr [IO] optional
|
12
|
-
# @param
|
13
|
-
def initialize(stdout: $stdout, stderr: $stderr,
|
12
|
+
# @param url [String] optional
|
13
|
+
def initialize(stdout: $stdout, stderr: $stderr, url: nil)
|
14
14
|
@stdout = stdout
|
15
15
|
@stderr = stderr
|
16
|
-
@
|
16
|
+
@url = url
|
17
17
|
end
|
18
18
|
|
19
19
|
def run
|
20
|
-
|
21
|
-
|
22
|
-
|
23
|
-
|
24
|
-
|
20
|
+
if @url
|
21
|
+
process(url: @url)
|
22
|
+
else
|
23
|
+
sitemap = Facility.sitemap
|
24
|
+
@stdout.puts("count=#{sitemap.links.count}")
|
25
|
+
@stdout.puts
|
26
|
+
sitemap.links.each { |link| process(url: link.loc) }
|
27
|
+
end
|
25
28
|
end
|
26
29
|
|
27
30
|
def process(url:)
|
data/lib/cubesmart/crawler.rb
CHANGED
@@ -24,10 +24,10 @@ module CubeSmart
|
|
24
24
|
@connection ||= begin
|
25
25
|
config = CubeSmart.config
|
26
26
|
|
27
|
-
connection = HTTP.persistent(HOST)
|
28
|
-
connection = connection.headers(
|
29
|
-
connection = connection.timeout(config.timeout) if config.timeout
|
30
|
-
connection = connection.via(*config.
|
27
|
+
connection = HTTP.use(:auto_deflate).use(:auto_inflate).persistent(HOST)
|
28
|
+
connection = connection.headers(config.headers) if config.headers?
|
29
|
+
connection = connection.timeout(config.timeout) if config.timeout?
|
30
|
+
connection = connection.via(*config.via) if config.proxy?
|
31
31
|
|
32
32
|
connection
|
33
33
|
end
|
data/lib/cubesmart/dimensions.rb
CHANGED
@@ -3,23 +3,23 @@
|
|
3
3
|
module CubeSmart
|
4
4
|
# The dimensions (width + depth + sqft) of a price.
|
5
5
|
class Dimensions
|
6
|
-
DEFAULT_HEIGHT = 8 # feet
|
6
|
+
DEFAULT_HEIGHT = 8.0 # feet
|
7
7
|
|
8
8
|
# @attribute [rw] depth
|
9
|
-
# @return [
|
9
|
+
# @return [Float]
|
10
10
|
attr_accessor :depth
|
11
11
|
|
12
12
|
# @attribute [rw] width
|
13
|
-
# @return [
|
13
|
+
# @return [Float]
|
14
14
|
attr_accessor :width
|
15
15
|
|
16
16
|
# @attribute [rw] height
|
17
|
-
# @return [
|
17
|
+
# @return [Float]
|
18
18
|
attr_accessor :height
|
19
19
|
|
20
|
-
# @param depth [
|
21
|
-
# @param width [
|
22
|
-
# @param height [
|
20
|
+
# @param depth [Float]
|
21
|
+
# @param width [Float]
|
22
|
+
# @param height [Float]
|
23
23
|
def initialize(depth:, width:, height: DEFAULT_HEIGHT)
|
24
24
|
@depth = depth
|
25
25
|
@width = width
|
@@ -48,7 +48,7 @@ module CubeSmart
|
|
48
48
|
|
49
49
|
# @return [String] e.g. "10' × 10' (100 sqft)"
|
50
50
|
def text
|
51
|
-
"#{format('%g', @width)}' × #{format('%g', @depth)}' (#{
|
51
|
+
"#{format('%g', @width)}' × #{format('%g', @depth)}' (#{sqft} sqft)"
|
52
52
|
end
|
53
53
|
|
54
54
|
# @param element [Nokogiri::XML::Element]
|
data/lib/cubesmart/version.rb
CHANGED
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: cubesmart
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.
|
4
|
+
version: 1.0.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Kevin Sylvestre
|
8
8
|
autorequire:
|
9
9
|
bindir: exe
|
10
10
|
cert_chain: []
|
11
|
-
date: 2024-12-
|
11
|
+
date: 2024-12-18 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: http
|
@@ -115,8 +115,9 @@ licenses:
|
|
115
115
|
metadata:
|
116
116
|
rubygems_mfa_required: 'true'
|
117
117
|
homepage_uri: https://github.com/ksylvest/cubesmart
|
118
|
-
source_code_uri: https://github.com/ksylvest/cubesmart
|
119
|
-
changelog_uri: https://github.com/ksylvest/cubesmart
|
118
|
+
source_code_uri: https://github.com/ksylvest/cubesmart/tree/v1.0.0
|
119
|
+
changelog_uri: https://github.com/ksylvest/cubesmart/releases/tag/v1.0.0
|
120
|
+
documentation_uri: https://cubesmart.ksylvest.com/
|
120
121
|
post_install_message:
|
121
122
|
rdoc_options: []
|
122
123
|
require_paths:
|