lcbo 1.4.0 → 1.5.0
Sign up to get free protection for your applications and to get access to all the features.
- data/CHANGELOG.md +6 -0
- data/Gemfile +1 -1
- data/lcbo.gemspec +4 -4
- data/lib/lcbo.rb +7 -1
- data/lib/lcbo/crawlkit.rb +0 -6
- data/lib/lcbo/crawlkit/request.rb +10 -13
- data/lib/lcbo/version.rb +1 -1
- data/spec/spec_helper.rb +1 -1
- metadata +13 -13
data/CHANGELOG.md
CHANGED
@@ -1,3 +1,9 @@
|
|
1
|
+
Version 1.5.0
|
2
|
+
|
3
|
+
* Updated Gemfile to use an explicit source URL.
|
4
|
+
* Updated dependencies to ensure compatability with Ruby 2.0.0.
|
5
|
+
* Updated default user-agent string to be more informative.
|
6
|
+
|
1
7
|
Version 1.4.0
|
2
8
|
|
3
9
|
* Fixed issue with store list spec not using stubbed HTML data.
|
data/Gemfile
CHANGED
data/lcbo.gemspec
CHANGED
@@ -13,10 +13,10 @@ Gem::Specification.new do |s|
|
|
13
13
|
|
14
14
|
s.rubyforge_project = 'lcbo'
|
15
15
|
|
16
|
-
s.add_dependency 'typhoeus', '~> 0.
|
17
|
-
s.add_dependency 'nokogiri', '~> 1.5.
|
18
|
-
s.add_dependency 'unicode_utils', '~> 1.
|
19
|
-
s.add_dependency 'stringex', '~> 1.
|
16
|
+
s.add_dependency 'typhoeus', '~> 0.6.3'
|
17
|
+
s.add_dependency 'nokogiri', '~> 1.5.9'
|
18
|
+
s.add_dependency 'unicode_utils', '~> 1.4.0'
|
19
|
+
s.add_dependency 'stringex', '~> 1.5.1'
|
20
20
|
|
21
21
|
s.files = `git ls-files`.split(?\n)
|
22
22
|
s.test_files = `git ls-files -- {test,spec}/*`.split(?\n)
|
data/lib/lcbo.rb
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
module LCBO
|
2
2
|
|
3
3
|
DEFAULT_CONFIG = {
|
4
|
-
:user_agent => nil, # Use the default User-Agent
|
4
|
+
:user_agent => nil, # Use the default User-Agent
|
5
5
|
:max_retries => 8, # Number of times to retry a request that fails
|
6
6
|
:timeout => 2 # Seconds to wait for a request before timing out
|
7
7
|
}.freeze
|
@@ -15,6 +15,12 @@ module LCBO
|
|
15
15
|
@config = DEFAULT_CONFIG.dup
|
16
16
|
end
|
17
17
|
|
18
|
+
def self.user_agent
|
19
|
+
config[:user_agent] ||
|
20
|
+
ENV['LCBO_USER_AGENT'] ||
|
21
|
+
"LCBO #{VERSION} - http://github.com/heycarsten/lcbo"
|
22
|
+
end
|
23
|
+
|
18
24
|
end
|
19
25
|
|
20
26
|
require 'lcbo/version'
|
data/lib/lcbo/crawlkit.rb
CHANGED
@@ -7,12 +7,6 @@ require 'uri'
|
|
7
7
|
|
8
8
|
module LCBO
|
9
9
|
module CrawlKit
|
10
|
-
USER_AGENT ||= begin
|
11
|
-
LCBO.config[:user_agent] ||
|
12
|
-
ENV['LCBO_USER_AGENT'] ||
|
13
|
-
Typhoeus::USER_AGENT
|
14
|
-
end
|
15
|
-
|
16
10
|
class MalformedError < StandardError; end
|
17
11
|
class NotFoundError < StandardError; end
|
18
12
|
class RedirectedError < StandardError; end
|
@@ -27,15 +27,17 @@ module LCBO
|
|
27
27
|
def config
|
28
28
|
opts = {}
|
29
29
|
opts[:method] = request_prototype.http_method
|
30
|
-
opts[:headers] = { 'User-Agent' =>
|
31
|
-
opts[:body] =
|
30
|
+
opts[:headers] = { 'User-Agent' => LCBO.user_agent }
|
31
|
+
opts[:body] = body_params if (body_params && body_params.any?) && !gettable?
|
32
32
|
opts
|
33
33
|
end
|
34
34
|
|
35
35
|
def uri
|
36
|
-
|
37
|
-
|
38
|
-
|
36
|
+
@uri ||= begin
|
37
|
+
template = request_prototype.uri_template.dup
|
38
|
+
query_params.reduce(template) do |mem, (key, value)|
|
39
|
+
mem.gsub("{#{key}}", value.to_s)
|
40
|
+
end
|
39
41
|
end
|
40
42
|
end
|
41
43
|
|
@@ -45,19 +47,14 @@ module LCBO
|
|
45
47
|
|
46
48
|
protected
|
47
49
|
|
48
|
-
def _body
|
49
|
-
traversal = Typhoeus::Utils.traverse_params_hash(body_params)
|
50
|
-
Typhoeus::Utils.traversal_to_param_string(traversal)
|
51
|
-
end
|
52
|
-
|
53
50
|
def _run(tries = 0)
|
54
51
|
response = Timeout.timeout(LCBO.config[:timeout]) do
|
55
|
-
Typhoeus::Request.
|
52
|
+
Typhoeus::Request.new(uri, config).run
|
56
53
|
end
|
57
54
|
Response.new \
|
58
55
|
:code => response.code,
|
59
|
-
:uri =>
|
60
|
-
:http_method => response.
|
56
|
+
:uri => uri,
|
57
|
+
:http_method => (response.options[:method] || :get),
|
61
58
|
:time => response.time,
|
62
59
|
:query_params => query_params,
|
63
60
|
:body_params => body_params,
|
data/lib/lcbo/version.rb
CHANGED
data/spec/spec_helper.rb
CHANGED
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: lcbo
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 1.
|
4
|
+
version: 1.5.0
|
5
5
|
prerelease:
|
6
6
|
platform: ruby
|
7
7
|
authors:
|
@@ -9,7 +9,7 @@ authors:
|
|
9
9
|
autorequire:
|
10
10
|
bindir: bin
|
11
11
|
cert_chain: []
|
12
|
-
date:
|
12
|
+
date: 2013-05-03 00:00:00.000000000 Z
|
13
13
|
dependencies:
|
14
14
|
- !ruby/object:Gem::Dependency
|
15
15
|
name: typhoeus
|
@@ -18,7 +18,7 @@ dependencies:
|
|
18
18
|
requirements:
|
19
19
|
- - ~>
|
20
20
|
- !ruby/object:Gem::Version
|
21
|
-
version: 0.
|
21
|
+
version: 0.6.3
|
22
22
|
type: :runtime
|
23
23
|
prerelease: false
|
24
24
|
version_requirements: !ruby/object:Gem::Requirement
|
@@ -26,7 +26,7 @@ dependencies:
|
|
26
26
|
requirements:
|
27
27
|
- - ~>
|
28
28
|
- !ruby/object:Gem::Version
|
29
|
-
version: 0.
|
29
|
+
version: 0.6.3
|
30
30
|
- !ruby/object:Gem::Dependency
|
31
31
|
name: nokogiri
|
32
32
|
requirement: !ruby/object:Gem::Requirement
|
@@ -34,7 +34,7 @@ dependencies:
|
|
34
34
|
requirements:
|
35
35
|
- - ~>
|
36
36
|
- !ruby/object:Gem::Version
|
37
|
-
version: 1.5.
|
37
|
+
version: 1.5.9
|
38
38
|
type: :runtime
|
39
39
|
prerelease: false
|
40
40
|
version_requirements: !ruby/object:Gem::Requirement
|
@@ -42,7 +42,7 @@ dependencies:
|
|
42
42
|
requirements:
|
43
43
|
- - ~>
|
44
44
|
- !ruby/object:Gem::Version
|
45
|
-
version: 1.5.
|
45
|
+
version: 1.5.9
|
46
46
|
- !ruby/object:Gem::Dependency
|
47
47
|
name: unicode_utils
|
48
48
|
requirement: !ruby/object:Gem::Requirement
|
@@ -50,7 +50,7 @@ dependencies:
|
|
50
50
|
requirements:
|
51
51
|
- - ~>
|
52
52
|
- !ruby/object:Gem::Version
|
53
|
-
version: 1.
|
53
|
+
version: 1.4.0
|
54
54
|
type: :runtime
|
55
55
|
prerelease: false
|
56
56
|
version_requirements: !ruby/object:Gem::Requirement
|
@@ -58,7 +58,7 @@ dependencies:
|
|
58
58
|
requirements:
|
59
59
|
- - ~>
|
60
60
|
- !ruby/object:Gem::Version
|
61
|
-
version: 1.
|
61
|
+
version: 1.4.0
|
62
62
|
- !ruby/object:Gem::Dependency
|
63
63
|
name: stringex
|
64
64
|
requirement: !ruby/object:Gem::Requirement
|
@@ -66,7 +66,7 @@ dependencies:
|
|
66
66
|
requirements:
|
67
67
|
- - ~>
|
68
68
|
- !ruby/object:Gem::Version
|
69
|
-
version: 1.
|
69
|
+
version: 1.5.1
|
70
70
|
type: :runtime
|
71
71
|
prerelease: false
|
72
72
|
version_requirements: !ruby/object:Gem::Requirement
|
@@ -74,7 +74,7 @@ dependencies:
|
|
74
74
|
requirements:
|
75
75
|
- - ~>
|
76
76
|
- !ruby/object:Gem::Version
|
77
|
-
version: 1.
|
77
|
+
version: 1.5.1
|
78
78
|
description: Request and parse product, store, inventory, and product search pages
|
79
79
|
directly from the official LCBO website.
|
80
80
|
email:
|
@@ -168,7 +168,7 @@ required_ruby_version: !ruby/object:Gem::Requirement
|
|
168
168
|
version: '0'
|
169
169
|
segments:
|
170
170
|
- 0
|
171
|
-
hash:
|
171
|
+
hash: 2175933647770352577
|
172
172
|
required_rubygems_version: !ruby/object:Gem::Requirement
|
173
173
|
none: false
|
174
174
|
requirements:
|
@@ -177,10 +177,10 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
177
177
|
version: '0'
|
178
178
|
segments:
|
179
179
|
- 0
|
180
|
-
hash:
|
180
|
+
hash: 2175933647770352577
|
181
181
|
requirements: []
|
182
182
|
rubyforge_project: lcbo
|
183
|
-
rubygems_version: 1.8.
|
183
|
+
rubygems_version: 1.8.25
|
184
184
|
signing_key:
|
185
185
|
specification_version: 3
|
186
186
|
summary: A library for parsing HTML pages from http://lcbo.com
|