lcbo 0.9.8 → 0.9.9
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- data/CHANGELOG.md +6 -0
- data/lib/lcbo.rb +0 -1
- data/lib/lcbo/helpers.rb +8 -3
- data/lib/lcbo/pages.rb +2 -1
- data/lib/lcbo/pages/product_page.rb +4 -0
- data/lib/lcbo/pages/store_list_page.rb +50 -0
- data/lib/lcbo/version.rb +1 -1
- data/spec/pages/product_pages.yml +48 -0
- data/spec/pages/product_pages/8.html +1317 -0
- data/spec/pages/store_list_pages.yml +616 -0
- data/spec/pages/store_list_pages/1.html +7045 -0
- data/spec/pages_spec.rb +2 -1
- metadata +10 -8
- data/lib/lcbo/crawlers.rb +0 -4
- data/lib/lcbo/crawlers/inventories_crawler.rb +0 -15
- data/lib/lcbo/crawlers/product_lists_crawler.rb +0 -22
- data/lib/lcbo/crawlers/products_crawler.rb +0 -16
- data/lib/lcbo/crawlers/stores_crawler.rb +0 -25
data/spec/pages_spec.rb
CHANGED
@@ -4,7 +4,8 @@ require 'yaml'
|
|
4
4
|
{ :product_pages => LCBO::ProductPage,
|
5
5
|
:store_pages => LCBO::StorePage,
|
6
6
|
:inventory_pages => LCBO::InventoryPage,
|
7
|
-
:product_list_pages => LCBO::ProductListPage
|
7
|
+
:product_list_pages => LCBO::ProductListPage,
|
8
|
+
:store_list_pages => LCBO::StoreListPage
|
8
9
|
}.each_pair do |type, page|
|
9
10
|
|
10
11
|
describe(page) do
|
metadata
CHANGED
@@ -5,8 +5,8 @@ version: !ruby/object:Gem::Version
|
|
5
5
|
segments:
|
6
6
|
- 0
|
7
7
|
- 9
|
8
|
-
-
|
9
|
-
version: 0.9.
|
8
|
+
- 9
|
9
|
+
version: 0.9.9
|
10
10
|
platform: ruby
|
11
11
|
authors:
|
12
12
|
- Carsten Nielsen
|
@@ -14,7 +14,7 @@ autorequire:
|
|
14
14
|
bindir: bin
|
15
15
|
cert_chain: []
|
16
16
|
|
17
|
-
date: 2010-12-
|
17
|
+
date: 2010-12-06 00:00:00 -05:00
|
18
18
|
default_executable:
|
19
19
|
dependencies:
|
20
20
|
- !ruby/object:Gem::Dependency
|
@@ -76,11 +76,6 @@ files:
|
|
76
76
|
- Rakefile
|
77
77
|
- lcbo.gemspec
|
78
78
|
- lib/lcbo.rb
|
79
|
-
- lib/lcbo/crawlers.rb
|
80
|
-
- lib/lcbo/crawlers/inventories_crawler.rb
|
81
|
-
- lib/lcbo/crawlers/product_lists_crawler.rb
|
82
|
-
- lib/lcbo/crawlers/products_crawler.rb
|
83
|
-
- lib/lcbo/crawlers/stores_crawler.rb
|
84
79
|
- lib/lcbo/crawlkit.rb
|
85
80
|
- lib/lcbo/crawlkit/eventable.rb
|
86
81
|
- lib/lcbo/crawlkit/fastdate_helper.rb
|
@@ -96,6 +91,7 @@ files:
|
|
96
91
|
- lib/lcbo/pages/inventory_page.rb
|
97
92
|
- lib/lcbo/pages/product_list_page.rb
|
98
93
|
- lib/lcbo/pages/product_page.rb
|
94
|
+
- lib/lcbo/pages/store_list_page.rb
|
99
95
|
- lib/lcbo/pages/store_page.rb
|
100
96
|
- lib/lcbo/version.rb
|
101
97
|
- spec/crawlkit/eventable_spec.rb
|
@@ -121,6 +117,9 @@ files:
|
|
121
117
|
- spec/pages/product_pages/5.html
|
122
118
|
- spec/pages/product_pages/6.html
|
123
119
|
- spec/pages/product_pages/7.html
|
120
|
+
- spec/pages/product_pages/8.html
|
121
|
+
- spec/pages/store_list_pages.yml
|
122
|
+
- spec/pages/store_list_pages/1.html
|
124
123
|
- spec/pages/store_pages.yml
|
125
124
|
- spec/pages/store_pages/1.html
|
126
125
|
- spec/pages/store_pages/2.html
|
@@ -184,6 +183,9 @@ test_files:
|
|
184
183
|
- spec/pages/product_pages/5.html
|
185
184
|
- spec/pages/product_pages/6.html
|
186
185
|
- spec/pages/product_pages/7.html
|
186
|
+
- spec/pages/product_pages/8.html
|
187
|
+
- spec/pages/store_list_pages.yml
|
188
|
+
- spec/pages/store_list_pages/1.html
|
187
189
|
- spec/pages/store_pages.yml
|
188
190
|
- spec/pages/store_pages/1.html
|
189
191
|
- spec/pages/store_pages/2.html
|
data/lib/lcbo/crawlers.rb
DELETED
@@ -1,15 +0,0 @@
|
|
1
|
-
module LCBO
|
2
|
-
class InventoriesCrawler
|
3
|
-
|
4
|
-
def self.run(product_nos, &block)
|
5
|
-
raise ArgumentError, 'block expected' unless block_given?
|
6
|
-
product_nos.each do |product_no|
|
7
|
-
begin
|
8
|
-
yield InventoryRequest.parse(:product_no => product_no).as_hash
|
9
|
-
rescue
|
10
|
-
end
|
11
|
-
end
|
12
|
-
end
|
13
|
-
|
14
|
-
end
|
15
|
-
end
|
@@ -1,22 +0,0 @@
|
|
1
|
-
module LCBO
|
2
|
-
class ProductListsCrawler
|
3
|
-
|
4
|
-
MAX_RETRIES = 10
|
5
|
-
|
6
|
-
class EpicTimeoutError < StandardError; end
|
7
|
-
|
8
|
-
def self.run(params = {}, tries = 0, &block)
|
9
|
-
raise ArgumentError, 'block expected' unless block_given?
|
10
|
-
begin
|
11
|
-
payload = LCBO.product_list(params[:page] || 1)
|
12
|
-
yield(payload)
|
13
|
-
run(:page => payload[:next_page], &block) if payload[:next_page]
|
14
|
-
rescue Errno::ETIMEDOUT, Timeout::Error
|
15
|
-
# On timeout, try again.
|
16
|
-
raise EpicTimeoutError if tries > MAX_RETRIES
|
17
|
-
run(params, (tries + 1), &block)
|
18
|
-
end
|
19
|
-
end
|
20
|
-
|
21
|
-
end
|
22
|
-
end
|
@@ -1,16 +0,0 @@
|
|
1
|
-
module LCBO
|
2
|
-
class ProductsCrawler
|
3
|
-
|
4
|
-
def self.run(product_nos, &block)
|
5
|
-
raise ArgumentError, 'block expected' unless block_given?
|
6
|
-
product_nos.each do |product_no|
|
7
|
-
begin
|
8
|
-
yield ProductRequest.parse(:product_no => product_no).as_hash
|
9
|
-
rescue CrawlKit::MissingResourceError, Errno::ETIMEDOUT, Timeout::Error
|
10
|
-
# Ignore products with no data, and timeouts.
|
11
|
-
end
|
12
|
-
end
|
13
|
-
end
|
14
|
-
|
15
|
-
end
|
16
|
-
end
|
@@ -1,25 +0,0 @@
|
|
1
|
-
module LCBO
|
2
|
-
class StoresCrawler
|
3
|
-
|
4
|
-
MAX_STORE_NO = 850
|
5
|
-
MAX_RETRIES = 10
|
6
|
-
|
7
|
-
class EpicTimeoutError < StandardError; end
|
8
|
-
|
9
|
-
def self.run(params, tries = 0, &block)
|
10
|
-
begin
|
11
|
-
payload = LCBO.store(params[:store_no])
|
12
|
-
yield payload
|
13
|
-
params[:store_no] = params[:store_nos].pop
|
14
|
-
run(params, &block)
|
15
|
-
rescue Errno::ETIMEDOUT, Timeout::Error
|
16
|
-
raise EpicTimeoutError if tries > MAX_RETRIES
|
17
|
-
run(params, (tries + 1), &block)
|
18
|
-
rescue LCBO::StorePage::MissingResourceError
|
19
|
-
params[:store_no] = params[:store_nos].pop
|
20
|
-
run(params, &block)
|
21
|
-
end
|
22
|
-
end
|
23
|
-
|
24
|
-
end
|
25
|
-
end
|