lcbo 0.9.9 → 0.10.0

data/CHANGELOG.md

@@ -1,3 +1,12 @@
+Version 0.10.0
+
+ * Moved `CrawlKit` related errors into the `CrawlKit` namespace.
+ * Added `:timeout` and `:max_retries` to configuration options and enabled
+   auto _n_-retries for timed-out requests.
+ * Added `LCBO::CrawlKit::Crawler` mixin as a helper for making crawlers.
+ * Added example crawlers for inventories, products, stores, and product list
+   pages.
+
 Version 0.9.9
 
  * Added `ProductPage#is_kosher` to designate Kosher products.

data/README.md

@@ -1,8 +1,8 @@
 # LCBO: The Ruby Gem
 
 This library is used to gather data for [LCBO API](http://lcboapi.com). It
-allows you to request and parse store, product, inventory, and product list
-pages directly from the [LCBO](http://lcbo.com) website.
+allows you to request and parse store, product, inventory, product list, and
+store list pages directly from the [LCBO](http://lcbo.com) website.
 
 ## Synopsis
 
@@ -12,7 +12,7 @@ pages directly from the [LCBO](http://lcbo.com) website.
     # => { :store_no => 511, :name => "King & Spadina", ... }
 
     LCBO.product(18)
-    # => { :product_no => 11, :name => "Heineken Lager", ... }
+    # => { :product_no => 18, :name => "Heineken Lager", ... }
 
     LCBO.inventory(18)
     # => { :product_no => 18, :inventory_count => 40398, :inventories => [ ... ] }
@@ -20,6 +20,17 @@ pages directly from the [LCBO](http://lcbo.com) website.
     LCBO.products_list(1)
     # => { :page => 1, :final_page => 108, ..., :product_nos => [ ... ] }
 
+    LCBO.store_list
+    # => { :store_nos => [1, 2, 3, 4, 5, 6, 8, 9, 10, 11, ...] }
+
+## Crawlers
+
+Some examples of crawlers exist
+[here](http://github.com/heycarsten/lcbo/blob/master/examples). You can also
+check out the
+[crawler spec](http://github.com/heycarsten/lcbo/blob/master/spec/crawlkit/crawler_spec.rb)
+to see how to interact with them.
+
 ## Installation
 
 Use RubyGems: `gem install lcbo`

data/examples/crawlers/inventories_crawler.rb

@@ -0,0 +1,22 @@
+class InventoriesCrawler
+
+  include LCBO::CrawlKit::Crawler
+
+  def enum
+    ProductListsCrawler.run
+  end
+
+  def request(product_no)
+    LCBO.inventory(product_no)
+  end
+
+  def failure(error, product_no)
+    case error
+    when LCBO::CrawlKit::NotFoundError
+      puts "[missing] Skipped inventory for product ##{product_no}"
+    else
+      raise error
+    end
+  end
+
+end

data/examples/crawlers/product_lists_crawler.rb

@@ -0,0 +1,17 @@
+class ProductListsCrawler
+
+  include CrawlKit::Crawler
+
+  def request(params)
+    LCBO.product_list(params[:next_page] || 1)
+  end
+
+  def continue?(current_params)
+    current_params[:next_page] ? true : false
+  end
+
+  def reduce
+    requests.map { |params| params[:product_nos] }.flatten
+  end
+
+end

data/examples/crawlers/products_crawler.rb

@@ -0,0 +1,22 @@
+class ProductsCrawler
+
+  include LCBO::CrawlKit::Crawler
+
+  def enum
+    ProductListsCrawler.run
+  end
+
+  def request(product_no)
+    LCBO.product(product_no)
+  end
+
+  def failure(error, product_no)
+    case error
+    when LCBO::CrawlKit::NotFoundError
+      puts "[missing] Skipped product ##{product_no}"
+    else
+      raise error
+    end
+  end
+
+end

data/examples/crawlers/products_queue_crawler.rb

@@ -0,0 +1,23 @@
+class ProductsQueueCrawler
+
+  include LCBO::CrawlKit::Crawler
+
+  def pop
+    $redis.rpop('lcbo.products.queue')
+  end
+
+  def request(product_no)
+    LCBO.product(product_no)
+  end
+
+  def failure(error, product_no)
+    case error
+    when LCBO::CrawlKit::NotFoundError
+      puts "[missing] Skipped product ##{product_no}"
+      $redis.rpush('lcbo.products.missing', product_no)
+    else
+      raise error
+    end
+  end
+
+end

data/examples/crawlers/stores_crawler.rb

@@ -0,0 +1,22 @@
+class StoresCrawler
+
+  include LCBO::CrawlKit::Crawler
+
+  def enum
+    LCBO.store_list[:store_nos]
+  end
+
+  def request(store_no)
+    LCBO.store(store_no)
+  end
+
+  def failure(error, store_no)
+    case error
+    when LCBO::CrawlKit::NotFoundError
+      puts "[missing] Skipped store ##{store_no}"
+    else
+      raise error
+    end
+  end
+
+end

data/lib/lcbo.rb

@@ -1,8 +1,10 @@
 module LCBO
 
   DEFAULT_CONFIG = {
-    :user_agent => nil,
-  }
+    :user_agent  => nil, # Use the default User-Agent by default
+    :max_retries => 8,   # Number of times to retry a request that fails
+    :timeout     => 2    # Seconds to wait for a request before timing out
+  }.freeze
 
   def self.config
     reset_config! unless @config

data/lib/lcbo/crawlkit.rb

@@ -10,6 +10,11 @@ module LCBO
       ENV['LCBO_USER_AGENT'] ||
       Typhoeus::USER_AGENT
     end
+
+    class MalformedError < StandardError; end
+    class NotFoundError < StandardError; end
+    class RequestFailedError < StandardError; end
+    class TimeoutError < StandardError; end
   end
 end
 
@@ -19,5 +24,6 @@ require 'lcbo/crawlkit/page'
 require 'lcbo/crawlkit/request'
 require 'lcbo/crawlkit/response'
 require 'lcbo/crawlkit/request_prototype'
+require 'lcbo/crawlkit/crawler'
 require 'lcbo/crawlkit/titlecase_helper'
 require 'lcbo/crawlkit/volume_helper'

data/lib/lcbo/crawlkit/crawler.rb

@@ -0,0 +1,79 @@
+module LCBO
+  module CrawlKit
+    module Crawler
+
+      MAX_RETRIES = 8
+
+      class NotImplementedError < StandardError; end
+
+      def self.included(host)
+        host.extend(ClassMethods)
+        host.instance_eval { include InstanceMethods }
+      end
+
+      module ClassMethods
+        def run(params = {}, &emitter)
+          crawler = new(&emitter)
+          result  = crawler.run(params)
+          crawler.respond_to?(:reduce) ? crawler.reduce : result
+        end
+      end
+
+      module InstanceMethods
+        attr_reader :responses
+
+        def initialize(&emitter)
+          @emitter = emitter
+          @responses = []
+        end
+
+        def run(params = {})
+          case
+          when params.is_a?(Array) && params.any?
+            runeach(params)
+          when respond_to?(:pop)
+            runpop
+          when respond_to?(:enum)
+            runeach(enum)
+          else
+            _request(params)
+          end
+        end
+
+        def failure(error, params)
+          raise error
+        end
+
+        def continue?(response)
+          false
+        end
+
+        def request(params = {})
+          raise NotImplementedError, "#{self.class} must implement #request"
+        end
+
+        protected
+
+        def runpop
+          while (params = pop)
+            _request(params)
+          end
+        end
+
+        def runeach(params)
+          params.each { |p| _request(p) }
+        end
+
+        def _request(params = {})
+          response = request(params)
+          @responses << response if respond_to?(:reduce)
+          @emitter.(response) if @emitter
+          continue?(response) ? run(response) : response
+        rescue => error
+          failure(error, params)
+        end
+      end
+
+    end
+  end
+end

data/lib/lcbo/crawlkit/page.rb

@@ -2,11 +2,6 @@ module LCBO
   module CrawlKit
     module Page
 
-      class Error < StandardError; end
-      class MalformedDocumentError < Error; end
-      class MissingResourceError < Error; end
-      class RequestFailedError < Error; end
-
       def self.included(mod)
         mod.module_eval do
           include Eventable

data/lib/lcbo/crawlkit/request.rb

@@ -2,6 +2,8 @@ module LCBO
   module CrawlKit
     class Request
 
+      MAX_RETRIES = 8
+
       attr_reader :request_prototype, :query_params, :body_params
 
       def initialize(request_prototype, query_p = {}, body_p = {})
@@ -38,7 +40,15 @@ module LCBO
       end
 
       def run
-        response = Typhoeus::Request.run(uri, config)
+        _run
+      end
+
+      protected
+
+      def _run(tries = 0)
+        response = Timeout.timeout(LCBO.config[:timeout]) do
+          Typhoeus::Request.run(uri, config)
+        end
         Response.new \
           :code         => response.code,
           :uri          => response.request.url,
@@ -47,6 +57,11 @@ module LCBO
           :query_params => query_params,
           :body_params  => body_params,
           :body         => response.body
+      rescue Errno::ETIMEDOUT, Timeout::Error
+        if tries > LCBO.config[:max_retries]
+          raise TimeoutError, "Request failed after timing out #{tries} times"
+        end
+        _run(tries + 1)
       end
 
     end

data/lib/lcbo/pages/product_page.rb

@@ -286,26 +286,26 @@ module LCBO
 
     def verify_third_info_cell
       return unless has_package? && info_cell_lines[2][0,1] != '|'
-      raise CrawlKit::MalformedDocumentError,
+      raise CrawlKit::MalformedError,
         "Expected third line in info cell to begin with bar. LCBO No: " \
         "#{product_no}, Dump: #{info_cell_lines[2].inspect}"
     end
 
     def verify_response_not_blank
       return unless html.strip == ''
-      raise CrawlKit::MissingResourceError,
+      raise CrawlKit::NotFoundError,
         "product #{product_no} does not appear to exist"
     end
 
     def verify_product_name
       return unless product_details_form('itemName').strip == ''
-      raise CrawlKit::MissingResourceError,
+      raise CrawlKit::NotFoundError,
         "can not locate name for product #{product_no}"
     end
 
     def verify_product_details_form
       return unless doc.css('form[name="productdetails"]').empty?
-      raise CrawlKit::MalformedDocumentError,
+      raise CrawlKit::MalformedError,
         "productdetails form not found in doc for product #{product_no}"
     end
 

data/lib/lcbo/pages/store_list_page.rb

@@ -42,7 +42,7 @@ module LCBO
 
     def verify_number_of_stores
       return if STORE_COUNT_RANGE.include?(store_nos.length)
-      raise CrawlKit::MalformedDocumentError,
+      raise CrawlKit::MalformedError,
         "Store count (#{total_stores}) not in range: #{STORE_COUNT_RANGE}"
     end
 

data/lib/lcbo/pages/store_page.rb

@@ -53,8 +53,8 @@ module LCBO
     emits :address_line_1 do
       data = info_nodes[2].content.strip.split(',')[0]
       unless data
-        raise MalformedDocumentError,
-          "unable to locate address for store #{store_no}"
+        raise CrawlKit::MalformedError,
+        "unable to locate address for store #{store_no}"
       end
       CrawlKit::TitleCaseHelper[data.gsub(/[\n\r\t]+/, ' ').strip]
     end
@@ -72,7 +72,7 @@ module LCBO
     emits :postal_code do
       data = info_nodes[3].content.strip.split(',')[1]
       unless data
-        raise MalformedDocumentError,
+        raise CrawlKit::MalformedError,
         "unable to locate postal code for store #{store_no}"
       end
       data.gsub(/[\n\r\t]+/, ' ').strip.upcase
@@ -174,18 +174,18 @@ module LCBO
 
     def verify_store_returned
       return if !@html.include?('No stores were located using your criteria.')
-      raise MissingResourceError, "store #{store_no} does not exist"
+      raise CrawlKit::NotFoundError, "store #{store_no} does not exist"
     end
 
     def verify_telephone_number
       return if telephone
-      raise MalformedDocumentError,
+      raise CrawlKit::MalformedError,
         "unable to locate telephone number for store #{store_no}"
     end
 
     def verify_node_count
       return if expected_node_count == info_nodes.size
-      raise MalformedDocumentError,
+      raise CrawlKit::MalformedError,
         "Expected #{expected_node_count} nodes for store #{store_no} but found " \
         "#{info_nodes.size} instead."
     end

data/lib/lcbo/version.rb

@@ -1,3 +1,3 @@
 module LCBO
-  VERSION = '0.9.9'
+  VERSION = '0.10.0'
 end

data/spec/crawlkit/crawler_spec.rb

@@ -0,0 +1,196 @@
+require 'spec_helper'
+
+module CrawlerSpec
+  module TheInternets
+
+    RESOURCES = {
+      'list?page=1' => {
+        :ids => [1, 2],
+        :page => 1,
+        :next_page => 2
+      },
+      'list?page=2' => {
+        :ids => [3, 4],
+        :page => 2,
+        :next_page => 3
+      },
+      'list?page=3' => {
+        :ids => [5, 6],
+        :page => 3,
+        :next_page => nil
+      },
+      'books/1' => {
+        :id => 1,
+        :title => 'book_1'
+      },
+      'books/2' => {
+        :id => 2,
+        :title => 'book_2'
+      },
+      'books/3' => {
+        :id => 3,
+        :title => 'book_3'
+      },
+      'books/4' => {
+        :id => 4,
+        :title => 'book_4'
+      },
+      'books/5' => {
+        :id => 5,
+        :title => 'book_5'
+      },
+      'books/6' => {
+        :id => 6,
+        :title => 'book_6'
+      }
+    }
+
+    def self.get(uri)
+      RESOURCES.fetch(uri) do
+        raise LCBO::CrawlKit::NotFoundError, "#{uri} does not exist"
+      end
+    end
+
+  end
+
+
+  class BookCrawler
+
+    include LCBO::CrawlKit::Crawler
+
+    def request(book_id)
+      TheInternets.get("books/#{book_id}")
+    end
+
+  end
+
+
+  class BookListsCrawler
+
+    include LCBO::CrawlKit::Crawler
+
+    def request(params)
+      TheInternets.get("list?page=#{params[:next_page] || 1}")
+    end
+
+    def continue?(page)
+      page[:next_page] ? true : false
+    end
+
+    def reduce
+      responses.map { |page| page[:ids] }.flatten
+    end
+
+  end
+
+
+  class EnumBooksCrawler
+
+    include LCBO::CrawlKit::Crawler
+
+    def enum
+      BookListsCrawler.run
+    end
+
+    def request(book_id)
+      TheInternets.get("books/#{book_id}")
+    end
+
+  end
+
+
+  class QueueBooksCrawler
+
+    QUEUE = [1, 2, 3, 4, 0, 5, 6]
+    MISSING = []
+
+    include LCBO::CrawlKit::Crawler
+
+    def pop
+      QUEUE.pop
+    end
+
+    def request(book_id)
+      TheInternets.get("books/#{book_id}")
+    end
+
+    def failure(error, book_id)
+      case error
+      when LCBO::CrawlKit::NotFoundError
+        MISSING.push(book_id)
+      else
+        raise error
+      end
+    end
+
+  end
+end
+
+describe CrawlerSpec::TheInternets do
+  it 'should let you get a resource' do
+    CrawlerSpec::TheInternets.get('books/1')[:title].must_equal 'book_1'
+  end
+
+  it 'should throw an error when a resource does not exist' do
+    -> { CrawlerSpec::TheInternets.get('books/0') }.must_raise LCBO::CrawlKit::NotFoundError
+  end
+end
+
+describe CrawlerSpec::BookCrawler do
+  it 'should return a book' do
+    CrawlerSpec::BookCrawler.run(1)[:title].must_equal 'book_1'
+  end
+
+  it 'should yield a book' do
+    title = nil
+    CrawlerSpec::BookCrawler.run(1) { |page| title = page[:title] }
+    title.must_equal 'book_1'
+  end
+
+  it 'should raise an error if a book does not exist' do
+    -> { CrawlerSpec::BookCrawler.run(0) }.must_raise LCBO::CrawlKit::NotFoundError
+  end
+end
+
+describe CrawlerSpec::BookListsCrawler do
+  it 'should return all the book ids' do
+    CrawlerSpec::BookListsCrawler.run.must_equal [1, 2, 3, 4, 5, 6]
+  end
+
+  it 'should emit all the pages' do
+    pages = []
+    CrawlerSpec::BookListsCrawler.run { |page| pages << page }
+    pages.size.must_equal 3
+  end
+
+  it 'should consider provided params' do
+    pages = []
+    CrawlerSpec::BookListsCrawler.run(:next_page => 2) { |page| pages << page }
+    pages.size.must_equal 2
+  end
+end
+
+describe CrawlerSpec::EnumBooksCrawler do
+  it 'should emit all the books' do
+    books = []
+    CrawlerSpec::EnumBooksCrawler.run { |book| books << book }
+    books.size.must_equal 6
+  end
+
+  it 'should emit books when provided with their params' do
+    books = []
+    CrawlerSpec::EnumBooksCrawler.run([1, 2, 3]) { |book| books << book }
+    books.size.must_equal 3
+  end
+end
+
+describe CrawlerSpec::QueueBooksCrawler do
+  it 'should emit all the books that did not fail' do
+    CrawlerSpec::QueueBooksCrawler::QUEUE.size.must_equal 7
+    books = []
+    CrawlerSpec::QueueBooksCrawler.run { |book| books << book }
+    books.size.must_equal 6
+    CrawlerSpec::QueueBooksCrawler::QUEUE.size.must_equal 0
+    CrawlerSpec::QueueBooksCrawler::MISSING.size.must_equal 1
+  end
+end

metadata

@@ -4,9 +4,9 @@ version: !ruby/object:Gem::Version
   prerelease: false
   segments: 
   - 0
-  - 9
-  - 9
-  version: 0.9.9
+  - 10
+  - 0
+  version: 0.10.0
 platform: ruby
 authors: 
 - Carsten Nielsen
@@ -14,7 +14,7 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2010-12-06 00:00:00 -05:00
+date: 2010-12-09 00:00:00 -05:00
 default_executable: 
 dependencies: 
 - !ruby/object:Gem::Dependency 
@@ -74,9 +74,15 @@ files:
 - LICENSE
 - README.md
 - Rakefile
+- examples/crawlers/inventories_crawler.rb
+- examples/crawlers/product_lists_crawler.rb
+- examples/crawlers/products_crawler.rb
+- examples/crawlers/products_queue_crawler.rb
+- examples/crawlers/stores_crawler.rb
 - lcbo.gemspec
 - lib/lcbo.rb
 - lib/lcbo/crawlkit.rb
+- lib/lcbo/crawlkit/crawler.rb
 - lib/lcbo/crawlkit/eventable.rb
 - lib/lcbo/crawlkit/fastdate_helper.rb
 - lib/lcbo/crawlkit/page.rb
@@ -94,6 +100,7 @@ files:
 - lib/lcbo/pages/store_list_page.rb
 - lib/lcbo/pages/store_page.rb
 - lib/lcbo/version.rb
+- spec/crawlkit/crawler_spec.rb
 - spec/crawlkit/eventable_spec.rb
 - spec/crawlkit/fastdate_helper_spec.rb
 - spec/crawlkit/page_spec.rb
@@ -160,6 +167,7 @@ signing_key:
 specification_version: 3
 summary: A library for parsing HTML pages from http://lcbo.com
 test_files: 
+- spec/crawlkit/crawler_spec.rb
 - spec/crawlkit/eventable_spec.rb
 - spec/crawlkit/fastdate_helper_spec.rb
 - spec/crawlkit/page_spec.rb