lcbo 0.9.0

data/lib/lcbo/crawlkit/request.rb

@@ -0,0 +1,51 @@
+module LCBO
+  module CrawlKit
+    class Request
+
+      attr_reader :request_prototype, :query_params, :body_params
+
+      def initialize(request_prototype, query_p = {}, body_p = {})
+        @request_prototype = request_prototype
+        self.query_params  = query_p
+        self.body_params   = body_p
+      end
+
+      def query_params=(value)
+        @query_params = (value || {})
+      end
+
+      def body_params=(value)
+        @body_params = request_prototype.body_params.merge(value || {})
+      end
+
+      def gettable?
+        [:head, :get].include?(request_prototype.http_method)
+      end
+
+      def config
+        opts = {}
+        opts[:method]     = request_prototype.http_method
+        opts[:user_agent] = USER_AGENT
+        opts[:params]     = body_params unless gettable?
+        opts
+      end
+
+      def uri
+        request_prototype.uri_template.expand(query_params).to_s
+      end
+
+      def run
+        response = Typhoeus::Request.run(uri, config)
+        Response.new \
+          :code         => response.code,
+          :uri          => response.request.url,
+          :http_method  => response.request.method,
+          :time         => response.time,
+          :query_params => query_params,
+          :body_params  => body_params,
+          :body         => response.body
+      end
+
+    end
+  end
+end

data/lib/lcbo/crawlkit/request_prototype.rb

@@ -0,0 +1,31 @@
+module LCBO
+  module CrawlKit
+    class RequestPrototype
+
+      attr_reader :http_method, :uri_template, :body_params
+
+      def initialize(uri_template = nil, http_method = :get, body_params = {})
+        self.uri_template = uri_template
+        self.http_method  = http_method
+        self.body_params  = body_params
+      end
+
+      def http_method=(value)
+        @http_method = value ? value.to_s.downcase.to_sym : :get
+      end
+
+      def uri_template=(value)
+        @uri_template = Addressable::Template.new(value) if value
+      end
+
+      def body_params=(value)
+        @body_params = value ? HashExt.symbolize_keys(value) : {}
+      end
+
+      def request(query_params = {}, body_params = {})
+        Request.new(self, query_params, body_params).run
+      end
+
+    end
+  end
+end

data/lib/lcbo/crawlkit/response.rb

@@ -0,0 +1,48 @@
+module LCBO
+  module CrawlKit
+    class Response
+
+      attr_reader :response, :body, :query_params, :body_params, :uri,
+        :code, :time, :http_method
+
+      def initialize(response)
+        params        = HashExt.symbolize_keys(response)
+        @code         = params[:code]
+        @uri          = params[:uri]
+        @http_method  = params[:http_method]
+        @time         = params[:time]
+        @query_params = params[:query_params]
+        @body_params  = params[:body_params]
+        @body         = self.class.normalize_encoding(params[:body])
+        ensure_success!
+      end
+
+      def self.normalize_encoding(html)
+        if html.valid_encoding?
+          html
+        else
+          html.force_encoding('ISO-8859-1')
+          html.encode('UTF-8')
+        end.gsub("\r\n", "\n")
+      end
+
+      def as_hash
+        { :code         => code,
+          :uri          => uri,
+          :http_method  => http_method,
+          :time         => time,
+          :query_params => query_params,
+          :body_params  => body_params,
+          :body         => body }
+      end
+
+      protected
+
+      def ensure_success!
+        return if @code == 200
+        raise RequestFailedError, "<#{@uri}> failed with status: #{@code}"
+      end
+
+    end
+  end
+end

data/lib/lcbo/crawlkit/titlecase_helper.rb

@@ -0,0 +1,97 @@
+# coding: utf-8
+# TODO: This is an ugly piece of ass that should burn and die!
+module LCBO
+  module CrawlKit
+    class TitleCaseHelper
+
+      UPPER_CHARS = 'ABCDEFGHIJKLMNOPQRSTUVWXYZÀÁÂÃÄÅÆÇÈÉÊËÌÍÎÏÐÑÒÓÔÕÖØÙÚÛÜÝ'
+      LOWER_CHARS = 'abcdefghijklmnopqrstuvwxyzàáâãäåæçèéêëìíîïðñòóôõöøùúûüý'
+      ALPHA_RANGE = "[#{UPPER_CHARS}#{LOWER_CHARS}]"
+      UPPER_RANGE = "[#{UPPER_CHARS}]"
+      LOWER_RANGE = "[#{LOWER_CHARS}]"
+      FIRST_CHAR_RE = /#{ALPHA_RANGE}{1}/u
+      ALPHA_RE = /#{ALPHA_RANGE}.*/u
+      SMALL_WORDS = %w[ a an and as at but by en for if in of del de on or the to v v. via vs vs. ]
+      ACRONYMS = %w[ vqa vsop xo nq5 vs xxx igt xiii xi xoxo srl bdb cvbg ocb lcbo i ii iii ]
+
+      attr_reader :input
+
+      def self.[](string)
+        titleize(string)
+      end
+
+      def self.upcase(string)
+        string.tr(LOWER_CHARS, UPPER_CHARS)
+      end
+
+      def self.downcase(string)
+        string.tr(UPPER_CHARS, LOWER_CHARS)
+      end
+
+      def self.preclean(string)
+        # Strip useless bracketed crap: Some Product Name (Some Redundant Stuff)**
+        string.gsub(/\(.+\Z/, '').
+        # Strip trailing stars.
+        gsub(/\*+\Z/, '')
+      end
+
+      def self.capitalize(string)
+        first_letter = string.scan(FIRST_CHAR_RE)[0]
+        if first_letter
+          uchar = upcase(first_letter)
+          string.sub(/#{first_letter}/u, uchar)
+        else
+          string
+        end
+      end
+
+      def self.titleize(string)
+        phrases(preclean(downcase(string))).map do |phrase|
+          words = phrase.split
+          words.map do |word|
+            def word.capitalize
+              self.sub(ALPHA_RE) { |subword| TitleCaseHelper.capitalize(subword) }
+            end
+            case word
+            when *(ACRONYMS + ACRONYMS.map { |ac| capitalize(ac) })
+              upcase(word)
+            when /#{ALPHA_RANGE}\&#{ALPHA_RANGE}/u # words with &, like E&J
+              word.split(/\&/).map { |w| capitalize(w) }.join('&')
+            when /#{ALPHA_RANGE}\-#{ALPHA_RANGE}/u # words with dashes, like "Smith-Weston"
+              word.split(/\-/).map { |w| capitalize(w) }.join('-')
+            when /#{ALPHA_RANGE}\/#{ALPHA_RANGE}/u # words with slashes
+              word.split(/\//).map { |w| capitalize(w) }.join(' / ')
+            when /#{ALPHA_RANGE}\.#{ALPHA_RANGE}/u # words with dots, like "example.com"
+              capitalized = word.split(/\./u).map { |w| capitalize(w) }.join('.')
+              '.' == word[-1, 1] ? capitalized + '.' : capitalized
+            when /^#{ALPHA_RANGE}.*#{UPPER_RANGE}/u # non-first letter capitalized already
+              word
+            when words.first, words.last
+              word.capitalize
+            when *(SMALL_WORDS + SMALL_WORDS.map { |small| capitalize(small) })
+              word.downcase
+            else
+              word.capitalize
+            end
+          end.join(' ')
+        end.join(' ').
+        # Special case for Word'S
+        gsub(/(['’])S\b/, '\1s')
+      end
+
+      def self.phrases(title)
+        phrases = title.scan(/.+?(?:[:.;?!] |$)/u).map { |phrase| phrase.strip }
+        # rejoin phrases that were split on the '.' from a small word
+        if phrases.size > 1
+          phrases[0..-2].each_with_index do |phrase, index|
+            if SMALL_WORDS.include?(phrase.split.last.downcase)
+              phrases[index] << " " + phrases.slice!(index + 1)
+            end
+          end
+        end
+        phrases
+      end
+
+    end
+  end
+end

data/lib/lcbo/crawlkit/volume_helper.rb

@@ -0,0 +1,46 @@
+module LCBO
+  module CrawlKit
+    class VolumeHelper
+
+      attr_reader :package_volume, :unit_volume, :total_units, :unit_type
+
+      def initialize(input)
+        @input = input
+        @package_volume = 0
+        @unit_volume = 0
+        @total_units = 0
+        calculate
+        self
+      end
+
+      def self.[](input_string)
+        new(input_string).as_milliliters
+      end
+
+      def as_milliliters
+        @package_volume
+      end
+
+      private
+
+      def calculate
+        return unless @input
+        match = @input.match(/([0-9]+|[0-9]+x[0-9]+) (mL) ([a-z]+)/)
+        return unless match
+        captures = match.captures
+        return unless captures.size == 3
+
+        if captures[0].include?('x')
+          @total_units, @unit_volume = *captures[0].split('x').map(&:to_i)
+        else
+          @total_units = 1
+          @unit_volume = captures[0].to_i
+        end
+
+        @unit_type = captures[2]
+        @package_volume = @total_units * @unit_volume
+      end
+
+    end
+  end
+end

data/lib/lcbo/ext.rb

@@ -0,0 +1,13 @@
+module LCBO
+  module HashExt
+
+    def self.symbolize_keys(input)
+      input.reduce({}) { |hsh, (key, value)| hsh.merge(key.to_sym => value) }
+    end
+
+    def self.stringify_keys(input)
+      input.reduce({}) { |hsh, (key, value)| hsh.merge(key.to_s => value) }
+    end
+
+  end
+end

data/lib/lcbo/helpers.rb

@@ -0,0 +1,34 @@
+module LCBO
+
+  PAGE_TYPES = {
+    :product => 'ProductPage',
+    :product_list => 'ProductListPage',
+    :store => 'StorePage',
+    :inventory => 'InventoryPage'
+  }
+
+  def self.page(type)
+    Object.const_get(PAGE_TYPES[type.to_sym])
+  end
+
+  def self.parse(page_type, response)
+    page[page_type].parse(response)
+  end
+
+  def self.product(product_no)
+    ProductPage.process(:product_no => product_no).as_hash
+  end
+
+  def self.store(store_no)
+    StorePage.process(:store_no => store_no).as_hash
+  end
+
+  def self.inventory(product_no)
+    InventoryPage.process(:product_no => product_no).as_hash
+  end
+
+  def self.product_list(page_number)
+    ProductListPage.process({}, { :page => page_number }).as_hash
+  end
+
+end

data/lib/lcbo/pages.rb

@@ -0,0 +1,4 @@
+require 'lcbo/pages/inventory_page'
+require 'lcbo/pages/product_page'
+require 'lcbo/pages/product_list_page'
+require 'lcbo/pages/store_page'

data/lib/lcbo/pages/inventory_page.rb

@@ -0,0 +1,60 @@
+module LCBO
+  class InventoryPage
+
+    include CrawlKit::Page
+
+    uri 'http://www.lcbo.com/lcbo-ear/lcbo/product/inventory/searchResults.do' \
+        '?language=EN&itemNumber={product_no}'
+
+    emits :product_no do
+      query_params[:product_no].to_i
+    end
+
+    emits :inventory_count do
+      inventories.reduce(0) { |sum, inv| sum + inv[:quantity] }
+    end
+
+    emits :inventories do
+      # [updated_on, store_no, quantity]
+      inventory_table_rows.reduce([]) do |ary, node|
+        h = {}
+        h[:updated_on] = begin
+          CrawlKit::FastDateHelper[
+          node.
+          css('td[width="17%"]')[-1].
+          text.
+          strip]
+        end
+        h[:store_no] = begin
+          node.
+          css('td[width="38%"] a.item-details-col2').
+          attribute('href').
+          value.
+          match(/\?STORE=([0-9]{1,3})\&/).
+          captures[0].
+          to_s.
+          to_i
+        end
+        h[:quantity] = begin
+          node.
+          css('td[width="13%"]')[0].
+          content.
+          strip.
+          to_i
+        end
+        ary << h
+      end
+    end
+
+    private
+
+    def inventory_table
+      doc.css('table[cellpadding="3"]')
+    end
+
+    def inventory_table_rows
+      inventory_table.css('tr[bgcolor]')
+    end
+
+  end
+end

data/lib/lcbo/pages/product_list_page.rb

@@ -0,0 +1,85 @@
+module LCBO
+  class ProductListPage
+
+    include CrawlKit::Page
+
+    PER_PAGE = 100
+    http_method :post
+    uri 'http://www.lcbo.com/lcbo-ear/lcbo/product/searchResults.do'
+
+    default_body_params \
+      :STOCK_TYPE_NAME    => 'All',
+      :ITEM_NAME          => '',
+      :KEYWORDS           => '',
+      :ITEM_NUMBER        => '',
+      :productListingType => '',
+      :LIQUOR_TYPE_SHORT_ => '*',
+      :CATEGORY_NAME      => '*',
+      :SUB_CATEGORY_NAME  => '*',
+      :PRODUCING_CNAME    => '*',
+      :PRODUCING_REGION_N => '*',
+      :UNIT_VOLUME        => '*',
+      :SELLING_PRICE      => '*',
+      :LTO_SALES_CODE     => 'N',
+      :VQA_CODE           => 'N',
+      :KOSHER_CODE        => 'N',
+      :VINTAGES_CODE      => 'N',
+      :VALUE_ADD_SALES_CO => 'N',
+      :AIR_MILES_SALES_CO => 'N',
+      :language           => 'EN',
+      :style              => 'LCBO.css',
+      :sort               => 'sortedProduct',
+      :order              => '1',
+      :resultsPerPage     => PER_PAGE.to_s,
+      :page               => '1',
+      :action             => 'result',
+      :sortby             => 'sortedProduct',
+      :orderby            => '',
+      :numPerPage         => PER_PAGE.to_s
+
+    emits :page do
+      body_params[:page].to_i
+    end
+
+    emits :final_page do
+      @final_page ||= begin
+        count = total_products / PER_PAGE
+        0 == (total_products % PER_PAGE) ? count : count + 1
+      end
+    end
+
+    emits :next_page do
+      @next_page ||= begin
+        page < final_page ? page + 1 : nil
+      end
+    end
+
+    emits :total_products do
+      @total_products ||= begin
+        doc.css('td[width="42%"] font.main_font b')[0].
+        text.
+        gsub(/\s+/, ' ').
+        strip.
+        to_i
+      end
+    end
+
+    emits :product_nos do
+      product_anchors.reduce([]) do |ary, a|
+        if (match = a.attribute('href').value.match(/\&itemNumber=([0-9]+)/))
+          ary << (match.captures[0].to_i)
+        else
+          next ary
+        end
+      end
+    end
+    alias_method :as_array, :product_nos
+
+    protected
+
+    def product_anchors
+      doc.css('td[style="padding: 5 5 5 0;"] a.item-details-col2')
+    end
+
+  end
+end