lcbo 0.9.0

data/lib/lcbo/pages/product_page.rb

@@ -0,0 +1,296 @@
+module LCBO
+  class ProductPage
+
+    include CrawlKit::Page
+
+    uri 'http://lcbo.com/lcbo-ear/lcbo/product/details.do?' \
+        'language=EN&itemNumber={product_no}'
+
+    on :before_parse, :verify_response_not_blank
+    on :after_parse,  :verify_product_details_form
+    on :after_parse,  :verify_product_name
+    on :after_parse,  :verify_third_info_cell
+
+    emits :product_no do
+      query_params[:product_no].to_i
+    end
+
+    emits :name do
+      CrawlKit::TitleCaseHelper[product_details_form('itemName')]
+    end
+
+    emits :price_in_cents do
+      (product_details_form('price').to_f * 100).to_i
+    end
+
+    emits :regular_price_in_cents do
+      if has_limited_time_offer
+        info_cell_line_after('Was:').sub('$ ', '').to_f * 100
+      else
+        price_in_cents
+      end
+    end
+
+    emits :limited_time_offer_savings_in_cents do
+      regular_price_in_cents - price_in_cents
+    end
+
+    emits :limited_time_offer_ends_on do
+      if has_limited_time_offer
+        CrawlKit::FastDateHelper[info_cell_line_after('Until')]
+      else
+        nil
+      end
+    end
+
+    emits :bonus_reward_miles do
+      if has_bonus_reward_miles
+        info_cell_line_after('Earn').to_i
+      else
+        0
+      end
+    end
+
+    emits :bonus_reward_miles_ends_on do
+      if has_bonus_reward_miles
+        CrawlKit::FastDateHelper[info_cell_line_after('Until')]
+      else
+        nil
+      end
+    end
+
+    emits :stock_type do
+      product_details_form('stock type')
+    end
+
+    emits :primary_category do
+      if stock_category
+        cat = stock_category.split(',')[0]
+        cat ? cat.strip : cat
+      end
+    end
+
+    emits :secondary_category do
+      if stock_category
+        cat = stock_category.split(',')[1]
+        cat ? cat.strip : cat
+      end
+    end
+
+    emits :origin do
+      match = find_info_line(/\AMade in: /)
+      if match
+        place = match.
+          gsub('Made in: ', '').
+          gsub('/Californie', '').
+          gsub('Bosnia\'Hercegovina', 'Bosnia and Herzegovina').
+          gsub('Is. Of', 'Island of').
+          gsub('Italy Quality', 'Italy').
+          gsub('Usa-', '').
+          gsub(', Rep. Of', '').
+          gsub('&', 'and')
+        place.split(',').map { |s| s.strip }.uniq.join(', ')
+      end
+    end
+
+    emits :package do
+      @package ||= begin
+        string = info_cell_lines[2]
+        string.include?('Price: ') ? nil : string.sub('|','').strip
+      end
+    end
+
+    emits :package_unit_type do
+      volume_helper.unit_type
+    end
+
+    emits :package_unit_volume_in_milliliters do
+      volume_helper.unit_volume
+    end
+
+    emits :total_package_units do
+      volume_helper.total_units
+    end
+
+    emits :total_package_volume_in_milliliters do
+      volume_helper.package_volume
+    end
+
+    emits :volume_in_milliliters do
+      CrawlKit::VolumeHelper[package]
+    end
+
+    emits :alcohol_content do
+      match = find_info_line(/ Alcohol\/Vol.\Z/)
+      if match
+        ac = match.gsub(/%| Alcohol\/Vol./, '').to_f
+        ac.zero? ? nil : (ac * 100).to_i
+      end
+    end
+
+    emits :sugar_content do
+      match = match = find_info_line(/\ASugar Content : /)
+      if match
+        match.gsub('Sugar Content : ', '')
+      end
+    end
+
+    emits :producer_name do
+      match = find_info_line(/\ABy: /)
+      if match
+        CrawlKit::TitleCaseHelper[
+          match.gsub(/By: |Tasting Note|Serving Suggestion|NOTE:/, '')
+        ]
+      end
+    end
+
+    emits :released_on do
+      if html.include?('Release Date:')
+        date = info_cell_line_after('Release Date:')
+        date == 'N/A' ? nil : CrawlKit::FastDateHelper[date]
+      else
+        nil
+      end
+    end
+
+    emits :is_discontinued do
+      html.include?('PRODUCT DISCONTINUED')
+    end
+
+    emits :has_limited_time_offer do
+      html.include?('<B>Limited Time Offer</B>')
+    end
+
+    emits :has_bonus_reward_miles do
+      html.include?('<B>Bonus Reward Miles Offer</B>')
+    end
+
+    emits :is_seasonal do
+      html.include?('<font color="#ff0000">SEASONAL/LIMITED QUANTITIES</font>')
+    end
+
+    emits :is_vqa do
+      html.include?('This is a <B>VQA</B> wine')
+    end
+
+    emits :description do
+      if html.include?('<B>Description</B>')
+        match = html.match(/<B>Description<\/B><\/font><BR>\n\t\t\t(.*)<BR>\n\t\t\t<BR>/m)
+        match ? match.captures[0] : nil
+      else
+        nil
+      end
+    end
+
+    emits :serving_suggestion do
+      if html.include?('<B>Serving Suggestion</B>')
+        match = html.match(/<B>Serving Suggestion<\/B><\/font><BR>\n\t\t\t(.*)<BR><BR>/m)
+        match ? match.captures[0] : nil
+      else
+        nil
+      end
+    end
+
+    emits :tasting_note do
+      if html.include?('<B>Tasting Note</B>')
+        match = html.match(/<B>Tasting Note<\/B><\/font><BR>\n\t\t\t(.*)<BR>\n\t\t\t<BR>/m)
+        match ? match.captures[0] : nil
+      else
+        nil
+      end
+    end
+
+    private
+
+    def volume_helper
+      @volume_helper ||= CrawlKit::VolumeHelper.new(package)
+    end
+
+    def has_package?
+      !info_cell_lines[2].include?('Price:')
+    end
+
+    def stock_category
+      cat = get_info_lines_at_offset(12).reject do |line|
+        l = line.strip
+        l == '' ||
+        l.include?('Price:') ||
+        l.include?('Bonus Reward Miles Offer') ||
+        l.include?('Value Added Promotion') ||
+        l.include?('Limited Time Offer') ||
+        l.include?('NOTE:')
+      end.first
+      cat ? cat.strip : nil
+    end
+
+    def product_details_form(name)
+      doc.css("form[name=\"productdetails\"] input[name=\"#{name}\"]")[0].
+        attributes['value'].to_s
+    end
+
+    def get_info_lines_at_offset(offset)
+      raw_info_cell_lines.select do |line|
+        match = line.scan(/\A[\s]+/)[0]
+        match ? offset == match.size : false
+      end
+    end
+
+    def info_cell_text
+      @info_cell_text ||= info_cell_lines.join("\n")
+    end
+
+    def find_info_line(regexp)
+      info_cell_lines.select { |l| l =~ regexp }.first
+    end
+
+    def raw_info_cell_lines
+      @raw_info_cell_lines ||= info_cell_element.content.split(/\n/)
+    end
+
+    def info_cell_lines
+      @info_cell_lines ||= begin
+        raw_info_cell_lines.map { |l| l.strip }.reject { |l| l == '' }
+      end
+    end
+
+    def info_cell_line_after(item)
+      i = info_cell_lines.index(item)
+      return unless i
+      info_cell_lines[i + 1]
+    end
+
+    def info_cell_html
+      @info_cell_html ||= info_cell_element.inner_html
+    end
+
+    def info_cell_element
+      doc.css('table[width="478"] td[height="271"] td[colspan="2"].main_font')[0]
+    end
+
+    def verify_third_info_cell
+      return unless has_package? && info_cell_lines[2][0,1] != '|'
+      raise CrawlKit::MalformedDocumentError,
+        "Expected third line in info cell to begin with bar. LCBO No: " \
+        "#{product_no}, Dump: #{info_cell_lines[2].inspect}"
+    end
+
+    def verify_response_not_blank
+      return unless html.strip == ''
+      raise CrawlKit::MissingResourceError,
+        "product #{product_no} does not appear to exist"
+    end
+
+    def verify_product_name
+      return unless product_details_form('itemName').strip == ''
+      raise CrawlKit::MissingResourceError,
+        "can not locate name for product #{product_no}"
+    end
+
+    def verify_product_details_form
+      return unless doc.css('form[name="productdetails"]').empty?
+      raise CrawlKit::MalformedDocumentError,
+        "productdetails form not found in doc for product #{product_no}"
+    end
+
+  end
+end

data/lib/lcbo/pages/store_page.rb

@@ -0,0 +1,196 @@
+require 'cgi'
+
+module LCBO
+  class StorePage
+
+    include CrawlKit::Page
+
+    uri 'http://www.lcbo.com/lcbo-ear/jsp/storeinfo.jsp?' \
+        'STORE={store_no}&language=EN'
+
+    DAY_NAMES = %w[
+      monday
+      tuesday
+      wednesday
+      thursday
+      friday
+      saturday
+      sunday ]
+
+    DETAIL_FIELDS = {
+      :has_wheelchair_accessability => 'wheelchair',
+      :has_bilingual_services       => 'bilingual',
+      :has_product_consultant       => 'consultant',
+      :has_tasting_bar              => 'tasting',
+      :has_beer_cold_room           => 'cold',
+      :has_special_occasion_permits => 'permits',
+      :has_vintages_corner          => 'vintages',
+      :has_parking                  => 'parking',
+      :has_transit_access           => 'transit' }
+
+    on :before_parse, :verify_store_returned
+    on :after_parse,  :verify_node_count
+    on :after_parse,  :verify_telephone_number
+
+    emits :store_no do
+      query_params[:store_no].to_i
+    end
+
+    DAY_NAMES.each do |day|
+      emits :"#{day}_open" do
+        time_open_close(day)[0]
+      end
+
+      emits :"#{day}_close" do
+        time_open_close(day)[1]
+      end
+    end
+
+    emits :name do
+      CrawlKit::TitleCaseHelper[info_nodes[1].content.strip]
+    end
+
+    emits :address_line_1 do
+      data = info_nodes[2].content.strip.split(',')[0]
+      unless data
+        raise MalformedDocumentError,
+          "unable to locate address for store #{store_no}"
+      end
+      CrawlKit::TitleCaseHelper[data.gsub(/[\n\r\t]+/, ' ').strip]
+    end
+
+    emits :address_line_2 do
+      data = info_nodes[2].content.strip.split(',')[1]
+      CrawlKit::TitleCaseHelper[data.gsub(/[\n\r\t]+/, ' ').strip] if data
+    end
+
+    emits :city do
+      data = info_nodes[3].content.strip.split(',')[0]
+      CrawlKit::TitleCaseHelper[data.gsub(/[\n\r\t]+/, ' ').strip] if data
+    end
+
+    emits :postal_code do
+      data = info_nodes[3].content.strip.split(',')[1]
+      unless data
+        raise MalformedDocumentError,
+        "unable to locate postal code for store #{store_no}"
+      end
+      data.gsub(/[\n\r\t]+/, ' ').strip.upcase
+    end
+
+    emits :telephone do
+      info_nodes[4].content.
+        gsub(/[\n\r\t]+/, ' ').
+        gsub('Telephone:', '').
+        strip
+    end
+
+    emits :fax do
+      if has_fax?
+        info_nodes[5].content.gsub(/[\n\r\t]+/, ' ').gsub('Fax:', '').strip
+      end
+    end
+
+    emits :latitude do
+      location['latitude'][0].to_f
+    end
+
+    emits :longitude do
+      location['longitude'][0].to_f
+    end
+
+    DETAIL_FIELDS.keys.each do |field|
+      emits(field) { details[field] }
+    end
+
+    protected
+
+    def detail_rows
+      @detail_rows ||= begin
+        doc.css('input[type="checkbox"]').map { |e| e.parent.parent.inner_html }
+      end
+    end
+
+    def details
+      @details ||= begin
+        DETAIL_FIELDS.reduce({}) do |hsh, (field, term)|
+          row   = detail_rows.detect { |row| row.include?(term) }
+          value = row.include?('checked')
+          hsh.merge(field => value)
+        end
+      end
+    end
+
+    def map_anchor_href
+      info_nodes[has_fax? ? 6 : 5].css('a').first.attributes['href'].to_s
+    end
+
+    def location
+      CGI.parse(URI.parse(map_anchor_href).query)
+    end
+
+    def has_fax?
+      info_nodes.to_s.include?('Fax: ')
+    end
+
+    def time_open_close(day)
+      open_close_times[day.to_s.downcase]
+    end
+
+    def open_close_times
+      @open_close_times ||= begin
+        time_cells.inject({}) do |hsh, td|
+          text = td.text.gsub(/\s+/, ' ')
+          day = text.match(/[MTWTFS]{1}[a-z]+/).to_s.downcase
+          times = text.scan(/[0-9]{1,2}:[0-9]{2}/)
+          open, close = *times.map { |time|
+            hour, min = *time.split(':').map { |t| t.to_i }
+            (hour * 60) + min
+          }
+          hsh.merge(day => (open == close ? [nil, nil] : [open, close]))
+        end
+      end
+    end
+
+    def container_table
+      @doc.css('table.border[width="478"]')
+    end
+
+    def hours_table
+      container_table.css('table[width="100%"]')
+    end
+
+    def info_nodes
+      container_table.css('td[width="48%"]')
+    end
+
+    def time_cells
+      hours_table.
+        css('td[width="50%"] tr').
+        select { |td| td.to_s =~ /[MTWTFS]{1}[onuesdhriat]{2,5}day/ }
+    end
+
+    def expected_node_count
+      has_fax? ? 8 : 7
+    end
+
+    def verify_store_returned
+      return if !@html.include?('No stores were located using your criteria.')
+      raise MissingResourceError, "store #{store_no} does not exist"
+    end
+
+    def verify_telephone_number
+      return if telephone
+      raise MalformedDocumentError,
+        "unable to locate telephone number for store #{store_no}"
+    end
+
+    def verify_node_count
+      return if expected_node_count == info_nodes.size
+      raise MalformedDocumentError,
+        "Expected #{expected_node_count} nodes for store #{store_no} but found " \
+        "#{info_nodes.size} instead."
+    end
+
+  end
+end