lcbo 0.10.0 → 0.10.1

data/CHANGELOG.md

@@ -1,3 +1,8 @@
+Version 0.10.1
+
+ * Added `#tags` attribute to `ProductPage` and `StorePage` to provide simple
+   stems for full-text search.
+
 Version 0.10.0
 
  * Moved `CrawlKit` related errors into the `CrawlKit` namespace.

data/Gemfile

@@ -3,3 +3,4 @@ source 'http://rubygems.org'
 gem 'typhoeus'
 gem 'nokogiri'
 gem 'unicode_utils'
+gem 'stringex'

data/Gemfile.lock

@@ -2,6 +2,7 @@ GEM
   remote: http://rubygems.org/
   specs:
     nokogiri (1.4.4)
+    stringex (1.1.0)
     typhoeus (0.2.0)
     unicode_utils (1.0.0)
 
@@ -10,5 +11,6 @@ PLATFORMS
 
 DEPENDENCIES
   nokogiri
+  stringex
   typhoeus
   unicode_utils

data/README.md

@@ -26,9 +26,9 @@ store list pages directly from the [LCBO](http://lcbo.com) website.
 ## Crawlers
 
 Some examples of crawlers exist
-[here](http://github.com/heycarsten/lcbo/blob/master/examples). You can also
-check out the
-[crawler spec](http://github.com/heycarsten/lcbo/blob/master/spec/crawlkit/crawler_spec.rb)
+[here](https://github.com/heycarsten/lcbo/tree/master/examples/crawlers).
+You can also check out the
+[crawler spec](https://github.com/heycarsten/lcbo/blob/master/spec/crawlkit/crawler_spec.rb)
 to see how to interact with them.
 
 ## Installation

data/examples/crawlers/product_lists_crawler.rb

@@ -1,6 +1,6 @@
 class ProductListsCrawler
 
-  include CrawlKit::Crawler
+  include LCBO::CrawlKit::Crawler
 
   def request(params)
     LCBO.product_list(params[:next_page] || 1)
@@ -11,7 +11,7 @@ class ProductListsCrawler
   end
 
   def reduce
-    requests.map { |params| params[:product_nos] }.flatten
+    responses.map { |params| params[:product_nos] }.flatten
   end
 
 end

data/lcbo.gemspec

@@ -17,6 +17,7 @@ Gem::Specification.new do |s|
   s.add_dependency 'typhoeus'
   s.add_dependency 'nokogiri'
   s.add_dependency 'unicode_utils'
+  s.add_dependency 'stringex'
 
   s.files         = `git ls-files`.split(?\n)
   s.test_files    = `git ls-files -- {test,spec}/*`.split(?\n)

data/lib/lcbo/crawlkit.rb

@@ -1,6 +1,7 @@
 require 'nokogiri'
 require 'typhoeus'
 require 'unicode_utils'
+require 'stringex'
 require 'uri'
 
 module LCBO
@@ -23,6 +24,7 @@ require 'lcbo/crawlkit/fastdate_helper'
 require 'lcbo/crawlkit/page'
 require 'lcbo/crawlkit/request'
 require 'lcbo/crawlkit/response'
+require 'lcbo/crawlkit/tag_helper'
 require 'lcbo/crawlkit/request_prototype'
 require 'lcbo/crawlkit/crawler'
 require 'lcbo/crawlkit/titlecase_helper'

data/lib/lcbo/crawlkit/tag_helper.rb

@@ -0,0 +1,41 @@
+module LCBO
+  module CrawlKit
+    module TagHelper
+      DELETION_RE   = /\'|\"|\\|\/|\(|\)|\[|\]|\./
+      WHITESPACE_RE = /\*|\+|\&|\_|\,|\s/
+
+      def self.flatten(values)
+        TitleCaseHelper.downcase(values.flatten.join(' ')).
+          gsub(DELETION_RE, '').
+          gsub(WHITESPACE_RE, ' ').
+          strip
+      end
+
+      def self.split(str)
+        [str, str.to_ascii].
+          join(' ').
+          split.
+          map { |word| stem(word) }.
+          flatten.
+          uniq
+      end
+
+      def self.stem(word)
+        if word.include?('-')
+          parts = word.split('-')
+          a = parts.dup
+          a << parts.join
+          a
+        else
+          word
+        end
+      end
+
+      def self.[](*values)
+        return [] if values.any? { |val| '' == val.to_s.strip }
+        split(flatten(values))
+      end
+
+    end
+  end
+end

data/lib/lcbo/pages/product_page.rb

@@ -19,6 +19,17 @@ module LCBO
       CrawlKit::TitleCaseHelper[product_details_form('itemName')]
     end
 
+    emits :tags do
+      CrawlKit::TagHelper[
+        name,
+        primary_category,
+        secondary_category,
+        origin,
+        producer_name,
+        package_unit_type
+      ]
+    end
+
     emits :price_in_cents do
       (product_details_form('price').to_f * 100).to_i
     end

data/lib/lcbo/pages/store_page.rb

@@ -36,20 +36,20 @@ module LCBO
       query_params[:store_no].to_i
     end
 
-    DAY_NAMES.each do |day|
-      emits :"#{day}_open" do
-        time_open_close(day)[0]
-      end
-
-      emits :"#{day}_close" do
-        time_open_close(day)[1]
-      end
-    end
-
     emits :name do
       CrawlKit::TitleCaseHelper[info_nodes[1].content.strip]
     end
 
+    emits :tags do
+      CrawlKit::TagHelper[
+        name,
+        address_line_1,
+        address_line_2,
+        city,
+        postal_code
+      ]
+    end
+
     emits :address_line_1 do
       data = info_nodes[2].content.strip.split(',')[0]
       unless data
@@ -99,6 +99,16 @@ module LCBO
       location['longitude'][0].to_f
     end
 
+    DAY_NAMES.each do |day|
+      emits :"#{day}_open" do
+        time_open_close(day)[0]
+      end
+
+      emits :"#{day}_close" do
+        time_open_close(day)[1]
+      end
+    end
+
     DETAIL_FIELDS.keys.each do |field|
       emits(field) { details[field] }
     end

data/lib/lcbo/version.rb

@@ -1,3 +1,3 @@
 module LCBO
-  VERSION = '0.10.0'
+  VERSION = '0.10.1'
 end

data/spec/crawlkit/tag_helper_spec.rb

@@ -0,0 +1,18 @@
+# coding: utf-8
+require 'spec_helper'
+
+describe LCBO::CrawlKit::TagHelper do
+  @expectations = {
+    ['Hello World']         => %w[hello world],
+    ['Éve Picard']          => %w[éve picard eve],
+    ['Hello Hello World']   => %w[hello world],
+    ['Hello', 'Éve Picard'] => %w[hello éve picard eve],
+    ['Hello', 'Éve-Picard'] => %w[hello éve picard évepicard eve evepicard]
+  }
+
+  @expectations.each_pair do |input, expectation|
+    it "should tagify: #{input.inspect} to: #{expectation.inspect}" do
+      LCBO::CrawlKit::TagHelper[*input].must_equal expectation
+    end
+  end
+end

data/spec/pages/product_pages.yml

@@ -7,6 +7,7 @@
   :expectation:
     :product_no: 122408
     :name: Pascual Toso Malbec Rose
+    :tags: []
     :price_in_cents: 1195
     :regular_price_in_cents: 1195
     :limited_time_offer_savings_in_cents: 0
@@ -47,6 +48,7 @@
   :expectation:
     :product_no: 18
     :name: Heineken Lager
+    :tags: ["heineken", "lager", "beer", "netherlands", "heinekens", "brouwerijen", "nederland", "bv", "bottle"]
     :price_in_cents: 1250
     :regular_price_in_cents: 1350
     :limited_time_offer_savings_in_cents: 100
@@ -87,6 +89,7 @@
   :expectation:
     :product_no: 280461
     :name: Dom Pérignon Brut Champagne
+    :tags: ["dom", "pérignon", "brut", "champagne", "wine", "france", "moet", "chandon", "sa", "bottle", "perignon"]
     :price_in_cents: 21995
     :regular_price_in_cents: 21995
     :limited_time_offer_savings_in_cents: 0
@@ -127,6 +130,7 @@
   :expectation:
     :product_no: 479154
     :name: Floris Ninkeberry Gardenbeer
+    :tags: ["floris", "ninkeberry", "gardenbeer", "beer", "ale", "belgium", "brouwerij", "huyghe", "bottle"]
     :price_in_cents: 250
     :regular_price_in_cents: 250
     :limited_time_offer_savings_in_cents: 0
@@ -167,6 +171,7 @@
   :expectation:
     :product_no: 148528
     :name: Bodegas Ruconia Rey Don Garcia El de Najera Crianza 2005
+    :tags: ["bodegas", "ruconia", "rey", "don", "garcia", "el", "de", "najera", "crianza", "2005", "wine", "red", "rioja", "spain", "sl", "bottle"]
     :price_in_cents: 1595
     :regular_price_in_cents: 1595
     :limited_time_offer_savings_in_cents: 0
@@ -207,6 +212,7 @@
   :expectation:
     :product_no: 582973
     :name: Bombay Sapphire
+    :tags: ["bombay", "sapphire", "spirits", "gin", "england", "united", "kingdom", "company", "ltd", "bottle"]
     :price_in_cents: 5545
     :regular_price_in_cents: 5545
     :limited_time_offer_savings_in_cents: 0
@@ -247,6 +253,7 @@
   :expectation:
     :product_no: 2183079
     :name: Lake & River Series Cabernet Franc VQA
+    :tags: []
     :price_in_cents: 1095
     :regular_price_in_cents: 1095
     :limited_time_offer_savings_in_cents: 0
@@ -287,6 +294,7 @@
   :expectation:
     :product_no: 114173
     :name: 05 Sauv Blanc Weinstock Cellar K
+    :tags: []
     :price_in_cents: 2295
     :regular_price_in_cents: 2295
     :limited_time_offer_savings_in_cents: 0

data/spec/pages/store_pages.yml

@@ -7,6 +7,7 @@
   :expectation:
     :store_no: 444
     :name: Kennedy & Sheppard
+    :tags: ["kennedy", "sheppard", "2356", "road", "agincourt", "mall", "toronto", "scarborough", "torontoscarborough", "m1t3h1"]
     :address_line_1: 2356 Kennedy Road
     :address_line_2: Agincourt Mall
     :city: Toronto-Scarborough
@@ -61,6 +62,7 @@
     :sunday_open: 720
     :sunday_close: 960
     :name: Flora & Clinton Street
+    :tags: ["flora", "clinton", "street", "1", "industrial", "road", "unit", "b", "teeswater", "n0g2s0"]
     :address_line_1: 1 Industrial Road
     :address_line_2: Unit B
     :city: Teeswater

metadata

@@ -5,8 +5,8 @@ version: !ruby/object:Gem::Version
   segments: 
   - 0
   - 10
-  - 0
-  version: 0.10.0
+  - 1
+  version: 0.10.1
 platform: ruby
 authors: 
 - Carsten Nielsen
@@ -14,7 +14,7 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2010-12-09 00:00:00 -05:00
+date: 2010-12-30 00:00:00 -05:00
 default_executable: 
 dependencies: 
 - !ruby/object:Gem::Dependency 
@@ -56,6 +56,19 @@ dependencies:
         version: "0"
   type: :runtime
   version_requirements: *id003
+- !ruby/object:Gem::Dependency 
+  name: stringex
+  prerelease: false
+  requirement: &id004 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        segments: 
+        - 0
+        version: "0"
+  type: :runtime
+  version_requirements: *id004
 description: Request and parse product, store, inventory, and product search pages directly from the official LCBO website.
 email: 
 - heycarsten@gmail.com
@@ -89,6 +102,7 @@ files:
 - lib/lcbo/crawlkit/request.rb
 - lib/lcbo/crawlkit/request_prototype.rb
 - lib/lcbo/crawlkit/response.rb
+- lib/lcbo/crawlkit/tag_helper.rb
 - lib/lcbo/crawlkit/titlecase_helper.rb
 - lib/lcbo/crawlkit/volume_helper.rb
 - lib/lcbo/ext.rb
@@ -107,6 +121,7 @@ files:
 - spec/crawlkit/request_prototype_spec.rb
 - spec/crawlkit/request_spec.rb
 - spec/crawlkit/response_spec.rb
+- spec/crawlkit/tag_helper_spec.rb
 - spec/crawlkit/titlecase_helper_spec.rb
 - spec/crawlkit/volume_helper_spec.rb
 - spec/crawlkit_spec.rb
@@ -174,6 +189,7 @@ test_files:
 - spec/crawlkit/request_prototype_spec.rb
 - spec/crawlkit/request_spec.rb
 - spec/crawlkit/response_spec.rb
+- spec/crawlkit/tag_helper_spec.rb
 - spec/crawlkit/titlecase_helper_spec.rb
 - spec/crawlkit/volume_helper_spec.rb
 - spec/crawlkit_spec.rb