lcbo 0.9.0

data/.gitignore

@@ -0,0 +1 @@
+pkg

data/CHANGELOG.md

@@ -0,0 +1,3 @@
+Version 0.9.0
+
+ * Initial public release

data/Gemfile

@@ -0,0 +1,6 @@
+source 'http://rubygems.org'
+
+gem 'rspec',       '1.3.0'
+gem 'typhoeus',    '0.1.29'
+gem 'addressable', '2.1.2'
+gem 'nokogiri',    '1.4.3.1'

data/Gemfile.lock

@@ -0,0 +1,18 @@
+GEM
+  remote: http://rubygems.org/
+  specs:
+    addressable (2.1.2)
+    nokogiri (1.4.3.1)
+    rack (1.2.1)
+    rspec (1.3.0)
+    typhoeus (0.1.29)
+      rack
+
+PLATFORMS
+  ruby
+
+DEPENDENCIES
+  addressable (= 2.1.2)
+  nokogiri (= 1.4.3.1)
+  rspec (= 1.3.0)
+  typhoeus (= 0.1.29)

data/LICENSE

@@ -0,0 +1,18 @@
+Copyright (c) 2010 Carsten Nielsen
+
+Permission is hereby granted, free of charge, to any person obtaining a copy of
+this software and associated documentation files (the "Software"), to deal in
+the Software without restriction, including without limitation the rights to
+use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies
+of the Software, and to permit persons to whom the Software is furnished to do
+so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS
+FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.md

@@ -0,0 +1,29 @@
+# LCBO: The Ruby Gem
+
+This library is used to gather data for [LCBO API](http://lcboapi.com). It allows you to request and parse store, product, inventory, and product list pages directly from the [LCBO](http://lcbo.com) website.
+
+## Synopsis
+
+    require 'lcbo'
+
+    LCBO.store(511)
+    # => { :store_no => 511, :name => "King & Spadina", ... }
+
+    LCBO.product(18)
+    # => { :product_no => 11, :name => "Heineken Lager", ... }
+
+    LCBO.inventory(18)
+    # => { :product_no => 18, :inventory_count => 40398, :inventories => [ ... ] }
+
+    LCBO.products_list(1)
+    # => { :page => 1, :final_page => 108, ..., :product_nos => [ ... ] }
+
+## Installation
+
+Use Ruby Gems: `gem install lcbo`
+
+## Links
+
+ * [Issue tracker](http://github.com/heycarsten/lcbo/issues)
+ * [Source code](http://github.com/heycarsten/lcbo)
+ * [License](http://github.com/heycarsten/lcbo/blob/master/LICENSE)

data/Rakefile

@@ -0,0 +1,62 @@
+gem 'rspec', '1.3.0'
+require 'rubygems/specification' unless defined?(Gem::Specification)
+require 'spec/rake/spectask'
+require 'rake/gempackagetask'
+
+task :default => :spec
+
+def gemspec
+  @gemspec ||= begin
+    Gem::Specification.load(File.expand_path('lcbo.gemspec'))
+  end
+end
+
+desc 'Start an irb console'
+task :console do
+  system 'irb -I lib -r lcbo'
+end
+
+desc 'Validates the gemspec'
+task :gemspec do
+  gemspec.validate
+end
+
+desc 'Displays the current version'
+task :version do
+  puts "Current version: #{gemspec.version}"
+end
+
+desc 'Installs the gem locally'
+task :install => :package do
+  sh "gem install pkg/#{gemspec.name}-#{gemspec.version}"
+end
+
+desc 'Release the gem'
+task :release => :package do
+  sh "gem push pkg/#{gemspec.name}-#{gemspec.version}.gem"
+end
+
+Rake::GemPackageTask.new(gemspec) do |pkg|
+  pkg.gem_spec = gemspec
+end
+task :gem => :gemspec
+task :package => :gemspec
+
+desc 'Run the specs'
+Spec::Rake::SpecTask.new do |t|
+  t.spec_files = FileList['spec/**/*_spec.rb']
+  t.spec_opts  = %w[-fs --color]
+end
+
+desc 'Download all HTML indicated in YAML assertion files'
+task :download_support do
+  require 'yaml'
+  require 'open-uri'
+  product_pages = YAML.load_file('./spec/support/product_pages.yml')
+  product_pages.each do |spec|
+    html = open(spec[:uri]).read
+    File.open("./spec/support/product_pages/#{spec[:file]}", ?w) { |file|
+      file.print(html)
+    }
+  end
+end

data/lcbo.gemspec

@@ -0,0 +1,29 @@
+# coding: utf-8
+require File.expand_path("../lib/lcbo/version", __FILE__)
+
+Gem::Specification.new do |s|
+  s.name        = 'lcbo'
+  s.version     = LCBO::VERSION
+  s.platform    = Gem::Platform::RUBY
+  s.authors     = ['Carsten Nielsen']
+  s.email       = ['heycarsten@gmail.com']
+  s.homepage    = 'http://github.com/heycarsten/lcbo'
+  s.summary     = %q{A library for parsing HTML pages from http://lcbo.com}
+  s.description = %q{Request and parse product, store, inventory, and product search pages directly from the official LCBO website.}
+
+  s.required_rubygems_version = '>= 1.3.6'
+  s.rubyforge_project = 'lcbo'
+
+  s.add_dependency 'typhoeus'
+  s.add_dependency 'addressable'
+  s.add_dependency 'nokogiri'
+
+  s.add_development_dependency 'rspec', '1.3.0'
+
+  s.files         = `git ls-files`.split(?\n)
+  s.test_files    = `git ls-files -- {test,spec}/*`.split(?\n)
+  s.require_paths = ['lib']
+
+  # If you need an executable, add it here
+  # s.executables = ['lcbo']
+end

data/lib/lcbo.rb

@@ -0,0 +1,23 @@
+module LCBO
+
+  DEFAULT_CONFIG = {
+    :user_agent => nil,
+  }
+
+  def self.config
+    reset_config! unless @config
+    @config
+  end
+
+  def self.reset_config!
+    @config = DEFAULT_CONFIG.dup
+  end
+
+end
+
+require 'lcbo/version'
+require 'lcbo/ext'
+require 'lcbo/helpers'
+require 'lcbo/crawlkit'
+require 'lcbo/pages'
+require 'lcbo/crawlers'

data/lib/lcbo/crawlers.rb

@@ -0,0 +1,4 @@
+require 'lcbo/crawlers/inventories_crawler'
+require 'lcbo/crawlers/products_crawler'
+require 'lcbo/crawlers/product_lists_crawler'
+require 'lcbo/crawlers/stores_crawler'

data/lib/lcbo/crawlers/inventories_crawler.rb

@@ -0,0 +1,15 @@
+module LCBO
+  class InventoriesCrawler
+
+    def self.run(product_nos, &block)
+      raise ArgumentError, 'block expected' unless block_given?
+      product_nos.each do |product_no|
+        begin
+          yield InventoryRequest.parse(:product_no => product_no).as_hash
+        rescue
+        end
+      end
+    end
+
+  end
+end

data/lib/lcbo/crawlers/product_lists_crawler.rb

@@ -0,0 +1,23 @@
+module LCBO
+  class ProductListsCrawler
+
+    MAX_RETRIES = 10
+
+    class Error < StandardError; end
+    class EpicTimeoutError < Error; end
+
+    def self.run(params = {}, tries = 0, &block)
+      raise ArgumentError, 'block expected' unless block_given?
+      begin
+        payload = ProductListRequest.parse(params).as_hash
+        yield(payload)
+        run(:page => payload[:next_page], &block) if payload[:next_page]
+      rescue Errno::ETIMEDOUT, Timeout::Error
+        # On timeout, try again.
+        raise EpicTimeoutError if tries > MAX_RETRIES
+        run(params, (tries + 1), &block)
+      end
+    end
+
+  end
+end

data/lib/lcbo/crawlers/products_crawler.rb

@@ -0,0 +1,16 @@
+module LCBO
+  class ProductsCrawler
+
+    def self.run(product_nos, &block)
+      raise ArgumentError, 'block expected' unless block_given?
+      product_nos.each do |product_no|
+        begin
+          yield ProductRequest.parse(:product_no => product_no).as_hash
+        rescue CrawlKit::MissingResourceError, Errno::ETIMEDOUT, Timeout::Error
+          # Ignore products with no data, and timeouts.
+        end
+      end
+    end
+
+  end
+end

data/lib/lcbo/crawlers/stores_crawler.rb

@@ -0,0 +1,16 @@
+module LCBO
+  class StoresCrawler
+
+    def self.run(&block)
+      raise ArgumentError, 'block expected' unless block_given?
+      (1..720).each do |store_no|
+        begin
+          yield StoreRequest.parse(:store_no => store_no).as_hash
+        rescue CrawlKit::MissingResourceError, Errno::ETIMEDOUT, Timeout::Error
+          # Ignore stores that don't exist and timeouts.
+        end
+      end
+    end
+
+  end
+end

data/lib/lcbo/crawlkit.rb

@@ -0,0 +1,24 @@
+require 'addressable/template'
+require 'addressable/uri'
+require 'nokogiri'
+require 'typhoeus'
+require 'uri'
+
+module LCBO
+  module CrawlKit
+    USER_AGENT ||= begin
+      LCBO.config[:user_agent] ||
+      ENV['LCBO_USER_AGENT'] ||
+      Typhoeus::USER_AGENT
+    end
+  end
+end
+
+require 'lcbo/crawlkit/eventable'
+require 'lcbo/crawlkit/fastdate_helper'
+require 'lcbo/crawlkit/page'
+require 'lcbo/crawlkit/request'
+require 'lcbo/crawlkit/response'
+require 'lcbo/crawlkit/request_prototype'
+require 'lcbo/crawlkit/titlecase_helper'
+require 'lcbo/crawlkit/volume_helper'

data/lib/lcbo/crawlkit/eventable.rb

@@ -0,0 +1,56 @@
+module LCBO
+  module CrawlKit
+    module Eventable
+
+      EVENT_TYPES = %[
+        before_request
+        after_request
+        before_parse
+        after_parse ]
+
+      def self.included(mod)
+        mod.extend(ClassMethods)
+      end
+
+      def fire(event_type)
+        self.class.callbacks.
+          select { |callback| callback.is_for?(event_type) }.
+          each   { |callback| callback.call_on(self) }
+      end
+
+      module ClassMethods
+        def on(event_type, *method_names)
+          @callbacks ||= []
+          method_names.each do |method_name|
+            @callbacks << Callback.new(event_type, method_name)
+          end
+        end
+
+        def callbacks
+          @callbacks || []
+        end
+      end
+
+      class Callback
+        attr_reader :event_type, :method_name
+
+        def initialize(event_type, method_name)
+          @event_type = event_type.to_sym
+          @method_name = method_name.to_sym
+        end
+
+        def is_for?(event_sym)
+          unless EVENT_TYPES.include?(event_sym.to_s)
+            raise ArgumentError, "event_type: #{event_sym} is not valid"
+          end
+          event_type == event_sym.to_sym
+        end
+
+        def call_on(object)
+          object.send(method_name)
+        end
+      end
+
+    end
+  end
+end

data/lib/lcbo/crawlkit/fastdate_helper.rb

@@ -0,0 +1,40 @@
+module LCBO
+  module CrawlKit
+    class FastDateHelper
+
+      MONTH_NAMES_TO_NUMBERS = {
+        'Jan' => '01',
+        'Feb' => '02',
+        'Mar' => '03',
+        'Apr' => '04',
+        'May' => '05',
+        'Jun' => '06',
+        'Jul' => '07',
+        'Aug' => '08',
+        'Sep' => '09',
+        'Oct' => '10',
+        'Nov' => '11',
+        'Dec' => '12' }
+
+      attr_reader :input
+
+      def initialize(input_string)
+        @input = input_string
+      end
+
+      def self.[](input_string)
+        new(input_string).as_sql_date
+      end
+
+      def as_sql_date
+        return nil unless input
+        parts = input.gsub(',', '').split
+        month = MONTH_NAMES_TO_NUMBERS[parts[0]]
+        return nil unless month
+        day = parts[1].rjust(2, '0')
+        "#{parts[2]}-#{month}-#{day}"
+      end
+
+    end
+  end
+end

data/lib/lcbo/crawlkit/page.rb

@@ -0,0 +1,141 @@
+module LCBO
+  module CrawlKit
+    module Page
+
+      class Error < StandardError; end
+      class MalformedDocumentError < Error; end
+      class MissingResourceError < Error; end
+      class RequestFailedError < Error; end
+
+      def self.included(mod)
+        mod.module_eval do
+          include Eventable
+          attr_reader :html, :query_params, :body_params, :response
+          instance_variable_set :@request_prototype, RequestPrototype.new
+          instance_variable_set :@fields, []
+        end
+        mod.extend(ClassMethods)
+      end
+
+      module ClassMethods
+        def uri(value = nil)
+          if value
+            @request_prototype.uri_template = value
+          else
+            @request_prototype.uri_template
+          end
+        end
+
+        def default_body_params(value = nil)
+          if value
+            @request_prototype.body_params = value
+          else
+            @request_prototype.body_params
+          end
+        end
+
+        def http_method(value = nil)
+          if value
+            @request_prototype.http_method = value
+          else
+            @request_prototype.http_method
+          end
+        end
+
+        def emits(field, &block)
+          fields << field.to_sym
+          define_method(field) { instance_exec(field, &block) } if block_given?
+        end
+
+        def request(query_params = {}, body_params = {})
+          new(query_params, body_params).request
+        end
+
+        def parse(response)
+          new(nil, nil, response).parse
+        end
+
+        def process(query_params = {}, body_params = {})
+          new(query_params, body_params).process
+        end
+
+        def fields
+          @fields
+        end
+
+        def request_prototype
+          @request_prototype
+        end
+      end
+
+      def initialize(query_params = {}, body_params = {}, response = nil)
+        if response
+          @response     = response.is_a?(Hash) ? Response.new(response) : response
+          @query_params = @response.query_params
+          @body_params  = @response.body_params
+          @html         = @response.body
+        else
+          @query_params = query_params
+          @body_params  = body_params
+        end
+      end
+
+      def [](field)
+        as_hash[field.to_sym]
+      end
+
+      def request_prototype
+        self.class.request_prototype
+      end
+
+      def fields
+        self.class.fields
+      end
+
+      def http_method
+        self.class.http_method
+      end
+
+      def process
+        request
+        parse
+        self
+      end
+
+      def request
+        return if @html
+        fire :before_request
+        @response = request_prototype.request(query_params, body_params)
+        @html     = @response.body
+        fire :after_request
+        self
+      end
+
+      def parse
+        return if is_parsed?
+        return unless @html
+        fire :before_parse
+        @doc = Nokogiri::HTML(@html)
+        fire :after_parse
+        self
+      end
+
+      def is_parsed?
+        doc ? true : false
+      end
+
+      def as_hash
+        @as_hash ||= begin
+          fields.reduce({}) { |hsh, field| hsh.merge(field => send(field)) }
+        end
+      end
+
+      protected
+
+      def doc
+        @doc
+      end
+
+    end
+  end
+end