bazaar_sources 0.2.1.1.1.4

data/lib/api_helpers/external_url.rb

@@ -0,0 +1,127 @@
+module ExternalUrl
+  require 'net/http'
+  require 'uri'
+
+  REQUEST_HEADERS = {
+    'User-Agent' => 'Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.8.1.11) Gecko/20071127 Firefox/2.0.0.11'
+  }
+
+  # Note: This method is only used now by the Validator, and is only suitable for the Validator.
+  def self.just_page_content(page)
+    # No longer used
+    justbody = /.*?<body.*?>(.*)<\/body>/im
+    comments = /(<!--.*?-->)(.{1,40})/m
+    nostyle = /<style.*?<\/style>/im
+    notags = /<.*?>/im
+    noentities = /&.*?;/
+    noextrawhitespace = /(\s)+/im
+    # Remove comments, unless inside of JavaScript (because frequently JavaScript has good matches for model numbers, etc.)
+    page.gsub(comments) do |c|
+      comment = $1
+      post = $2
+      if post =~ /<\/script/
+        comment + post
+      else
+        post
+      end
+    end.gsub(nostyle,' ').gsub(notags,'').gsub(noentities,' ').gsub(noextrawhitespace,'\1')
+  end
+
+  # returns a hash containing :success flag; if true, you'll have the :response (thus response.body) and :final_uri
+  # (e.g. if redirected) If false a :message is set and :final_uri
+  def self.fetch_response(url, limit = 10, debug = false)
+    begin
+      if limit == 0
+        return {:success => false, :response => nil, :message => "Redirected too many times", :final_uri => url}
+      end
+
+      message = self.invalid_uri(url)
+      if message
+        return {:success => false, :response => nil, :message => message, :final_uri => url}
+      end
+      uri = URI.safe_parse(url.to_s)
+      http_request = Net::HTTP.new(uri.host)
+      if debug
+        puts "http request: #{http_request.inspect}"
+      end
+      # Adding user agent header helps some merchants feel more comfortable with our bot
+      no_host_url = uri.to_s.gsub(/.*?#{uri.host}(.*)/,'\1')
+      if debug
+        puts "http request to: #{no_host_url}"
+      end
+      response = http_request.get(no_host_url, REQUEST_HEADERS)
+      if debug
+        puts "http response: #{response.inspect}"
+      end
+
+      case response
+      when Net::HTTPSuccess then
+        if debug
+          puts "Success, final url: #{url}"
+          ExternalUrl.to_file(url, response.body, "html")
+          ExternalUrl.to_file(url, ExternalUrl.just_page_content(response.body), "txt")
+        end
+        {:success => true, :response => response, :final_uri => url}
+      when Net::HTTPRedirection then
+        redirect_url = to_absolute_url(response['location'], url)
+        if debug
+          puts "Redirecting to #{redirect_url}"
+        end
+        self.fetch_response(redirect_url, limit - 1, debug)
+      else
+        {:success => false, :response => response, :final_uri => url}
+      end
+    rescue Exception => exp
+      {:success => false, :response => nil, :message => exp.message, :final_uri => url}
+    end
+  end
+
+  private
+
+  def self.to_file(url, content, ext = "html")
+    uri = URI.safe_parse(url.to_s)
+    filename_base = "#{uri.host}.#{uri.path}?#{uri.query}"
+    filename = filename_base.gsub(/(\W)+/,"-") + ".#{ext}"
+    f = File.new(filename, "w")
+    f.write(content)
+    f.close
+    puts "Wrote content to file #{filename} in #{Dir.pwd}"
+  end
+
+  # Returns nil if the URL is a valid URI, else a message.  In this case, scheme, host and path are all required.
+  def self.invalid_uri(url)
+    return 'No URL' if url.nil? || url.empty?
+    begin
+      uri = URI.safe_parse(url)
+      if uri.nil? || uri.scheme.nil? || uri.host.nil? || uri.path.nil?
+        if uri.nil?
+          return "URL is not well formed"
+        else
+          return "URL incomplete: scheme is #{uri.scheme.nil? ? "missing" : uri.scheme}, host is #{uri.host.nil? ? "missing" : uri.host} and path is #{uri.path.nil? ? "missing" : uri.path}"
+        end
+      end
+    rescue Exception => exp
+      return "URI improperly formed: #{exp.message}"
+    end
+    return nil
+  end
+
+  def self.to_absolute_url(url, current_url)
+    unless url.is_a? URI
+      url = URI.safe_parse(url)
+    end
+
+    # construct an absolute url
+    if url.relative?
+      unless current_url.is_a? URI
+        current_url = URI.safe_parse(current_url)
+      end
+
+      url.scheme = current_url.scheme
+      url.host = current_url.host
+      url.port = current_url.port unless current_url.port == 80
+    end
+
+    return url.to_s
+  end
+end

data/lib/api_helpers/httparty_nokogiri_parser.rb

@@ -0,0 +1,14 @@
+require 'nokogiri'
+
+class HttpartyNokogiriParser < HTTParty::Parser
+
+  protected
+
+  def xml
+    Nokogiri::XML(body)
+  end
+
+  def html
+    Nokogiri::HTML(body)
+  end
+end

data/lib/api_helpers/reseller_ratings_api.rb

@@ -0,0 +1,174 @@
+require 'ostruct'
+require 'hpricot'
+require 'api_helpers/external_url'
+
+module ResellerRatingsAPI
+  def self.alt_code_from_merchant_source_page_url(merchant_source_page_url)
+    alt_code = nil
+    if res = merchant_source_page_url.match(/resellerratings\.com\/store\/([^\/\?#]*)/)
+      alt_code = res[1]
+    end
+    alt_code
+  end
+
+  def self.fetch_suggestions(search_text, limit)
+    rr_search_url = "http://www.resellerratings.com/reseller_list.pl?keyword_search=#{URI.escape(search_text)}"
+    doc_and_final_uri = open_doc(rr_search_url)
+    result = []
+    unless doc_and_final_uri.nil? || doc_and_final_uri[:final_uri].nil? || doc_and_final_uri[:final_uri].empty?
+      final_uri = URI.safe_parse(doc_and_final_uri[:final_uri])
+      if final_uri.path == '/reseller_list.pl'
+        # got the search results page with more than one result
+        result = convert_search_results_page_to_merchant_array(doc_and_final_uri[:doc], limit)
+      elsif final_uri.path.match(/^\/store\/(.+)$/)
+        # got merchant page back
+        result << { :merchant_page_url => final_uri, :merchant_code => $1, :merchant_name => $1.gsub('_', ' ') }
+      else
+        # don't know where we ended up
+      end
+    end
+    result
+  end
+
+  def self.search_for_merchant_source(search_text, limit=15)
+    merchant_sources = []
+    rr_search_url = "http://www.resellerratings.com/reseller_list.pl?keyword_search=#{URI.escape(search_text)}"
+    doc_and_final_uri = open_doc(rr_search_url)
+    unless doc_and_final_uri.nil?
+      if merchant_page_url?(doc_and_final_uri[:final_uri])
+        merchant_sources << convert_merchant_page_to_merchant_source(doc_and_final_uri)
+      else
+        doc_and_final_uri[:doc].search('tr/td/font/a[text() = "Read Reviews"]/../../..').each do |result_row|
+          element = result_row.at('td//a')
+          name = element.inner_text.strip
+          alt_merchant_code = element.attributes['href'].match(/\/store\/(.+)$/)[1]
+          existing_merchant_source = MerchantSource.find_by_source_and_alt_code(Source.reseller_ratings_source, alt_merchant_code)
+          if existing_merchant_source.nil?
+            merchant_sources << OpenStruct.new({:source => Source.reseller_ratings_source, :name => name, :alt_code => alt_merchant_code})
+          else
+            merchant_sources << existing_merchant_source
+          end
+          break if merchant_sources.length >= limit
+        end
+      end
+    end
+    merchant_sources
+  end
+
+  def self.search_for_merchant_source_best_match(search_text)
+    rr_search_url = "http://www.resellerratings.com/reseller_list.pl?keyword_search=#{URI.escape(search_text)}"
+    fetch_merchant_source(rr_search_url)
+  end
+
+  def self.fetch_merchant_source(merchant_source_page_url)
+    doc_and_final_uri = open_doc(merchant_source_page_url)
+    if !doc_and_final_uri.nil?
+      convert_merchant_page_to_merchant_source(doc_and_final_uri)
+    else
+      nil
+    end
+  end
+
+  def self.fetch_merchant_source_by_alt_merchant_code(alt_merchant_code)
+    merchant_source_page_url = "http://www.resellerratings.com/store/#{alt_merchant_code}"
+    fetch_merchant_source(merchant_source_page_url)
+  end
+
+  def self.merchant_page_url?(url)
+    !url.nil? && url.match(/\/store\/.+$/) != nil
+  end
+
+  private
+
+  def self.convert_merchant_page_to_merchant_source(doc_and_final_uri)
+    return nil if doc_and_final_uri.nil?
+    merchant_source = OpenStruct.new
+    merchant_source.source = Source.reseller_ratings_source
+    doc = doc_and_final_uri[:doc]
+
+    # Merchant Code
+    elements = doc.search('img[@src="http://images.resellerratings.com/images/write_a_review.gif"]/..')
+    unless elements.empty?
+      code = elements.first[:href].match(/^.*?([0-9]+).*?$/)[1]
+      merchant_source.code = code
+    end
+
+    # Use a blank 'code' to indicate we didn't find the merchant page
+    if merchant_source.code.nil? || merchant_source.code.empty?
+      return nil
+    end
+
+    # Alternative Merchant Code
+    unless doc_and_final_uri[:final_uri].nil? || doc_and_final_uri[:final_uri].empty?
+      merchant_source.alt_code = alt_code_from_merchant_source_page_url(doc_and_final_uri[:final_uri])
+    end
+
+    # Merchant Name
+    elements = doc.search('img[@src="http://images.resellerratings.com/images/small-storefront-rev.gif"]/../..')
+    unless elements.empty?
+      name = elements.first.inner_text.strip
+      merchant_source.name = name
+    end
+
+    # Merchant Homepage
+    elements = doc.search('font[text() *= "Homepage:"]/a/font')
+    unless elements.empty?
+      homepage = elements.first.inner_text.strip
+      merchant_source.homepage = homepage
+    end
+
+    # Merchant Rating
+    elements = doc.search('font[text() *= "Six-Month Rating:"]/../font[2]')
+    unless elements.empty?
+      merchant_rating = elements.first.inner_text.match(/\s*(.*?)\s*\/.*?/)[1]
+      merchant_source.merchant_rating = (merchant_rating.to_f * 10.0).round unless merchant_rating.nil?
+    end
+
+    # Num Merchant Reviews
+    elements = doc.search('font[text() *= "Six-Month Reviews:"]/../../td[2]')
+    unless elements.empty?
+      num_merchant_reviews = elements.first.inner_text.strip
+      merchant_source.num_merchant_reviews = num_merchant_reviews
+    end
+
+    # Merchant Rating Lifetime
+    elements = doc.search('font[text() *= "Lifetime Rating:"]/../font[2]')
+    unless elements.empty?
+      merchant_rating_lifetime = elements.first.inner_text.match(/\s*(.*?)\s*\/.*?/)[1]
+      merchant_source.merchant_rating_lifetime = (merchant_rating_lifetime.to_f * 10.0).round unless merchant_rating_lifetime.nil?
+    end
+
+    # Num Merchant Reviews Lifetime
+    elements = doc.search("font[text() *= 'Lifetime\nReviews:']/../../td[2]")
+    unless elements.empty?
+      num_merchant_reviews_lifetime = elements.first.inner_text.strip
+      merchant_source.num_merchant_reviews_lifetime = num_merchant_reviews_lifetime
+    end
+
+    merchant_source
+  end
+
+  def self.convert_search_results_page_to_merchant_array(search_results_doc, limit)
+    result = []
+    merchant_links = search_results_doc.search('tr[/td/font/a/font/b[text() = "Store Name"]]../tr/td/a')
+    merchant_links.each_with_index do |merchant_link, index|
+      break if index > limit-1
+      merchant_link.attributes['href'].match(/^.*\/store\/(.+)$/)
+      result << { :merchant_page_url => merchant_link.attributes['href'],
+                  :merchant_code => $1,
+                  :merchant_name => merchant_link.inner_text.strip }
+    end
+    result
+  end
+
+  def self.open_doc(url)
+    response = ExternalUrl.fetch_response(url)
+    if response[:success]
+      doc = Hpricot(response[:response].body)
+      final_uri = response[:final_uri]
+      return {:doc => doc, :final_uri => final_uri}
+    else
+      return nil
+    end
+  end
+end

data/lib/api_helpers/shopping.rb

@@ -0,0 +1,224 @@
+require File.join(File.dirname(__FILE__), 'api_helper')
+require 'rubygems'
+gem 'httparty', '>= 0.5.0'
+require 'httparty'
+require File.join(File.dirname(__FILE__), 'httparty_nokogiri_parser')
+
+module Shopping
+  class Error < StandardError
+    attr_reader :code
+    def initialize(message, code)
+      super(message)
+      @code = code
+    end
+  end
+
+  class Publisher
+    include ApiHelper
+    include HTTParty
+    parser HttpartyNokogiriParser
+    format :xml
+    base_uri 'http://publisher.api.shopping.com/publisher/3.0/rest/'
+    default_params 'trackingId' => '3068547', 'apiKey' => SHOPPING_API_KEY
+
+    MAX_OFFERS = 20
+    PUTS_API_URL = false
+
+    def fetch_product(product_id, include_specs=false, include_offers=false)
+      # Won't get psuedo-redirected to new product ID unless we request at least one offer (very strange)
+      query = {'numItems' => include_offers ? MAX_OFFERS : 1, 'productId' => product_id.strip, 'showProductSpecs' => include_specs ? 'true' : 'false'}
+      call_api('/GeneralSearch', {:query => query}) do |doc|
+        product_node = doc.at('GeneralSearchResponse/categories/category/items/product')
+        convert_product_node(product_node, include_offers)
+      end
+    end
+
+    def fetch_offers(product_id)
+      query = {'numItems' => MAX_OFFERS, 'showOffersOnly' => 'true', 'productId' => product_id.strip}
+      call_api('/GeneralSearch', {:query => query}) do |doc|
+        items_node = doc.at('GeneralSearchResponse/categories/category/items')
+        convert_offers_collection_node(items_node)
+      end
+    end
+
+    def search_for_product(keyword, max_results=10)
+      query = {'doSkipping' => 'false', 'showProductOffers' => 'false', 'numAttributes' => 0, 'numItems' => max_results, 'keyword' => keyword.strip}
+      call_api('/GeneralSearch', {:query => query}) do |doc|
+        product_nodes = doc.search('GeneralSearchResponse/categories/category/items/product')
+        product_nodes.collect{|product_node| convert_product_node(product_node)}
+      end
+    end
+
+    protected
+
+    def call_api(path, options, &block)
+      if PUTS_API_URL
+        merged_options = self.class.default_options.dup.merge(options)
+        puts "Shopping.com API URL: #{HTTParty::Request.new(Net::HTTP::Get, path, merged_options).uri}"
+      end
+      doc = self.class.get(path, options)
+      errors = get_errors(doc)
+      if errors.empty?
+        yield doc
+      else
+        raise_exception(errors)
+      end
+    end
+
+    def get_errors(doc)
+      errors = []
+      doc.search('GenericResponse/exceptions/exception') do |exception_node|
+        message = exception_mode.at('message').text
+        code = exception_node.at('code').text.to_i
+        errors << Shopping::Error.new(message, code)
+      end
+      errors
+    end
+
+    def raise_exception(errors)
+      raise errors.first
+    end
+
+    def convert_product_node(product_node, include_offers=false)
+      product = {}
+      product[:product_id] = product_node['id']
+      product[:name] = product_node.at('name').text
+
+      description = product_node.at('fullDescription').text
+      if description.nil? || description.empty?
+        description = product_node.at('shortDescription').text
+      end
+      product[:description] = (description.nil? || description.empty?) ? '' : description
+
+      image_nodes = product_node.search('images/image[@available="true"]')
+      images = image_nodes.collect{|x|
+        {
+          :width => x['width'].to_i,
+          :height => x['height'].to_i,
+          :url => x.at('sourceURL').text
+        }
+      }.sort_by{|x| x[:width] * x[:height] }
+
+      product[:images] = {
+        :small_image => images[0],
+        :medium_image => images[1],
+        :large_image => images[2]
+      }
+
+      # possible_manufacturers = (product / 'offer > manufacturer').collect{|x| x.text}.compact.uniq
+      #
+      # if possible_manufacturers.length == 1
+      #   product[:manufacturer] = possible_manufacturers.first # easy peasy lemon squezy
+      # elsif possible_manufacturers.length > 1
+      #   # figure out which manufacturer is the most popular
+      #   manufacturers_popularity_index = possible_manufacturers.inject({}) {|ha, manufacturer| ha[manufacturer] ||= 0; ha[manufacturer] += 1; ha }
+      #   product[:manufacturer] = manufacturers_popularity_index.sort_by{|key, val| val }.last.first
+      # else
+      #   product[:manufacturer] = nil # zip, zero, doodad :(
+      # end
+
+      # rating
+      review_count_node = product_node.at('rating/reviewCount')
+      product[:num_reviews] = review_count_node.nil? ? 0 : review_count_node.text.to_i
+      rating_value_node = product_node.at('rating/rating')
+      product[:rating] = rating_value_node.nil? ? nil : normalize_product_rating(rating_value_node.text.to_f)
+
+      # offers
+      if include_offers
+        offers_node = product_node.at('offers')
+        product[:offers] = convert_offers_collection_node(offers_node) unless offers_node.nil?
+      end
+
+      # specifications
+      specifications_node = product_node.at('specifications')
+      product[:specifications] = convert_specifications_node(specifications_node) unless specifications_node.nil?
+
+      product
+    end
+
+    def convert_offers_collection_node(offers_collection_node)
+      offer_nodes = offers_collection_node.nil? ? nil : offers_collection_node.search('offer')
+      return [] if offer_nodes.nil?
+      offers = {}
+      offer_nodes.each_with_index do |offer, offer_index|
+        # in-stock
+        stock_status = offer.at('stockStatus').text
+        in_stock = stock_status != 'out-of-stock' && stock_status != 'back-order'
+
+        if in_stock
+          store = offer.at('store')
+          store_hash = {
+            :id => store['id'],
+            :name => store.at('name').text,
+            :trusted => store['trusted'] == "true",
+            :authorized_reseller => store['authorizedReseller'] == 'true'
+          }
+          store_logo = store.at('logo')
+          if store_logo['available'] == 'true'
+            store_hash[:logo] = {
+              :width => store_logo['width'],
+              :height => store_logo['height'],
+              :url => store_logo.at('sourceURL').text
+            }
+          else
+            store_hash[:logo] = nil
+          end
+
+          # store rating
+          store_rating = store.at('ratingInfo')
+          store_hash[:rating] = {
+            :number => store_rating.at('rating').nil? ? nil : normalize_merchant_rating(store_rating.at('rating').text.to_f),
+            :count => store_rating.at('reviewCount').text.to_i,
+            :url =>  store_rating.at('reviewURL').nil? ? nil : store_rating.at('reviewURL').text
+          }
+
+          # prices
+          cpc = offer.at('cpc').nil? ? nil : (offer.at('cpc').text.to_f*100).to_i
+          base_price = to_d_or_nil(offer.at('basePrice').text)
+          shipping_cost = offer.at('shippingCost')['checkSite'] == 'true' ? nil : to_d_or_nil(offer.at('shippingCost').text)
+
+          # skip this offer if we already have one from same merchant and it has a lower total price
+          existing_offer = offers[store_hash[:id]]
+          unless existing_offer.nil?
+            next if existing_offer[:price] + (existing_offer[:shipping] || 0.0) < base_price + (shipping_cost || 0.0)
+          end
+
+          offers[store_hash[:id]] = { :original_index => offer_index,
+                                      :merchant_code => store_hash[:id],
+                                      :merchant_name => store_hash[:name],
+                                      :merchant_logo_url => store_hash[:logo].nil? ? nil : store_hash[:logo][:url],
+                                      :cpc => cpc,
+                                      :price => base_price,
+                                      :shipping => shipping_cost,
+                                      :offer_url => offer.at('offerURL').text,
+                                      :offer_tier => 1,
+                                      :merchant_rating => store_hash[:rating][:number],
+                                      :num_merchant_reviews => store_hash[:rating][:count] }
+        end
+      end
+      offers.values.sort_by{|x| x[:price] + (x[:shipping] || 0) }
+    end
+
+    def convert_specifications_node(specifications_node)
+      specifications = {}
+      specifications_node.search('feature').each do |feature_node|
+        feature_name = feature_node.at('name').text
+        value_nodes = feature_node.search('value')
+        if value_nodes.length > 1
+          specifications[feature_name] = value_nodes.collect{|value_node| value_node.text}
+        elsif value_nodes.length == 1
+          specifications[feature_name] = value_nodes.first.text
+        end
+      end
+      specifications
+    end
+
+    def normalize_product_rating(product_rating)
+      product_rating.nil? ? nil : (product_rating * 20.0).round
+    end
+
+    def normalize_merchant_rating(merchant_rating)
+      merchant_rating.nil? ? nil : (merchant_rating * 20.0).round
+    end
+  end
+end