bazaar_sources 0.2.1.1.1.4

data/lib/sources/price_grabber_source.rb

@@ -0,0 +1,94 @@
+require 'ostruct'
+
+class PriceGrabberSource < Source
+  def initialize
+    super(:name => 'PriceGrabber',
+          :homepage => ' http://www.pricegrabber.com/',
+          :cpc => 0,
+          :offer_enabled => false,
+          :offer_ttl_seconds => 0,
+          :use_for_merchant_ratings => true,
+          :offer_affiliate => false,
+          :supports_lifetime_ratings => false,
+          :batch_fetch_delay => 2,
+          :product_code_regexp => /^\d{6,9}$/,
+          :product_code_examples => ['716698181', '563043'],
+          :product_page_link_erb => "http://reviews.pricegrabber.com/-/m/<%= product_code %>/")
+  end
+  
+  def url_for_merchant_source_page(merchant_source_code)
+    "http://www.pricegrabber.com/info_retailer.php/r=#{merchant_source_code}"
+  end
+
+  def code_from_merchant_source_page_url(merchant_source_page_url)
+    merchant_source_page_url.match(/pricegrabber\.com.*\/r[\/=](\d+)/)[1]
+  end
+
+  def fetch_merchant_source(merchant_source_page_url)
+    delay_fetch
+    doc = Hpricot(open(merchant_source_page_url))
+
+    merchant_source = OpenStruct.new
+    merchant_source.source = self
+
+    merchant_identity_block = doc.at('div#merchantIdentityBlock')
+
+    unless merchant_identity_block.nil?
+      # merchant name
+      element = merchant_identity_block.at('/h4')
+      unless element.nil?
+        name = element.inner_text.strip
+        merchant_source.name = name
+      end
+
+      # merchant logo
+      element = merchant_identity_block.at('/img')
+      unless element.nil?
+        logo_url = element.attributes['src']
+        merchant_source.logo_url = logo_url
+      end
+    end
+
+    # merchant code
+    merchant_source.code = code_from_merchant_source_page_url(merchant_source_page_url)
+
+    # merchant rating
+    ratings = doc.search('table#scoreTable/tr/th[text() = "Avg Rating"]/../td')
+    unless ratings.nil?
+      unless ratings[0].nil?
+        three_month_rating = ratings[0].inner_text.strip.to_f
+        merchant_source.merchant_rating = (three_month_rating * 20).round
+      end
+
+      unless ratings[2].nil?
+        lifetime_month_rating = ratings[2].inner_text.strip.to_f
+        merchant_source.merchant_rating_lifetime = (lifetime_month_rating * 20).round
+      end
+    end
+
+    # Num Merchant Reviews
+    num_reviews = doc.search('table#scoreTable/tr/th[text() = "Total Reviews"]/../td')
+    unless num_reviews.nil?
+      unless num_reviews[0].nil?
+        merchant_source.num_merchant_reviews = num_reviews[0].inner_text.strip.to_i
+      end
+
+      unless num_reviews[2].nil?
+        merchant_source.num_merchant_reviews_lifetime = num_reviews[2].inner_text.strip.to_i
+      end
+    end
+
+    # Homepage
+    element = doc.at('table#contactTable//th[text() = "Website:"]/../td/a')
+    unless element.nil?
+      homepage = element.inner_text.strip.downcase
+      merchant_source.homepage = homepage
+    end
+
+    merchant_source
+  end
+
+  def format_rating(merchant_source)
+    '%01.2f/5.0' % (merchant_source.get_merchant_rating.to_f / 20.0)
+  end
+end

data/lib/sources/reseller_ratings_source.rb

@@ -0,0 +1,47 @@
+require 'api_helpers/reseller_ratings_api'
+
+class ResellerRatingsSource < Source
+  def initialize
+    super(:name => 'ResellerRatings.com',
+          :homepage => 'http://www.resellerratings.com/',
+          :cpc => 0,
+          :offer_enabled => false,
+          :offer_ttl_seconds => 0,
+          :use_for_merchant_ratings => true,
+          :offer_affiliate => false,
+          :supports_lifetime_ratings => true,
+          :batch_fetch_delay => 5,
+          :product_code_regexp => /^\d{9}$/,
+          :product_code_examples => ['652196596', '676109333'],
+          :product_page_link_erb => "http://resellerratings.nextag.com/<%= product_code %>/resellerratings/prices-html")
+  end
+  
+  def url_for_merchant_source_page(merchant_source_code)
+    "http://www.resellerratings.com/seller#{merchant_source_code}.html"
+  end
+
+  def url_for_merchant_source_page_alt(merchant_source_alt_code)
+    "http://www.resellerratings.com/store/#{merchant_source_alt_code}"
+  end
+
+  def code_from_merchant_source_page_url(merchant_source_page_url)
+    nil
+  end
+
+  def fetch_merchant_source(merchant_source_page_url)
+    delay_fetch
+    ResellerRatingsAPI.fetch_merchant_source(merchant_source_page_url)
+  end
+
+  def search_for_merchant_source(search_text)
+    ResellerRatingsAPI.search_for_merchant_source(search_text)
+  end
+
+  def search_for_merchant_source_best_match(search_text)
+    ResellerRatingsAPI.search_for_merchant_source_best_match(search_text)
+  end
+
+  def format_rating(merchant_source)
+    '%01.1f/10' % (merchant_source.get_merchant_rating.to_f / 10.0)
+  end
+end

data/lib/sources/shopping_source.rb

@@ -0,0 +1,136 @@
+require 'ostruct'
+require 'api_helpers/shopping'
+
+class ShoppingSource < Source
+  def initialize
+    super(:name => 'Shopping.com',
+          :homepage => 'http://www.shopping.com/',
+          :cpc => 50,
+          :offer_enabled => true,
+          :offer_ttl_seconds => 1800,
+          :use_for_merchant_ratings => true,
+          :offer_affiliate => false,
+          :supports_lifetime_ratings => false,
+          :batch_fetch_delay => 2,
+          :product_code_regexp => /^\d{4,10}$/,
+          :product_code_examples => ['44393573', '37469715'])
+  end
+
+  def api
+    @api ||= Shopping::Publisher.new
+  end
+
+  def url_for_merchant_source_page(merchant_source_code)
+    "http://www.shopping.com/xMR-~MRD-#{merchant_source_code}"
+  end
+
+  def fetch_merchant_source(merchant_source_page_url)
+    delay_fetch
+    doc = nil
+    4.times do |i|
+      # This is a workaround for some weirdness with Shopping.com.
+      # About one in ten requests for a merchant's info page results
+      # in a page with zeros for everything (as if the merchant has
+      # never been reviewed).  One indication that we received the
+      # bogus page is the the title will look like:
+      # Shopping.com: null - Compare Prices &amp; Read Reviews
+      # If we see that 'null' in the title, try fetching the page
+      # again (up to 4 times).
+      doc = Hpricot(open(merchant_source_page_url))
+      page_title = doc.at('head/title').inner_text
+      break if page_title.match(/ null /).nil?
+    end
+
+    merchant_source = OpenStruct.new
+    merchant_source.source = self
+
+    # merchant name
+    element = doc.at('h1[@class = "pageTitle"]')
+    unless element.nil?
+      name = element.inner_text.strip
+      merchant_source.name = name
+    end
+
+    # merchant logo
+    element = doc.at('img[@class = "logoBorder1"]')
+    unless element.nil?
+      logo_url = element.attributes['src']
+      merchant_source.logo_url = logo_url
+
+      # merchant code
+      code = logo_url.match(/merch_logos\/(.+)\.gif/)[1]
+      merchant_source.code = code
+    end
+
+    # merchant rating
+    element = doc.at('td[@id = "image"]/img')
+    unless element.nil?
+      merchant_rating = element.attributes['title'].match(/((\d|,)*\.?\d)/)[1]
+      merchant_source.merchant_rating = merchant_rating.delete(',').to_f * 20.0 unless merchant_rating.nil?
+    end
+
+    # Num Merchant Reviews
+    element = doc.at('table[@class = "boxTableTop"]//h3[@class = "boxTitleNB"]')
+    unless element.nil?
+      num_merchant_reviews = element.inner_text.match(/of\s+((\d|,)+)/)[1]
+      merchant_source.num_merchant_reviews = num_merchant_reviews.delete(',').to_i unless num_merchant_reviews.nil?|| num_merchant_reviews.empty?
+    end
+
+    merchant_source
+  end
+
+  def search_for_merchant_source(search_text)
+    merchant_search_url = "http://www.shopping.com/xSD-#{CGI::escape(search_text.strip)}"
+    doc = Hpricot(open(merchant_search_url))
+    merchant_sources = []
+    element = doc.search('div[@class*="contentContainer"]/div[@class="boxMid"]')[1]
+
+    # Do we have any results?
+    unless element.nil?
+      element.search('tr[td/div/ul/li/a/span[text() = "See Store Info"]]').each do |result_row|
+        element = result_row.at('/td/a')
+        name = element.inner_text.strip
+        merchant_code = element.attributes['href'].match(/~MRD-(\d+)/)[1]
+        element = result_row.at('/td[@class = "smallTxt"]/img')
+        logo_url = element.attributes['src'] unless element.nil?
+
+        existing_merchant_source = MerchantSource.find_by_source_and_code(self, merchant_code)
+        if existing_merchant_source.nil?
+          merchant_sources << OpenStruct.new({:source => self, :name => name, :code => merchant_code, :logo_url => logo_url})
+        else
+          merchant_sources << existing_merchant_source
+        end
+      end
+    end
+    merchant_sources
+  end
+
+  def format_rating(merchant_source)
+    '%01.1f/5.0' % (merchant_source.get_merchant_rating.to_f / 20.0)
+  end
+
+  def nullify_offer_url(offer_url)
+    offer_url.gsub(/3068547/, '8039098')
+  end
+
+  def fetch_street_price(product_code)
+    delay_fetch
+    offers = fetch_offers(product_code)
+    num_offers = 0
+    total_prices = 0.0
+    offers.each do |offer|
+      if !offer.merchant_rating.nil? &&
+         offer.merchant_rating >= 55 &&
+         !offer.price.nil? &&
+         !offer.shipping.nil?
+        total_prices += offer.total_price
+        num_offers += 1
+      end
+    end
+    num_offers.zero? ? nil : (total_prices / num_offers)
+  end
+
+  def fetch_offers(product_code)
+    api.fetch_offers(product_code)
+  end
+end

data/lib/sources/shopzilla_source.rb

@@ -0,0 +1,108 @@
+require 'ostruct'
+require 'api_helpers/shopzilla_api'
+
+class ShopzillaSource < Source
+  def initialize
+    super(:name => 'Shopzilla',
+          :homepage => 'http://www.shopzilla.com/',
+          :cpc => 39,
+          :offer_enabled => false,
+          :offer_ttl_seconds => 1800,
+          :use_for_merchant_ratings => true,
+          :offer_affiliate => false,
+          :supports_lifetime_ratings => false,
+          :batch_fetch_delay => 1,
+          :product_code_regexp => /^\d{7,11}$/,
+          :product_code_examples => ['1028968032', '852926140'],
+          :product_page_link_erb => "http://www.shopzilla.com/-/<%= product_code %>/shop")
+  end
+  
+  def url_for_merchant_source_page(merchant_source_code)
+    "http://www.shopzilla.com/6E_-_mid--#{merchant_source_code}"
+  end
+
+  def code_from_merchant_source_page_url(merchant_source_page_url)
+    merchant_source_page_url.match /6[A-Z](\-\-.*)?_\-_mid\-\-(\d+)/i
+    $2
+  end
+
+  def fetch_merchant_source(merchant_source_page_url)
+    delay_fetch
+    merchant_source = OpenStruct.new
+    merchant_source.source = self
+
+    merchant_code = code_from_merchant_source_page_url(merchant_source_page_url)
+    merchant_source_detail = api.merchant_source_detail(merchant_code)
+
+    unless merchant_source_detail.nil?
+      merchant_source.code = merchant_source_detail[:code]
+      merchant_source.name = merchant_source_detail[:name]
+      merchant_source.logo_url = merchant_source_detail[:logo_url]
+      merchant_source.merchant_rating = merchant_source_detail[:merchant_rating]
+      merchant_source.homepage = merchant_source_detail[:homepage]
+      merchant_source.num_merchant_reviews = merchant_source_detail[:num_merchant_reviews]
+    end
+    merchant_source
+  end
+
+  SHOPZILLA_SEARCH_PAGE = 'http://www.bizrate.com/ratings_guide/guide.html'
+  SHOPZILLA_SEARCH_ACTION = 'http://www.bizrate.com/merchant/results.xpml'
+  def search_for_merchant_source(search_text)
+    merchant_sources = []
+
+    agent = WWW::Mechanize.new
+    agent.html_parser = Nokogiri::HTML
+    agent.user_agent_alias = 'Windows IE 7'
+    agent.follow_meta_refresh = true
+
+    search_page = agent.get(SHOPZILLA_SEARCH_PAGE)
+    if form = search_page.form_with(:action => /superfind/)
+      # Must switch the action given in the form, because BizRate does exactly this in JavaScript
+      form.action = SHOPZILLA_SEARCH_ACTION
+      form['SEARCH_GO'] = 'Find it!'
+      form.keyword = search_text
+      if result = form.submit
+        if single_store = result.at('table[id="merchant_overview"]')
+          if store = single_store.at('div[class="certified"] strong a')
+            add_merchant_source_from_store(merchant_sources, store)
+          end
+        elsif stores_rated_list = result.at('div[class="storesRatedList"]')
+          if stores = stores_rated_list.search('th a')
+            stores.each do |store|
+              add_merchant_source_from_store(merchant_sources, store)
+            end
+          end
+        end
+      end
+    end
+    merchant_sources
+  end
+  
+  def add_merchant_source_from_store(merchant_sources, store)
+    name = store.text
+    merchant_code = CGI.parse(URI.parse(store['href']).query)['mid']
+    logo_url = api.verified_logo_url(merchant_code)
+    existing_merchant_source = MerchantSource.find_by_source_and_code(self, merchant_code)
+    if existing_merchant_source.nil?
+      merchant_sources << OpenStruct.new({:source => self, :name => name, :code => merchant_code, :logo_url => logo_url})
+    else
+      merchant_sources << existing_merchant_source
+    end
+  end
+  
+  def format_rating(merchant_source)
+    '%01.1f/10' % (merchant_source.get_merchant_rating.to_f / 10.0)
+  end
+
+  def nullify_offer_url(offer_url)
+    offer_url.gsub(/af_id=3973/, 'af_id=3233')
+  end
+
+  def api
+    @api ||= ShopzillaAPI.new
+  end
+
+  def fetch_offers(product_code)
+    api.find_offers_by_product_id(product_code)
+  end
+end

data/lib/sources/simple_sources.yml

@@ -0,0 +1,71 @@
+circuit-city:
+  name: Circuit City
+  mappable: true
+  for_offers: false
+  for_product_info: false
+  for_review_aggregates: true
+  search_url:
+  search_token_separator:
+  product_code_regexp: ^\d{7}$
+  product_code_examples: ['4497835', '5007270']
+  product_page_link_erb: http://www.circuitcity.com/applications/SearchTools/item-details.asp?EdpNo=<%= product_code %>
+
+cnet:
+  name: CNET
+  mappable: true
+  for_offers: false
+  for_product_info: false
+  for_review_aggregates: true
+  search_url: http://cnet.search.com/search?chkpt=astg.cnet.fd.search.cnet&q=!placeholder!
+  search_token_separator: +
+  product_code_regexp: ^\d{4}-\d{4}_\d-\d{7,8}$
+  product_code_examples: ['4852-6501_7-9666631', '4505-6501_7-33765912']
+  product_page_link_erb: http://www.cnet.com/<%= product_code %>.html
+
+newegg:
+  name: Newegg
+  mappable: true
+  for_offers: false
+  for_product_info: false
+  for_review_aggregates: true
+  search_url: http://www.newegg.com/Product/ProductList.asp?DEPA=0&type=&Description=!placeholder!&Submit=ENE&N=0&Ntk=all
+  search_token_separator: +
+  product_code_regexp: ^[A-Z0-9]{4}\d{11}$
+  product_code_examples: ['N82E16830111260', 'N82E16830144222']
+  product_page_link_erb: http://www.newegg.com/Product/Product.aspx?Item=<%= product_code %>
+
+nextag:
+  name: NexTag
+  mappable: true
+  for_offers: false
+  for_product_info: false
+  for_review_aggregates: true
+  search_url: http://www.nextag.com/blah/search-html?search=!placeholder!
+  search_token_separator: +
+  product_code_regexp: ^\d{8,9}$
+  product_code_examples: ['560328520', '656751324']
+  product_page_link_erb: http://www.nextag.com/<%= product_code %>/prices-html
+
+photography-review:
+  name: Photography Review
+  mappable: false
+  for_offers: false
+  for_product_info: false
+  for_review_aggregates: false
+  search_url: http://www.photographyreview.com/befid-449/keyword-!placeholder!/searchcrx.aspx
+  search_token_separator: %20
+  product_code_regexp:
+  product_code_examples: ['casio_exz550', 'canon_eos1dmkiv']
+  product_page_link_erb: http://www.dpreview.com/reviews/specs/Canon/<%= product_code %>.asp
+
+yahoo-shopping:
+  name: Yahoo! Shopping
+  mappable: true
+  for_offers: false
+  for_product_info: false
+  for_review_aggregates: true
+  search_url: http://shopping.yahoo.com/search?p=!placeholder!&did=59
+  search_token_separator: +
+  product_code_regexp: ^\d{10}$
+  product_code_examples: ['1993713221', '2002987863']
+  product_page_link_erb: http://shopping.yahoo.com/p:-:<%= product_code %>

data/lib/sources/source.rb

@@ -0,0 +1,242 @@
+require 'set'
+require 'erb'
+
+class Source
+  class GeneralError < StandardError
+    attr_reader :source_keyname
+    def initialize(message, source_keyname)
+      super(message)
+      @source_keyname = source_keyname
+    end
+  end
+
+  class ProductNotFoundError < GeneralError
+    attr_reader :product_code
+    def initialize(message, source_keyname, product_code)
+      super(message, source_keyname)
+      @product_code = product_code
+    end
+  end
+
+  class ProductFatalError < GeneralError
+    attr_reader :product_code
+    def initialize(message, source_keyname, product_code)
+      super(message, source_keyname)
+      @product_code = product_code
+    end
+  end
+
+  attr_reader :name
+  attr_reader :homepage
+  attr_reader :cpc
+  attr_reader :offer_enabled
+  alias :offer_enabled? :offer_enabled
+  alias :for_offers :offer_enabled
+  attr_reader :offer_ttl_seconds
+  attr_reader :use_for_merchant_ratings
+  alias :use_for_merchant_ratings? :use_for_merchant_ratings
+  attr_reader :offer_affiliate
+  alias :offer_affiliate? :offer_affiliate
+  attr_reader :supports_lifetime_ratings
+  alias :supports_lifetime_ratings? :supports_lifetime_ratings
+  attr_reader :batch_fetch_delay
+  attr_reader :product_code_regexp
+  attr_reader :product_code_examples
+  attr_reader :product_page_link_erb
+
+  # properties from the legacy Source table in DA/DCHQ
+  attr_reader :mappable
+  attr_reader :for_product_info
+  attr_reader :for_review_aggregates
+  attr_reader :search_url
+  attr_reader :search_token_separator
+
+  @@subclasses = []
+  @@sources = Set.new
+  @@sources_map = {}
+  @@offer_sources = []
+  @@affiliate_sources = []
+  @@merchant_rating_sources = []
+
+  SIMPLE_SOURCES_YAML_FILE = File.join(File.dirname(__FILE__), 'simple_sources.yml')
+
+  class << self
+    @keyname = nil
+  end
+
+  def self.keyname
+    if @keyname.nil? && !self.name.nil?
+      matches = self.name.match(/(.+)Source/)
+      @keyname = matches[1].gsub(/([a-z\d])([A-Z])/,'\1-\2').downcase unless matches.nil?
+    end
+    @keyname
+  end
+
+  def self.keyname=(keyname)
+    @keyname = keyname
+  end
+
+  def self.inherited(child)
+    @@subclasses << child
+    set_source_keyname_const(child.keyname)
+    super
+  end
+
+  def initialize(attributes)
+    attributes.each {|k, v| instance_variable_set("@#{k}", v)}
+  end
+
+  def keyname
+    self.class.keyname
+  end
+
+  def self.source(source_keyname)
+    load_sources
+    @@sources_map[source_keyname]
+  end
+
+  def self.sources
+    load_sources
+    @@sources
+  end
+
+  def self.offer_sources
+    load_sources
+    if @@offer_sources.empty?
+      @@offer_sources = @@sources.select{|source| source.offer_enabled?}
+    end
+    @@offer_sources
+  end
+
+  def self.affiliate_sources
+    load_sources
+    if @@affiliate_sources.empty?
+      @@affiliate_sources = @@sources.select{|source| source.offer_affiliate?}
+    end
+    @@affiliate_sources
+  end
+
+  def self.merchant_rating_sources
+    load_sources
+    if @@merchant_rating_sources.empty?
+      @@merchant_rating_sources = @@sources.select{|source| source.use_for_merchant_ratings?}
+    end
+    @@merchant_rating_sources
+  end
+
+  def self.method_missing(meth)
+    source = nil
+    if matches = meth.to_s.match(/(.+)_source$/)
+      source_keyname = matches[1].gsub('_', '-')
+      source = send(:source, source_keyname)
+    end
+    source.nil? ? super : source
+  end
+
+  def product_code_valid?(product_code)
+    product_code_regexp.nil? ? (!product_code.nil? && product_code.length > 0) : !(product_code =~ product_code_regexp).nil?
+  end
+
+  def product_page_link(product_code)
+    ERB.new(product_page_link_erb).result(binding) unless product_page_link_erb.nil?
+  end
+
+  def url_for_merchant_source_page(merchant_source_code)
+    nil
+  end
+
+  def url_for_merchant_source_page_alt(merchant_source_alt_code)
+    nil
+  end
+
+  def code_from_merchant_source_page_url(merchant_source_page_url)
+    nil
+  end
+
+  def fetch_merchant_source(merchant_source_page_url)
+    nil
+  end
+
+  def format_rating(merchant_source)
+    "#{merchant_source.get_merchant_rating}%"
+  end
+
+  def nullify_offer_url(offer_url)
+    offer_url
+  end
+
+  def fetch_offers(product_code)
+    nil
+  end
+
+  def hash
+    self.class.hash
+  end
+
+  def eql?(other)
+    self.class == other.class
+  end
+
+  def to_s
+    keyname
+  end
+
+  protected
+
+  def self.set_source_keyname_const(source_keyname)
+    unless source_keyname.nil? || source_keyname.empty?
+      const_name = source_keyname.gsub(/([A-Z]+)([A-Z][a-z])/,'\1_\2').gsub(/([a-z\d])([A-Z])/,'\1_\2').tr('-~', '_').upcase + '_KEYNAME'
+      const_set(const_name.to_sym, source_keyname)
+    end
+  end
+
+  def self.load_sources
+    if @@sources.empty?
+      @@subclasses.each do |source_class|
+        source_instance = source_class.new
+        @@sources << source_instance
+        @@sources_map[source_instance.keyname] = source_instance
+      end
+      load_simple_sources.each do |source_instance|
+        @@sources << source_instance
+        @@sources_map[source_instance.keyname] = source_instance
+      end
+      @@sources.sort{|a,b| a.name <=> b.name}
+    end
+    nil
+  end
+
+  def self.load_simple_sources
+    simple_sources = []
+    simple_sources_map = YAML.load_file(SIMPLE_SOURCES_YAML_FILE)
+    simple_sources_map.each do |source_keyname, source_attributes|
+      const_name = source_keyname.gsub(/(?:^|[-_~])(.)/) { $1.upcase } + 'Source'
+      simple_source_class = Object.const_set(const_name, Class.new(Source))
+      simple_source_class.keyname = source_keyname
+      set_source_keyname_const(source_keyname)
+      product_code_regexp = source_attributes['product_code_regexp']
+      product_code_regexp = Regexp.new(product_code_regexp) unless product_code_regexp.nil?
+      source = simple_source_class.new(:name => source_attributes['name'],
+                                       :offer_enabled => source_attributes['for_offers'] == 'true',
+                                       :mappable => source_attributes['mappable'] == 'true',
+                                       :for_product_info => source_attributes['for_product_info'] == 'true',
+                                       :for_review_aggregates => source_attributes['for_review_aggregates'] == 'true',
+                                       :search_url => source_attributes['search_url'],
+                                       :search_token_separator => source_attributes['search_token_separator'],
+                                       :product_code_regexp => product_code_regexp,
+                                       :product_code_examples => source_attributes['product_code_examples'],
+                                       :product_page_link_erb => source_attributes['product_page_link_erb'])
+      simple_sources << source
+    end
+    simple_sources
+  end
+
+  def delay_fetch
+    if !@last_fetched_at.nil? &&
+       batch_fetch_delay > 0 &&
+       @last_fetched_at > batch_fetch_delay.seconds.ago
+      sleep(batch_fetch_delay)
+    end
+    @last_fetched_at = Time.now
+  end
+end