wedding_registry_scraper 0.0.1 → 0.0.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 4574355a920b3e4e74e24c5515895efe2cb49113
-  data.tar.gz: bd33c074a9562b32a6ca198b642951e64d87c8e6
+  metadata.gz: 1d95783063ce46d1a28af98b6c3330bd8d33ba95
+  data.tar.gz: 56298d31eba3630e7a1f28bfd4565f655f110708
 SHA512:
-  metadata.gz: ffe729b27d8ecb0f3c4061a06d66d8f9fcda08e441e23850f9122aba1c670dd93b2907ed6f581c487032e205689b7572ddeb8cd4bec235f88518744fec82a017
-  data.tar.gz: 7d358609b18515b869bd028e850426a921ee1d9659eab9a3884ab267d2c29f8158e9a5bfff0209b9dae1151d778e60889f7f37632104a028ec00ae4f78f61037
+  metadata.gz: 0f82473b27cf53ccdd6d000b2543256fe72ae7d35322e06a4eef9b71cf2b30bf780cf2a00d169adb7bbc38b0dc8e2a771f904768e02b85ac3caa125c1c433d14
+  data.tar.gz: b29414b56a8faab31165d8fece59f57b2a5bc568aae8ce9809ad8cfadd14149d98befab78b7fa8d388d58ad58ef0ab55bfcdf798af0783a1a63e5a776b6eeebd

data/LICENSE

@@ -0,0 +1,22 @@
+The MIT License (MIT)
+
+Copyright (c) 2015 Andrew Pariser
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.
+

data/README.md

@@ -0,0 +1,53 @@
+# Wedding Registry Scraper
+
+## Why?
+
+Want to consolidate multiple registries onto one website?
+
+Want to regularly update your fulfillment counts?
+
+This (work in progress) is for you!
+
+## Which Registries?
+
+Right now, works with:
+
+* Zola — https://www.zola.com
+* REI — http://www.rei.com
+* Heath Ceramics — http://www.heathceramics.com
+* Crate & Barrel — http://www.crateandbarrel.com
+* Williams Sonoma — http://www.williams-sonoma.com
+
+I intend to get this working with:
+
+* Sur la Table
+* Pottery Barn
+
+## Usage
+
+Add to your Gemfile
+
+    gem 'wedding-registry-scraper'
+
+Use `WeddingRegistryScraper#scrape`:
+
+    require 'wedding_registry_scraper'
+
+    WeddingRegistryScraper.scrape([
+      "http://www.heathceramics.com/giftregistry/view/index/id/YOUR_UNIQUE_REGISTRY_ID",
+      "http://www.crateandbarrel.com/Gift-Registry/YOUR_NAMES/YOUR_UNIQUE_REGISTRY_ID",
+      "https://www.zola.com/registry/YOUR_REGISTRY_SLUG",
+      "http://www.rei.com/GiftRegistryDetails/YOUR_UNIQUE_REGISTRY_ID",
+      "https://secure.williams-sonoma.com/registry/YOUR_UNIQUE_REGISTRY_ID/registry-list.html",
+    ])
+
+
+## Want to Help?
+
+It should be simple to add another registry by cloning a `Regsitry` inside `lib/wedding_registry_scraper/registries`.
+
+## Like this project and planning on registering?
+
+Why don't you register for Zola using my invite code: https://www.zola.com/invite/pariser
+
+Thanks!

data/lib/wedding_registry_scraper.rb

@@ -1,3 +1,6 @@
+require 'active_support/all'
+require_relative 'wedding_registry_scraper/version'
+
 module WeddingRegistryScraper
   autoload :Registry, 'wedding_registry_scraper/registry'
   autoload :Registries, 'wedding_registry_scraper/registries'

data/lib/wedding_registry_scraper/registries.rb

@@ -0,0 +1,28 @@
+module WeddingRegistryScraper::Registries
+  class << self
+    def registry_classes
+      @registry_classes ||= begin
+        self.constants.map do |const|
+          obj = self.const_get(const)
+          obj if obj.is_a?(Class) && obj < WeddingRegistryScraper::Registry
+        end.compact
+      end
+      @registry_classes
+    end
+
+    def registry_class_from_url(url)
+      registry_classes.detect do |klass|
+        /^https?:\/\/[^\/]*#{klass.domain}/.match(url)
+      end
+    end
+
+    def initialize_registry(url, options={})
+      klass = registry_class_from_url(url)
+      klass ? klass.new(url, options) : nil
+    end
+  end
+end
+
+# Load all registries
+
+Dir[File.dirname(__FILE__) + '/registries/*.rb'].each { |file| require file }

data/lib/wedding_registry_scraper/registries/crate_and_barrel.rb

@@ -0,0 +1,45 @@
+class WeddingRegistryScraper::Registries::CrateAndBarrel < WeddingRegistryScraper::Registry
+  @display_name = "Crate & Barrel"
+  @domain = "crateandbarrel.com"
+private
+  def get_products(doc)
+    doc.css('.jsItemRow:not(.emptyCategoryRow)')
+  end
+
+  def get_name(product)
+    product.css('.itemTitle').text.strip
+  end
+
+  def get_sku(product)
+    sku = product.css('.skuNum').text.strip.match(/SKU (\S+)/)[1]
+    "c&b:#{sku}"
+  end
+
+  def get_url(product)
+    # TODO pop up modal? set proper anchor (they're not unique!)
+    @url
+  end
+
+  def get_image_url(product)
+    thumb_src = product.css('img')[0]['src']
+    thumb_src.sub(/\$web_itembasket\$/, '&$web_popup_zoom$&wid=379&hei=379')
+  end
+
+  def get_remaining(product)
+    desired = product.css('td')[4].css('.itemHas').text.strip.to_i
+    fulfilled = product.css('td')[5].css('.itemHas').text.strip.to_i
+    desired - fulfilled
+  end
+
+  def get_desired(product)
+    product.css('td')[4].css('.itemHas').text.strip.to_i
+  end
+
+  def get_price(product)
+    if (sale_price = product.css('.salePrice')).any?
+      /\$?(\d+(\.\d+)?)/.match(sale_price.text) { |match| match[1].to_f }
+    elsif (regular_price = product.css('.regPrice')).any?
+      /\$?(\d+(\.\d+)?)/.match(regular_price.text) { |match| match[1].to_f }
+    end
+  end
+end

data/lib/wedding_registry_scraper/registries/heath_ceramics.rb

@@ -0,0 +1,90 @@
+require 'uri'
+
+ActiveSupport::Inflector.inflections(:en) do |inflector|
+  inflector.uncountable 'heath_ceramics'
+end
+
+class WeddingRegistryScraper::Registries::HeathCeramics < WeddingRegistryScraper::Registry
+  @display_name = "Heath Ceramics"
+  @domain = "heathceramics.com"
+private
+  def get_products(doc)
+    doc.css('table#shopping-cart-table tbody tr')
+  end
+
+  def get_name(product)
+    product.css('.attentionText').text.strip
+  end
+
+  def get_sku(product)
+    text = product.css('.ctxProductCol .tinyText').map(&:text).detect { |t| t =~ /SKU/ }
+    sku = text.match(/SKU:\s+(\S+)/)[1]
+    "heath:#{sku}"
+  end
+
+  def get_url(product)
+    @url
+  end
+
+  def get_image_url(product)
+    details_url = product.css('[data-url]')[0]['data-url']
+
+    request_params = {
+      :headers => {
+        'X-Requested-With' => 'XMLHttpRequest'
+      }
+    }
+
+    result = with_lax_heath_uri_parsing do
+      puts "GET #{details_url.inspect} with params #{request_params.to_json}" if @debug
+      Unirest.get(details_url, request_params)
+    end
+
+    doc = Nokogiri::HTML(result.body)
+
+    if doc.css('ul.mixMatchList').any?
+      doc.css('ul.mixMatchList li:last-child img[src]')[0]['src']
+    else
+      doc.css('.imagesScrollable img[src]')[0]['src']
+    end
+  end
+
+  def get_remaining(product)
+    fulfillment = product.css('.fulfilled').text.strip
+    fulfillment.match(/^(\d+)/)[0].to_i
+  end
+
+  def get_desired(product)
+    fulfillment = product.css('.fulfilled').text.strip
+    fulfillment.match(/(\d+)$/)[0].to_i
+  end
+
+  def get_price(product)
+      product.css('.price').text.strip.gsub(/[^\d\.]/, '').to_f
+  end
+
+  def with_lax_heath_uri_parsing(&block)
+    URI::RFC3986_Parser.class_eval do
+      alias_method :old_split, :split
+
+      def split(uri)
+        # scheme, userinfo, host, port, registry, path, opaque, query, fragment
+        [ "http", nil, "www.heathceramics.com", nil, nil, uri.sub(/^https?:\/\/www.heathceramics.com/, ''), nil, nil, nil ]
+      end
+    end
+
+    exception = nil
+    begin
+      result = yield
+    rescue Exception => e
+      exception = e
+    ensure
+      URI::RFC3986_Parser.class_eval do
+        alias_method :split, :old_split
+      end
+    end
+
+    raise exception if exception
+    result
+  end
+end

data/lib/wedding_registry_scraper/registries/rei.rb

@@ -0,0 +1,49 @@
+class WeddingRegistryScraper::Registries::Rei < WeddingRegistryScraper::Registry
+  @display_name = "REI"
+  @domain = "rei.com"
+private
+  def get_product_details_url(product)
+    href = product.css('a[name=prod]')[0]['href'].sub(/^\/?/, '')
+    "https://www.rei.com/#{href.sub(/^\//, '')}"
+  end
+
+  def get_products(doc)
+    doc.css('table.registryList').first.css('tr.tr0')
+  end
+
+  def get_name(product)
+    product.css('a[name=prod]').text.strip
+  end
+
+  def get_sku(product)
+    sku = get_product_details_url(product).match(/\/product\/(\d+)/)[1].to_i
+    "rei-#{sku}"
+  end
+
+  def get_url(product)
+    @url
+  end
+
+  def get_image_url(product)
+    details_url = get_product_details_url(product)
+
+    puts "GET #{details_url.inspect}" if @debug
+    result = Unirest.get(details_url)
+    doc = Nokogiri::HTML(result.body)
+
+    image_url = doc.css('#zoomLink')[0]['href']
+    "https://www.rei.com/#{image_url.sub(/^\//,'')}"
+  end
+
+  def get_remaining(product)
+    product.css('td')[5].text.strip.to_i
+  end
+
+  def get_desired(product)
+    product.css('td')[4].text.strip.to_i
+  end
+
+  def get_price(product)
+    product.css('td')[3].text.strip.sub('$','').to_f
+  end
+end

data/lib/wedding_registry_scraper/registries/williams_sonoma.rb

@@ -0,0 +1,42 @@
+class WeddingRegistryScraper::Registries::WilliamsSonoma < WeddingRegistryScraper::Registry
+  @display_name = "Williams-Sonoma"
+  @domain = "williams-sonoma.com"
+private
+  def get_products(doc)
+    doc.css('table.registry-category-list tbody tr')
+  end
+
+  def get_name(product)
+    product.css('.product-detail .product-info .title a').text.strip
+  end
+
+  def get_sku(product)
+    sku = product.css('.product-detail .product-info .item-number').text.strip.match(/: (\d+)/)[1]
+    "williams-sonoma:#{sku}"
+  end
+
+  def get_url(product)
+    # product.css('.product-detail .product-info .title a')[0]['href']
+    @url
+  end
+
+  def get_image_url(product)
+    product.css('img')[0]['src'].sub(/f\.jpg$/, 'c.jpg')
+  end
+
+  def get_remaining(product)
+    product.css('td.still-needs').text.strip.to_i
+  end
+
+  def get_desired(product)
+    product.css('td.requested').text.strip.to_i
+  end
+
+  def get_price(product)
+    if (sale_price = product.css('td.price .price-state.price-special')).any?
+      sale_price.css('.currencyUSD .price-amount').text.strip.to_f
+    else
+      product.css('td.price .price-state.price-standard .currencyUSD .price-amount').text.strip.to_f
+    end
+  end
+end

data/lib/wedding_registry_scraper/registries/zola.rb

@@ -0,0 +1,55 @@
+class WeddingRegistryScraper::Registries::Zola < WeddingRegistryScraper::Registry
+  @display_name = "Zola"
+  @domain = "zola.com"
+private
+  def get_products(doc)
+    doc.css('#all-panel .product-tile')
+  end
+
+  def get_name(product)
+    product.css('.single-product-name').text.strip
+  end
+
+  def get_sku(product)
+    sku = product.css('.single-product')[0]['id']
+    "zola:#{sku}"
+  end
+
+  def get_url(product)
+    href = product.css('.content a')[0]['href'].sub(/^\/?/, '')
+    "https://www.zola.com/#{href}"
+  end
+
+  def get_image_url(product)
+    product.css('[data-image-url]')[0]['data-image-url']
+  end
+
+  def get_remaining(product)
+    product.css('.needed').text.strip.gsub(/[^\d]+/, '').to_i
+  end
+
+  def get_desired(product)
+  end
+
+  def get_price(product)
+    product.css('[data-price]')[0]['data-price'].gsub(/[$,]/, '').to_f
+  end
+
+  def price_type(product)
+    product_price = product.css('.product-price')
+
+    if product_price.length > 0 && product_price[0].text.strip == 'Contribute what you wish'
+      WeddingRegistryScraper::Registry::VARIABLE_PRICE
+    else
+      WeddingRegistryScraper::Registry::FIXED_PRICE
+    end
+  end
+
+  def fulfilled?(product)
+    if price_type(product) == WeddingRegistryScraper::Registry::VARIABLE_PRICE
+      get_price(product) <= 0
+    else
+      get_remaining(product) <= 0
+    end
+  end
+end

data/lib/wedding_registry_scraper/registry.rb

@@ -0,0 +1,120 @@
+require 'unirest'
+require 'nokogiri'
+
+class WeddingRegistryScraper::Registry
+  PRICE_TYPES = [
+    FIXED_PRICE = 'Fixed price',
+    VARIABLE_PRICE = 'Variable price',
+  ]
+
+  class << self
+    attr_reader :display_name
+    attr_reader :domain
+  end
+
+  def initialize(url, params={})
+    params.symbolize_keys!
+    @url = url
+    @debug = params[:debug] == true
+  end
+
+  def get_items
+    doc = get_registry
+
+    get_products(doc).reduce({}) do |products, product|
+      sku = get_sku(product)
+
+      details = {
+        :name => get_name(product),
+        :remaining => get_remaining(product),
+        :desired => get_desired(product),
+        :url => get_url(product),
+        :image_url => get_image_url(product),
+        :registry_name => self.class.display_name,
+        :fulfilled => fulfilled?(product),
+        :price_type => price_type(product),
+        :price_value => get_price(product),
+      }
+
+      products.merge! sku => details
+    end
+  end
+
+private
+
+  def get_products(doc)      ; raise NotImplementedError; end
+
+  def get_name(product)      ; raise NotImplementedError; end
+  def get_sku(product)       ; raise NotImplementedError; end
+  def get_url(product)       ; raise NotImplementedError; end
+  def get_image_url(product) ; raise NotImplementedError; end
+  def get_remaining(product) ; raise NotImplementedError; end
+  def get_desired(product)   ; raise NotImplementedError; end
+  def get_price(product)     ; raise NotImplementedError; end
+
+  def price_type(product)
+    FIXED_PRICE
+  end
+
+  def fulfilled?(product)
+    get_remaining(product) == 0
+  end
+
+  def get_registry
+    result, _ = make_request(:get, @url)
+    Nokogiri::HTML(result.body)
+  end
+
+  def make_request(method, url, params={})
+    json = params.delete(:json)
+    request_params = {}
+
+    if cookies = params.delete(:cookies)
+      request_params[:headers] = {
+        'Cookie' => dump_cookies(cookies)
+      }
+    end
+
+    if json
+      request_params[:headers] = {
+        'Accept' => "application/json"
+      }
+    end
+
+    request_params[:parameters] = json ? params.to_json : params
+
+    puts "#{method.to_s.upcase} #{url} with params #{request_params.to_json}" if @debug
+    result = Unirest.send(method, url, request_params)
+
+    puts "RESULT #{result.code} with headers #{result.headers.inspect}" if @debug
+
+    open_result_in_browser(result) if @debug
+
+    cookies = load_cookies(result.headers[:set_cookie])
+
+    [ result, cookies ]
+  end
+
+  def open_result_in_browser(result)
+    file = Tempfile.new(['weddding-scraper','.html'])
+    file << result.body
+    file.close
+
+    `open "#{file.path}"`
+
+    sleep 1
+    file.unlink
+  end
+
+  def dump_cookies(cookies)
+    cookies.map { |k, v| "#{k}=#{v}" }.join("; ")
+  end
+
+  def load_cookies(set_cookie)
+    set_cookie ||= []
+    set_cookie.reduce({}) do |cookies, cookie|
+      key, value = cookie.split(';')[0].split('=')
+      cookies.merge!(key => value || "")
+    end
+  end
+end

data/lib/wedding_registry_scraper/registry_scraper.rb

@@ -0,0 +1,21 @@
+class WeddingRegistryScraper::RegistryScraper
+  class << self
+    def scrape(registry_urls=[], options={})
+      all_products = {}
+
+      registry_urls.each do |url|
+        unless registry = WeddingRegistryScraper::Registries.initialize_registry(url)
+          raise "Could not initialize registry from url #{url.inspect}"
+        end
+
+        puts "* Loading items from #{registry.class.name.demodulize}..." if options[:log_messages]
+        products = registry.get_items
+        puts "  Loaded %d items\n" % products.count if options[:log_messages]
+
+        all_products.merge!(products)
+      end
+
+      all_products
+    end
+  end
+end

data/lib/wedding_registry_scraper/version.rb

@@ -0,0 +1,7 @@
+module WeddingRegistryScraper
+  VERSION = Gem::Version.new("0.0.2")
+
+  def self.version
+    VERSION
+  end
+end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: wedding_registry_scraper
 version: !ruby/object:Gem::Version
-  version: 0.0.1
+  version: 0.0.2
 platform: ruby
 authors:
 - Andrew Pariser
@@ -44,48 +44,20 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '1.6'
-- !ruby/object:Gem::Dependency
-  name: colored
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '1.2'
-  type: :runtime
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '1.2'
-- !ruby/object:Gem::Dependency
-  name: safe_yaml
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '1.0'
-  type: :runtime
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '1.0'
 - !ruby/object:Gem::Dependency
   name: activesupport
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '4.0'
+        version: '4.2'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '4.0'
+        version: '4.2'
 description: Look at a bunch of various retailers' wedding registries and consolidate
   information in one place.
 email: pariser@gmail.com
@@ -93,7 +65,18 @@ executables: []
 extensions: []
 extra_rdoc_files: []
 files:
+- LICENSE
+- README.md
 - lib/wedding_registry_scraper.rb
+- lib/wedding_registry_scraper/registries.rb
+- lib/wedding_registry_scraper/registries/crate_and_barrel.rb
+- lib/wedding_registry_scraper/registries/heath_ceramics.rb
+- lib/wedding_registry_scraper/registries/rei.rb
+- lib/wedding_registry_scraper/registries/williams_sonoma.rb
+- lib/wedding_registry_scraper/registries/zola.rb
+- lib/wedding_registry_scraper/registry.rb
+- lib/wedding_registry_scraper/registry_scraper.rb
+- lib/wedding_registry_scraper/version.rb
 homepage: https://github.com/pariser/wedding-registry-scraper
 licenses:
 - MIT
@@ -106,7 +89,7 @@ required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
-      version: '0'
+      version: '2.2'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="