amazon_wish_miner 0.0.1 → 0.1.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 320443f261d0231425f425704d2990492afedc27820e0a2b78ae42d17d4f8c20
-  data.tar.gz: 1b9f85e4fe95da69ee320ecc06d1acb96b4a6410f9233ffc2ab925930698eba4
+  metadata.gz: dc6ba9d094477598ed88ae8290160153086fd048ace3c90af8ac107265154e8e
+  data.tar.gz: 0afab73d342d5b7b12c004fbb88e8710ae09f3076fde0d664e627805a0a48463
 SHA512:
-  metadata.gz: 25ca33d4e161e75ccc44a9084b276350f3fec5bc6b85150b9885484229df4641f8c24a98ddf327a98169697cb39504eb34b1ccf5ae32ed1445b92841a888db17
-  data.tar.gz: f476836203c819b1f1a0f27ae1147aad51f61f88d9f9b93d6019288a8f05b21f80425f4abb53a8af8a6758a5218f046a81e46ad8a99bd7e2570916587a54321d
+  metadata.gz: 55a49aa00e70fd5e5719dc9f169113ecec79a74f666e265be6c2912589f2cbba9ea82443ef8a68f6a51c2dec90b0439fdbfa324f786ccb20605ed4a770ae08b7
+  data.tar.gz: c639720d3b144945655cb3f119ebf3d39203ca2f57d1719fdce2f710496ec1e2976aeb96f551b1c7e2f575ecfb37799aa8c4cfd5c68c6ac1ef481bc042b8bd77

data/lib/amazon_wish_miner.rb

@@ -1 +1,4 @@
-puts 'hello world'
+require 'nokogiri'
+require 'rest-client'
+require_relative 'amazon_wish_miner/amazon_wish_list.rb'
+require_relative 'amazon_wish_miner/amazon_wish.rb'

data/lib/amazon_wish_miner/amazon_wish.rb

@@ -0,0 +1,67 @@
+class AmazonWish
+
+  attr_reader: :title, :id
+
+  def initialize(id, title)
+    @title = title
+    @id = id
+  end
+
+  def self.parse_wishes_from_pages(page_responses)
+    list_items = self.list_items_from_response(page_responses)
+    wish_ids = self.draps_from_list_items(list_items)
+    # wishes_from_ids(wish_ids)
+  end
+
+  def self.list_items_from_response(page_responses)
+    page_responses.each_with_object(Array.new) do |response, list_items|
+      page = Nokogiri::HTML(response)
+      page.css('ul#g-items li').each do |li|
+        list_items << li
+      end
+    end
+  end
+
+  def self.draps_from_list_items(list_items)
+    list_items.each_with_object(Array.new) do |li, wish_ids|
+      drap = li['data-reposition-action-params']
+      wish_ids << external_id_from_drap(drap)
+    end
+  end
+
+  def self.external_id_from_drap(drap)
+    attrs = drap.split(',')
+    attr_substrings = attrs.map { |elem| elem.split(':') }
+    ied_attr = attr_substrings.find { |ss| ss.include?("{\"itemExternalId\"")}
+    id_string = ied_attr.last
+    ids_arr = id_string.split('|')
+    ids_arr.first
+  end
+
+  # parsing item info from the item's own url rather than from the wishlist
+  #=> means that we can reuse the method below to scrape item info
+
+  def self.wishes_from_ids(ids)
+    ids.map do |id|
+      self.item_from_id(id)
+    end
+  end
+
+  def self.item_from_id(id)
+    item_url = 'https://www.amazon.com/dp/' + id
+    response = RestClient.get(item_url)
+    page = Nokogiri::HTML(response)
+    title = page.css('span[id$="roductTitle"]') # not a typo, css selectors are
+    #=> case sensetive, and we need to capture e.g. both "productTitle" and "ebookProductTitle"
+    # price = page.css('priceblock_ourprice')
+    # TODO: parse prices
+    # description = parse_feature_bullets(page.css('div#feature-bullets'))
+    # TODO: get description parsing to work for different types of items
+    AmazonWish.new(id, title)
+  end
+
+  def self.parse_feature_bullets(feature_bullets_div)
+    bullets = feature_bullets_div.css('ul li')
+  end
+
+end

data/lib/amazon_wish_miner/amazon_wish_list.rb

@@ -0,0 +1,60 @@
+class AmazonWishList
+
+  REVEAL_OPTIONS = [:all, :purchased, :unpurchased].freeze
+  SORT_OPTIONS = {date_added: "date-added", title: 'universal-title',
+    price_high: 'universal-price-desc', price_low: 'universal-price',
+    date_updated: 'last-updated', priority: 'priority'}.freeze
+
+  def initialize
+  end
+
+  # TODO: https://www.amazon.com/hz/wishlist/ls/2WHUDN1UIDVUT/ref=cm_sw_r_cp_ep_ws_8xNVBb731TTMS,
+  #=> https://www.amazon.com/gp/registry/wishlist/2WHUDN1UIDVUT/ref=cm_sw_r_cp_ep_ws_8xNVBb731TTMS,
+  #=> and https://www.amazon.com/registry/wishlist/2WHUDN1UIDVUT/ref=cm_sw_r_cp_ep_ws_8xNVBb731TTMS
+  #=> appear to be functionally the same. Code should reflect this when it is
+  #=> given links as arguments.
+
+  def self.get_wishlist(amazon_list_id, reveal = :all, sort = :date_added, tld = 'com')
+    raise "invalid reveal" unless REVEAL_OPTIONS.include?(reveal)
+    raise "invalid sort" unless SORT_OPTIONS[sort]
+
+    query_params = {reveal: reveal.to_s, sort_string: SORT_OPTIONS[sort]}
+    # lek is nil for the first page
+    url_without_qstring = "http://www.amazon.#{tld}/hz/wishlist/ls/#{amazon_list_id}"
+
+    pages = self.get_all_wishlist_pages(url_without_qstring, query_params)
+  end
+
+  def self.get_all_wishlist_pages(url_without_qstring, query_params)
+    responses = Array.new
+    loop do
+      response = self.get_wishlist_page(url_without_qstring, query_params)
+      responses << response
+      return responses if response.body.include?("Find a gift") # as of the
+      #=> time this was written, this phrase appears only on the last page
+      lek = self.find_lek_from_response(response)
+      query_params[:lek] = lek # the rest of the query_params hash stays the same
+    end
+  end
+
+  def self.get_wishlist_page(url_without_qstring, query_params)
+    query_string = self.page_query_string(query_params)
+    RestClient.get(url_without_qstring + query_string)
+  end
+
+  def self.page_query_string(query_params)
+    "?reveal=#{query_params[:reveal]}&layout=standard&sort=#{query_params[:sort_string]})" +
+    (query_params[:lek] ? "&lek=#{query_params[:lek]}&type=wishlist&ajax=true" : '')
+  end
+
+  def self.find_lek_from_response(response)
+    # As of the time of writing this, "lastEvaluatedKey", abbreviated as "lek",
+    # is used to keep track of what portions of the wishlist have already been
+    # loaded, and is sent in the query string of ajax calls to get the next page
+    start_of_lek = response.body.split('name="lastEvaluatedKey" value="')[1]
+    start_of_lek.split('" class="lastEvaluatedKey"')[0]
+  end
+
+
+
+end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: amazon_wish_miner
 version: !ruby/object:Gem::Version
-  version: 0.0.1
+  version: 0.1.1
 platform: ruby
 authors:
 - Alexander V. Trujillo
@@ -11,7 +11,7 @@ cert_chain: []
 date: 2018-10-03 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
-  name: HTTParty
+  name: rest-client
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
@@ -30,14 +30,14 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '2'
+        version: '1.8'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '2'
+        version: '1.8'
 description: Modeled after Amazon Wish Lister http://doitlikejustin.github.io/amazon-wish-lister/
 email: 
 executables: []
@@ -45,6 +45,8 @@ extensions: []
 extra_rdoc_files: []
 files:
 - lib/amazon_wish_miner.rb
+- lib/amazon_wish_miner/amazon_wish.rb
+- lib/amazon_wish_miner/amazon_wish_list.rb
 homepage: https://github.com/avtrujillo/amazon_wish_miner
 licenses:
 - MIT