amazoned 0.1.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: c4ff0dac060161cb2ba774302dcadfa145332a1f
+  data.tar.gz: 30a81104eb62ef099d4515a3b6effbdde23520b3
+SHA512:
+  metadata.gz: 134f051f0e192d2490422a3ee8b061787234a2b0c1cbd2b9af2b3a0d3d9d5745133f2b166e63458289b960667e21cdc58a30bdcca6539dcae7253365a24df71d
+  data.tar.gz: 7d88efe96872019f2258cd95b9e869c58a8d59dde6028db3bf91a55794f59a578605151941f64b2d66ec5ec10c168f976adf437ed158eb4ecdc806ac66790b28

data/.gitignore

@@ -0,0 +1,11 @@
+/.bundle/
+/.yardoc
+/_yardoc/
+/coverage/
+/doc/
+/pkg/
+/spec/reports/
+/tmp/
+
+# rspec failure tracking
+.rspec_status

data/.rspec

@@ -0,0 +1,3 @@
+--format documentation
+--color
+--require spec_helper

data/.travis.yml

@@ -0,0 +1,5 @@
+sudo: false
+language: ruby
+rvm:
+  - 2.5.1
+before_install: gem install bundler -v 1.16.2

data/Gemfile

@@ -0,0 +1,6 @@
+source "https://rubygems.org"
+
+git_source(:github) {|repo_name| "https://github.com/#{repo_name}" }
+
+# Specify your gem's dependencies in amazonian.gemspec
+gemspec

data/LICENSE.txt

@@ -0,0 +1,21 @@
+The MIT License (MIT)
+
+Copyright (c) 2018 kelseydh
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.

data/README.md

@@ -0,0 +1,39 @@
+# Amazoned
+
+Welcome to your new gem! In this directory, you'll find the files you need to be able to package up your Ruby library into a gem. Put your Ruby code in the file `lib/amazoned`. To experiment with that code, run `bin/console` for an interactive prompt.
+
+TODO: Delete this and the text above, and describe your gem
+
+## Installation
+
+Add this line to your application's Gemfile:
+
+```ruby
+gem 'amazoned'
+```
+
+And then execute:
+
+    $ bundle
+
+Or install it yourself as:
+
+    $ gem install amazoned
+
+## Usage
+
+TODO: Write usage instructions here
+
+## Development
+
+After checking out the repo, run `bin/setup` to install dependencies. Then, run `rake spec` to run the tests. You can also run `bin/console` for an interactive prompt that will allow you to experiment.
+
+To install this gem onto your local machine, run `bundle exec rake install`. To release a new version, update the version number in `version.rb`, and then run `bundle exec rake release`, which will create a git tag for the version, push git commits and tags, and push the `.gem` file to [rubygems.org](https://rubygems.org).
+
+## Contributing
+
+Bug reports and pull requests are welcome on GitHub at https://github.com/[USERNAME]/amazoned.
+
+## License
+
+The gem is available as open source under the terms of the [MIT License](https://opensource.org/licenses/MIT).

data/Rakefile

@@ -0,0 +1,6 @@
+require "bundler/gem_tasks"
+require "rspec/core/rake_task"
+
+RSpec::Core::RakeTask.new(:spec)
+
+task :default => :spec

data/amazoned.gemspec

@@ -0,0 +1,32 @@
+
+lib = File.expand_path("../lib", __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require "amazoned/version"
+
+Gem::Specification.new do |spec|
+  spec.name          = "amazoned"
+  spec.version       = Amazoned::VERSION
+  spec.authors       = ["kelseydh"]
+  spec.email         = ["kelseyh@gmail.com"]
+
+  spec.summary       = %q{A manual scraper for Amazon ASIN product data}
+  spec.description   = %q{This gem allows you to scrap product information from Amazon without the need to register for Amazon's API}
+  spec.homepage      = "http://twitter.com/kelsoh"
+  spec.license       = "MIT"
+
+
+  # Specify which files should be added to the gem when it is released.
+  # The `git ls-files -z` loads the files in the RubyGem that have been added into git.
+  spec.files         = Dir.chdir(File.expand_path('..', __FILE__)) do
+    `git ls-files -z`.split("\x0").reject { |f| f.match(%r{^(test|spec|features)/}) }
+  end
+  spec.bindir        = "exe"
+  spec.executables   = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
+  spec.require_paths = ["lib"]
+
+  spec.add_development_dependency "bundler", "~> 1.16"
+  spec.add_development_dependency "rake", "~> 10.0"
+  spec.add_development_dependency "rspec", "~> 3.0"
+  spec.add_development_dependency "mechanize", '~> 2.7', '>= 2.7.6'
+  spec.add_development_dependency "activesupport", '~> 5.2', '>= 4.2.0'
+end

data/bin/console

@@ -0,0 +1,14 @@
+#!/usr/bin/env ruby
+
+require "bundler/setup"
+require "amazoned"
+
+# You can add fixtures and/or initialization code here to make experimenting
+# with your gem easier. You can also use a different console, if you like.
+
+# (If you use this, don't forget to add pry to your Gemfile!)
+# require "pry"
+# Pry.start
+
+require "irb"
+IRB.start(__FILE__)

data/bin/setup

@@ -0,0 +1,8 @@
+#!/usr/bin/env bash
+set -euo pipefail
+IFS=$'\n\t'
+set -vx
+
+bundle install
+
+# Do any other automated setup that you need to do here

data/lib/amazoned.rb

@@ -0,0 +1,26 @@
+require 'active_support/core_ext/string'
+require 'amazoned/version'
+require 'amazoned/client'
+require 'amazoned/parser'
+require 'mechanize'
+require 'nokogiri'
+require 'byebug'
+
+module Amazoned
+  @max_network_retries = 3
+  @max_network_retry_delay = 2.3
+  @initial_network_retry_delay = 1.1 # requests under a second could flag us as a scraping bot
+
+
+  class << self
+    attr_reader :max_network_retry_delay, :initial_network_retry_delay
+  end
+
+  def self.max_network_retries
+    @max_network_retries
+  end
+
+  def self.max_network_retries=(val)
+    @max_network_retries = val.to_i
+  end
+end

data/lib/amazoned/client.rb

@@ -0,0 +1,76 @@
+cled Amazoned::ProductNotFoundError < StandardError; end
+class Amazoned::BotDeniedAccessError < StandardError; end
+class Amazoned::Client
+  attr_reader :asin
+
+  def initialize(asin)
+    @asin = asin
+  end
+
+  def call
+    response = get_product
+    Amazoned::Parser.new(response).call
+  end
+
+  def get_product(num_retries = 1)
+    agent = Mechanize.new.tap do |web|
+      web.html_parser = HtmlParser # Avoid encoding issues: https://stackoverflow.com/a/20666246/3448554
+      web.user_agent_alias = (Mechanize::AGENT_ALIASES.keys - ['Mechanize']).sample # spoof every request with a random User Agent as a way to hit fewer CAPTCHA walls
+    end
+
+    begin
+      # Start GET request of Amazon page using ASIN.
+      response = agent.get("https://www.amazon.com/dp/#{asin}")
+      if request_failed(response)
+        puts "Request failed!  Trying again..."
+        # On failure, recursively try again to be resilient against one-off failures
+        if num_retries <= Amazoned.max_network_retries
+          sleep self.class.sleep_time(num_retries)
+          get_product(num_retries += 1)
+        else
+          handle_failed_request!(response)
+        end
+      else
+        response
+      end
+    rescue Mechanize::ResponseCodeError => e
+      raise Amazoned::ProductNotFoundError
+    end
+  end
+
+  def request_failed(response)
+    return true if response.xpath('//p[contains(text(), "Sorry, we just need to make sure")]').any? # captcha hit
+    false
+  end
+
+  def handle_failed_request!(response)
+    # Raise this error when we can't penetrate Amazon's CAPTCHA wall
+    raise Amazoned::BotDeniedAccessError if response.xpath('//p[contains(text(), "Sorry, we just need to make sure")]').any?
+  end
+
+  # Taken from Stripe API
+  # Stripe uses jitter to smooth server load; we use it to obfuscate timing detection of our scraper bot
+  # https://github.com/stripe/stripe-ruby/blob/ec66c3f0f44274f885de8d13de5dce2657932121/lib/stripe/stripe_client.rb#L80
+  def self.sleep_time(num_retries)
+    # Apply exponential backoff with initial_network_retry_delay on the
+    # number of num_retries so far as inputs. Do not allow the number to exceed
+    # max_network_retry_delay.
+    sleep_seconds = [Amazoned.initial_network_retry_delay * (2**(num_retries - 1)), Amazoned.max_network_retry_delay].min
+
+    # Apply some jitter by randomizing the value in the range of (sleep_seconds
+    # / 2) to (sleep_seconds).
+    sleep_seconds *= (0.5 * (1 + rand))
+
+    # But never sleep less than the base sleep seconds.
+    sleep_seconds = [Amazoned.initial_network_retry_delay, sleep_seconds].max
+
+    sleep_seconds
+  end
+end
+
+class HtmlParser
+  def self.parse(body, url, encoding)
+    body.encode!('UTF-8', encoding, invalid: :replace, undef: :replace, replace: '')
+    Nokogiri::HTML::Document.parse(body, url, 'UTF-8')
+  end
+end

data/lib/amazoned/parser.rb

@@ -0,0 +1,109 @@
+class Amazoned::Parser
+  attr_accessor :product_hash
+  attr_reader :html_doc, :response
+
+  def initialize(response)
+    @product_hash = Hash.new
+    @response = response
+    @html_doc = Nokogiri::HTML(response.body)
+  end
+
+  def call
+    parse_response_for_product_details( response )
+  end
+
+  def parse_response_for_product_details(response)
+    product_hash[:best_sellers_rank] = []
+
+    ########
+    # # Parent category Seller Rank Parser
+    ########
+    parsed_parent_category = html_doc.css('#SalesRank').text.partition("(").first.chop.partition("#").last.partition("in").map(&:strip) - ["in"]
+    product_hash[:rank] = parsed_parent_category.first.delete(',').to_i # "903,610" -> 903610
+    product_hash[:category] = parsed_parent_category.last
+
+    ########
+    # # Subcategory Seller Rank Parser
+    ########
+    extract_subcategory_rankings( html_doc.css('.zg_hrsr_item') )
+
+    ########
+    # # Package Dimension Parser
+    ########
+    # Package Dimension Parsing Strategy 1:
+    product_hash[:package_dimensions] = html_doc.css('.size-weight').children.map{|r| r.text}.reject{|r| !r.match?("inches")}.first
+
+    # Package Dimension Parsing Strategy 2:
+    if product_hash[:package_dimensions].blank?
+
+      # Find an index for the string "Package Dimensions" within a string text extraction of the page
+      str_index = html_doc.inner_text.index("Package Dimensions")
+
+      unless str_index.nil?
+
+        # Reduce string representing the html page down to a smaller target string including "Package Dimensions" and the weights
+        str = html_doc.inner_text[str_index .. str_index + 150]
+
+        # Find within target string an index for where the word "inches" appears, then grab characters around it
+        product_hash[:package_dimensions] = str[str.index("inches")- 20.. str.index("inches")+8].strip
+      end
+    end
+
+    # Package Dimension Parsing Strategy 3:
+      response.search('.//*[@class="a-color-secondary a-size-base prodDetSectionEntry"]').map{|n| n.parent}.each do |n|
+
+        # Parse html in each row of Amazon's product details table to get back a string. E.g:  "\n    \n        Best Sellers Rank\n    \n    \n         \n              \n #63 in Toys & Games (See Top 100 in Toys & Games)\n        \n              \n #3 in Toys & Games > Baby & Toddler Toys > Teethers\n        \n              \n        \n    \n    "
+        str = n.children.inner_text
+        if product_hash[:best_sellers_rank].blank?
+          str.match("Best Sellers Rank") do |m|
+            # Gnarly string manipulation extracts the array: ["63", "in", "Toys & Games"]
+            parsed_parent_category = str.partition("(").first.chop.partition("#").last.partition("in").map(&:strip)
+
+            # From ["63", "in", "Toys & Games"] we only care about first & last parts of this array
+            product_hash[:rank] = parsed_parent_category.first.delete(',').to_i
+            product_hash[:category] = parsed_parent_category.last
+
+            parsed_category = str.partition(")").last.partition("in").map(&:strip).map{|i| i.gsub("#", "")} - ["in"]
+
+            hsh = {}
+            hsh[:rank] = parsed_category.first.delete(',').to_i
+            hsh[:ladder] = parsed_category.last
+            product_hash[:best_sellers_rank] << hsh
+          end
+        end
+
+        if product_hash[:product_dimensions].blank?
+
+          # Use pattern matching to extract the product details we care about
+          str.match("Product Dimensions") do |m|
+            product_hash[:package_dimensions] = str[str.index("inches")- 20.. str.index("inches")+8].strip
+          end
+        end
+      end
+    product_hash
+  end
+
+  def extract_subcategory_rankings(nokogiri_html)
+    # Below is gnarly string manipulation to parse text strings like:
+    # "\n    #2\n    in Baby > Baby Care > Health\n    \n    #2\n    in Baby > Baby Care > Pacifiers, Teethers & Teething Relief > Teethers\n    "
+    # into:
+    # [["2", "Baby > Baby Care > Health"], ["2", "Baby > Baby Care > Pacifiers, Teethers & Teething Relief > Teethers"]]
+    nokogiri_html
+    .map{|i| i.text}
+    .map{|i| i.partition("in")
+    .map(&:strip)}
+    .map{|i| i - ["in"] }
+    .map{|i|
+      i.map{|ii|
+        ii.gsub("#", "") # remove '#' from '#2'
+        .gsub("\u00A0", "") # remove No-Break Space Unicode characters (U+00A0) since Ruby's .strip command won't remove them
+      }
+    }.each do |i|
+      hsh = {}
+      hsh[:rank] = i.first.to_i
+      hsh[:ladder] = i.last
+      product_hash[:best_sellers_rank] << hsh
+    end
+  end
+
+end

data/lib/amazoned/version.rb

@@ -0,0 +1,3 @@
+module Amazoned
+  VERSION = "0.1.0"
+end

metadata

@@ -0,0 +1,141 @@
+--- !ruby/object:Gem::Specification
+name: amazoned
+version: !ruby/object:Gem::Version
+  version: 0.1.0
+platform: ruby
+authors:
+- kelseydh
+autorequire: 
+bindir: exe
+cert_chain: []
+date: 2018-07-08 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.16'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.16'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '10.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '10.0'
+- !ruby/object:Gem::Dependency
+  name: rspec
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.0'
+- !ruby/object:Gem::Dependency
+  name: mechanize
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.7'
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 2.7.6
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.7'
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 2.7.6
+- !ruby/object:Gem::Dependency
+  name: activesupport
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '5.2'
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 4.2.0
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '5.2'
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 4.2.0
+description: This gem allows you to scrap product information from Amazon without
+  the need to register for Amazon's API
+email:
+- kelseyh@gmail.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- ".gitignore"
+- ".rspec"
+- ".travis.yml"
+- Gemfile
+- LICENSE.txt
+- README.md
+- Rakefile
+- amazoned.gemspec
+- bin/console
+- bin/setup
+- lib/amazoned.rb
+- lib/amazoned/client.rb
+- lib/amazoned/parser.rb
+- lib/amazoned/version.rb
+homepage: http://twitter.com/kelsoh
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.6.13
+signing_key: 
+specification_version: 4
+summary: A manual scraper for Amazon ASIN product data
+test_files: []