amazon-search 1.2.1 → 1.3.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 9d34643d6bb26f18ac331af2887673000fce1d8e
-  data.tar.gz: aff9e145005c2d5362f4c8465bc16bddefa418a9
+  metadata.gz: d3bd73fc88e578c6c9572913876b6ccb4fca5f5c
+  data.tar.gz: 51d26171b63240b005b54494eab74abc78fdf484
 SHA512:
-  metadata.gz: 8d80a1fb04a7bfe50baa173255942713b84c160c3e0d73a094b8d635ec2ef2017cfebc72514a49a3b6e8187a4fff9dd1487747eb86018383119bad012fe26491
-  data.tar.gz: bd413ecd5459d464c551fb621d073550c45002c67f01d2be50ce1c2136ff1767cf79ca9d9a8a9d9c318a2647602cc5bb79f68cd55eeda5e21de702c6052beb18
+  metadata.gz: b6886d46b6347fab8dc5f0671716d03892f69f85aac71762257a428ecaf79ed0d456683f2e5667d1f5ce507d9612a55b47025a9766bfeef572ffcf14135266ea
+  data.tar.gz: d7ffb27e2956701acdb4ec441c01943b835a241aa765a31a7f706b879f55a3b47912c272c8100f117a6430bba45d6cacb762c9607325adb73e8d773e5227ee4f

data/Readme.rdoc

@@ -12,17 +12,9 @@ This is a tool that does not require configuration of Amazon's API.  The functio
 
     require 'amazon-search'
     
-    # Search for products by keyword string and store results
-    Amazon::Search.find_products "ruby"
-    Amazon::Search.find_products "books"
-    Amazon::Search.find_products "games"
-
-    # After a search is complete, run this command
-    Amazon::Search.display_results
-
-== GOTCHAS
-
-Sometimes a 503 error occurs due to the pagination scanning too quickly and overloading the server.  An option will come out in the future to customize how long to sleep before requesting next page of search results.  In the meantime, a simple retry will often work.
+    # Search for products by string
+    
+    Amazon::search "ruby"
     
 == MIT LICENSE
 

data/amazon-search.gemspec

@@ -2,8 +2,8 @@
 
 Gem::Specification.new do |gem|
   gem.name        = %q{amazon-search}
-  gem.version     = '1.2.1'
-  gem.date        = '2015-09-18'
+  gem.version     = '1.3.0'
+  gem.date        = '2015-09-19'
   gem.platform = Gem::Platform::RUBY
   gem.required_ruby_version = '>= 1.8'
 

data/lib/amazon-search/form.rb

@@ -0,0 +1,26 @@
+require 'mechanize'
+require_relative './scan'
+require_relative './products'
+
+module Amazon
+    class << self
+        # prepares Mechanize
+        def set_agent
+            $agent = Mechanize.new{ |a|  a.user_agent_alias = "Mac Safari"}
+        end
+
+        # finds Amazon search box
+        def find_form
+            $main_page = $agent.get("http://amazon.com")
+            $search_form = $main_page.form_with :name => "site-search"
+        end
+
+        # submits Amazon search box
+        def submit_form
+            $search_form.field_with(:name => "field-keywords").value = $keywords # sets value of search box
+            $current_page =  $agent.submit $search_form # submits form
+
+        end
+    end
+
+end

data/lib/amazon-search/products.rb

@@ -0,0 +1,59 @@
+require 'mechanize'
+require_relative './scan'
+require_relative './form'
+
+module Amazon
+    class << self
+        # extract product data
+        def extract_product_data
+            # nokogiri syntax is needed when iterating...not mechanize!
+            $current_divs.each do |html|
+                title = html.at_css(".s-access-title")
+                seller = html.at_css(".a-row > .a-spacing-none")
+                price = html.at_css(".s-price")
+                stars = html.at_css(".a-icon-star")
+                reviews = html.at_css("span+ .a-text-normal") 
+                image_href = html.at_css(".s-access-image")
+                url = html.at_css(".a-row > a") 
+
+                break if title == nil # if it's nil it's prob an ad 
+                break if price == nil # no price? prob not worthy item 
+                break if stars == nil # no stars? not worth it
+
+                if seller == nil # sometimes seller is nil on movies, etc.
+                    seller = "Unknown"
+                else
+                    seller = seller.text
+                end
+                 
+                # extract text and set variables for puts       
+                title = title.text      
+                price = price.text
+                stars = stars.text
+                reviews = reviews.text
+                image_href = image_href['src'] 
+                url = url['href']
+
+                Product.new(title, price, stars, reviews, image_href, url, html)
+
+            end
+        end
+
+
+        # currently not being used and needs adjusting  
+        def display_product
+            STDOUT.puts "--"*50
+            STDOUT.puts "title: \t\t#{title}"
+            STDOUT.puts "seller: \t#{seller}"
+            STDOUT.puts "price: \t\t#{price}"
+            STDOUT.puts "stars: \t\t#{stars}"
+            STDOUT.puts "reviews: \t#{reviews}"
+            STDOUT.puts "image url: \t#{image}"
+            STDOUT.puts "product url: \t#{url}"
+        end
+
+
+
+    end
+
+end

data/lib/amazon-search/scan.rb

@@ -0,0 +1,56 @@
+require 'mechanize'
+require_relative './products'
+require_relative './form'
+
+module Amazon
+    class << self
+        # examine current_pagenum
+        def examine_current_pagenum
+            $current_pagenum = $current_page.search '//*[contains(concat( " ", @class, " " ), concat( " ", "pagnCur", " " ))]'
+            $current_pagenum = $current_pagenum.text.to_i # need integer for checks
+        end
+
+        # find last page number
+        def find_last_pagenum
+            $last_pagenum = $current_page.search '//*[contains(concat( " ", @class, " " ), concat( " ", "pagnDisabled", " " ))]'
+            $last_pagenum = $last_pagenum.text.to_i # need integer for checks
+        end
+
+
+        # load next page
+        def load_next_page
+
+            examine_current_pagenum # does this need to be here?
+
+            $next_page_link = $current_page.link_with text: /Next Page/ # find next page link
+            $next_page = $next_page_link.click unless $current_pagenum == $last_pagenum # click to next page unless on last page
+
+            $current_page = $agent.get($next_page.uri)
+
+        end
+
+
+        # cycle through search result pages and store product html
+        def scan
+            $pages = {}
+
+            find_last_pagenum
+
+            $last_pagenum.times do # paginate until on last page.
+
+                examine_current_pagenum
+                puts "\nscanning page #{$current_pagenum} of #{$last_pagenum}..."
+
+                $current_divs = $current_page.search('//li[starts-with(@id, "result")]')
+                $pages[$page_num] = $current_divs # store page results
+
+                load_next_page
+
+            end
+
+            puts "\n(scan complete.)"
+            $pages
+        end 
+    end
+
+end

data/lib/amazon-search.rb

@@ -1,95 +1,22 @@
 #!/usr/bin/env ruby
 
 require 'mechanize'
+require './amazon-search/form'
+require './amazon-search/scan'
+require './amazon-search/products'
 
 module Amazon
-
-class Search
-    def self.find_products(keywords)
-        #--------- submit the search form with keywords ---------------------
-        agent = Mechanize.new{ |a|  a.user_agent_alias = "Mac Safari"} # set browser
-
-        main_page = agent.get("http://amazon.com")
-        search_form = main_page.form_with :name => "site-search" # find the search form in Amazon
-
-        search_form.field_with(:name => "field-keywords").value = keywords # sets value of search box
-        search_results = agent.submit search_form # submits form
-
-        #--------- scan each page and store the results ---------------------
-        $product_divs = []
-        page_num = 0
-        next_page = agent.get(search_results.uri) # initial search results are the first page
-
-        last_page_num = search_results.search '//*[contains(concat( " ", @class, " " ), concat( " ", "pagnDisabled", " " ))]'
-        last_page_num = last_page_num.text.to_i # change to int for upcoming iteration instructions
-
-        last_page_num.times do # cycle all pages and stop on last page
-            page_num += 1
-            page = agent.get(next_page.uri) # load the next page
-            
-            $product_divs << page.search('//li[starts-with(@id, "result")]') # store the div of each product
-           
-            next_page_link = page.link_with text: /Next Page/ # find the next page link   
-            next_page = next_page_link.click unless page_num == last_page_num # click to next page unless on last page
-        end # ends pagination loop
-
-        puts "\n\n(end of search results)"
-        sleep(1) # don't overload the servers
+    class << self
+        def search(keywords)
+            $keywords = keywords 
+
+            set_agent
+            find_form
+            submit_form
+            scan
+        end
     end
+end
 
 
-    def self.display_results
-        # nokogiri syntax is needed when iterating...not mechanize!
-        product_divs.each do |product|
-            
-            #--------- nokogiri select html sections from css ---------------------
-            title = product.at_css(".s-access-title")
-            seller = product.at_css(".a-row > .a-spacing-none") #".a-spacing-small .a-spacing-none"
-            price = product.at_css(".s-price")
-            stars = product.at_css(".a-icon-star")
-            reviews = product.at_css("span+ .a-text-normal") # ".a-span-last .a-spacing-mini > span+ .a-text-normal"
-            image = product.at_css(".s-access-image")
-            url = product.at_css(".a-row > a") 
-            
-            #--------- avoid the related items gotchas ---------------------  
-            if title == nil # if it's nil it's prob an ad 
-                break
-            else
-                title = title.text
-                
-                if seller == nil # if seller is nil put unknown
-                    seller = "Unknown"
-                else
-                    seller = seller.text
-                    if price == nil # no price? prob not worthy item
-                    break
-                    
-                    else
-                        price = price.text
-                        if stars == nil
-                            break
-                            
-                        else
-                            stars = stars.text
-                            reviews = reviews.text
-                            image = image['src'] 
-                            url = url['href']
 
-                            # errors properly avoided, now puts the results   
-                            STDOUT.puts "--"*50
-                            STDOUT.puts "title: \t\t#{title}"
-                            STDOUT.puts "seller: \t#{seller}"
-                            STDOUT.puts "price: \t\t#{price}"
-                            STDOUT.puts "stars: \t\t#{stars}"
-                            STDOUT.puts "reviews: \t#{reviews}"
-                            STDOUT.puts "image url: \t#{image}"
-                            STDOUT.puts "product url: \t#{url}"
-                            
-                        end # ends nil price
-                    end # ends nil stars
-                end # ends nil seller 
-            end # ends nil product 
-        end # ends each product div iteration (page is finished)    
-    end # ends display_results
-end # ends Search Class
-end # ends Amazon Module

data/test/lib/amazon-search/form.rb

@@ -0,0 +1,26 @@
+require 'mechanize'
+require_relative './pages'
+require_relative './products'
+
+module Amazon
+    class << self
+        # prepares Mechanize
+        def set_agent
+            $agent = Mechanize.new{ |a|  a.user_agent_alias = "Mac Safari"}
+        end
+
+        # finds Amazon search box
+        def find_form
+            $main_page = $agent.get("http://amazon.com")
+            $search_form = $main_page.form_with :name => "site-search"
+        end
+
+        # submits Amazon search box
+        def submit_form
+            $search_form.field_with(:name => "field-keywords").value = $keywords # sets value of search box
+            $current_page =  $agent.submit $search_form # submits form
+
+        end
+    end
+
+end

data/test/lib/amazon-search/pages.rb

@@ -0,0 +1,113 @@
+require 'mechanize'
+require_relative './products'
+require_relative './form'
+
+module Amazon
+    class << self
+        # examine current_pagenum
+        def examine_current_pagenum
+            $current_pagenum = $current_page.search '//*[contains(concat( " ", @class, " " ), concat( " ", "pagnCur", " " ))]'
+            $current_pagenum = $current_pagenum.text.to_i # need integer for checks
+        end
+
+        # find last page number
+        def find_last_pagenum
+            $last_pagenum = $current_page.search '//*[contains(concat( " ", @class, " " ), concat( " ", "pagnDisabled", " " ))]'
+            $last_pagenum = $last_pagenum.text.to_i # need integer for checks
+        end
+
+
+        # load next page
+        def load_next_page
+            puts "***started load_next_page method***"
+            puts "ready to examine the page number?"
+            gets
+
+            examine_current_pagenum
+
+            puts "page number is..."
+            puts $current_pagenum
+            puts "continue?"
+            gets
+
+            $next_page_link = $current_page.link_with text: /Next Page/ # find next page link
+
+            puts "found next page..."
+            puts "this is link:"
+            puts $main_page.uri+$next_page_link.uri
+            puts "continue?"
+            gets
+
+            $next_page = $next_page_link.click unless $current_pagenum == $last_pagenum # click to next page unless on last page
+
+            puts "next step is to load the next page..."
+            puts "page will load to:"
+            puts $agent.get($next_page.uri).uri
+            puts "continue?"
+            gets
+
+            $current_page = $agent.get($next_page.uri)
+            examine_current_pagenum
+
+            puts "====current_page has changed===="
+            puts "this is uri:"
+            puts $current_page.uri
+            puts "this is page_num"
+            puts $current_pagenum
+
+            puts "\ncontinue and exit loading method?"
+            gets
+
+            puts "***ending load_next_page method***"
+        end
+
+
+        # cycle through search result pages and store product html
+        def scan
+            puts "***started scan method***"
+            $pages = {}
+
+            find_last_pagenum
+
+            $last_pagenum.times do # paginate until on last page.
+                puts "***started pagination block***"
+
+                puts "Enter 'html' if you want to puts pages array, other press RETURN to continue"
+                answer = gets.chomp
+
+                if answer == "html"
+                    if $pages.empty?
+                        puts "pages array is empty"
+                    else
+                        $pages.each {|x| puts x}
+                    end
+                end
+
+                examine_current_pagenum
+
+                $current_divs = $current_page.search('//li[starts-with(@id, "result")]')
+                $pages[$page_num] = $current_divs # store page results
+
+
+                puts "--"*50
+                puts "\nlast page number is #{$last_pagenum}"
+                puts "we're on #{$current_pagenum}"
+                puts "this is current hyperlink:"
+                puts $current_page.uri
+
+
+                puts "ready to go to #{$current_pagenum+1}?"
+                gets
+
+                load_next_page
+                puts "scanning is ready to restart loop."
+                puts "continue?"
+                gets
+                puts "***ending pagination block***"
+            end
+
+            puts "***ending scan method***"
+        end 
+    end
+
+end

data/test/lib/amazon-search/products.rb

@@ -0,0 +1,59 @@
+require 'mechanize'
+require_relative './pages'
+require_relative './form'
+
+module Amazon
+    class << self
+        # extract product data
+        def extract_product_data
+            # nokogiri syntax is needed when iterating...not mechanize!
+            $current_divs.each do |html|
+                title = html.at_css(".s-access-title")
+                seller = html.at_css(".a-row > .a-spacing-none")
+                price = html.at_css(".s-price")
+                stars = html.at_css(".a-icon-star")
+                reviews = html.at_css("span+ .a-text-normal") 
+                image_href = html.at_css(".s-access-image")
+                url = html.at_css(".a-row > a") 
+
+                break if title == nil # if it's nil it's prob an ad 
+                break if price == nil # no price? prob not worthy item 
+                break if stars == nil # no stars? not worth it
+
+                if seller == nil # sometimes seller is nil on movies, etc.
+                    seller = "Unknown"
+                else
+                    seller = seller.text
+                end
+                 
+                # extract text and set variables for puts       
+                title = title.text      
+                price = price.text
+                stars = stars.text
+                reviews = reviews.text
+                image_href = image_href['src'] 
+                url = url['href']
+
+                Product.new(title, price, stars, reviews, image_href, url, html)
+
+            end
+        end
+
+
+        # currently not being used and needs adjusting  
+        def display_product
+            STDOUT.puts "--"*50
+            STDOUT.puts "title: \t\t#{title}"
+            STDOUT.puts "seller: \t#{seller}"
+            STDOUT.puts "price: \t\t#{price}"
+            STDOUT.puts "stars: \t\t#{stars}"
+            STDOUT.puts "reviews: \t#{reviews}"
+            STDOUT.puts "image url: \t#{image}"
+            STDOUT.puts "product url: \t#{url}"
+        end
+
+
+
+    end
+
+end

data/test/lib/amazon-search.rb

@@ -0,0 +1,22 @@
+#!/usr/bin/env ruby
+
+require 'mechanize'
+require './amazon-search/form'
+require './amazon-search/pages'
+require './amazon-search/products'
+
+module Amazon
+    class << self
+        def search(keywords)
+            $keywords = keywords 
+
+            set_agent
+            find_form
+            submit_form
+            scan
+        end
+    end
+end
+
+
+

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: amazon-search
 version: !ruby/object:Gem::Version
-  version: 1.2.1
+  version: 1.3.0
 platform: ruby
 authors:
 - John Mason
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2015-09-18 00:00:00.000000000 Z
+date: 2015-09-19 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: mechanize
@@ -34,6 +34,13 @@ files:
 - Readme.rdoc
 - amazon-search.gemspec
 - lib/amazon-search.rb
+- lib/amazon-search/form.rb
+- lib/amazon-search/products.rb
+- lib/amazon-search/scan.rb
+- test/lib/amazon-search.rb
+- test/lib/amazon-search/form.rb
+- test/lib/amazon-search/pages.rb
+- test/lib/amazon-search/products.rb
 homepage: https://github.com/m8ss/amazon-search
 licenses:
 - MIT
@@ -58,4 +65,8 @@ rubygems_version: 2.4.6
 signing_key: 
 specification_version: 4
 summary: A simple screenscraper to search Amazon
-test_files: []
+test_files:
+- test/lib/amazon-search.rb
+- test/lib/amazon-search/form.rb
+- test/lib/amazon-search/pages.rb
+- test/lib/amazon-search/products.rb