scrapers 2.0.2 → 2.1.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 0fe614f84b4dd26d8bb7d314e7e13a08125fa022
-  data.tar.gz: 06bf20ac2efe3bbf90a0fa5f1a35b4c2d92b0a5d
+  metadata.gz: 7a122441c7a4d715eded98e92a58d31d6b00f21c
+  data.tar.gz: 74e3ad669f233d43542155819d2224499c062d5e
 SHA512:
-  metadata.gz: 138916948ac182b0a0fb5998b5b4929493df0e70aee9530cb978f989dfa36dc02b8577f524c9712470d83eec48abca5c5aca976f4272308086e4219417333a37
-  data.tar.gz: e8dcfc8be2dd0a391364f7a3c182241467dddf79a2b77514f90616b6b5f5b964f58394ccc00ae87644e6a49c0a3d10c991ef229bd90e057b8ced9e1f7ca62c50
+  metadata.gz: c7fe23236b2a325eec855f865aa687b329d0cc3b470cad66f56623df7b4833f11fc6167871cfa6b10351da1d0c3747b40d44b64bc0057bb563735b234cf15a56
+  data.tar.gz: a22303c8d58b65795a5811c6f997b47ff3a4bc64d60849e83e43bc5794650222718ee106f60891c344acd210524e00c15a2074958732d14b50a54bd8b2d3e57c

data/bin/manning_books

@@ -1,6 +1,5 @@
 #!/usr/bin/env ruby
 require 'thor'
-require 'netrc'
 require 'scrapers/manning_books'
 require 'awesome_print'
 
@@ -15,22 +14,16 @@ require 'awesome_print'
 
 class ManningBooks < Thor
 
-  MANNING = 'manning'
-
   desc "download", "Downloads all the editions of all the books on your dashboard"
   method_option :destination, :aliases => %w{-d --dest}, :desc => "Destination to store the downloads", :default => "."
   method_option :user, :aliases => %w{-u -U}, :desc => "Manning user. Default is read from $HOME/.netrc"
   method_option :password, :aliases => %w{-p -pw}, :desc => "Manning password. Default is read from $HOME/.netrc"
+  method_option :delay, :aliases => %w{-t}, :type => :numeric, :desc => "delay time between requests", :default => Scrapers::ManningBooks::DELAY_TIME
+  method_option :dry_run, :aliases => %w[-n], :type => :boolean, :desc => "dry run, do not download and save books", :default => false
 
   def download
-    netrc = Netrc.read
-    user, pw = netrc[MANNING]
-    user = options.fetch("user", user)
-    pw = options.fetch("password", pw)
-    destination = options.fetch("destination", nil)
-    STDERR.puts "destination: #{destination}, user: #{user}, pw: #{pw.length}"
     Signal.trap('INT', proc { STDERR.puts "Download Interrupted"; exit(-1)})
-    results = Scrapers::ManningBooks.scrape destination, user, pw
+    results = Scrapers::ManningBooks::Scraper.new(options).scrape
     ap results
   end
 

data/lib/netrc_reader.rb

@@ -0,0 +1,12 @@
+require 'netrc'
+
+module Scrapers  
+  class NetrcReader
+    attr_accessor :user, :pw
+
+    def initialize(section)
+      netrc = Netrc.read
+      @user, @pw = netrc[section]      
+    end
+  end
+end

data/lib/scrapers/manning_books.rb

@@ -1,55 +1,84 @@
 # -*- ruby -*-
 require 'mechanize'
-# require 'pry'
+require 'netrc_reader'
 
 module Scrapers
   module ManningBooks
 
+    NETRC_MANNING_ENTRY = 'manning'
     DASHBOARD_URL = "https://account.manning.com/dashboard"
-    
-    def self.scrape(dest=".", user=nil, pw=nil)
-      results = Array.new
+    DELAY_TIME = 5 # seconds
 
-      Mechanize.start do |m|
-        m.get DASHBOARD_URL
-        unless m.current_page.uri == DASHBOARD_URL
-          # log in
-          m.current_page.form.field_with(:type => 'email').value= user
-          m.current_page.form.field_with(:type => 'password').value= pw
-          m.current_page.form.submit
-          sleep 2
-          raise "could not log in" unless m.current_page.uri.to_s == DASHBOARD_URL
-        end
+    class Scraper
+      attr_accessor :user, :pw, :delay_time, :destination, :dry_run
 
-        book_downloads = m.current_page.links_with(:href => %r{/account/bookProduct/download})
-
-        Dir.chdir(dest) do |dir|
-          book_downloads.each do |book|
-            puts "Downloading #{book.href}"
-            m.get book.href
-            results << [m.current_page.filename, m.current_page.uri.to_s]
-            puts "Saving #{m.current_page.filename}"
-            m.current_page.save! # overwrite!
+      def initialize(options={})
+        netrc_reader = ::Scrapers::NetrcReader.new(NETRC_MANNING_ENTRY)
+        @user = options.fetch("user", netrc_reader.user)
+        @pw = options.fetch("pw", netrc_reader.pw)
+        @delay_time = options.fetch("delay", DELAY_TIME)
+        @destination = options.fetch("destination", ".")
+        @dry_run = options.fetch("dry_run", false)
+      end
 
-            wait_a_bit 5
+      def scrape
+        Mechanize.start do |m|
+          login(m) do |m|
+            book_downloads = m.current_page.links_with(:href => %r{/account/bookProduct/download})
+            Dir.chdir(destination) do |dir|
+              @results = download_books(m, book_downloads)
+            end
           end
-
         end
 
+        Hash[@results]
       end
+      
+      def login(agent, &block)
+        raise "Must provide a block to execute after logged in to site" unless block_given?
 
-      Hash[results]
-    end
+        agent.get DASHBOARD_URL
+        unless agent.current_page.uri == DASHBOARD_URL
+          # log in
+          agent.current_page.form.field_with(:type => 'email').value= user
+          agent.current_page.form.field_with(:type => 'password').value= pw
+          agent.current_page.form.submit
+          sleep 2
+          raise "could not log in" unless agent.current_page.uri.to_s == DASHBOARD_URL
+        end
+        yield agent
+      end
 
-    def self.wait_a_bit(delay)
-      puts "delaying for #{delay} second(s)"
-      %w[- \ | /].cycle(delay) do |c|
-        print "\r#{c}"
-        sleep 1
+      def wait_a_bit(delay)
+        puts "delaying for #{delay} second(s)"
+        %w[- * | +].cycle do |c|
+          print "\r#{c}"
+          sleep 1
+          delay -= 1
+          break if delay < 1
+        end
+        print "\r"
       end
-      print "\r"
-    end
 
 
+      def download_books(agent, books)
+        books.map do |book|
+          bookname = book.node.parent.parent.parent.parent.at_css('h1').text
+          puts "Downloading #{bookname} from #{book.href}"
+          if dry_run
+            warn "dry run, not saving"
+          else
+            agent.get book.href
+            puts "Saving #{agent.current_page.filename}"
+            agent.current_page.save! # overwrite!
+          end
+          
+          wait_a_bit delay_time
+          [agent.current_page.filename, agent.current_page.uri.to_s]
+        end
+      end
+
+    end
   end
 end
+

data/lib/scrapers/version.rb

@@ -2,8 +2,8 @@ module Scrapers
   module Version
 
     MAJOR = 2
-    MINOR = 0
-    BUILD = 2
+    MINOR = 1
+    BUILD = 0
 
   end
   

data/spec/scrapers/manning_books_spec.rb

@@ -1,20 +1,113 @@
 # -*- ruby -*-
 require 'spec_helper'
 require 'scrapers/manning_books'
+require 'ostruct'
 
-module Scrapers
+RSpec.describe Scrapers::ManningBooks::Scraper do
+  describe "verify Class method signatures" do
+    it "responds to :new" do
+      expect(Scrapers::ManningBooks::Scraper).to respond_to(:new)
+    end
+  end
+  describe "verify instance method signatures" do
+    subject { Scrapers::ManningBooks::Scraper.new }
+    it { is_expected.to respond_to :scrape }
+    it { is_expected.to respond_to :login }
+    it { is_expected.to respond_to :wait_a_bit }
+    it { is_expected.to respond_to :download_books }
+  end
+  describe "#login" do
+    let(:scraper) { Scrapers::ManningBooks::Scraper.new } 
+    let(:agent) { double('agent') }
 
-  describe ManningBooks do
-    it{should respond_to :scrape}
-    context "scraping" do
-      before(:all) do
-        @comic = VCR.use_cassette('manning_books') do
-          @result = Scrapers::ManningBooks.scrape
-        end
+    before do
+      allow(Scrapers::NetrcReader).to receive(:new) do
+        OpenStruct.new(user: "joe@example.com", pw: "password")
+      end
+    end
+
+    it "verify user" do
+      expect(scraper.user).to eq("joe@example.com")
+    end
+    it "verify pw" do
+      expect(scraper.pw).to eq("password")
+    end
+
+    context "when login is passed a block" do
+      it "logs in and yields the block" do
+        expect(agent).to receive(:get).and_return(agent)
+        expect(agent).to receive(:current_page).at_least(5).times.and_return(agent)
+        expect(agent).to receive(:uri)
+        expect(agent).to receive(:form).exactly(3).times.and_return(agent)
+        expect(agent).to receive(:field_with).exactly(2).times.and_return(agent)
+        expect(agent).to receive(:value=).exactly(2).times.and_return(agent)
+        expect(agent).to receive(:submit).and_return(agent)
+        expect(agent).to receive(:uri).and_return(Scrapers::ManningBooks::DASHBOARD_URL)
+        scraper.login(agent) { |m| @result = "in yield" }
+        expect(@result).to eq("in yield")
       end
       
-      it {expect(@result).to_not be_nil}
-      
     end
+
+    context "when login is not passed a block" do
+      it "raises an exception" do
+        expect{ scraper.login(agent) }.to raise_error("Must provide a block to execute after logged in to site")
+      end
+    end
+
+  end
+
+  describe "#download_books" do
+    let(:scraper) {Scrapers::ManningBooks::Scraper.new}
+    let(:agent) {double('agent')}
+    let(:books) do
+      3.times.map do |i|
+        OpenStruct.new(href: "http://#{Scrapers::ManningBooks::DASHBOARD_URL}/#{i}")
+      end
+    end
+
+
+    before do
+      allow(Scrapers::NetrcReader).to receive(:new) do
+        OpenStruct.new(user: "joe@example.com", pw: "password")
+      end
+
+      allow(scraper).to receive(:wait_a_bit).at_least(:once)
+    end
+
+    it "downloads the books" do
+      save_stdout = $stdout
+      $stdout = double('output').as_null_object
+      expect(agent).to receive(:get).exactly(3).times
+      expect(agent).to receive(:current_page).exactly(3*4).times.and_return(agent)
+      expect(agent).to receive(:filename).exactly(3*2).times.and_return("FILENAME")
+      expect(agent).to receive(:save!).exactly(3).times
+      expect(agent).to receive(:uri).exactly(3).times
+      results = scraper.download_books(agent, books)
+      $stdout = save_stdout
+      expect(results.size).to eq(3)
+    end
+
+  end
+
+  # Saving the best for last
+  describe "#scrape" do
+    let(:scraper) {Scrapers::ManningBooks::Scraper.new}
+    let(:agent) {double('agent').as_null_object}
+    let(:netrc_reader) {double('netrc_reader').as_null_object}
+    let(:book_list) {[['book1','url1'],['book2','url2']]}
+
+    before do
+      allow(Scrapers::NetrcReader).to receive(:new).and_return(netrc_reader)
+      allow(scraper).to receive(:wait_a_bit).at_least(:once)
+      allow(scraper).to receive(:login).and_yield(agent)
+    end
+
+    it "scrapes the dashboard" do
+      expect(Mechanize).to receive(:start).and_yield(agent)
+      expect(scraper).to receive(:download_books).and_return(book_list)
+      scraper.scrape
+    end
+
   end
 end

data/spec/spec_helper.rb

@@ -1,4 +1,3 @@
-#require 'webmock/rspec'
 require 'vcr'
 
 # This file was generated by the `rspec --init` command. Conventionally, all
@@ -8,7 +7,6 @@ require 'vcr'
 #
 # See http://rubydoc.info/gems/rspec-core/RSpec/Core/Configuration
 RSpec.configure do |config|
-  config.treat_symbols_as_metadata_keys_with_true_values = true
   config.run_all_when_everything_filtered = true
   config.filter_run :focus
 
@@ -23,9 +21,3 @@ VCR.configure do |c|
   c.cassette_library_dir = 'vcr_cassettes'
   c.hook_into :webmock
 end
-
-
-
-
-require 'scrapers.rb'
-

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: scrapers
 version: !ruby/object:Gem::Version
-  version: 2.0.2
+  version: 2.1.0
 platform: ruby
 authors:
 - Tamara Temple
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2014-10-09 00:00:00.000000000 Z
+date: 2014-12-24 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: mechanize
@@ -215,6 +215,7 @@ files:
 - bin/new_scraper
 - bin/rubytapas
 - bin/wunderground
+- lib/netrc_reader.rb
 - lib/scrapers.rb
 - lib/scrapers/allrecipes.rb
 - lib/scrapers/discoverynews.rb
@@ -297,3 +298,4 @@ test_files:
 - spec/scrapers_spec.rb
 - spec/spec_helper.rb
 - spec/wunderground_thor_spec.rb
+has_rdoc: