scrapers 1.4.0 → 1.5.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 4896b8aea0625256112a05ca839b9b9775a4ff85
-  data.tar.gz: 994a36cc3a5b97eda32c53838c2f519a008e8f0f
+  metadata.gz: 003f7ffd1d5c6ccc6df7470045cf46b28fc5256e
+  data.tar.gz: 621f3354ac4af8a765495973ba7e204cde20b570
 SHA512:
-  metadata.gz: 94d1e19c786ed926aaaa15c56d72fffcdf0157e25ec60cea8b4dd76313b72cf112da0297f39aefed7ab41a26c18589e96706815de0469795bc749450056171ad
-  data.tar.gz: 7368f28e2bf91a5063acc6e37a68285a87f3cbbf8de534d66218acde300f5e6fc28b7e7935daf5c4232d6a2056fbb9b087034aa5e324ed0b1e6f3f1ccf9e9de5
+  metadata.gz: 8b5c40acbe36d4b90d0f06db30f79acdae8c4ab173c6f572f0a7a0bb96400560038187cdbaf8030ee1bec801d822ac9ec7e0ff4cbc5937839f6853a26173fc68
+  data.tar.gz: 35e373c156d79a39f7661453475b5a3cfcfbac855c0d09bdb0a0fa913cae89fc047e29605e98d3189a46672a64b9781e76e972f024e9b2d0bc287dca45b4e5f2

data/Rakefile

@@ -2,46 +2,3 @@ require "bundler/gem_tasks"
 require "highline/import"
 require "active_support/core_ext/string/inflections"
 
-desc "Create the basis for a new scraper"
-task :new, [:module_name] do |t, args|
-  if args.module_name
-    module_name = args.module_name
-  else
-    module_name = ask("<%= color('What is the name of your new scraper module?', YELLOW) %>") {|q| q.default = "NewScraper" }
-  end
-
-  file_name = module_name.underscore + ".rb"
-  dir_name = File.join(File.dirname(__FILE__),'lib','scrapers')
-
-  new_scraper_path = File.join(dir_name, file_name)
-
-  template = <<-EOT
-module Scrapers
-  module #{module_name}
-
-    def self.scrape(url)
-      results = Hash.new
-
-      Mechanize.start(url) do |m|
-
-      end
-
-      results
-    end
-
-  end
-end
-EOT
-  
-  if File.exist?(new_scraper_path)
-    if agree("<%= color('#{file_name}', BLUE); color('already exists. Do you want to overwrite it?', YELLOW) %>", true)
-      File.unlink(new_scraper_path)
-    else
-      exit 0
-    end
-  end
-  
-  File.write(new_scraper_path, template)
-  say("<%= color('New scraper in', YELLOW) %> <%= color('#{new_scraper_path}', BLUE) %>")
-  
-end

data/bin/manning_books

@@ -0,0 +1,43 @@
+#!/usr/bin/env ruby
+require 'rubygems'
+require 'bundler/setup'
+require 'thor'
+require 'netrc'
+require 'scrapers/manning_books'
+require 'awesome_print'
+require 'pry'
+
+################################################################################
+#
+# Scraper for Manning Books I've purchased
+#
+# (Note: you need to log into Manning and have acutally purchased e-books.
+# They will appear on your dashboard.)
+#
+################################################################################
+
+class ManningBooks < Thor
+
+  MANNING = 'manning'
+
+  desc "download", "Downloads all the editions of all the books on your dashboard"
+  method_option :destination, :aliases => %w{-d --dest}, :desc => "Destination to store the downloads", :default => "."
+  method_option :user, :aliases => %w{-u -U}, :desc => "Manning user. Default is read from $HOME/.netrc"
+  method_option :password, :aliases => %w{-p -pw}, :desc => "Manning password. Default is read from $HOME/.netrc"
+
+  def download
+    netrc = Netrc.read
+    user, pw = netrc[MANNING]
+    user = options.fetch("user", user)
+    pw = options.fetch("password", pw)
+    destination = options.fetch("destination", nil)
+    STDERR.puts "destination: #{destination}, user: #{user}, pw: #{pw.length}"
+    binding.pry
+    Signal.trap('INT', proc { STDERR.puts "Download Interrupted"; exit(-1)})
+    results = Scrapers::ManningBooks.scrape destination, user, pw
+    ap results
+  end
+
+end
+
+ManningBooks.start

data/lib/scrapers/manning_books.rb

@@ -0,0 +1,55 @@
+# -*- ruby -*-
+require 'mechanize'
+require 'pry'
+
+module Scrapers
+  module ManningBooks
+
+    DASHBOARD_URL = "https://account.manning.com/dashboard"
+    
+    def self.scrape(dest=".", user=nil, pw=nil)
+      results = Array.new
+
+      Mechanize.start do |m|
+        m.get DASHBOARD_URL
+        unless m.current_page.uri == DASHBOARD_URL
+          # log in
+          m.current_page.form.field_with(:type => 'email').value= user
+          m.current_page.form.field_with(:type => 'password').value= pw
+          m.current_page.form.submit
+          sleep 2
+          raise "could not log in" unless m.current_page.uri.to_s == DASHBOARD_URL
+        end
+
+        book_downloads = m.current_page.links_with(:href => %r{/account/bookProduct/download})
+
+        Dir.chdir(dest) do |dir|
+          book_downloads.each do |book|
+            puts "Downloading #{book.href}"
+            m.get book.href
+            results << [m.current_page.filename, m.current_page.uri.to_s]
+            puts "Saving #{m.current_page.filename}"
+            m.current_page.save! # overwrite!
+
+            wait_a_bit 5
+          end
+
+        end
+
+      end
+
+      Hash[results]
+    end
+
+    def self.wait_a_bit(delay)
+      puts "delaying for #{delay} second(s)"
+      %w[- \ | /].cycle(delay) do |c|
+        print "\r#{c}"
+        sleep 1
+      end
+      print "\r"
+    end
+
+
+  end
+end

data/lib/scrapers/version.rb

@@ -2,7 +2,7 @@ module Scrapers
   module Version
 
     MAJOR = 1
-    MINOR = 4
+    MINOR = 5
     BUILD = 0
 
   end

data/spec/scrapers/manning_books_spec.rb

@@ -0,0 +1,20 @@
+# -*- ruby -*-
+require 'spec_helper'
+require 'scrapers/manning_books'
+
+module Scrapers
+
+  describe ManningBooks do
+    it{should respond_to :scrape}
+    context "scraping" do
+      before(:all) do
+        @comic = VCR.use_cassette('manning_books') do
+          @result = Scrapers::ManningBooks.scrape
+        end
+      end
+      
+      it {expect(@result).to_not be_nil}
+      
+    end
+  end
+end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: scrapers
 version: !ruby/object:Gem::Version
-  version: 1.4.0
+  version: 1.5.0
 platform: ruby
 authors:
 - Tamara Temple
@@ -169,6 +169,7 @@ description: A library of web site scrapers utilizing mechanize and other goodie
 email:
 - tamouse@gmail.com
 executables:
+- manning_books
 - new_scraper
 - rubytapas
 extensions: []
@@ -181,6 +182,7 @@ files:
 - LICENSE.txt
 - README.md
 - Rakefile
+- bin/manning_books
 - bin/new_scraper
 - bin/rubytapas
 - lib/scrapers.rb
@@ -190,6 +192,7 @@ files:
 - lib/scrapers/esod.rb
 - lib/scrapers/gocomics.rb
 - lib/scrapers/imgur.rb
+- lib/scrapers/manning_books.rb
 - lib/scrapers/manning_dashboard.rb
 - lib/scrapers/nasa_apod.rb
 - lib/scrapers/rubytapas.rb
@@ -202,6 +205,7 @@ files:
 - spec/scrapers/download_spec.rb
 - spec/scrapers/gocomics_spec.rb
 - spec/scrapers/imgur_spec.rb
+- spec/scrapers/manning_books_spec.rb
 - spec/scrapers/nasa_apod_spec.rb
 - spec/scrapers/sinfest_spec.rb
 - spec/scrapers/xkcd_spec.rb
@@ -247,9 +251,9 @@ test_files:
 - spec/scrapers/download_spec.rb
 - spec/scrapers/gocomics_spec.rb
 - spec/scrapers/imgur_spec.rb
+- spec/scrapers/manning_books_spec.rb
 - spec/scrapers/nasa_apod_spec.rb
 - spec/scrapers/sinfest_spec.rb
 - spec/scrapers/xkcd_spec.rb
 - spec/scrapers_spec.rb
 - spec/spec_helper.rb
-has_rdoc: