scrapers 1.5.1 → 1.5.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: eb80fb9fb6be0a4c94bba33715ea5bafef20f941
-  data.tar.gz: 066bec449349a457301154834289f3db74e257fe
+  metadata.gz: 86724e63b99e28cc9e82a1c6806dce93515f9e3e
+  data.tar.gz: 7f1789114ac38d02a006e99bfb00294fa8c3c5b1
 SHA512:
-  metadata.gz: 1683f98572977ddbff0d92252f58ee53966a8cef7b362374ef5e04194771718cc4b84156dfac740f86e113f4cff7c112bdc1a4b1ab872e0bf488020cedf72d97
-  data.tar.gz: cab26c4fd6438c5743dad7dfbcd7cd14d005c5f43d8fef40fc7cd4f47efabfca6d61543aa0a433cd9f8b585c26b4ab4d732927fcb251b47abd103b6ef6564f74
+  metadata.gz: c60d23ca256369982c57e8ad8099403042f411d9989694b3f9d32e65fa1307ba6452bf64d93b895059a457c44916aa2c90149903e075bdb76e0fe33fe640c0e5
+  data.tar.gz: 692ca67d599d2e846dc191827c5f77199f4c2de5d5b5415243995569f153681b80c72a72d910716af92d10c76c87775e705b3d0bc5b76191382092196ace3a70

data/bin/rubytapas

@@ -41,6 +41,35 @@ class RubyTapasDownload < Thor
     Scrapers::RubyTapas.scrape url, user, pw, destination
   end
 
+  desc "all", "Download all rubytapas episodes"
+  method_option :destination, :aliases => %w{-d --dest}, :desc => "Destination of dowload", :default => '.'
+  method_option :url, :desc => "url of showlist", :default => 'https://rubytapas.dpdcart.com/subscriber/content'
+  method_option :user, :aliases => %w{-u -U}, :desc => "dpdcart user. Default is read from $HOME/.netrc"
+  method_option :password, :aliases => %w{-p -pw}, :desc => "dpdcart password. Default is read from $HOME/.netrc"
+
+  def all
+    STDERR.puts options.inspect
+
+    netrc = Netrc.read
+    user, pw = netrc[RUBYTAPAS]
+    user = options.fetch("user", user)
+    pw = options.fetch("password", pw)
+    url = options.fetch("url", nil)
+    destination = options.fetch("destination", nil)
+    STDERR.puts "destination: #{destination}, url: #{url}, user: #{user}, pw: #{pw.length}"
+    raise "Must give url" unless url
+
+    showlist_urls = Scrapers::RubyTapas.showlist(url, user, pw)
+
+    showlist_urls.each do |url|
+      Scrapers::RubyTapas.scrape url, user, pw, destination
+      print "pausing..."
+      sleep 5
+      puts "."
+    end
+
+  end
+
 end
 
 RubyTapasDownload.start

data/lib/scrapers.rb

@@ -1,9 +1,16 @@
 require 'mechanize'
-
+require 'uri'
 Dir[File.join(File.expand_path('../', __FILE__),'**','*.rb')].each {|file| require file}
 
 module Scrapers
   def self.agent()
     @agent ||= Mechanize.new
   end
+
+  def self.base(url)
+    u = URI.parse(url)
+    u.path=''
+    u.to_s
+  end
+
 end

data/lib/scrapers/rubytapas.rb

@@ -1,6 +1,7 @@
 require 'fileutils'
 require 'ostruct'
 require 'mechanize'
+require 'uri'
 
 module Scrapers
   
@@ -32,20 +33,13 @@ module Scrapers
 
         tapas = OpenStruct.new
 
-        # First time, we will get redirected to the login page
-        m.get url
-        m.current_page.form.field_with(:name => "username").value = user
-        m.current_page.form.field_with(:name => "password").value = pw
-        m.current_page.form.submit
-
-        # Second time, we should land on episode page
-        m.get url
-        raise "Not where I expected. #{m.current_page.uri} is not #{url}" unless m.current_page.uri != url
+        m = self.login(m, url, user, pw)
 
         m.current_page.tap do |page|
           tapas.title = page.title.strip
           tapas.episode_dir = File.join(dest,tapas.title.split("|").first.strip.downcase.gsub(%r{\s+},'-'))
           tapas.attachments = page.links_with(:href => %r{\bdownload\b})
+          puts "Fetching and saving #{tapas.title} into #{tapas.episode_dir}"
           FileUtils.mkdir(tapas.episode_dir)
           Dir.chdir(tapas.episode_dir) do |dir|
             tapas.attachments.each do |att|
@@ -61,6 +55,34 @@ module Scrapers
         
       end
     end
+    
+    # retrieve a list of URLs for shows from the showlist
+    def self.showlist(showlist_url, user=nil, pw=nil)
+      raise "Must give showlist url, user, and password" if showlist_url.to_s.empty? || user.to_s.empty? || pw.to_s.empty?
+
+      Mechanize.start do |m|
+        m = self.login(m, showlist_url, user, pw)
+        links = m.current_page.links_with(:text => "Read More")
+        s = URI.parse(showlist_url)
+        s.path = ''
+        links.map{|l| "#{s}#{l.href}" }
+      end
+
+      
+    end
+
+    def self.login(m, url, user, pw)
+      # First time, we will get redirected to the login page
+      m.get url
+      m.current_page.form.field_with(:name => "username").value = user
+      m.current_page.form.field_with(:name => "password").value = pw
+      m.current_page.form.submit
+
+      # Second time, we should land on episode page
+      m.get url
+      raise "Not where I expected. #{m.current_page.uri} is not #{url}" unless m.current_page.uri != url
+      m
+    end
+    
   end
-  
 end

data/lib/scrapers/version.rb

@@ -3,7 +3,7 @@ module Scrapers
 
     MAJOR = 1
     MINOR = 5
-    BUILD = 1
+    BUILD = 2
 
   end
   

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: scrapers
 version: !ruby/object:Gem::Version
-  version: 1.5.1
+  version: 1.5.2
 platform: ruby
 authors:
 - Tamara Temple
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2014-02-21 00:00:00.000000000 Z
+date: 2014-02-25 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: mechanize