scrapers 3.0.0 → 3.1.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 6a3e0b5e2fa58fb8686e8c350b1619dce0e85e0f
-  data.tar.gz: 9b2f65ddff548a914a2b5e9d8155f725568c5708
+  metadata.gz: b09c6be6c6c50e75b02dcc6e889d577f1f3c4d1c
+  data.tar.gz: c35d853aad8bfab4880d445f7db11e7532537c76
 SHA512:
-  metadata.gz: e67a79dccb7e436e9b629a63157ed44c0bdf76e83f99bbe8bc2d33f4edab9c0b68401d1c1da554077871b7f0affde40bb4a2f33508db20b6726c1c2f7781d549
-  data.tar.gz: d8f62b7d23d09253ecaba8f37f26b0852019e0b4e8403feafaf109791896dd89d491edb99eeffec87b8fdf1946a0e4b8a40f3038f6706686b79c44e61554c2a1
+  metadata.gz: 945df36bd1df9f8d91dcf86ee3e7e51796e50c604da0c9f135339c9be5bc54aa0170c6f8af83aa40ad7f23a4ee4d7c5c840b7f2585cd58170b7a936206543b51
+  data.tar.gz: 245fc91742ca186713c28f6ccfb5b591604fdf0e4573a7bf50c32e723a5697f4e8f24f61ce57421830dbff58f6677bbba5f293a810526d66261c393833fcec34

data/lib/netrc_reader.rb

@@ -1,12 +1,14 @@
 require 'netrc'
 
-module Scrapers  
+module Scrapers
   class NetrcReader
     attr_accessor :user, :pw
 
     def initialize(section)
       netrc = Netrc.read
-      @user, @pw = netrc[section]      
+      @user, @pw = netrc[section]
+    rescue NoMethodError => e
+      fail "Could not find credentials for #{section}"
     end
   end
 end

data/lib/scrapers/rubytapas/cli.rb

@@ -4,7 +4,7 @@ require 'scrapers/rubytapas'
 
 module Scrapers
   module RubyTapas
-    
+
     # Thor script that handles things with Avdi Grimm's RubyTapas
     class CLI < Thor
 
@@ -15,6 +15,7 @@ module Scrapers
       method_option :destination, :aliases => %w{-d}, :desc => "Destination to store the downloads. Default is the current working directory.", :default => "."
       method_option :user, :aliases => %w{-u}, :desc => "dpdcart user. Default is read from $HOME/.netrc"
       method_option :pw, :aliases => %w{-p}, :desc => "dpdcart password. Default is read from $HOME/.netrc"
+      method_option :subscription, default: 'rubytapas'
       def download(episode)
         Scrapers::RubyTapas::Scraper.new(episode, options).scrape!
       end
@@ -23,6 +24,7 @@ module Scrapers
       desc "list", "Show a list of the available episodes"
       method_option :user, :aliases => %w{-u}, :desc => "dpdcart user. Default is read from $HOME/.netrc"
       method_option :pw, :aliases => %w{-p}, :desc => "dpdcart password. Default is read from $HOME/.netrc"
+      method_option :subscription, default: 'rubytapas'
       def list()
         Scrapers::RubyTapas::Scraper.new(nil, options).list!
       end

data/lib/scrapers/rubytapas/config.rb

@@ -1,11 +1,5 @@
 module Scrapers
   module RubyTapas
 
-    RUBYTAPAS_HOST="rubytapas.dpdcart.com"
-    CONTENTS_PATH = "/subscriber/content"
-    POST_PATH = "/post"
-    FEED_PATH = "/feed"
-    POST_QUERY = "id"
-
   end
 end

data/lib/scrapers/rubytapas/dpdcart.rb

@@ -1,5 +1,5 @@
-require 'netrc'
-require 'mechanize'
+require "mechanize"
+require "netrc"
 
 module Scrapers
   module RubyTapas
@@ -8,17 +8,29 @@ module Scrapers
     # that provides a connection to rubytapas.dpdcart.com where the
     # RubyTapas episodes and download files are available, as well as
     # the episode feed.
+
     class DpdCart
 
-      RUBYTAPAS_HOST = 'rubytapas.dpdcart.com'
-      ENV_RUBYTAPAS_USER = 'RUBYTAPAS_USER'
-      ENV_RUBYTAPAS_PASSWORD = 'RUBYTAPAS_PASSWORD'
-      LOGIN_PATH = '/subscriber/login'
-      LOGIN_URL = "https://#{RUBYTAPAS_HOST}#{LOGIN_PATH}"
-      FEED_PATH = '/feed'
-      FEED_URL = "https://#{RUBYTAPAS_HOST}#{FEED_PATH}"
-      CONTENT_PATH = "/subscriber/content"
-      CONTENT_URL = "https://#{RUBYTAPAS_HOST}#{CONTENT_PATH}"
+      # NOTE: Updating this since now I have *two* subscriptions that
+      # use DPD Cart, rubytapas and elixirsips. Generalizing this
+      # accordingly. :)
+
+      # The subscription name will be filled in depending on which
+      # subscription I'm downloading from. This is a stock sprintf-type
+      # fill in where you pass in the subscription parameter with a
+      # value, thusly:
+      #
+      #     DPDCART_HOST % {subscription: "rubytapas"}
+      #
+      DPDCART_HOST_FORMAT         = "%{subscription}.dpdcart.com"
+      ENV_DPDCART_USER_FORMAT     = "%{subscription}_USER"
+      ENV_DPDCART_PASSWORD_FORMAT = "%{subscription}_USER"
+      LOGIN_PATH                  = '/subscriber/login'
+      FEED_PATH                   = '/feed'
+      CONTENT_PATH                = "/subscriber/content"
+
+      # Subscription at dbdcart
+      attr_accessor :subscription
 
       # User name for dpdcart account
       attr_accessor :user
@@ -26,7 +38,13 @@ module Scrapers
       # Password for dpdcart acount
       attr_accessor :password
 
-      attr_accessor :dry_run, :debug
+      def dpdcart_host         ; @dpdcart_host         ||= DPDCART_HOST_FORMAT         % {subscription: subscription} ; end
+      def env_dpdcart_user     ; @env_dpdcart_user     ||= ENV_DPDCART_PASSWORD_FORMAT % {subscription: subscription} ; end
+      def env_dpdcart_password ; @env_dpdcart_password ||= ENV_DPDCART_PASSWORD_FORMAT % {subscription: subscription} ; end
+      def debug                ; @debug                ||= options[:debug]                                            ; end
+      def dry_run              ; @dry_run              ||= options[:dry_run]                                          ; end
+      def feed_url             ; @feed_url             ||= URI("https://#{dpdcart_host}#{FEED_PATH}")                 ; end
+      def login_url            ; @login_url            ||= URI("https://#{dpdcart_host}#{LOGIN_PATH}")                ; end
 
       # Create a new instance of the DpdCart gateway.
       #
@@ -34,63 +52,47 @@ module Scrapers
       # email address.
       # @param password [String] - password associated with the
       # account.
+      # @param subscription [String] - subscription name at DPD Cart
+      # (e.g. 'rubytapas' or 'elixirsips')
       #
       # If the user and password are empty, the information will be
       # obtained in the following order:
       #
-      # - reading the environment variables `RUBYTAPAS_USER` and
-      # `RUBYTAPAS_PASSWORD`
+      # - reading the environment variables `<subscriptiion>_USER` and
+      # `<subscription>_PASSWORD`
+      #
+      #   Note that <subscription> will be the subscription passed in
+      #   above.
       #
       # - reading the user's `$HOME/.netrc` file and pulling the
-      # credentials that match the host name for the rubytapas
+      # credentials that match the host name for the subscription
       # account.
       #
-      # If no credentials can be found, it will raise and error:
+      # If no credentials can be found, it will raise an error:
       # `NoCredentialsError`.
-      def initialize(user=nil, password=nil, options={})
-        self.dry_run = options[:dry_run]
-        self.debug = options[:debug]
-        if user && password
-          @user = user
-          @password = password
-        else
-          @user, @password = get_credentials_from_environment
-          unless user && password
-            @user, @password = get_credentials_from_netrc
-          end
-        end
+      #
+      def initialize(user=nil, password=nil, subscription='rubytapas', options={})
+        self.options = options
+        self.subscription = subscription
+        set_user_and_password(user, password)
         self.agent = Mechanize.new
       end
 
-      # Return the episode feed from dpdcart
+      # Retreive the episode feed from dpdcart
       def feed!
-        uri = URI(FEED_URL)
-        request = Net::HTTP::Get.new(uri)
-        request.basic_auth user, password
-        Net::HTTP.start(uri.host, uri.port, {:use_ssl => true}) {|http| http.request(request)}.body
-      end
-
-      # Login to dpdcart before downloading
-      def login!
-        page = agent.get LOGIN_URL
-        page.form.field_with(name: "username").value = user
-        page.form.field_with(name: "password").value = password
-        page.form.submit
-        unless agent.page.title.match(/Subscription Content/)
-          raise "Could not log in"
-        end
-        agent
+        http_fetch(feed_url)
       end
 
       # Download the file from dpdcart
       def download!(file)
+        login
         warn "DEBUG: downloading #{file}" if debug
         if dry_run
           warn "DEBUG: download skipped for dry run" if dry_run
           filename = file
           body = "no body"
         else
-          page = agent.get(file) unless dry_run
+          page = agent.get(file)
           filename = page.filename
           body = page.body
         end
@@ -101,15 +103,49 @@ module Scrapers
 
       attr_accessor :options, :agent
 
+      def set_user_and_password(user, password)
+        if user && password
+          @user = user
+          @password = password
+        else
+          @user, @password = get_credentials_from_environment
+          unless user && password
+            @user, @password = get_credentials_from_netrc
+          end
+        end
+      end
+
       def get_credentials_from_environment
-        [ ENV[ENV_RUBYTAPAS_USER], ENV[ENV_RUBYTAPAS_PASSWORD] ]
+        [ ENV[env_dpdcart_user], ENV[env_dpdcart_password] ]
       end
 
       def get_credentials_from_netrc
-        creds = Netrc.read[RUBYTAPAS_HOST]
+        creds = Netrc.read[dpdcart_host]
+        if creds.nil?
+          warn "Could not find credentials for #{dpdcart_host}"
+          exit -1
+        end
         [ creds.login, creds.password ]
       end
 
+      # Login to dpdcart before downloading
+      def login
+        page = agent.get login_url
+        page.form.field_with(name: "username").value = user
+        page.form.field_with(name: "password").value = password
+        page.form.submit
+        unless agent.page.title.match(/Subscription Content/)
+          raise "Could not log in"
+        end
+        agent
+      end
+
+      def http_fetch(uri)
+        request = Net::HTTP::Get.new(uri)
+        request.basic_auth user, password
+        Net::HTTP.start(uri.host, uri.port, {:use_ssl => true}) {|http| http.request(request)}.body
+      end
+
     end
   end
 end

data/lib/scrapers/rubytapas/scraper.rb

@@ -8,11 +8,12 @@ module Scrapers
   module RubyTapas
 
     # Scraper provides the methods to download, extract and build a collection
-    # of RubyTapas episodes from the RubyTapas RSS feed.
+    # of DPD cart subscription episode from the RubyTapas RSS feed.
     class Scraper
 
-      attr_accessor :user, :pw, :destination, :episode_number, :netrc_reader, :dry_run, :debug
+      attr_accessor :subscription, :user, :pw, :destination, :episode_number, :netrc_reader, :dry_run, :debug
       attr_reader :dpdcart
+      attr_reader :episodes
 
       # *episode_number* is the RubyTapas episode number (note! not the post id!) of the
       # episode to download. If the episode number is the symbol :all, then all episodes
@@ -26,6 +27,7 @@ module Scrapers
       # - "pw": the password of the RubyTapas account
       # - "destination": the root destination of the episode downloads
       def initialize(episode_number, options)
+        self.subscription = options.fetch("subscription") # let this fail if no subscription given.
         self.episode_number = episode_number
         self.user = options["user"]
         self.pw = options["pw"]
@@ -33,13 +35,12 @@ module Scrapers
         self.dry_run = options["dry_run"]
         self.debug = options["debug"]
         @dpdcart = Scrapers::RubyTapas::DpdCart.
-                   new(user, pw, {dry_run: dry_run, debug: debug})
-        warn "DEBUG: episode_number: #{episode_number}, options: #{options.inspect}" if debug
+          new(user, pw, subscription, {dry_run: dry_run, debug: debug})
+        @episodes ||= fetch_episodes
       end
 
       # Perform the scraping operation
       def scrape!
-        dpdcart.login!
         if all_episodes?
           episodes.each do |episode|
 
@@ -70,11 +71,6 @@ module Scrapers
         end
       end
 
-      # Returns the collection of episodes.
-      def episodes
-        @episodes ||= fetch_episodes
-      end
-
       # Retrieves the episode associated with *episode number*.
       def find_by_episode(episode_number)
         episodes.detect {|e| e.number == episode_number}

data/lib/scrapers/version.rb

@@ -2,11 +2,11 @@ module Scrapers
   module Version
 
     MAJOR = 3
-    MINOR = 0
+    MINOR = 1
     BUILD = 0
 
   end
-  
+
   VERSION = [Version::MAJOR,Version::MINOR,Version::BUILD].map(&:to_s).join(".")
 
   DESCRIPTION = "A library of web site scrapers utilizing mechanize and other goodies. Helpful in gathering images, moving things, saving things, etc."

data/spec/lib/scrapers/rubytapas/dpdcart_spec.rb

@@ -8,7 +8,6 @@ describe Scrapers::RubyTapas::DpdCart do
 
   describe "method signatures" do
     it { is_expected.to respond_to(:feed!) }
-    it { is_expected.to respond_to(:login!)}
     it { is_expected.to respond_to(:download!)}
   end
 
@@ -22,17 +21,6 @@ describe Scrapers::RubyTapas::DpdCart do
     end
   end
 
-  describe "#login!" do
-    it "shows the subscriber content page" do
-      VCR.use_cassette('rubytapas_login', record: :new_episodes,
-                       match_requests_on: [:method, :host, :path]
-                      ) do
-        expect(gateway.login!.page.title).to eq("Subscription Content | RubyTapas")
-      end
-    end
-  end
-
-
   describe "#download!" do
     let(:file) { "https://rubytapas.dpdcart.com/subscriber/download?file_id=26" }
     let(:name) { "001-binary-literals.html" }
@@ -44,7 +32,6 @@ describe Scrapers::RubyTapas::DpdCart do
       VCR.use_cassette('rubytapas_download', record: :new_episodes,
                        match_requests_on: [:method, :host, :path,
                                            :query]) do
-        gateway.login!
         filename, body = gateway.download! file
         expect(filename).to eq(name)
         expect(body.size).to eq(5744)
@@ -55,7 +42,6 @@ describe Scrapers::RubyTapas::DpdCart do
       VCR.use_cassette('rubytapas_download_twice', record: :new_episodes,
                        match_requests_on: [:method, :host, :path,
                                            :query]) do
-        gateway.login!
         filename, body = gateway.download! file
         expect(filename).to eq(name)
         filename, body = gateway.download! file2

data/spec/lib/scrapers/rubytapas/rubytapas_spec.rb

@@ -13,7 +13,6 @@ RSpec.describe "RubyTapas Thor Script", :type => :integration do
 
   let(:cart) do
     instance_spy("Scrapers::RubyTapas::DpdCart",
-                 :login! => "Subscription Content | RubyTapas",
                  :feed! => feed,
                  :download! => download
                 )
@@ -35,12 +34,12 @@ RSpec.describe "RubyTapas Thor Script", :type => :integration do
       it "retrieves one episode" do
         expect_any_instance_of(Scrapers::RubyTapas::Scraper).to receive(:scrape!).once.and_call_original
         expect(cart).to receive(:download!).exactly(3).times
-        
+
         VCR.use_cassette('rubytapas-download-1', :match_requests_on => [:method, :host, :path, :query]) do
           Scrapers::RubyTapas::CLI.start(%w[download 001 --destination=. --user=joan@example.com --pw=password])
         end
       end
-      
+
     end
 
     context "when scraping all episodes" do
@@ -48,7 +47,7 @@ RSpec.describe "RubyTapas Thor Script", :type => :integration do
       it "retrieves all episodes" do
         expect_any_instance_of(Scrapers::RubyTapas::Scraper).to receive(:scrape!).once.and_call_original
         expect(cart).to receive(:download!).exactly(933).times
-        
+
         VCR.use_cassette('rubytapas-download-all', :match_requests_on => [:method, :host, :path, :query]) do
           save_stdout = $stdout
           # $stdout = output
@@ -56,9 +55,9 @@ RSpec.describe "RubyTapas Thor Script", :type => :integration do
           $stdout = save_stdout
         end
       end
-      
+
     end
-    
+
   end
 
   describe "list command" do
@@ -73,7 +72,7 @@ RSpec.describe "RubyTapas Thor Script", :type => :integration do
       Scrapers::RubyTapas::CLI.start(%w[list --user=joan@example.com --pw=password])
     end
   end
-  
+
   describe "version command" do
     it "prints the version numbers for rubytapas and scrapers" do
       save_stdout = $stdout

data/spec/lib/scrapers/rubytapas/scraper_spec.rb

@@ -13,12 +13,11 @@ describe Scrapers::RubyTapas::Scraper do
   let(:options) do
     {
       "destination" => '.',
+      "subscription" => 'rubytapas'
     }
   end
   let(:cart) {instance_spy("Scrapers::RubyTapas::DpdCart",
                            :feed! => feed,
-                           :login! =>
-                           double("Subscription Content | RubyTapas"),
                            :download! => [ 'filename',
                                            'body' ]
                           )}
@@ -28,7 +27,6 @@ describe Scrapers::RubyTapas::Scraper do
 
   describe "#episodes" do
     it "gets a collection of episodes" do
-      expect(scraper).to receive(:fetch_episodes).and_call_original
       expect(scraper.episodes.size).to eq(267)
     end
   end
@@ -53,7 +51,6 @@ describe Scrapers::RubyTapas::Scraper do
     context "when scraping one episode" do
       it "scrapes one episode" do
         expect(scraper).to receive(:find_by_episode).with(episode_number).and_call_original
-        expect(scraper).to receive(:fetch_episodes).once.and_call_original
         scraper.scrape!
       end
 
@@ -63,7 +60,6 @@ describe Scrapers::RubyTapas::Scraper do
       let(:scraper) { Scrapers::RubyTapas::Scraper.new(:all, options) }
 
       it "scrapes all the episodes" do
-        expect(scraper).to receive(:fetch_episodes).once.and_call_original
         scraper.scrape!
       end
     end