scrapers 2.1.0 → 3.0.0

data/lib/scrapers/rubytapas/episode.rb

@@ -0,0 +1,86 @@
+require 'nokogiri'
+require 'date'
+require 'stringex_lite'
+
+module Scrapers
+  module RubyTapas
+
+    class Episode 
+
+      FileLink = Struct.new :filename, :href
+
+      attr_accessor :number, :title, :link, :description, :guid, :pub_date, :file_list, :slug
+
+      def initialize(*args)
+        if args.size == 1
+          case args[0]
+          when String
+            parse_item(Nokogiri::XML.parse(args[0]){|c| c.noblanks}.children.first)
+          when Nokogiri::XML::Element
+            parse_item(args[0])
+          when Hash
+            parse_options(args[0])
+          else
+          end
+        elsif args.size > 1
+          assign_from_args(*args)
+        end
+      end
+
+      def number_from_title
+        title.scan(/\w+/).first
+      end
+
+      def slug_from_title
+        title.to_s.to_url
+      end
+
+      def file_list_from_description
+        find_file_list(description)
+      end
+      
+      private
+
+      def parse_item(item)
+        self.title = item.xpath("title").text
+        self.number = number_from_title
+        self.slug = slug_from_title
+        self.link = item.xpath("link").text
+        self.description = item.xpath("description").text
+        self.guid = item.xpath("guid").text
+        self.pub_date = DateTime.parse(item.xpath("pubDate").text)
+        self.file_list = file_list_from_description
+      end
+
+      def parse_options(options)
+        self.number = options[:number]
+        self.title = options[:title]
+        self.slug = options[:slug]
+        self.link = options[:link]
+        self.description = options[:description]
+        self.guid = options[:guid]
+        self.pub_date = options[:pub_date]
+        self.file_list = options[:file_list]
+      end
+
+      def assign_from_args(*args)
+        self.number,
+        self.title,
+        self.slug,
+        self.link,
+        self.description,
+        self.guid,
+        self.pub_date,
+        self.file_list =
+          *args
+      end
+
+      def find_file_list(content)
+        Nokogiri::HTML.parse(content).css("a").
+          select {|link| link['href'] =~ /download\?file_id=/ }.
+          map { |link| FileLink.new(link.child.text, link['href']) }
+      end
+    end
+
+  end
+end

data/lib/scrapers/rubytapas/scraper.rb

@@ -0,0 +1,142 @@
+require 'nokogiri'
+require 'fileutils'
+require 'scrapers/rubytapas/config'
+require 'scrapers/rubytapas/episode'
+require 'scrapers/rubytapas/dpdcart'
+
+module Scrapers
+  module RubyTapas
+
+    # Scraper provides the methods to download, extract and build a collection
+    # of RubyTapas episodes from the RubyTapas RSS feed.
+    class Scraper
+
+      attr_accessor :user, :pw, :destination, :episode_number, :netrc_reader, :dry_run, :debug
+      attr_reader :dpdcart
+
+      # *episode_number* is the RubyTapas episode number (note! not the post id!) of the
+      # episode to download. If the episode number is the symbol :all, then all episodes
+      # will be retrieved. Note that if any of the episodes have been previously retrieved
+      # to the destination, i.e., the associated directory already exists, that episode
+      # download will be skipped.
+      #
+      # *options* contains the options from the cli, which include:
+      #
+      # - "user": the username of the RubyTapas account
+      # - "pw": the password of the RubyTapas account
+      # - "destination": the root destination of the episode downloads
+      def initialize(episode_number, options)
+        self.episode_number = episode_number
+        self.user = options["user"]
+        self.pw = options["pw"]
+        self.destination = options.fetch("destination", Dir.pwd)
+        self.dry_run = options["dry_run"]
+        self.debug = options["debug"]
+        @dpdcart = Scrapers::RubyTapas::DpdCart.
+                   new(user, pw, {dry_run: dry_run, debug: debug})
+        warn "DEBUG: episode_number: #{episode_number}, options: #{options.inspect}" if debug
+      end
+
+      # Perform the scraping operation
+      def scrape!
+        dpdcart.login!
+        if all_episodes?
+          episodes.each do |episode|
+
+            begin
+              download(episode)
+              friendly_pause unless dry_run
+            rescue Errno::EEXIST
+              warn "Episode previously downloaded. Skipping."
+            end
+
+          end
+        else
+          episode = find_by_episode(episode_number)
+          if episode.nil?
+            raise "Unknown episode for #{episode_number}"
+          else
+            download(episode)
+          end
+        end
+      end
+
+      # Print a list of episodes
+      def list!
+        with_pager do |pager|
+          episodes.each do |episode|
+            pager.puts format_episode(episode)
+          end
+        end
+      end
+
+      # Returns the collection of episodes.
+      def episodes
+        @episodes ||= fetch_episodes
+      end
+
+      # Retrieves the episode associated with *episode number*.
+      def find_by_episode(episode_number)
+        episodes.detect {|e| e.number == episode_number}
+      end
+
+      private
+
+      def all_episodes?
+        episode_number.to_s.downcase.to_sym == :all
+      end
+
+      # Builds a collection of all the episodes listed in the feed
+      def fetch_episodes
+        feed = Nokogiri::XML.parse(dpdcart.feed!)
+        feed.xpath("//item").map do |item|
+          Episode.new(item)
+        end
+      end
+
+      def download(episode)
+        download_directory = make_download_directory(episode.slug)
+        episode.file_list.each do |file|
+          download_file(download_directory, file.href)
+        end
+      end
+
+      def download_file(dir, url)
+        warn "fetching #{url}" if debug
+        name, body = dpdcart.download!(url)
+        File.binwrite(File.join(dir,name), body) unless dry_run
+        warn "saved #{name} to #{dir}" if debug
+      end
+
+
+      def make_download_directory(slug)
+        dir = File.join(File.realpath(destination), slug)
+        warn "Downloading to #{dir}" if debug
+        if dry_run
+          "no dir for dry run"
+        else
+          FileUtils.mkdir(dir).first
+        end
+      end
+
+      def friendly_pause(delay=5)
+        puts
+        print "Sleeping #{delay} seconds"
+        delay.downto(1) { sleep 1; print "." }
+        puts "\n"
+      end
+
+      def with_pager(&block)
+        raise "Must be called with block" unless block_given?
+        pager = open("|more","w")
+        yield pager
+        pager.close
+      end
+
+      def format_episode(episode)
+        "%-5s\t%-40s\t%-15s" % [episode.number, episode.title, episode.pub_date.strftime("%Y-%b-%d")]
+      end
+
+    end
+  end
+end

data/lib/scrapers/version.rb

@@ -1,8 +1,8 @@
 module Scrapers
   module Version
 
-    MAJOR = 2
-    MINOR = 1
+    MAJOR = 3
+    MINOR = 0
     BUILD = 0
 
   end

data/scrapers.gemspec

@@ -24,6 +24,7 @@ Gem::Specification.new do |spec|
   spec.add_dependency "activesupport", "~> 4.1"
   spec.add_dependency "highline"
   spec.add_dependency "awesome_print"
+  spec.add_dependency "stringex"
 
   spec.add_development_dependency "bundler"
   spec.add_development_dependency "rake"
@@ -32,5 +33,7 @@ Gem::Specification.new do |spec|
   spec.add_development_dependency "guard-rspec"
   spec.add_development_dependency "webmock"
   spec.add_development_dependency "vcr"
-  
+  spec.add_development_dependency "pry"
+  spec.add_development_dependency "pry-byebug"
+
 end

data/spec/lib/scrapers/rubytapas/dpdcart_spec.rb

@@ -0,0 +1,68 @@
+require 'spec_helper'
+require 'support/use_vcr'
+require 'scrapers/rubytapas/dpdcart'
+require 'nokogiri'
+
+describe Scrapers::RubyTapas::DpdCart do
+  let(:gateway) { Scrapers::RubyTapas::DpdCart.new }
+
+  describe "method signatures" do
+    it { is_expected.to respond_to(:feed!) }
+    it { is_expected.to respond_to(:login!)}
+    it { is_expected.to respond_to(:download!)}
+  end
+
+  describe "#feed!" do
+    it "returns an rss feed" do
+      VCR.use_cassette('rubytapas_feed', record: :new_episodes,
+                       match_requests_on: [:method, :host, :path]
+                      ) do
+        expect(Nokogiri::XML.parse(gateway.feed!)).to be_a(Nokogiri::XML::Document)
+      end
+    end
+  end
+
+  describe "#login!" do
+    it "shows the subscriber content page" do
+      VCR.use_cassette('rubytapas_login', record: :new_episodes,
+                       match_requests_on: [:method, :host, :path]
+                      ) do
+        expect(gateway.login!.page.title).to eq("Subscription Content | RubyTapas")
+      end
+    end
+  end
+
+
+  describe "#download!" do
+    let(:file) { "https://rubytapas.dpdcart.com/subscriber/download?file_id=26" }
+    let(:name) { "001-binary-literals.html" }
+
+    let(:file2){ "https://rubytapas.dpdcart.com/subscriber/download?file_id=27" }
+    let(:name2) { "001-binary-literals.rb" }
+
+    it "returns the downloaded file" do
+      VCR.use_cassette('rubytapas_download', record: :new_episodes,
+                       match_requests_on: [:method, :host, :path,
+                                           :query]) do
+        gateway.login!
+        filename, body = gateway.download! file
+        expect(filename).to eq(name)
+        expect(body.size).to eq(5744)
+      end
+    end
+
+    it "can download multiple files with a single login" do
+      VCR.use_cassette('rubytapas_download_twice', record: :new_episodes,
+                       match_requests_on: [:method, :host, :path,
+                                           :query]) do
+        gateway.login!
+        filename, body = gateway.download! file
+        expect(filename).to eq(name)
+        filename, body = gateway.download! file2
+        expect(filename).to eq(name2)
+      end
+    end
+
+  end
+
+end

data/spec/lib/scrapers/rubytapas/episode_spec.rb

@@ -0,0 +1,140 @@
+require 'spec_helper'
+require 'scrapers/rubytapas/episode'
+
+describe Scrapers::RubyTapas::Episode do
+
+  let(:number) { "001" }
+  let(:title)  { "001 Binary Literals" }
+  let(:slug)   { "001-binary-literals" }
+  let(:link)   { "https://rubytapas.dpdcart.com/subscriber/post?id=18" }
+  let(:description) {<<-DESC
+<div class="blog-entry">
+<div class="blog-content"><p>In this inaugural episode, a look at a handy syntax for writing out binary numbers.</p>
+</div>
+<h3>Attached Files</h3>
+<ul>
+<li><a href="https://rubytapas.dpdcart.com/subscriber/download?file_id=25">RubyTapas001.mp4</a></li>
+<li><a href="https://rubytapas.dpdcart.com/subscriber/download?file_id=26">001-binary-literals.html</a></li>
+<li><a href="https://rubytapas.dpdcart.com/subscriber/download?file_id=27">001-binary-literals.rb</a></li>
+</ul></div>
+DESC
+  }
+  let(:guid)  { "dpd-89e8004c8242e7ad548833bef1e18a5b575c92c1" }
+  let(:pub_date) { DateTime.new(2012,9,24,9,0,0,'-4') }
+  let(:file_list) do
+    [
+      Scrapers::RubyTapas::Episode::FileLink.new("RubyTapas001.mp4", "https://rubytapas.dpdcart.com/subscriber/download?file_id=25"),
+      Scrapers::RubyTapas::Episode::FileLink.new("001-binary-literals.html", "https://rubytapas.dpdcart.com/subscriber/download?file_id=26"),
+      Scrapers::RubyTapas::Episode::FileLink.new("001-binary-literals.rb", "https://rubytapas.dpdcart.com/subscriber/download?file_id=27")
+    ]
+  end
+
+  let(:xml_string) do
+    <<-ITEM
+<item>
+<title><![CDATA[001 Binary Literals]]></title>
+<link>https://rubytapas.dpdcart.com/subscriber/post?id=18</link>
+<description><![CDATA[<div class="blog-entry">
+<div class="blog-content"><p>In this inaugural episode, a look at a handy syntax for writing out binary numbers.</p>
+</div>
+<h3>Attached Files</h3>
+<ul>
+<li><a href="https://rubytapas.dpdcart.com/subscriber/download?file_id=25">RubyTapas001.mp4</a></li>
+<li><a href="https://rubytapas.dpdcart.com/subscriber/download?file_id=26">001-binary-literals.html</a></li>
+<li><a href="https://rubytapas.dpdcart.com/subscriber/download?file_id=27">001-binary-literals.rb</a></li>
+</ul></div>]]></description>
+<guid isPermaLink="false">dpd-89e8004c8242e7ad548833bef1e18a5b575c92c1</guid>
+<pubDate>Mon, 24 Sep 2012 09:00:00 -0400</pubDate>
+<enclosure url="https://rubytapas.dpdcart.com/feed/download/25/RubyTapas001.mp4" length="12502397" type="video/mp4"/>
+<itunes:image href="https://getdpd.com/uploads/ruby-tapas.png"/>
+</item>
+ITEM
+  end
+
+  let(:xml_item) do
+    Nokogiri::XML.parse(xml_string) {|c| c.noblanks }.children.first
+  end
+
+  describe "#initialize" do
+
+    shared_examples "initialize episode" do
+      it "is episode number 001" do
+        expect(episode.number).to eq(number)
+      end
+
+      it "is episode '001 Binary Literals'" do
+        expect(episode.title).to eq(title)
+      end
+
+      it "has slug" do
+        expect(episode.slug).to eq(slug)
+      end
+
+      it "has link" do
+        expect(episode.link).to eq(link)
+      end
+
+      it "has guid" do
+        expect(episode.guid).to eq(guid)
+      end
+
+      it "has publication date" do
+        expect(episode.pub_date).to eq(pub_date)
+      end
+
+      it "has file list" do
+        expect(episode.file_list).to match_array(file_list)
+      end
+      
+    end
+
+    context "when given an xml string" do
+      include_examples "initialize episode" do
+        let(:episode) {Scrapers::RubyTapas::Episode.new(xml_string)}
+      end
+
+    end
+
+    context "when given a Nokogiri::XML::Element" do
+      include_examples "initialize episode" do
+        let(:episode) {Scrapers::RubyTapas::Episode.new(xml_item)}
+      end
+    end
+
+    context "when given a Hash" do
+      include_examples "initialize episode" do
+        let(:episode) do
+          Scrapers::RubyTapas::Episode.new(
+            :number => number,
+            :title => title,
+            :slug => slug,
+            :link => link,
+            :description => description,
+            :guid => guid,
+            :pub_date => pub_date,
+            :file_list => file_list
+          )
+        end
+      end
+    end
+
+    context "when given a list of arguments" do
+      include_examples "initialize episode" do
+        let(:episode) do
+          Scrapers::RubyTapas::Episode.new(
+            number,
+            title,
+            slug,
+            link,
+            description,
+            guid,
+            pub_date,
+            file_list
+          )
+        end
+      end
+    end
+
+  end
+end
+