abcde-at-the-movies 0.1.0

data/README.markdown

@@ -0,0 +1,33 @@
+# at-the-movies
+
+## Description:
+
+Scraper for http://abc.net.au/atthemovies
+
+## Usage (I guess):
+
+    review = AtTheMovies::Parsers.for('http://www.abc.net.au/atthemovies/txt/s1533013.htm')
+    review.title # "Brokeback Mountain"
+    
+    latest = AtTheMovies::Review.latest
+    latest.first.title # "Public Enemies"
+
+## License:
+
+(The MIT License)
+
+Copyright (c) 2009 Dylan Egan
+
+Permission is hereby granted, free of charge, to any person obtaining a copy of
+this software and associated documentation files (the 'Software'), to deal in
+the Software without restriction, including without limitation the rights to use,
+copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the
+Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED 'AS IS', WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED,
+INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
+TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/Rakefile

@@ -0,0 +1,15 @@
+begin
+  require 'jeweler'
+  Jeweler::Tasks.new do |gemspec|
+    gemspec.name = "at-the-movies"
+    gemspec.summary = "Pulling that shit in yo!"
+    gemspec.description = "Scraper for http://abc.net.au/atthemovies"
+    gemspec.email = "dylanegan@gmail.com"
+    gemspec.homepage = "http://github.com/abcde/at-the-movies"
+    gemspec.authors = ["Dylan Egan"]
+    gemspec.files = %w(README.markdown Rakefile VERSION) + Dir.glob("{lib,spec}/**/*")
+    gemspec.add_dependency 'mechanize'
+  end
+rescue LoadError
+  puts "Jeweler not available. Install it with: sudo gem install technicalpickles-jeweler -s http://gems.github.com"
+end

data/VERSION

@@ -0,0 +1 @@
+0.1.0

data/lib/at_the_movies.rb

@@ -0,0 +1,7 @@
+require 'mechanize'
+
+$:.unshift File.dirname(__FILE__) + '/at_the_movies'
+
+%w( interview parser parsers review ).each { |lib| require lib }
+
+require File.dirname(__FILE__) + '/core_ext'

data/lib/at_the_movies/interview.rb

@@ -0,0 +1,13 @@
+module AtTheMovies
+  class Interview
+    attr_reader :title, :date, :interviewee, :interviewer, :url
+
+    def initialize(title, date, interviewee, interviewer, url)
+      @title = title
+      @date = date
+      @interviewee = interviewee
+      @interviewer = interviewer
+      @url = url
+    end
+  end
+end

data/lib/at_the_movies/parser.rb

@@ -0,0 +1,19 @@
+module AtTheMovies
+  class ParserError < StandardError; end
+  class Parser
+    attr_reader :page
+
+    def self.parse(page)
+      parser = new(page)
+      parser.parse
+    end
+
+    def initialize(page)
+      @page = page
+    end
+
+    def parse
+      raise NotImplementedError, "where am I?"
+    end
+  end
+end

data/lib/at_the_movies/parsers.rb

@@ -0,0 +1,24 @@
+%w( interview review ).each { |parser| require File.dirname(__FILE__) + "/parsers/#{parser}" }
+
+module AtTheMovies
+  module Parsers
+    MAP = {
+      "review" => Review
+    }
+
+    def self.for(url, options = {})
+      tries = 0
+      begin
+        page = WWW::Mechanize.new.get(url)
+      rescue => e
+        retry if (tries += 1) < (options[:tries] || 5)
+        raise AtTheMovies::ParserError,
+          "Failed to parse #{url}. #{e.message}"
+      end
+      type = page.search('meta[@name="ABC-Author"]').first['content']
+      return if options[:only] and type != options[:only]
+      parser = MAP[type] || raise(ArgumentError, "Couldn't find a Parser class to parse a #{type.inspect} page")
+      parser.parse(page)
+    end
+  end
+end

data/lib/at_the_movies/parsers/interview.rb

@@ -0,0 +1,24 @@
+module AtTheMovies
+  module Parsers
+    class Interview < Parser
+      def parse
+        interviewer, interviewee = details
+        if interviewer && interviewee
+          AtTheMovies::Interview.new(title, date, interviewee, interviewer, @page.uri.to_s)
+        end
+      end
+
+      def details
+        @page.body.scan(/(Margaret|David) interviews (.*).<\/p>\r/).flatten
+      end
+
+      def title
+        @page.title.strip[15..-1]
+      end
+
+      def date
+        Date.parse(@page.search('meta[@name="Date"]').first['content'].gsub('/', '-'))
+      end
+    end
+  end
+end

data/lib/at_the_movies/parsers/review.rb

@@ -0,0 +1,58 @@
+module AtTheMovies
+  module Parsers
+    class Review < Parser
+      RATINGS = { 
+        "zero" => 0,
+        "half" => 0.5,
+        "one" => 1,
+        "one-and-a-half" => 1.5,
+        "two" => 2,
+        "two-and-a-half" => 2.5,
+        "three" => 3,
+        "three-and-a-half" => 3.5,
+        "four" => 4,
+        "four-and-a-half" => 4.5,
+        "five" => 5
+      }
+
+      def parse
+        return unless @page.body[/Review by/]
+        AtTheMovies::Review.new(title, classification, date, duration, genre, ratings, @page.uri.to_s)
+      end
+
+      def details
+        @details ||= @page.search('p.moviedetails').inner_html.strip
+      end
+
+      def title
+        @page.title.strip[15..-1]
+      end
+
+      def classification
+        details.scan(/Classification:<\/strong> ([[:alnum:][:punct:][:space:]]{1,10})<br>/).flatten.first
+      end
+
+      def date
+        Date.parse(@page.search('meta[@name="Date"]').first['content'].gsub('/', '-'))
+      end
+
+      def duration
+        details.scan(/Duration:<\/strong> ([0-9]{1,3})/).flatten.first.to_i
+      end
+
+      def genre
+        details.scan(/Genre:<\/strong> ([[:alnum:][:space:][:punct:]]+)<br><strong>Director:/).flatten.first
+      end
+
+      def ratings
+        score = @page.search('p.score')
+        ratings = score.css('img').collect { |image| image['alt'][0..-7] }
+        ratings = score.css('p.score').text.scan(/Margaret|David/).inject_with_index({}) do |hash, person, index|
+          hash[person] = RATINGS[ratings[index]]
+          hash
+        end
+        ratings
+      end
+    end
+  end
+end

data/lib/at_the_movies/review.rb

@@ -0,0 +1,33 @@
+module AtTheMovies
+  class Review
+    attr_reader :title, :classification, :date, :duration, :genre, :ratings, :url
+
+    def self.latest
+      page = WWW::Mechanize.new.get('http://www.abc.net.au/atthemovies/review/')
+      page.search('table.sideRating').first.css('a').inject([]) do |array, a|
+        page = AtTheMovies::Parsers.for(a['href'], :only => "review")
+        array << page
+        array
+      end.compact
+    end
+
+    def initialize(title, classification, date, duration, genre, ratings, url)
+      @title = title
+      @classification = classification
+      @duration = duration
+      @genre = genre
+      @date = date
+      @ratings = ratings
+      @url = url
+    end
+
+    def rating(rater = nil)
+      return total_rating unless rater
+      @ratings[rater.to_s.capitalize]
+    end
+
+    def total_rating
+      @ratings.inject(0) { |i,p| i += p[1]; i }
+    end
+  end
+end

data/lib/at_the_movies/version.rb

@@ -0,0 +1,3 @@
+module AtTheMovies 
+  VERSION = "0.0.2"
+end

data/lib/core_ext.rb

@@ -0,0 +1 @@
+require File.dirname(__FILE__) + '/core_ext/enumerable'

data/lib/core_ext/enumerable.rb

@@ -0,0 +1,6 @@
+module Enumerable
+  def inject_with_index(injected)
+    each_with_index{ |obj, index| injected = yield(injected, obj, index) }
+    injected
+  end
+end

data/spec/at_the_movies/interview_spec.rb

@@ -0,0 +1,33 @@
+require File.dirname(__FILE__) + '/../spec_helper'
+
+describe AtTheMovies::Interview do
+  context "with a parsed interview" do
+    before do
+      url = 'http://www.abc.net.au/atthemovies/txt/s2642594.htm'
+      FakeWeb.register_uri(:get, url, :response => cached_page_for(url))
+      mech = WWW::Mechanize.new
+      page = mech.get(url)
+      @interview = AtTheMovies::Parsers::Interview.parse(page)
+    end
+
+    it "should have a title" do
+      @interview.title.should == "The Cove Interview"
+    end
+
+    it "should have a date" do
+      @interview.date.to_s.should == "2009-08-19"
+    end
+
+    it "should have an interviewee" do
+      @interview.interviewee.should == "filmmaker Louie Psihoyos"
+    end
+
+    it "should have an interviewer" do
+      @interview.interviewer.should == "David"
+    end
+
+    it "should have a url" do
+      @interview.url.should == "http://www.abc.net.au/atthemovies/txt/s2642594.htm"
+    end
+  end
+end

data/spec/at_the_movies/parser_spec.rb

@@ -0,0 +1,8 @@
+require File.dirname(__FILE__) + '/../spec_helper'
+
+describe AtTheMovies::Parser do
+  it "should act as a base class" do
+    parser = AtTheMovies::Parser.new(nil)
+    lambda { parser.parse }.should raise_error(NotImplementedError)
+  end
+end

data/spec/at_the_movies/parsers/interview_spec.rb

@@ -0,0 +1,17 @@
+require File.dirname(__FILE__) + '/../../spec_helper'
+
+describe AtTheMovies::Parsers::Interview do
+  describe "parse" do
+    before do
+      url = 'http://www.abc.net.au/atthemovies/txt/s2642594.htm'
+      FakeWeb.register_uri(:get, url, :response => cached_page_for(url))
+      mech = WWW::Mechanize.new
+      page = mech.get(url)
+      @interview = AtTheMovies::Parsers::Interview.parse(page)
+    end
+
+    it "should return an Interview object" do
+      @interview.should be_an_instance_of(AtTheMovies::Interview)
+    end
+  end
+end

data/spec/at_the_movies/parsers/review_spec.rb

@@ -0,0 +1,17 @@
+require File.dirname(__FILE__) + '/../../spec_helper'
+
+describe AtTheMovies::Parsers::Review do
+  describe "parse" do
+    before do
+      url = 'http://www.abc.net.au/atthemovies/txt/s1533013.htm'
+      FakeWeb.register_uri(:get, url, :response => cached_page_for(url))
+      mech = WWW::Mechanize.new
+      page = mech.get(url)
+      @review = AtTheMovies::Parsers::Review.parse(page)
+    end
+
+    it "should return a Review object" do
+      @review.should be_an_instance_of(AtTheMovies::Review)
+    end
+  end
+end

data/spec/at_the_movies/parsers_spec.rb

@@ -0,0 +1,36 @@
+require File.dirname(__FILE__) + '/../spec_helper'
+
+describe AtTheMovies::Parsers do
+  describe "for" do
+    context "with a review page" do
+      before do
+        url = 'http://www.abc.net.au/atthemovies/txt/s1533013.htm'
+        FakeWeb.register_uri(:get, url, :response => cached_page_for(url))
+        @parsed = AtTheMovies::Parsers.for(url)
+      end
+
+      it "should return a Review object" do
+        @parsed.should be_an_instance_of(AtTheMovies::Review)
+      end
+    end
+
+    context "using only" do
+      before do
+        url = 'http://www.abc.net.au/atthemovies/s2634329.htm'
+        FakeWeb.register_uri(:get, url, :response => cached_page_for(url))
+        @parsed = AtTheMovies::Parsers.for(url, :only => "review")
+      end
+
+      it "should return nothing for review when page is an interview" do
+        @parsed.should be_nil
+      end 
+    end
+
+    context "parsing issues" do
+      it "should try five times before reraising" do
+        WWW::Mechanize.should_receive(:new).exactly(5).times.and_raise(Errno::ECONNREFUSED)
+        lambda { AtTheMovies::Parsers.for('url') }.should raise_error(AtTheMovies::ParserError, "Failed to parse url. Connection refused")
+      end
+    end
+  end
+end

data/spec/at_the_movies/review_spec.rb

@@ -0,0 +1,66 @@
+require File.dirname(__FILE__) + '/../spec_helper'
+
+describe AtTheMovies::Review do
+  context "with a parsed review" do
+    before do
+      url = 'http://www.abc.net.au/atthemovies/txt/s1533013.htm'
+      FakeWeb.register_uri(:get, url, :response => cached_page_for(url))
+      mech = WWW::Mechanize.new
+      page = mech.get(url)
+      @review = AtTheMovies::Parsers::Review.parse(page)
+    end
+
+    it "should have a title" do
+      @review.title.should == "Brokeback Mountain"
+    end
+
+    it "should have a classification" do
+      @review.classification.should == "M"
+    end
+
+    it "should have a date" do
+      @review.date.to_s.should == "2005-12-18"
+    end
+
+    it "should have a duration" do
+      @review.duration.should == 134
+    end
+
+    it "should have a genre" do
+      @review.genre.should == "Drama"
+    end
+
+    it "should have a total rating" do
+      @review.rating.should == 10
+    end
+
+    it "should have a rating by Margaret" do
+      @review.rating(:margaret).should == 5
+    end
+
+    it "should have a rating by David" do
+      @review.rating(:david).should == 5
+    end
+
+    it "should have a url" do
+      @review.url.should == "http://www.abc.net.au/atthemovies/txt/s1533013.htm"
+    end
+  end
+
+  context "finding" do
+    context "latest" do
+      before do
+        url = 'http://www.abc.net.au/atthemovies/review/'
+        FakeWeb.register_uri(:get, url, :response => cached_page('reviews'))
+        %w( s2625733 s2625742 s2625654 s2625717 s2634329 s2631026 ).each do |uri|
+          FakeWeb.register_uri(:get, "http://www.abc.net.au/atthemovies/txt/#{uri}.htm", :response => cached_page("#{uri}.htm"))
+        end
+        @reviews = AtTheMovies::Review.latest
+      end
+
+      it "should contain the latest reviews only" do
+        @reviews.collect { |review| review.title }.should == ["Public Enemies", "Coraline", "My Sister's Keeper", "Cedar Boys"]
+      end
+    end
+  end
+end

data/spec/helpers/page.rb

@@ -0,0 +1,10 @@
+module PageHelperMethods
+  def cached_page_for(url)
+    page = url[/s[0-9]{1,}.htm/]
+    cached_page(page)
+  end
+
+  def cached_page(name)
+    SPEC_DIR + "/pages/#{name}"
+  end
+end