spotlite 0.8.2 → 0.8.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 42b6d7da0103a2f748b6ae6538dff22250514052
-  data.tar.gz: d85ae3e8043976bd0cffb516a76b2b1018588555
+  metadata.gz: c105673d29a0a0bae4ea7c336e5fdaedd5cc6340
+  data.tar.gz: 284715b2189e721d3355e905a02e7ac88ea5abf1
 SHA512:
-  metadata.gz: aa3593e02c8f945dc50da2f092ea92da9346fea75792561dba6677633b6498cae7c87c5135d2ddfa92c080f1fc8555559aac59058742f1fcb11660938e24b562
-  data.tar.gz: 5f058b249fa75382094d3e3a3429f14322611d699a21c40f2d9fe3f5640e9425f497ea8b41907c0677fe6d0da1a5b9edbece1c305eb97a714af7c08c8227a686
+  metadata.gz: bc9057597d1da13f16e58af1261c32952b5182ed11a125b9f85e85b9201cc624076bf358f3549ab09011cd2872e01ee14284f48174810cb30c28c99b63bb9125
+  data.tar.gz: 0125f5259dd412008c7ad8b22b48fca6f1c3cdf9c63d6ea3262a975d81b57f18f24ceb126c38e638dbec869a8f198b2223798d4f3915946c5e083e9ff6772ef9

data/CHANGELOG.md

@@ -1,3 +1,9 @@
+## v0.8.3 14-Dec-2014
+
+* Spotlite now uses HTTParty as HTTP client, adding `response` attribute to Movie instance
+* Fixed #parse_crew method failing when crew category was missing
+* Recommended movies will now include only feature films
+
 ## v0.8.2 18-Jul-2014
 
 * Clean some junk from credits text (particularly writers credits)

data/README.md

@@ -38,7 +38,7 @@ Or install it yourself as:
     => [#<Spotlite::Person:0x007f96a092be70 @imdb_id="0905152", @name="Andy Wachowski", @url="http://www.imdb.com/name/nm0905152/", @credits_category="Directed by", @credits_text="(as The Wachowski Brothers)">, #<Spotlite::Person:0x007f96a092bda8 @imdb_id="0905154", @name="Lana Wachowski", @url="http://www.imdb.com/name/nm0905154/", @credits_category="Directed by", @credits_text="(as The Wachowski Brothers)">]
     > movie.cast[0..2]
     => [#<Spotlite::Person:0x007f96a19521a0 @imdb_id="0000206", @name="Keanu Reeves", @url="http://www.imdb.com/name/nm0000206/", @credits_category="Cast", @credits_text="Neo">, #<Spotlite::Person:0x007f96a1951c28 @imdb_id="0000401", @name="Laurence Fishburne", @url="http://www.imdb.com/name/nm0000401/", @credits_category="Cast", @credits_text="Morpheus">, #<Spotlite::Person:0x007f96a1951a70 @imdb_id="0005251", @name="Carrie-Anne Moss", @url="http://www.imdb.com/name/nm0005251/", @credits_category="Cast", @credits_text="Trinity">]
-    
+
 ## Important notice
 
 Movie titles will be localized if movie has an alternative title specific to your country.
@@ -60,11 +60,11 @@ Sorry, there is nothing I can do about it at the moment.
 Spotlite uses RSpec as a test framework. So, first make sure you have it installed
 
     $ gem install rspec
-    
+
 Run the tests
 
     $ rake
-    
+
 Spotlite uses gem FakeWeb in order to stub out HTTP responses from IMDb. These
 stubs are located in `spec/fixtures` directory.
 
@@ -76,7 +76,7 @@ If you want to make a new feature that uses data from a page which is not stubbe
 
     $ cd spotlite
     $ curl -isH "Accept-Language: en-us" http://www.imdb.com/title/tt[IMDB_ID]/ > spec/fixtures/tt[IMDB_ID]/index
-    
+
 or, for example:
 
     $ curl -isH "Accept-Language: en-us" http://www.imdb.com/title/tt[IMDB_ID]/fullcredits > spec/fixtures/tt[IMDB_ID]/fullcredits
@@ -89,7 +89,7 @@ expected data, or more likely, methods will return nil or empty arrays.
 First, run tests with `LIVE_TEST=true` environment variable:
 
     $ LIVE_TEST=true rake
-   
+
 Adjust methods that are failing, according to the new layout. And refresh fixtures:
 
     $ rake refresh_fixtures

data/Rakefile

@@ -19,7 +19,7 @@ task :refresh_fixtures do
 
   IMDB_SAMPLES.each_pair do |url, fixture|
     page = `curl -isH "Accept-Language: en-us" '#{url}'`
-    
+
     File.open(File.expand_path(File.dirname(__FILE__) + "/spec/fixtures/#{fixture}"), 'w') do |f|
       f.write(page)
     end

data/lib/spotlite.rb

@@ -2,10 +2,11 @@ $:.unshift(File.dirname(__FILE__)) unless
   $:.include?(File.dirname(__FILE__)) || $:.include?(File.expand_path(File.dirname(__FILE__)))
 
 require "rubygems"
-require "open-uri"
+require "httparty"
 require "nokogiri"
 
 require "spotlite/version"
+require "spotlite/http_client"
 require "spotlite/movie"
 require "spotlite/person"
 require "spotlite/string_extensions"

data/lib/spotlite/http_client.rb

@@ -0,0 +1,18 @@
+class HtmlParserIncluded < HTTParty::Parser
+  SupportedFormats.merge!('text/html' => :html)
+
+  def html
+    Nokogiri::HTML(body)
+  end
+end
+
+module Spotlite
+  class Client
+    include HTTParty
+    parser HtmlParserIncluded
+
+    USER_AGENT = 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9_4) AppleWebKit/537.77.4 (KHTML, like Gecko) Version/7.0.5 Safari/537.77.4'
+
+    headers 'Accept-Language' => 'en-US,en;q=0.5', 'User-Agent' => USER_AGENT
+  end
+end

data/lib/spotlite/movie.rb

@@ -1,10 +1,8 @@
-require 'cgi'
-
 module Spotlite
   # Represents a movie on IMDb.com
   class Movie
-    attr_accessor :imdb_id, :url
-    
+    attr_accessor :imdb_id, :url, :response
+
     # Initialize a new movie object by its IMDb ID as a string
     #
     #   movie = Spotlite::Movie.new('0133093')
@@ -20,96 +18,97 @@ module Spotlite
       @year    = year
       @url     = "http://www.imdb.com/title/tt#{@imdb_id}/"
     end
-    
+
     # Returns a list of movies as an array of +Spotlite::Movie+ objects
     # Takes single parameter and searches for movies by title and alternative titles
     def self.find(query)
-      results = Nokogiri::HTML open("http://www.imdb.com/find?q=#{CGI::escape(query)}&s=tt&ttype=ft", 'Accept-Language' => 'en-us')
+      results = Spotlite::Client.get 'http://www.imdb.com/find', query: {q: query, s: 'tt', ttype: 'ft'}
       results.css('.result_text').map do |result|
         imdb_id = result.at('a')['href'].parse_imdb_id
         title   = result.at('a').text.strip
         year    = result.children.take(3).last.text.parse_year
-      
+
         [imdb_id, title, year]
       end.map do |values|
         self.new(*values)
       end
     end
-    
+
     # Returns a list of movies as an array of +Spotlite::Movie+ objects
     # Takes optional parameters as a hash
     # See https://github.com/defeed/spotlite/wiki/Advanced-movie-search for details
     def self.search(params = {})
       defaults = {
-        :title_type => 'feature',
-        :view       => 'simple',
-        :count      => 250,
-        :start      => 1,
-        :sort       => 'moviemeter,asc'
+        title_type: 'feature',
+        view: 'simple',
+        count: 250,
+        start: 1,
+        sort: 'moviemeter,asc'
       }
-      params = defaults.merge(params).map{ |k, v| "#{k}=#{v}" }.join('&')
-      results = Nokogiri::HTML open("http://www.imdb.com/search/title?#{params}", 'Accept-Language' => 'en-us')
+      params = defaults.merge(params)
+      results = Spotlite::Client.get 'http://www.imdb.com/search/title', query: params
       results.css('td.title').map do |result|
         imdb_id = result.at('a')['href'].parse_imdb_id
         title   = result.at('a').text.strip
         year    = result.at('.year_type').text.parse_year
-      
+
         [imdb_id, title, year]
       end.map do |values|
         self.new(*values)
       end
     end
-    
+
     # Returns title as a string
     def title
-      @title ||= details.at("h1.header span[itemprop='name']").text.strip
+      @title ||= details.at("h1.header span[itemprop='name']").text.strip rescue nil
     end
-    
+
     # Returns original non-english title as a string
     def original_title
       details.at("h1.header span.title-extra[itemprop='name']").children.first.text.gsub('"', '').strip rescue nil
     end
-    
+
     # Returns year of original release as an integer
     def year
       @year ||= details.at("h1.header a[href^='/year/']").text.parse_year rescue nil
     end
-    
+
     # Returns IMDb rating as a float
     def rating
       details.at("div.star-box-details span[itemprop='ratingValue']").text.to_f rescue nil
     end
-    
+
     # Returns Metascore rating as an integer
     def metascore
       details.at("div.star-box-details a[href^=criticreviews]").text.strip.split('/').first.to_i rescue nil
     end
-    
+
     # Returns number of votes as an integer
     def votes
       details.at("div.star-box-details span[itemprop='ratingCount']").text.gsub(/[^\d+]/, '').to_i rescue nil
     end
-    
+
     # Returns short description as a string
     def description
-      details.at("p[itemprop='description']").text.strip.clean_description rescue nil
+      desc = details.at("p[itemprop='description']").text.strip.clean_description rescue nil
+      (desc.nil? || desc.empty?) ? nil : desc
     end
-    
+
     # Returns storyline as a string. Often is the same as description
     def storyline
       details.at("#titleStoryLine div[itemprop='description'] p").text.strip.clean_description rescue nil
     end
-    
+
     # Returns content rating as a string
     def content_rating
       details.at(".infobar span[itemprop='contentRating']")['title'] rescue nil
     end
-    
+
     # Returns a list of genres as an array of strings
     def genres
       details.css("div.infobar a[href^='/genre/']").map { |genre| genre.text } rescue []
     end
-    
+
     # Returns a list of countries as an array of hashes
     # with keys: +code+ (string) and +name+ (string)
     def countries
@@ -117,10 +116,10 @@ module Spotlite
       details.css("div.txt-block a[href^='/country/']").each do |node|
         array << {:code => node['href'].clean_href, :name => node.text.strip}
       end
-      
+
       array
     end
-    
+
     # Returns a list of languages as an array of hashes
     # with keys: +code+ (string) and +name+ (string)
     def languages
@@ -128,47 +127,54 @@ module Spotlite
       details.css("div.txt-block a[href^='/language/']").each do |node|
         array << {:code => node['href'].clean_href, :name => node.text.strip}
       end
-      
+
       array
     end
-    
+
     # Returns runtime (length) in minutes as an integer
     def runtime
       details.at("time[itemprop='duration']").text.gsub(',', '').to_i rescue nil
     end
-    
+
     # Returns primary poster URL as a string
     def poster_url
       src = details.at('#img_primary img')['src'] rescue nil
-      
+
       if src =~ /^(http:.+@@)/ || src =~ /^(http:.+?)\.[^\/]+$/
         $1 + '.jpg'
       end
     end
-    
+
     # Returns an array of recommended movies as an array of initialized objects of +Movie+ class
     def recommended_movies
-      details.css('.rec-title').map do |node|
+      details.css('.rec-title').reject do |node|
+        # reject movies that don't have a release year yet
+        node.at('span').nil?
+      end.reject do |node|
+        # reject everything other than featured film
+        /\d{4}-\d{4}/.match(node.at('span').text) ||
+          /Series|Episode|Video|Documentary|Movie|Special|Short|Game|Unknown/.match(node.at('span').text)
+      end.map do |node|
         imdb_id = node.at("a[href^='/title/tt']")['href'].parse_imdb_id
         title   = node.at('a').text.strip
         year    = node.at('span').text.parse_year
-      
+
         [imdb_id, title, year]
       end.map do |values|
         Spotlite::Movie.new(*values)
       end
     end
-    
+
     # Returns a list of keywords as an array of strings
     def keywords
       plot_keywords.css("a[href^='/keyword/']").map { |keyword| keyword.text.strip } rescue []
     end
-    
+
     # Returns a list of trivia facts as an array of strings
     def trivia
       movie_trivia.css("div.sodatext").map { |node| node.text.strip } rescue []
     end
-    
+
     # Returns a list of movie alternative titles as an array of hashes
     # with keys +title+ (string) and +comment+ (string)
     def alternative_titles
@@ -177,37 +183,37 @@ module Spotlite
         cells = row.css('td')
         array << { :title => cells.last.text.strip, :comment => cells.first.text.strip }
       end
-      
+
       array
     end
-    
+
     # Returns a list of directors as an array of +Spotlite::Person+ objects
     def directors
       parse_crew('Directed by')
     end
-    
+
     # Returns a list of writers as an array of +Spotlite::Person+ objects
     def writers
       parse_crew('Writing Credits')
     end
-    
+
     # Returns a list of producers as an array of +Spotlite::Person+ objects
     def producers
       parse_crew('Produced by')
     end
-    
+
     # Returns a list of starred actors as an array of +Spotlite::Person+ objects
     def stars
       details.css("td#overview-top div[itemprop='actors'] a[href^='/name/nm']").map do |node|
         imdb_id = node['href'].parse_imdb_id
         name = node.text.strip
-        
+
         [imdb_id, name]
       end.map do |values|
         Spotlite::Person.new(*values)
       end
     end
-    
+
     # Returns a list of actors as an array +Spotlite::Person+ objects
     def cast
       full_credits.css('table.cast_list tr').reject do |row|
@@ -217,16 +223,16 @@ module Spotlite
         imdb_id = row.at('td:nth-child(2) a')['href'].parse_imdb_id
         name = row.at('td:nth-child(2) a').text.strip_whitespace
         credits_text = row.last_element_child.text.strip_whitespace
-        
+
         [imdb_id, name, 'Cast', credits_text]
       end.map do |values|
         Spotlite::Person.new(*values)
       end
     end
-    
+
     # Returns a list of crew members of a certain category as an array +Spotlite::Person+ objects
     def parse_crew(category)
-      table = full_credits.search("[text()^='#{category}']").first.next_element rescue []
+      table = full_credits.search("[text()^='#{category}']").first.next_element rescue nil
       if table && table.name == 'table'
         table.css('tr').reject do |row|
           # Skip empty table rows with one non-braking space
@@ -235,34 +241,36 @@ module Spotlite
           imdb_id = row.first_element_child.at('a')['href'].parse_imdb_id
           name = row.first_element_child.at('a').text.strip_whitespace
           credits_text = row.last_element_child.text.strip_whitespace.clean_credits_text
-          
+
           [imdb_id, name, category, credits_text]
         end.map do |values|
           Spotlite::Person.new(*values)
         end
+      else
+        []
       end
     end
-    
+
     # Combines all crew categories and returns an array of +Spotlite::Person+ objects
     def crew
       crew_categories.map{ |category| parse_crew(category) }.flatten
     end
-    
+
     # Returns combined `cast` and `crew` as an array of +Spotlite::Person+ objects
     def credits
       cast + crew
     end
-    
+
     # Returns available crew categories, e.g. "Art Department", "Writing Credits", or "Stunts", as an array of strings
     def crew_categories
       array = []
       full_credits.css('h4.dataHeaderWithBorder').reject{ |h| h['id'] == 'cast' }.map do |node|
         array << (node.children.size > 1 ? node.children.first.text.strip_whitespace : node.children.text.strip_whitespace)
       end
-      
+
       array
     end
-    
+
     # Returns a list of regions and corresponding release dates
     # as an array of hashes with keys:
     # region +code+ (string), +region+ name (string), +date+ (date), and +comment+ (string)
@@ -278,18 +286,18 @@ module Spotlite
         date = cells.at('.release_date').text.strip.parse_date
         comment = cells.last.text.strip.clean_release_comment
         comment = nil if comment.empty?
-        
+
         array << {:code => code, :region => region, :date => date, :comment => comment}
       end unless table.nil?
-      
+
       array
     end
-    
+
     # Returns original release date as a date
     def release_date
       release_dates.first[:date] rescue nil
     end
-    
+
     # Returns a list of critic reviews as an array of hashes
     # with keys: +source+ (string), +author+ (string), +excerpt+ (string), and +score+ (integer)
     def critic_reviews
@@ -299,32 +307,32 @@ module Spotlite
         author = review.at("span[itemprop='author'] span[itemprop='name']").text
         excerpt = review.at("div[itemprop='reviewbody']").text.strip
         score = review.at("span[itemprop='ratingValue']").text.to_i
-        
+
         array << {:source => source, :author => author, :excerpt => excerpt, :score => score}
       end
-      
+
       array
     end
-    
+
     # Returns URLs of movie still frames as an array of strings
     def images
       array = []
       still_frames.css('#media_index_thumbnail_grid img').map do |image|
         src = image['src'] rescue nil
-      
+
         if src =~ /^(http:.+@@)/ || src =~ /^(http:.+?)\.[^\/]+$/
           array << $1 + '.jpg'
         end
       end
-      
+
       array
     end
-    
+
     # Returns technical information like film length, aspect ratio, cameras, etc. as a hash of arrays of strings
     def technical
       hash = {}
       table = technical_info.at_css('#technical_content table') rescue nil
-      
+
       table.css('tr').map do |row|
         hash[row.css('td').first.text.strip] = row.css('td').last.children.
           map(&:text).
@@ -334,46 +342,47 @@ module Spotlite
           slice_before{|i| /^[^\(]/.match i}.
           map{|i| i.join(' ')}
       end unless table.nil?
-      
+
       hash
     end
-    
+
     private
-    
+
     def details # :nodoc:
       @details ||= open_page
     end
-    
+
     def release_info # :nodoc:
       @release_info ||= open_page('releaseinfo')
     end
-    
+
     def full_credits # :nodoc:
       @full_credits ||= open_page('fullcredits')
     end
-    
+
     def plot_keywords # :nodoc:
       @plot_keywords ||= open_page('keywords')
     end
-    
+
     def movie_trivia # :nodoc:
       @movie_trivia ||= open_page('trivia')
     end
-    
+
     def reviews
       @reviews ||= open_page('criticreviews')
     end
-    
+
     def still_frames # :nodoc:
-      @still_frames ||= open_page('mediaindex?refine=still_frame')
+      @still_frames ||= open_page('mediaindex', {refine: 'still_frame'})
     end
-    
-    def technical_info
+
+    def technical_info # :nodoc:
       @technical_info ||= open_page('technical')
     end
-    
-    def open_page(page = nil) # :nodoc:
-      Nokogiri::HTML open("#{@url}#{page}", 'Accept-Language' => 'en-us')
+
+    def open_page(page = nil, query = {}) # :nodoc:
+      response = Spotlite::Client.get "#{@url}#{page}", query: query
+      @response = { code: response.code, message: response.message } and response
     end
   end