movie_spider 0.0.1 → 0.0.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 5f678d1d4e3479f2fa48b889671b7bc2d915208c
-  data.tar.gz: bdf47ebf540ed0cb9374e46038d5f2913b236dd0
+  metadata.gz: 73521a5ab3ecc72f9db79b0c0a640190386b5d57
+  data.tar.gz: 7ea781c07b339f867494eeced1d9dbdde15df20d
 SHA512:
-  metadata.gz: fd34d70d9da3c88b3d38f2692949d2e07162d2a7fe8c860c98a580362ddf9c3670782a185f4dbbaf5f9ccabae310cc1f8af369a7bec01549e30225379a62e508
-  data.tar.gz: c75f433426d2f5425259a17d06238e8e3155ad2933d1a1ac37123c6ffd6e36a5a6a0b7cb82d9560112c40b87479d9f8e80a2c04d545ab5856e903d8c94ed1a4f
+  metadata.gz: a060107747831a6434ec4d418db9a774bf6138a40045b194c83a0ac2ea62c16be935152c0fb523e436467326274e7b318e26bb571e00fce4f86c3e2c15a2fd21
+  data.tar.gz: 702e792ad6d6fb8890a86d29727aa3e45ff0a36546692f3fe4acf0fd137772b498a87c9a00a5211c4e74549afc0b15b12e3413e3fde5feb9e936e9f5fadefa8b

data/README.md

@@ -1,6 +1,6 @@
 # MovieSpider
 
-TODO: Write a gem description
+单车网电影信息爬虫
 
 ## Installation
 
@@ -19,8 +19,19 @@ Or install it yourself as:
     $ gem install movie_spider
 
 ## Usage
+###从时光网获取影片信息、剧照、海报
+`MovieSpider.fetch_info_from_mtime("超能陆战队", "2014")`
+`MovieSpider.fetch_stills_from_mtime("超能陆战队", "2014")`
+`MovieSpider.fetch_posters_from_mtime("超能陆战队", "2014")`
+###从豆瓣网获取影片信息、剧照、海报
+`MovieSpider.fetch_info_from_douban("超能陆战队", "2014")`
+`MovieSpider.fetch_stills_from_douban("超能陆战队", "2014")`
+`MovieSpider.fetch_posters_from_douban("超能陆战队", "2014")`
+###通过豆瓣ID或时光网id获取以上信息，只使用id参数调用，不需要传影片年份，注意ID要对应好，使用豆瓣的ID获取豆瓣电影信息，使用时光网ID获取时光网电影信息。
+`MovieSpider.fetch_(info|posters|stills)_from_douban("11026735")`	
+`MovieSpider.fetch_(info|posters|stills)_from_mtime("160162")`
+以上都是获取电影木星上行的信息。
 
-TODO: Write usage instructions here
 
 ## Contributing
 

data/lib/movie_spider.rb

@@ -5,52 +5,101 @@ require 'json'
 require 'open-uri'
 
 module MovieSpider
-  def self.get_title_from_douban_id(douban_id)
-    DoubanMovie.new.get_subject(douban_id)["title"]
-  end
 
-  def self.fetch_stills(douban_id,source)
-    title = get_title_from_douban_id(douban_id)
+  def self.fetch_stills(id, source, img_size="img_1000")
     case source
     when /douban/
-      stills = DoubanMovie.new.douban_movie_stills(douban_id)
+      DoubanMovie.douban_movie_stills(id)
     when /mtime/
-      stills = MtimeMovie.new.get_mtime_stills(title)
+      MtimeMovie.mtime_movie_stills(id, img_size)
     when /all/
-      stills = DoubanMovie.new.douban_movie_stills(douban_id) + MtimeMovie.new.get_mtime_stills(title)
+      DoubanMovie.douban_movie_stills(id) + MtimeMovie.mtime_movie.stills(id, img_size)
     end
-    stills
   end
 
-  def self.fetch_posters(douban_id,source)
-    title = get_title_from_douban_id(douban_id)
+  def self.fetch_posters(id, source, img_size="img_1000")
     case source
     when /douban/
-      posters = DoubanMovie.new.douban_movie_posters(douban_id)
+      DoubanMovie.douban_movie_posters(id)
     when /mtime/
-      posters = MtimeMovie.new.get_mtime_posters(title)
+      MtimeMovie.mtime_movie_posters(id, img_size)
     when /all/
-      posters = DoubanMovie.new.douban_movie_posters(douban_id) + MtimeMovie.new.get_mtime_posters(title)
+      DoubanMovie.douban_movie_posters(id) + MtimeMovie.mtime_movie_posters(id, img_size)
+    end
+  end
+
+  def self.fetch_info(id, source)
+    case source
+    when /douban/
+      DoubanMovie.douban_movie_info(id)
+    when /mtime/
+      MtimeMovie.mtime_movie_info(id)
+    end
+  end
+
+  def self.search_movies(title, source)
+    case source
+    when /douban/
+      DoubanMovie.douban_search_movies(title)
+    when /mtime/
+      MtimeMovie.mtime_search_movies(title)
     end
-    posters
   end
 
-  def self.fetch_rating(douban_id, source)
-    title = get_title_from_douban_id(douban_id)
-    if source =~ /douban/
-      DoubanMovie.new.get_subject(douban_id)["rating"]["average"]
-    elsif source =~ /mtime/
-      MtimeMovie.new.get_mtime_rating(title)
-    else
-      "评分网站参数错误[douban|mtime]"
+  def self.get_id_from_title(title, year, source="douban")
+    year = year.to_i
+    subjects = search_movies(title, source)
+    match = []
+    unless subjects.empty?
+      subjects.each do |subject|
+        film_name = title.gsub(/\[.+\]/, "").gsub("（", "(").gsub("）",")")
+        if subject["year"].to_i == year and
+          string_similarity(film_name, subject["title"], 0.8)
+          match << subject['id']
+        end
+      end
     end
+    match[0]
   end
 
-  def self.get_subject(douban_id)
-    DoubanMovie.new.get_subject(douban_id)
+  def self.define_component(type, source)
+    define_singleton_method("fetch_#{type}_from_#{source}") do |id_or_title, year=Time.now.year|
+      if id_or_title.to_i.to_s == id_or_title
+        MovieSpider.send("fetch_#{type}", id_or_title, source)
+      else
+        id = get_id_from_title(id_or_title, year, source)
+        MovieSpider.send("fetch_#{type}", id, source)
+      end
+    end
   end
 
-  def self.douban_search_movie(movie_name)
-    DoubanMovie.new.douban_search_movie(movie_name)
+  define_component "stills", "douban"
+  define_component "stills", "mtime"
+  define_component "posters", "douban"
+  define_component "posters", "mtime"
+  define_component "info", "douban"
+  define_component "info", "mtime"
+
+  private
+
+  def self.string_similarity(origin, compare, score)
+    origin.downcase!
+    origin_pair = (0..origin.length-2).collect{|i| origin[i, 2]}.reject{|pair| pair.include? " "}
+    compare.downcase!
+    compare_pair = (0..compare.length-2).collect{|i| compare[i, 2]}.reject{|pair| pair.include? " "}
+
+    union = origin_pair.size + compare_pair.size
+    intersection = 0 
+    origin_pair.each do |op|
+      0.upto(compare_pair.size - 1) do |i|
+        if op == compare_pair[i]
+          intersection += 1
+          compare_pair.slice!(i)
+          break
+        end
+      end
+    end 
+    (2.0 * intersection) / union > score
   end
+
 end

data/lib/movie_spider/douban_fetcher.rb

@@ -1,21 +1,49 @@
 # coding: utf-8
 
 module MovieSpider
-  class DoubanMovie
+  module DoubanMovie
+
     UrlPrefix = "http://api.douban.com/v2/"
     Key = "0c89712b2297db4e259c538167c791ea"
-    def get_subject(movie_id)
+
+    def self.douban_movie_info(movie_id)
       path = "movie/subject/#{movie_id}?apikey=#{Key}"
       data = api_get(path)
+      subject ={}
+      subject['id'] = data['id']
+      subject['title'] = data['title']
+      subject['alt'] = data['alt']
+      subject['original_title'] = data['original_title']
+      subject['year'] = data['year']
+      subject['genres'] = data['genres']
+      subject['length'] = nil
+      subject['area'] = data['countries'].join("|")
+      subject['directors'] = data['directors'].map { |e| e['name'] }
+      subject['actors'] = data['casts'].map { |e| e['name'] }
+      subject['summary'] = data['summary'].gsub("©豆瓣", "")
+      subject['rating'] = data['rating']['average']
+      subject
     end
 
-    def douban_search_movie(movie_name)
+    def self.douban_search_movies(movie_name)
       movie_name = movie_name.gsub(/\[.+\]/, "")
       path = "movie/search?q=#{movie_name}&apikey=#{Key}"
-      data = api_get(path)["subjects"]
+      api_get(path)["subjects"].map do |subject|
+        subject.delete_if { |key,value| not %w|id year title|.include? key }
+      end
+    end
+
+    def self.douban_movie_stills(douban_id)
+      fetch_img("http://movie.douban.com/subject/#{douban_id}/photos?type=S&start=0&sortby=vote&size=a&subtype=o")
+    end
+
+    def self.douban_movie_posters(douban_id)
+      fetch_img("http://movie.douban.com/subject/#{douban_id}/photos?type=R&start=0&sortby=vote&size=a&subtype=a")
     end
 
-    def api_get(path)
+    private
+
+    def self.api_get(path)
       begin
         data = RestClient.get URI.encode(UrlPrefix + path)
         JSON.parse data
@@ -28,15 +56,7 @@ module MovieSpider
       end
     end
 
-    def douban_movie_stills(douban_id)
-      fetch_img("http://movie.douban.com/subject/#{douban_id}/photos?type=S&start=0&sortby=vote&size=a&subtype=o")
-    end
-
-    def douban_movie_posters(douban_id)
-      fetch_img("http://movie.douban.com/subject/#{douban_id}/photos?type=R&start=0&sortby=vote&size=a&subtype=a")
-    end
-
-    def fetch_img(url) 
+    def self.fetch_img(url) 
       begin
         doc = Nokogiri::HTML(open(url))
       rescue

data/lib/movie_spider/mtime_fetcher.rb

@@ -1,74 +1,90 @@
 # coding: utf-8
 module MovieSpider
-  class MtimeMovie
-    #获取官方剧照，三种参数选择img_200 img_235 img_1000,注意235的尺寸可能会fetch不到，期望使用220或1000
-    def get_mtime_stills(movie_title, img_size="img_1000")
-      parse_mtime(movie_title, "officialstageimage", img_size)
+  module MtimeMovie
+    # 时光剧照
+    def self.mtime_movie_stills(movie_id, img_size)
+      parse_mtime(movie_id, "officialstageimage", img_size)
     end
-    #获取正式海报
-    def get_general_posters(movie_title, img_size="img_1000")
-      parse_mtime(movie_title, "generalposter", img_size)
-    end
-    #获取预告海报
-    def get_forecast_posters(movie_title, img_size="img_1000")
-      parse_mtime(movie_title, "forecastposter", img_size)
-    end
-    #获取角色海报
-    def get_role_posters(movie_title, img_size="img_1000")
-      parse_mtime(movie_title, "roleposter", img_size)
-    end
-    #全部海报
-    def get_mtime_posters(movie_title, img_size="img_1000")
-      posters = []
-      posters += get_general_posters(movie_title, img_size) + get_forecast_posters(movie_title,img_size) + get_role_posters(movie_title, img_size)
-      posters.delete(nil);posters.delete("")
+    # 时光海报
+    def self.mtime_movie_posters(movie_id, img_size)
+      posters = parse_mtime(movie_id, "generalposter", img_size) \
+                + parse_mtime(movie_id, "forecastposter", img_size) \
+                + parse_mtime(movie_id, "roleposter", img_size)
+      posters.delete(nil)
+      posters.delete("")
       posters
     end
-    #获取时光网电影评分
-    def get_mtime_rating(movie_title)
-      fetch_result = parse_search(movie_title)
-      if fetch_result.is_a?(Hash)
-        movie_result = fetch_result['value']['movieResult']
-        if movie_result['directMovie'].empty?
-        #无法命中电影标题时
-          return "影片暂无评分" if movie_result['moreMovies'][0]['movieRating'].nil?
-          movie_result['moreMovies'][0]['movieRating']
-        else
-        #直接查找出电影
-          return "影片暂无评分" if movie_result['directMovie']['movieRating'].nil?
-          movie_result['directMovie']['movieRating']
-        end
+    # 时光信息
+    def self.mtime_movie_info(movie_id)
+      subject = {}
+      begin
+        html_doc = Nokogiri::HTML(open("http://movie.mtime.com/#{movie_id}/"))
+        plots = Nokogiri::HTML(open("http://movie.mtime.com/#{movie_id}/plots.html"))
+      rescue
+        return subject
+      end
+      subject['id'] = movie_id.to_s
+      subject['title'] = html_doc.css(".db_head h1").text
+      subject['alt'] = "http://movie.mtime.com/#{movie_id}/"
+      subject['original_title'] = html_doc.css(".db_head .db_enname").text
+      subject['year'] = html_doc.css(".db_head .db_year").text.gsub(/[\(\)]/, "")
+      subject['genres'] = html_doc.css(".db_head a[property='v:genre']").map { |e| e.text }
+      subject['length'] = html_doc.css(".db_head span[property='v:runtime']").text
+      subject['area'] = html_doc.css("dl.info_l dd")[2].css("a").text
+      subject['directors'] = html_doc.css(".db_contout a[rel='v:directedBy']").map { |e| e.text }
+      subject['actors'] = html_doc.css("dl.main_actor a[pan][title]").map { |e| e['title']}
+      subject['summary'] = plots.css("div.plots_box").text.strip
+      subject['rating'] = mtime_rating(movie_id)
+      subject
+    end
+    # 使用电影标题搜索时光网电影信息，返回数组
+    def self.mtime_search_movies(movie_title)
+      search_result = mtime_search(movie_title)['value']['movieResult']
+      direct_subject = search_result['directMovie']
+      subjects = search_result['moreMovies']
+      # 合并结果，去除空值
+      unless subjects.nil?
+        subjects.insert(0,direct_subject).delete({})
       else
-        "解析出错，暂无评分"
+        subjects = direct_subject.empty? ? [] : [direct_subject]
       end
+      subjects.map do |subject|
+        item = {}
+        subject['movieTitle'] =~ /\((.*)\)/
+        item['id'] = subject['movieId'].to_s
+        item['title'] = subject['movieTitle'].split(" ")[0]
+        item['year'] = $1
+        subject.clear
+        subject.update(item)
+      end
+      subjects
     end
-
-    private
-    #解析时光网search API返回
-    def parse_search(movie_title)
+    # 时光网评分
+    def self.mtime_rating(movie_id)
       begin
-        result_str = open(URI::encode("http://service.channel.mtime.com/Search.api?Ajax_CallBack=true&Ajax_CallBackType=Mtime.Channel.Services&Ajax_CallBackMethod=GetSearchResult&Ajax_CrossDomain=1&Ajax_CallBackArgument0=#{movie_title}")).read
+        result_str = open("http://service.library.mtime.com/Movie.api?Ajax_CallBack=true&Ajax_CallBackType=Mtime.Library.Services&Ajax_CallBackMethod=GetMovieOverviewRating&Ajax_CrossDomain=1&Ajax_RequestUrl=http%3A%2F%2Fmovie.mtime.com%2F189691%2F&t=20153418472369218&Ajax_CallBackArgument0=#{movie_id}").read
       rescue
+        puts "parse error"
         return ""
       end
-      result_str.scan(/{.*}/) {|match| return JSON.parse(match)}
+      result_str.scan(/{.*}/) {|match| return JSON.parse(match)['value']['movieRating']['RatingFinal'].to_i}
     end
-    def get_mtime_movie_id_by_title(movie_title)
-      movie_result = {}
+
+    private
+
+    # 时光网搜索
+    def self.mtime_search(movie_title)
       begin
-        movie_result = parse_search(movie_title)['value']['movieResult']
+        result_str = open(URI::encode("http://service.channel.mtime.com/Search.api?Ajax_CallBack=true&Ajax_CallBackType=Mtime.Channel.Services&Ajax_CallBackMethod=GetSearchResult&Ajax_CrossDomain=1&Ajax_RequestUrl=http%3A%2F%2Fsearch.mtime.com%2Fsearch%2F%3Fq%3D%25E8%25B6%2585%25E8%2583%25BD%25E9%2599%2586%25E6%2588%2598%25E9%2598%259F&t=20153414382422867&Ajax_CallBackArgument0=#{movie_title}&Ajax_CallBackArgument1=0&Ajax_CallBackArgument2=365&Ajax_CallBackArgument3=0&Ajax_CallBackArgument4=1")).read
       rescue
+        puts "parse error"
         return ""
       end
-      if movie_result['directMovie'].empty?
-        movie_result['moreMovies'][0]['movieId']
-      else
-        movie_result['directMovie']['movieId']
-      end
+      result_str.scan(/{.*}/) {|match| return JSON.parse(match)}
     end
-    #解析时光网JS返回变量
-    def parse_mtime(movie_title, type, img_size)
-      movie_id = get_mtime_movie_id_by_title(movie_title)
+
+    # 时光网图片解析
+    def self.parse_mtime(movie_id, type, img_size)
       begin
         html_doc = Nokogiri::HTML(open("http://movie.mtime.com/#{movie_id}/posters_and_images/posters/hot.html"))
         parser = ""

data/lib/movie_spider/version.rb

@@ -1,3 +1,3 @@
 module MovieSpider
-  VERSION = "0.0.1"
+  VERSION = "0.0.2"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: movie_spider
 version: !ruby/object:Gem::Version
-  version: 0.0.1
+  version: 0.0.2
 platform: ruby
 authors:
 - hzlu
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2015-03-04 00:00:00.000000000 Z
+date: 2015-03-05 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler