crawler-movie-imdb 0.1.0 → 0.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 80ae48bcce3f954c65f34ff2656db0bdb1e9c7cf365517f3a9a2f4dce474d1bb
-  data.tar.gz: 226b463a5513e82530a60d2c58a21b60f73321b5914bf15572aa647421aec839
+  metadata.gz: a261dfb1bad0cc3e1a762facb6167e2a67a907a592958300dbc4aaa81dfe7e84
+  data.tar.gz: b48fa5312d3c8a0cb44117fb91673b5cb35427f9bb21b13f4e3c4dca59091694
 SHA512:
-  metadata.gz: 5909892bf56d86b7d775d98f17df57f1c2e67dd3d94d1621e533fff9017ef80da51aa9170313c7f48f29b9810575eacac5418c61a614f0edf7b081a948880ecf
-  data.tar.gz: 764d83e319530d632ebe8cb0f49f64f51e674a32ef8e43b12d6728051526f1b841bf0dbd0d88472f976b11d6f9493c67bbbb96e814710127a028d042b2007264
+  metadata.gz: 268656d0ec96de093ce2deace8b106599ee5a0db4e0353d94f47ac256c1bfecc5acab27213a6055a3e3102c32946eba0bb4e7e0f3808c3cd254faf4cbddf6768
+  data.tar.gz: 772e2dff81f70c34df0f6cb6fa9068fefe0f3b9829a7c865927744816d4a42417190ca04aa62964ec305ccf9c36b2de4cb5b59ebf2c5320bab862b0fe900a98b

data/crawler-movie-imdb.gemspec

@@ -30,4 +30,5 @@ Gem::Specification.new do |spec|
   spec.add_development_dependency 'rake', '~> 10.0'
   spec.add_runtime_dependency 'faraday', '>= 0.15'
   spec.add_runtime_dependency 'nokogiri', '>= 1.4'
+  spec.add_runtime_dependency 'activesupport', '>= 3.0'
 end

data/lib/crawler/movie/providers/imdb.rb

@@ -2,11 +2,50 @@ require 'crawler/movie'
 require 'faraday'
 require 'nokogiri'
 require 'date'
+require 'active_support/core_ext/object/blank'
+require 'uri'
+require 'time'
 
 module Crawler
   module Movie
     module Providers
       module Imdb
+        def self.find(id)
+          response = Faraday.get("https://www.imdb.com/title/#{id}/")
+
+          return if !response.success? || !response.body
+
+          html = Nokogiri::HTML(response.body)
+          widget = html.css('#title-overview-widget')
+          title = widget.css('.title_wrapper h1 > text()').text.tr("\u00A0", '').strip
+          original_title = widget.css('.title_wrapper .originalTitle > text()').text.presence
+          genres = (html.css('#titleStoryLine .see-more:contains("Genres:") a') || []).map { |genre| genre.text.strip }
+          poster = widget.css('.poster img')
+          poster_url = poster.present? ? "#{poster.attr('src').value.split('@').first}@._V1_.jpg" : nil
+          overview = widget.css('.summary_text').text.sub(/See full (synopsis|summary) »/, '').strip
+          overview = nil if overview == 'Add a Plot »'
+          release_date_matches = html.css('#titleDetails .txt-block:contains("Release Date:") > text()').text.match(/(?<release_date>\d{1,2} \w+ \d{4})/)
+          release_date = release_date_matches && Date.parse(release_date_matches[:release_date])
+          original_languages = (html.css('#titleDetails .txt-block:contains("Language:") a') || []).map do |lang|
+            uri = URI(lang.attr('href'))
+            params = URI.decode_www_form(uri.query).to_h
+            params['primary_language']
+          end
+
+          {
+            id: id,
+            source: 'internet-movie-database',
+            title: title,
+            poster_url: poster_url,
+            backdrop_url: nil,
+            original_languages: original_languages,
+            original_titles: original_title,
+            genres: genres,
+            overview: overview,
+            release_date: release_date
+          }
+        end
+
         def self.search(query)
           movies = []
           current_start = 1
@@ -29,11 +68,15 @@ module Crawler
               overview = item_content.css('.text-muted')[2].text.sub(/See full (synopsis|summary) »/, '').strip
               overview = nil if overview == 'Add a Plot'
               year_matches = item_content.css('.lister-item-year').text.match(/\((?<year>\d+)\)/)
+              release_date = year_matches && Date.parse("#{year_matches[:year]}-1-1")
               item_image = movie.css('.lister-item-image img')
               id = item_image.attr('data-tconst').value
               poster_url = item_image.attr('loadlate').value.split('@').first
               poster_url = nil if poster_url.match?(%r{/nopicture/})
               poster_url += '@._V1_.jpg' if poster_url
+              details = find(id)
+
+              next details if details
 
               {
                 id: id,
@@ -45,7 +88,7 @@ module Crawler
                 original_title: nil,
                 genres: genres,
                 overview: overview,
-                release_date: year_matches && Date.parse("#{year_matches[:year]}-1-1")
+                release_date: release_date
               }
             end
 

data/lib/crawler/movie/providers/imdb/version.rb

@@ -2,7 +2,7 @@ module Crawler
   module Movie
     module Providers
       module Imdb
-        VERSION = '0.1.0'
+        VERSION = '0.2.0'
       end
     end
   end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: crawler-movie-imdb
 version: !ruby/object:Gem::Version
-  version: 0.1.0
+  version: 0.2.0
 platform: ruby
 authors:
 - Jonathan PHILIPPE
@@ -66,6 +66,20 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '1.4'
+- !ruby/object:Gem::Dependency
+  name: activesupport
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '3.0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '3.0'
 description: ''
 email:
 - jonathan@cinema.paris