movieDB 0.3.4 → 1.0.0

data/Rakefile

@@ -6,3 +6,6 @@ Rake::TestTask.new do |t|
   t.test_files = FileList['test/unit/test*.rb']
   t.verbose = true
 end
+
+require 'coveralls/rake/task'
+Coveralls::RakeTask.new

data/lib/movieDB.rb

@@ -1,23 +1,14 @@
-require "rubygems"
-require "open-uri"                # is an easy-to-use wrapper for net/http, net/https and net/ftp.
-require "nokogiri"                # is an HTML, XML, SAX, and Reader parser.
-require "themoviedb"              # Provides a simple, easy to use interface for the Movie Database API.
-require "imdb"                    # Easily use Ruby or the command line to find information on IMDB.com.
-require "spreadsheet"             # A library designed to read and write Spreadsheet Documents.
-require "MovieDB/base"
-require "MovieDB/data_analysis"
-require "MovieDB/secret"
-require "MovieDB/data_export"
-require "redis"
-require "json"
-require 'highline/import'
+require 'MovieDB/base'
+require 'MovieDB/data_store'
+require 'MovieDB/data_analysis/statistics'
+require 'MovieDB/relation/query_methods'
+require 'celluloid/current'
 
-unless defined? MovieDB::Movie
-  module MovieDB
+module MovieDB
   # Create a new movie record. The values are stored in the key-value data store.
   #
   # Default values are supplement during the instantiation of the class.
-  # Those values are overriden when you provide one.
+  # Those values are overridden when you provide one.
   #
   # You can use it like this:
   #
@@ -26,140 +17,30 @@ unless defined? MovieDB::Movie
   #
   # You can raise a MovieError like this:
   #
-  #   raise MovieError unless Movie.title_present?
-    class Movie < MovieDB::Base
-      include StatusChecker
+  class Movie < MovieDB::Base
+    include Celluloid
+    include MovieDB::Relation::QueryMethods
+    include MovieDB::DataAnalysis::Statistics
 
-      extend MovieDB::DataExport
-      extend MovieDB::Secret::Lock
+    def ids_to_array(ids)
+      arr ||= []
 
-      const_set("MovieError",  Class.new(StandardError))
-
-      attr_accessor :title,
-                    :cast_members,
-                    :cast_characters,
-                    :cast_member_ids,
-                    :cast_members_characters,
-                    :trailer_url,
-                    :director,
-                    :writers,
-                    :filming_locations,
-                    :company,
-                    :genres,
-                    :languages,
-                    :countries,
-                    :length,
-                    :plot,
-                    :poster,
-                    :rating,
-                    :votes,
-                    :mpaa_rating,
-                    :tagline,
-                    :year,
-                    :release_date,
-                    :revenue
-
-      DEFAULT_TITLE = "Method Missing 2: Rails Roars!"
-      DEFAULT_CAST_MEMBERS = ["David Black", "Paola Perotta", "Obie Fernandez", "David Chelimsky"]
-      DEFAULT_CAST_CHARACTERS = ["Developer", "Police Officer", "Hunter", "Hostage"]
-      DEFAULT_CAST_MEMBERS_CHARACTERS = ["David Black => Developer", "Paola Perotta => Police Officer",
-                                 "Obie Fernandez =>Hunter", "David Chelimsky =>Hostage"]
-      DEFAULT_CAST_MEMBER_IDS = ["nm3901234", "nm4901244", "nm5901235", "nm3601266"]
-      DEFAULT_TRAILER_URL = "http://imdb.com/video/screenplay/vi581042457/"
-      DEFAULT_DIRECTOR = "Yukihiro 'Matz' Matsumoto"
-      DEFAULT_WRITERS = 'David Heinemeier Hansson'
-      DEFAULT_FILMING_LOCATIONS = ["Manhattan, New York, USA"]
-      DEFAULT_COMPANY = "Open Source Community Film Corporation"
-      DEFAULT_GENRES = ["Bromantic", "Syfy"]
-      DEFAULT_LANGUAGES = ["English", "German", "Italian"]
-      DEFAULT_COUNTRIES = ["USA", "Germany", "Italy"]
-      DEFAULT_LENGTH = 146
-      DEFAULT_PLOT = ["David Black, a ruby developer, tries to write his flagship ruby book 'The Well-Grounded Rubyist vol. 186' only to find out that Ruby 9.0.2 and Rails 16.0.3 release dates have been postponed"]
-      DEFAULT_POSTER = "http://ia.media-imdb.com/images/M/MV5BMTY@@.jpg"
-      DEFAULT_RATING = 9.9
-      DEFAULT_VOTES = 110636
-      DEFAULT_MPAA_RATING = "Rated R for dynamic OOD language usage and private methods access (certificate 33087)"
-      DEFAULT_TAGLINE = 'Only One MVC Will Rule Them All.'
-      DEFAULT_YEAR = 2013
-      DEFAULT_RELEASE_DATE = "11 October 2013 (USA)"
-      DEFAULT_REVENUE = 456790
-
-      def initialize(attributes = {})
-        MovieDB::DataExport::IMDB_ATTRIBUTES_HEADERS.each do |attr|
-          self.send("#{attr}=", (attributes.has_key?(attr.to_sym) ? attributes[attr.to_sym] : self.class.const_get("DEFAULT_#{attr.upcase}")))
-        end
-      end
-
-      # Future release of this software will scrap IMDb data from boxofficemojoAPI.com
-      # https://github.com/skozilla/BoxOfficeMojo/tree/master/boxofficemojoAPI
-      #
-      # You can fetch IMDb movie data like this:
-      #   ids = ["2024544", "1800241" ]
-      #
-      #   MovieDB::Movie.find_imdb_id(ids)
-      #
-      #  The fetch data is stored in redis for 1800 seconds
-      #  And then written to a xls file.
-      def self.find_imdb_id(*args)
-        raise ArgumentError if args.empty?
-
-        get_imdb_movie_data(args)
-        get_tmdb_movie_data(args)
-        cache_movie_data_to_redis(args)
-        write_imdb_data_to_xls
-      end
-
-      def self.get_imdb_movie_data(*args) #:nodoc:
-        @imdb_id = []
-        @imdb_movie_data = []
-
-        args.flatten.each do |value|
-          @imdb_id << value
-
-          Movie.new   # Instantiate a movie object.
-          @imdb_movie_data << Imdb::Movie.new(value)
-        end
-        @imdb_movie_data
+      if ids.is_a? String
+        arr << ids
       end
 
-      def self.get_tmdb_movie_data(*args) #:nodoc:
-        @tmdb_movie_data = []
-        Tmdb::Api.key(Movie.key)
-
-        args.flatten.each do |value|
-          @tmdb_movie_data << Tmdb::Movie.detail("tt#{value}")
-        end
-        @tmdb_movie_data
+      if ids.is_a? Numeric
+        arr << ids.to_s
       end
 
-      def self.cache_movie_data_to_redis(*args) #:nodoc:
-        @db_redis ||= Redis.new
-        @db_redis.del "revenue"
-
-        args.flatten.each_with_index do |value, idx|
-          imdb_movie_data = @imdb_movie_data[idx]
-          tmdb_movie_data = @tmdb_movie_data[idx]
-
-          MovieDB::DataExport::IMDB_ATTRIBUTES_HEADERS.each do |attr_key|
-            begin imdb_movie_data.send(attr_key)
-              attr_value = imdb_movie_data.send(attr_key)
-            rescue
-              attr_value = tmdb_movie_data['revenue']
-            end
-
-            @db_redis.hset "movie:#{value}", "#{attr_key}", "#{attr_value}" # Adding a hash data type.
-            @db_redis.lpush "#{attr_key}", "#{attr_value}" if attr_value.is_a? Numeric # Adding a list data type.
-
-            @db_redis.expire "#{attr_key}", 1800
-            @db_redis.expire "movie:#{value}", 1800
-          end
+      if ids.is_a? Array
+        ids.each do |n|
+          arr << n
         end
-        @db_redis
       end
 
-      def self.write_imdb_data_to_xls #:nodoc:
-        Movie.export_movie_data(@db_redis, @imdb_id)
-      end
+      return arr.flatten
     end
   end
 end
+

data/lib/movieDB/base.rb

@@ -1,14 +1,7 @@
-require 'rubygems'
-require 'MovieDB/status_checker'
-require 'MovieDB/movie_error'
-
-module MovieDB #:nodoc
-# MoviesDB is not a datastore gem. Rather, it is a high-level statistical software that performs
+module MovieDB
+# MoviesDB is not a data store gem. Rather, it is a high-level statistical software that performs
 # mathematical computations for analyzing film data from imdb.
 # It is a solution to the common problem of deducing logical hypothesis based off movie data.
-  class Base
-    include StatusChecker
-    include MovieError
-  end
+  class Base; end
 end
 $:.unshift File.expand_path('..', __FILE__)

data/lib/movieDB/data_analysis/statistics.rb

@@ -0,0 +1,85 @@
+require 'daru'
+
+module MovieDB
+  module DataAnalysis
+    module Statistics
+      def numeric_vals
+        %w(votes budget rating revenue length year mpaa_rating popularity vote_count vote_average runtime)
+      end
+
+      module_function :numeric_vals
+
+      stats = [:mean, :std, :sum, :count, :max, :min, :min, :product, :standardize, :describe, :covariance, :correlation, :worksheet]
+
+      stats.each do |method_name|
+        define_method method_name do |**args|
+          dataframes_stats(method_name, args)
+        end
+      end
+
+      private
+
+        def dataframes_stats(method, filters = {})
+          raise ArgumentError, 'Please provide 2 or more IMDd ids.' if $movie_data.length <= 1
+
+          @data_key = {}
+          @index = []
+
+          if filters.empty?
+            $movie_data.each_with_index do |movie, _|
+              value_count = []
+
+              movie.each_pair do |k, v|
+                @data_key[(movie['title'].sub(" ", "_").downcase)] = value_count << (MovieDB::DataAnalysis::Statistics.numeric_vals.any? { |word| word == k } ? v.to_i : v.split(' ').count)
+                @index << k.to_sym
+              end
+            end
+          else
+            case filters.keys[0]
+            when :only
+              $movie_data.each_with_index do |movie, _|
+                value_count ||= []
+
+                filters.values.flatten.each do |filter|
+
+                  mr = movie.reject { |k, _| k != filter.to_s }
+
+                  mr.each_pair do |k, v|
+                    @data_key[(movie['title'].sub(" ", "_").downcase)] = value_count << (MovieDB::DataAnalysis::Statistics.numeric_vals.any? { |word| word == k } ? v.to_i : v.join(' ').split(' ').count)
+                    @index << k.to_sym
+                  end
+                end
+              end
+            when :except
+              $movie_data.each_with_index do |movie, _|
+
+                filters.values.flatten.each do |filter|
+                  mr = movie.reject { |k, _| k == filter.to_s }
+                  value_count = []
+
+                  mr.each_pair do |k, v|
+                    @data_key[(movie['title'].sub(" ", "_").downcase)] = value_count << (MovieDB::DataAnalysis::Statistics.numeric_vals.any? { |word| word == k } ? v.to_i : v.join(' ').split(' ').count)
+                    @index << k.to_sym
+                  end
+                end
+              end
+            else
+              raise ArgumentError, "#{filters.keys[0]} is not a valid filter."
+            end
+          end
+
+          index = @index.uniq
+
+          movie_numeric_vector = Hash[@data_key.map { |k, v| [k.to_s.gsub('-', '_').to_sym, v] }]
+          compute_stats(method, movie_numeric_vector, index )
+        end
+
+        def compute_stats(method, movie, index)
+          df = Daru::DataFrame.new(movie,
+                                       name: :movie, index: index)
+          method == :worksheet ? df : df.send(method)
+        end
+    end
+  end
+end
+

data/lib/movieDB/data_store.rb

@@ -0,0 +1,83 @@
+require 'redis'
+
+# Movie data fetched from IMDb is stored as a hash data type in redis.
+module MovieDB
+  module DataStore
+    # Create a redis instance
+    # with timeouts.
+    def self.initialize_redis
+      @redis_db ||= Redis.new(connect_timeout: 20, timeout: 20)
+    end
+
+    def imdb_methods
+      [:title, :also_known_as, :cast_members, :cast_characters, :cast_members_characters,
+       :director, :writers, :trailer_url, :genres, :languages, :countries, :length, :company, :plot, :plot_synopsis,
+       :plot_summary, :poster, :rating, :votes, :tagline, :mpaa_rating, :year, :release_date, :filming_locations]
+    end
+
+    module_function :imdb_methods
+
+    # The options returns with 3 keys
+    # options[:imdb_tmdb], contains the movie data
+    # options[:id], contains the IMDb id.
+    # options[:expire] contains the expiration time for redis.
+    #
+    # IMDb return a status code of 34 if the resource can not be found.
+
+    def self.write_data(**options)
+      if options[:imdb_tmdb].is_a? Hash
+
+        options.each_pair do |k, v|
+            if v.is_a? Hash
+              if v["status_code"] == "34"
+                puts "#{options[:id]} is an invalid IMDb id."
+              else
+                v.each_pair do |j, w|
+                    @redis_db.hsetnx "#{options[:id]}", "#{j}", "#{w}"
+                end
+              end
+            end
+        end
+      else
+        imdb_methods.each do |method|
+          @redis_db.hsetnx "#{options[:imdb_tmdb].id}", method.to_s, "#{options[:imdb_tmdb].send(method)}"
+        end
+      end
+
+      @redis_db.expire "#{options[:id]}", "#{options[:expire]}"
+    end
+
+    # You can fetch one data at at a time.
+    # Do not send an array of arguments.
+    #
+    # Example the following is accepted.
+    #
+    #   MovieDB::Movie.get_data('0369610')
+    #
+    # Not accepted:
+    #    MovieDB::Movie.get_data(['0369610', 3079380])
+    def self.get_data(method, id = nil)
+      initialize_redis
+
+      case method
+        when :all
+          return @redis_db.hgetall "#{id}"
+        when :hkeys
+          return @redis_db.hkeys "#{id}"
+        when :hvals
+          return @redis_db.hvals "#{id}"
+        when :scan
+          return @redis_db.scan 0
+        when :flushall
+          return @redis_db.flushall
+        when :get
+          return @redis_db.hgetall("#{id}")
+        when :ttl
+          return @redis_db.ttl("#{id}")
+      else
+        raise ArgumentError, "The method #{method} is invalid."
+      end
+    end
+  end
+end
+

data/lib/movieDB/relation/query_methods.rb

@@ -0,0 +1,139 @@
+require "MovieDB/secret"
+require "MovieDB/data_store"
+require "MovieDB/support/reporting"
+require "themoviedb"
+require "imdb"
+
+module MovieDB
+  module Relation
+    module QueryMethods
+      extend MovieDB::Secret
+
+      # Fetch data from IMDb.
+      # Default expiration time for stored object in redis is 1800 seconds.
+      # You can set this value to what ever you like.
+      #
+      # Example:
+      #
+      #   m = MovieDB::Movie.new
+      #
+      #   m.fetch("0369324", "0369662", expire: 84600)
+      #
+      def fetch(*ids, expire: 1800)
+        store_data(ids_to_array(ids), expire)
+
+        # Collect all fetched data and assign to global variable
+        arr = []
+
+        ids.each do |id|
+          arr << (hgetall(id))
+        end
+
+        $movie_data = arr
+      end
+
+      def store_data(ids, expire)
+        check_rate_limit(ids)
+
+        ids.each do |id|
+          movie_exists?(id) ? true : imdb_tmdb_lookup(id, expire)
+        end
+      end
+
+      # Modifying and manipulating redis objects.
+      #  Example:
+      #
+      #   m = MovieDB::Movie.new
+      #
+      #   m.fetch("0369610", "3079380", "0478970")
+      #
+      #   m.hgetall("0369610")
+      [:all, :hkeys, :hvals, :flushall, :ttl].each do |method_name|
+        define_method method_name do |arg|
+          MovieDB::DataStore.get_data(method_name, arg)
+        end
+      end
+
+      alias hgetall all
+
+      [:scan, :flushall].each do |method_name|
+        define_method method_name do
+         mn = MovieDB::DataStore.get_data(method_name)
+          mn.flatten.delete_if { |n| n == "0" } if method_name == :scan
+        end
+      end
+
+      alias all_ids scan
+      alias delete_all flushall
+
+      def mset(record, id, expire)
+        MovieDB::DataStore.write_data(imdb_tmdb: record, id: id, expire: expire)
+      end
+
+      def movie_exists?(id)
+        !hgetall(id).empty?
+      end
+
+      # Fetch the movie from both IMDb and TMDb repositories.
+      #
+      # Future release of this software will scrap IMDb data from boxofficemojoAPI.com
+      # using Mechanize gem.
+      #
+      # Reference https://github.com/skozilla/BoxOfficeMojo/tree/master/boxofficemojoAPI
+      # for the api.
+      def imdb_tmdb_lookup(id, expire) # :nodoc:
+        query_imdb(id, expire)
+        query_tmdb(id, expire)
+      end
+
+      def query_imdb(id, expire) # :nodoc:
+        # Query IMDb
+        imdb = Imdb::Movie.new(id)
+
+        raise NameError, "#{id} is an invalid IMDb id." if imdb.title.nil?
+
+        mset(imdb, id, expire)
+      end
+
+      def query_tmdb(id, expire) # :nodoc:
+        Tmdb::Api.key(MovieDB::Secret.key)
+
+        tmdb = Tmdb::Movie.detail("tt#{id}")
+
+        raise NameError, "#{id} is an invalid TMDb id." if tmdb.nil?
+
+        mset(tmdb, id, expire)
+      end
+
+      def fetch_data(method, ids = nil)
+        if ids.nil?
+          MovieDB::DataStore.get_data(method)
+        else
+          ids.each do |id|
+            MovieDB::DataStore.get_data(method, id)
+          end
+        end
+      end
+
+      private
+
+      def check_argument(method, ids) # :nodoc:
+        if ids.flatten!.empty?
+          raise ArgumentError, "The method #{method}() must contain arguments."
+        end
+      end
+
+      # IMDb current limits are 40 requests every 10
+      # seconds and are limited by IP address, not API key.
+      def check_rate_limit(ids)
+        if ids.length >= 40
+          MovieDB::Support::Reporting.warn(<<-MSG.strip!)
+            Reduce the amount of IMDb ids. \nYou have exceeded the rate limit.
+          MSG
+        else
+          MovieDB::Support::Reporting.silenced
+        end
+      end
+    end
+  end
+end