imdb-html 0.0.1

data/lib/imdb-html/const.rb

@@ -0,0 +1,5 @@
+module IMDB
+
+	BASE_URI = 'www.imdb.com'
+
+end

data/lib/imdb-html/data/credit.rb

@@ -0,0 +1,41 @@
+module IMDB
+	module Data
+		class Credit
+			include IMDB::REGEXP::HTML
+
+			def initialize noko_tr
+				@tr = noko_tr
+			end
+			attr_reader :tr
+
+			def name
+				tr.at('td.nm').text
+			end
+			def nm
+				tr.at('td.nm a')["href"][ NM ]
+			end
+			def roles
+				roles, parens = tr.at('td.char').inner_html.split( SPLIT_BEFORE_PAREN )
+				roles = "" unless roles
+				parens = "" unless parens
+
+				etc = parens.scan( IN_PARENS ).flatten
+				roles.split(" / ").map do |role|
+
+					name_of_role = role.gsub(%r{<.*?>},"").strip
+					role_ch = role[ CH ]
+
+					hash = {:char => name_of_role}
+					hash[:ch] = role_ch.to_i if role_ch
+					etc.each{ |thing| hash[ thing.to_sym ] = true }
+					hash
+				end
+			end
+
+
+
+
+
+		end
+	end
+end

data/lib/imdb-html/data/episode.rb

@@ -0,0 +1,63 @@
+module IMDB
+	module Data
+		class Episode
+			include IMDB::REGEXP
+
+			def self.parse_params noko_h4
+				noko_h4.episodes_h4.map do |h4|
+					self.new(h4).to_h
+				end
+			end
+
+
+
+			def initialize noko_h4
+				@h4 = noko_h4
+			end
+			attr_reader :h4
+
+			def season_number
+				num = h4.text[ CONTENT::SEASON ]
+				num ? num.rjust(2,'0') : "??"
+			end
+			def episode_number
+				num = h4.text[ CONTENT::EPISODE ]
+				num ? num.rjust(2,'0') : "??"
+			end
+			def se
+				["S", season_number, "E", episode_number].join
+			end
+			def title
+				h4.at('a').text
+			end
+			def tt
+				h4.at('a')["href"][ HTML::TT ]
+			end
+			def date
+				Date.parse h4.next_element.text
+			end
+			def description
+				h4.next_element.next_element.next.text
+			end
+			def cast
+				h4.next_element.next_element.next_element.next_element.cast.map do |elem|
+					credit = IMDB::Data::Credit.new elem
+					{:name => credit.name, :nm => credit.nm.nil? ? 0 : credit.nm.to_i, :roles => credit.roles}
+				end rescue nil
+			end
+			def to_h
+				hash = {:se => se, :title => title}
+				hash[:tt] = tt.to_i if tt
+				hash[:air_date] = date
+				hash[:description] = description.empty? ? nil : description
+				hash[:cast] = cast
+				hash
+			end
+
+
+
+
+
+		end
+	end
+end

data/lib/imdb-html/data/show.rb

@@ -0,0 +1,48 @@
+module IMDB
+	module Data
+		class Show
+			include HTTParty
+			base_uri 'www.imdb.com'
+			headers 'Content-Type' => 'utf-8'
+
+			def initialize tt
+				@tt = tt.to_i
+			end
+
+			def tt
+				@tt.to_s.rjust 7, '0'
+			end
+			def path *args
+				File.join *["/title/tt#{tt}", args].flatten
+			end
+			def http *args
+				self.class.get path *args
+			end
+			def noko *args
+				Nokogiri::HTML http *args
+			end
+
+
+			def episodes
+				Episode.parse_params( noko "epcast" )
+			end
+
+			def json_path
+				File.join "/home/mdt/g/imdb-memo/json/#{tt}.json"
+			end
+
+			def self.process tt
+				show = Show.new tt
+				File.open show.json_path, "w" do |j|
+					j.puts JSON.pretty_generate( show.episodes )
+				end
+			end
+
+
+
+
+
+
+		end
+	end
+end

data/lib/imdb-html/imdb.rb

@@ -0,0 +1,18 @@
+require 'nokogiri'
+require 'httparty'
+
+require 'imdb-html/const'
+require 'imdb-html/regexp/html'
+require 'imdb-html/regexp/content'
+
+require 'imdb-html/nokogiri/node'
+
+require 'imdb-html/data/credit'
+require 'imdb-html/data/episode'
+require 'imdb-html/data/show'
+
+
+module IMDB
+
+
+end

data/lib/imdb-html/nokogiri/node.rb

@@ -0,0 +1,29 @@
+module IMDB
+	class Nokogiri::XML::Node
+
+		def cast
+			pattern = %w{td.ddd td.ellipsis}.detect{ |css_pattern| at(css_pattern) }
+			css(pattern).map(&:parent) if pattern
+		end
+
+		def tn15content
+			at('div#tn15content')
+		end
+
+		def p_seasons
+			tn15content.at('p').css('a').map{ |a| a["href"][/#season-\K.+/] }.compact
+		end
+
+		def episodes_h4
+			name == "h4" ? [self] : css('h4')
+		end
+
+		def link
+			css('@href').count == 1 ? at('@href').value : nil
+		end
+
+
+
+
+	end
+end

data/lib/imdb-html/page/basic_page.rb

@@ -0,0 +1,32 @@
+module IMDB
+	module Page
+		class BasicPage
+			include HTTParty
+			base_uri IMDB::BASE_URI
+			include IMDB::REGEXP::HTML
+
+			def self.stub(stub_str)
+				self.base_uri File.join self.base_uri, stub_str
+			end
+
+
+
+			def target raw_html
+				raw_html[ DIV_ID_PAGECONTENT ][ DIV_ID_FOOTER ]
+				.sub(BILLBOARD, "")
+				.gsub(BR, "")
+			end
+
+			def noko
+				Nokogiri::HTML.parse target
+			end
+
+
+
+
+
+
+
+		end
+	end
+end

data/lib/imdb-html/regexp/content.rb

@@ -0,0 +1,14 @@
+	module IMDB
+		module REGEXP
+			module CONTENT
+
+				SEASON = /season \K[0-9]+/i
+				EPISODE = /episode \K[0-9]+/i
+				SxE = /[Ss]?([0-9]+)[\.EeXx]([0-9]+)/
+
+
+
+
+		end
+	end
+end

data/lib/imdb-html/regexp/html.rb

@@ -0,0 +1,36 @@
+module IMDB
+	module REGEXP
+		module HTML
+			DIV_ID_FOOTER = %r{^\s+<div id="footer".+}m
+			DIV_ID_PAGECONTENT = %r{^\s+<div id="pagecontent">.+}m
+			LINK_REL_CANONICAL = %r{^\s+<link rel="canonical" href="(.*?)">}
+			TITLE = %r{<title>(.*?)</title>}
+			META_CONTENT_TYPE = %r{<meta http-equiv="content-type" content="(.*?)">}
+
+			BILLBOARD = %r{<!-- begin injectable INJECTED_BILLBOARD.*?<!-- _get_ad_for_slot(INJECTED_BILLBOARD) -->}m
+			BR = %r{<br.*?>}
+
+
+			TT = %r{/title/tt\K[0-9]+}
+			NM = %r{/name/nm\K[0-9]+}
+			CH = %r{/character/ch\K[0-9]+}
+
+			TD_NM = %r{<td class="nm"><a href="/name/nm([0-9]+)/">(.*?)</a></td>}
+			TD_DDD = %r{<td class="ddd"> \.\.\. </td>}
+			IN_PARENS = %r{\((.*?)\)}
+			CHAR_CREDIT = %r{<a href="/character/ch([0-9]+)/">(.*?)</a>(?:\s+#{ IN_PARENS })*}
+			TD_CHAR = %r{<td class="char">(?:#{ CHAR_CREDIT })(?: / #{ CHAR_CREDIT })*</td> }
+
+			LINK = %r{<a href="/([a-z]+)/([a-z]{2})([0-9]+)/.*?".?>\s*?(\S.*?)</a>}
+			LINK_PLUS_PARENS = %r{#{LINK}(?:\s*?#{IN_PARENS})*}
+
+			SPLIT_BEFORE_PAREN = %r{ (?=\()}
+
+
+
+
+
+		end
+
+	end
+end

data/test/helper.rb

@@ -0,0 +1,34 @@
+require 'simplecov'
+
+module SimpleCov::Configuration
+  def clean_filters
+    @filters = []
+  end
+end
+
+SimpleCov.configure do
+  clean_filters
+  load_adapter 'test_frameworks'
+end
+
+ENV["COVERAGE"] && SimpleCov.start do
+  add_filter "/.rvm/"
+end
+require 'rubygems'
+require 'bundler'
+begin
+  Bundler.setup(:default, :development)
+rescue Bundler::BundlerError => e
+  $stderr.puts e.message
+  $stderr.puts "Run `bundle install` to install missing gems"
+  exit e.status_code
+end
+require 'test/unit'
+require 'shoulda'
+
+$LOAD_PATH.unshift(File.join(File.dirname(__FILE__), '..', 'lib'))
+$LOAD_PATH.unshift(File.dirname(__FILE__))
+require 'imdb-html'
+
+class Test::Unit::TestCase
+end

data/test/test_imdb-html.rb

@@ -0,0 +1,7 @@
+require 'helper'
+
+class TestImdbHtml < Test::Unit::TestCase
+  should "probably rename this file and start testing for real" do
+    flunk "hey buddy, you should probably rename this file and start testing for real"
+  end
+end

metadata

@@ -0,0 +1,190 @@
+--- !ruby/object:Gem::Specification
+name: imdb-html
+version: !ruby/object:Gem::Version
+  version: 0.0.1
+platform: ruby
+authors:
+- michael d. towle
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2014-06-13 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: mechanize
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.7'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.7'
+- !ruby/object:Gem::Dependency
+  name: nokogiri
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.6'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.6'
+- !ruby/object:Gem::Dependency
+  name: activesupport
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '4.1'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '4.1'
+- !ruby/object:Gem::Dependency
+  name: httparty
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.13'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.13'
+- !ruby/object:Gem::Dependency
+  name: shoulda
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.5'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.5'
+- !ruby/object:Gem::Dependency
+  name: rdoc
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.12'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.12'
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.6'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.6'
+- !ruby/object:Gem::Dependency
+  name: jeweler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.0'
+- !ruby/object:Gem::Dependency
+  name: simplecov
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.8'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.8'
+description: '2'
+email: gametheoretic@gmail.com
+executables: []
+extensions: []
+extra_rdoc_files:
+- LICENSE.txt
+- README.rdoc
+files:
+- ".document"
+- Gemfile
+- Gemfile.lock
+- LICENSE.txt
+- README.rdoc
+- Rakefile
+- lib/imdb-html.rb
+- lib/imdb-html/codes_todo.csv
+- lib/imdb-html/const.rb
+- lib/imdb-html/data/credit.rb
+- lib/imdb-html/data/episode.rb
+- lib/imdb-html/data/show.rb
+- lib/imdb-html/imdb.rb
+- lib/imdb-html/nokogiri/node.rb
+- lib/imdb-html/page/basic_page.rb
+- lib/imdb-html/regexp/content.rb
+- lib/imdb-html/regexp/html.rb
+- test/helper.rb
+- test/test_imdb-html.rb
+homepage: http://github.com/gametheoretic/imdb-html
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.2.2
+signing_key: 
+specification_version: 4
+summary: '1'
+test_files: []