rfilma 0.1.1

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 7647d761f2d69f1209da458110a6aff8315e770a
+  data.tar.gz: 11503bae8df3a9ad5c102d12ebed554278b4406e
+SHA512:
+  metadata.gz: a384e53fb27a1d3127a3ddb2e6879ad0bcdeebf66dc3c6bda41884f81273ce2ac7bce0c3317fccde60d519847c4745f7b779df707ff589689e5d3f3a588cb654
+  data.tar.gz: 146855783ab0c18329b0508ecfee91f9d8282cbe25f69f1d6f4c378c3ca24ef45f991a8591b77b010c3d4715d0996406398bfe2d6b974d6f040e341898c80d62

data/lib/config/mongoid.yml

@@ -0,0 +1,8 @@
+production:
+  sessions:
+    default:
+      database: filmadb
+      hosts:
+        - localhost:27017
+  options:
+    raise_not_found_error: false

data/lib/rfilma.rb

@@ -0,0 +1,42 @@
+require "rfilma/crawler"
+require "rfilma/crawlerdb"
+require "rfilma/pelicula"
+
+class RFilma
+
+	attr_accessor :crawler, :crawlerdb
+
+	def initialize
+		@crawler = Crawler.new
+		@crawlerdb = CrawlerDB.new
+	end
+
+	def buscar_por_titulo(titulo,web=false)
+		if web
+			result = @crawler.buscar_por_titulo(titulo)
+			result.each{|a| @crawlerdb.guardar_pelicula(a["id"])}
+		else
+			result = @crawlerdb.buscar_por_titulo(titulo)
+		end
+		return result
+	end
+
+	# Entrada: 1->(A-Z) 2->(0-9) 3->(*)
+	def actualizar_por_letra(caracter)
+		pelis = []
+		if caracter.upcase.match(/([A-Z])/)
+			pelis = @crawlerdb.procesar_paginas(caracter.upcase.match(/([A-Z])/)[1])
+		elsif caracter.match(/([0-9])/)
+			pelis = @crawlerdb.procesar_paginas("0-9")
+		else
+			pelis = @crawlerdb.procesar_paginas("*")
+		end
+		@crawlerdb.guardar_peliculas(pelis)
+	end
+
+	def actualizar_todo
+		@crawlerdb.procesar_todo
+	end
+
+
+end

data/lib/rfilma/crawler.rb

@@ -0,0 +1,70 @@
+require "mechanize"
+require "set"
+require "thread/pool"
+
+class Crawler
+
+	def initialize
+		@a = Mechanize.new{|op|
+			op.user_agent_alias = "Windows Mozilla"			
+		}
+	end
+
+
+	def obtener_pelicula(id)
+		data = {}
+		page = @a.get("http://www.filmaffinity.com/es/film#{id}.html").body
+		doc = Nokogiri::HTML(page)
+		data["id"] = id
+		data["titulo"] = doc.xpath("//h1[@id='main-title']/a/span").inner_html
+		data["puntuacion"] = doc.xpath('//div[@id="movie-rat-avg"]').text.strip.gsub(",",".").to_f
+		begin
+			data["portada"] = doc.xpath('//div[@id="movie-main-image-container"]/a')[0]["href"]
+		rescue
+			data["portada"] = doc.xpath('//div[@id="movie-main-image-container"]/img')[0]["src"]
+		end
+		doc.xpath('//dl[@class="movie-info"]/dt').each{|m|
+			dt = m.inner_html
+			case 
+			when dt.include?("Título original")
+				data["titulo_original"] = m.next_element.text
+			when dt.include?("Año")	
+				data["año"] = m.next_element.text.to_i		
+			when dt.include?("Duración")
+				data["duracion"] = m.next_element.text.match('(\d*)')[1].to_i
+			when dt.include?("País")
+				data["pais"] = m.next_element.at('img')['title']
+			when dt.include?("Director")		
+				data["director"] = m.next_element.search('a').map{|e| e.inner_html.strip}
+			when dt.include?("Guión")
+				data["guion"] = m.next_element.text.split(",").map{|e|e.strip}
+			when dt.include?("Música")
+				data["musica"] = m.next_element.text.split(",").map{|e|e.strip}
+			when dt.include?("Fotografía")
+				data["fotografia"] = m.next_element.text.split(",").map{|e|e.strip}
+			when dt.include?("Reparto")
+				data["reparto"] = m.next_element.text.split(",").map{|e|e.strip}
+			when dt.include?("Productora")
+				data["productora"] = m.next_element.text
+			when dt.include?("Género")		
+				data["genero"] = m.next_element.search('a').map{|e| e.inner_html}
+			when dt.include?("Web")	
+				data["web"] = m.next_element.text
+			when dt.include?("Sinopsis")	
+				data["sinopsis"] = m.next_element.text			
+			end
+		}
+		data
+	end
+
+	def buscar_por_titulo(titulo)
+		indices_pelis = []
+		p = @a.get("http://www.filmaffinity.com/es/search.php?stext=#{titulo.strip.gsub(" ","+")}&stype=title").body
+		doc = Nokogiri::HTML(p)
+		doc.xpath('//div[@class="movie-card movie-card-1"]').each{|mc|
+			indices_pelis << mc["data-movie-id"].to_i
+		}
+		indices_pelis.map{|i| obtener_pelicula(i)}
+	end
+	
+end

data/lib/rfilma/crawlerdb.rb

@@ -0,0 +1,66 @@
+require_relative "pelicula"
+require_relative "crawler"
+
+class CrawlerDB < Crawler 
+
+	def initialize
+		super
+	end
+
+	def obtener_pelicula(id)
+		Pelicula.where(id: id).as_json
+	end
+
+	def buscar_por_titulo(titulo)
+		Pelicula.where(titulo: /#{titulo}/i).as_json
+	end
+
+	def guardar_pelicula(id)		
+		p = Crawler.new.obtener_pelicula(id)		
+		m = Pelicula.new(p)
+		m.upsert
+	end
+
+	def guardar_peliculas(ids,nthread=5)
+		pool = Thread.pool(nthread)	
+		ids2 = Pelicula.find(ids).each.map{|idd| idd["id"]}
+		ids3 = (ids - ids2) + (ids2 - ids)	
+		ids3.each{|i|
+			pool.process{
+				guardar_pelicula(i)				
+			}
+		}
+		pool.shutdown
+	end
+
+	def procesar_paginas(letra)
+		pagina = 1
+		# Cualquier categoría tiene más de una página
+		r = ">>"		
+		indices_pelis = []
+		while r.include?(">>")
+			p = @a.get("http://www.filmaffinity.com/es/allfilms_#{letra}_#{pagina}.html").body
+			doc = Nokogiri::HTML(p)
+			r = doc.xpath('//div[@class="pager"]/a[contains(text(),">>")]').text
+			doc.xpath('//div[@class="movie-card movie-card-1"]').each{|mc|
+				indices_pelis << mc["data-movie-id"].to_i
+			}
+			pagina+=1
+		end
+		# Evitamos indices duplicados
+		Set.new(indices_pelis).to_a
+	end
+
+	def procesar_todo
+		cat = ('A'..'Z').to_a << "*" << "0-9"
+		pool = Thread.pool(5)
+		cat.each{|c|
+			pool.process{
+				ra = procesar_paginas(c)
+				guardar_peliculas(ra)
+			}
+		}
+		pool.shutdown
+	end
+end
+

data/lib/rfilma/pelicula.rb

@@ -0,0 +1,28 @@
+require "mongoid"
+
+mongoidyml = File.join(File.dirname(__FILE__),"..","config","mongoid.yml")
+Mongoid.load!(mongoidyml,:production)
+
+class Pelicula
+
+	include Mongoid::Document
+
+	field :id, type: Integer
+	field :titulo, type: String
+	field :titulo_original, type: String
+	field :año, type: Integer
+	field :duracion, type: Integer
+	field :pais, type: String
+	field :director, type: Array 
+	field :guion, type: Array 
+	field :musica, type: Array 
+	field :fotografia, type: Array 
+	field :reparto, type: Array 
+	field :productora, type: String 
+	field :genero, type: Array 
+	field :sinopsis, type: String
+	field :puntuacion, type: Float
+	field :web, type: String
+	field :portada, type: String	
+	field :_id, type: Integer, overwrite: true, default: ->{ id }
+end

metadata

@@ -0,0 +1,49 @@
+--- !ruby/object:Gem::Specification
+name: rfilma
+version: !ruby/object:Gem::Version
+  version: 0.1.1
+platform: ruby
+authors:
+- Jose Antonio PB
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2014-11-08 00:00:00.000000000 Z
+dependencies: []
+description: RFilma is a Ruby library for crawl data from FilmAffinity website
+email: aztuzeca@gmail.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- lib/config/mongoid.yml
+- lib/rfilma.rb
+- lib/rfilma/crawler.rb
+- lib/rfilma/crawlerdb.rb
+- lib/rfilma/pelicula.rb
+homepage: https://github.com/aztuzeca/rfilma
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.4.2
+signing_key: 
+specification_version: 4
+summary: FilmAffinity Crawler
+test_files: []
+has_rdoc: