movie_crawler 0.1.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 2d8c82453127b3fdba97d8ddbd5058ccb4ddc0f6
+  data.tar.gz: 0d5910dac4e714107db946f8f2b2768ea273c13d
+SHA512:
+  metadata.gz: 44a3bd6395d47b8ef7eae0ba323a657aef0db394700fb5175cf87590a2ed392ef5821b932a0a7df31a3229396f714576dc9870c0ee7799b40cc071326ecc0525
+  data.tar.gz: b9af1c5e8b28a244d0b86288b5fe0c026ad25a6264dd29fc1966db4695e75b7ef1ab4a3254bd79b2a91fcf988f318c0053408b0da730c3e54252bad2602ac4a0

data/Gemfile

@@ -0,0 +1,4 @@
+source 'https://rubygems.org'
+
+gem 'nokogiri'
+gem 'iconv'

data/README.md

@@ -0,0 +1,34 @@
+NiceSchedule
+===============
+
+NiceSchedule tries to grabs some information on the [**Niceday**](http://plan.niceday.tw).
+
+## About
+
+Once you have no idea on where to go on weekend, the application will give you some inspiration.
+
+## Usage
+
+Copy the following on command line.
+
+(Choose the directory you want to put it)
+````
+$ git clone git@github.com:ChenLiZhan/SOA-Crawler.git
+````
+
+Then type,
+````
+$ ruby app.rb
+````
+
+## Format
+
+*Ex :*
+
+**title:** 花蓮行(下雨版)
+
+**days:** 3
+
+**route:** 起點:陳記狀元粥鋪>松園別館>花蓮縣石雕博物館>公正包子店>一心泡泡冰>花蓮鐵道文化園區>周家蒸餃>賴桑壽司屋>時光二手書屋>林田山林業文化園區>滿妹豬腳>綠茶肉圓>瑞穗溫泉>慶修院>戴記扁食店>曾記麻糬>終點
+
+**link:** http://plan.niceday.tw/trip/view/id/20973

data/Rakefile

@@ -0,0 +1,8 @@
+require 'rake/testtask'
+
+task default: [:spec]
+
+desc 'Run specs'
+task :spec do
+  sh 'ruby -I lib spec/*_spec.rb'
+end

data/bin/app

@@ -0,0 +1,9 @@
+#!/usr/bin/env ruby
+require 'movie_crawler'
+
+puts MovieInfo.dvd_rank
+puts MovieInfo.us_weekend
+puts MovieInfo.taipei_weekend
+puts MovieInfo.movies('LATEST') # 'first_round','second_round
+puts MovieInfo.movies('FIRST_ROUND')
+puts MovieInfo.movies('SECOND_ROUND')

data/lib/movie_crawler/crawler.rb

@@ -0,0 +1,166 @@
+require 'nokogiri'
+require 'open-uri'
+require 'yaml'
+require 'iconv'
+
+module MovieCrawler
+  # get the info from atmovies
+  class MovieInfo
+    LATEST = 'http://www.atmovies.com.tw/movie/movie_new.html'
+    MOVIE_BASE_URL = 'http://www.atmovies.com.tw/movie/'
+    WHOLE_MOVIEWS_TITLES = "//div[@class = 'title']/a"
+    WHOLE_MOVIEWS_STORIES = "//div[@class = 'story']"
+    WHOLE_MOVIEWS_DATES = "//div[@class = 'date']/b"
+    WHOLE_MOVIEWS_CODES = "//div[@class = 'title']/a/@href"
+    REFLECTION_BASE = 'http://app.atmovies.com.tw/tool/good.cfm?type=film'
+    REFLECTION_FS = '&fs=2'
+    REFLECTION_CLASS = "//div[@class = 'act01']"
+    REFLECTION_SATITLE = '&satitle='
+    REFLECTION_SAID = '&said='
+    REFLECTION_NAME = "//span[@class = 'at21b']"
+    TRAILER_URL = 'http://app.atmovies.com.tw/movie/movie.cfm?action=trailer&film_id='
+    FIRST_ROUND = 'http://www.atmovies.com.tw/movie/movie_now-1.html'
+    SECOND_ROUND = 'http://www.atmovies.com.tw/movie/movie_now2-1.html'
+    ATMOVIES_MAIN_URL = 'http://www.atmovies.com.tw/home/movie_homepage.html'
+
+    # add three rank parser
+    def self.us_weekend
+      result = get_table('1')
+      to_yaml(result)
+    end
+
+    def self.taipei_weekend
+      result = get_table('2')
+      to_yaml(result)
+    end
+
+    def self.dvd_rank
+      result = get_table('3')
+      to_yaml(result)
+    end
+
+    # parse the ranktable info
+    def self.get_table(rankid)
+      doc = open_html(ATMOVIES_MAIN_URL)
+      table = doc.xpath("//*[@id = 'ranklist']/div[" + rankid + ']').text
+      table = table.gsub(' : ', ':').gsub(' ', '').split
+      table = table.each { |item| item.gsub(/[\t\r\n]/, '') }
+      table.pop
+      rankmix(table)
+    end
+
+    # mix the rank info
+    def self.rankmix(t)
+      t.each_with_index.map do |_, index|
+        {
+          index + 1 => t[index].to_s
+        }
+      end
+    end
+
+    # switch to different url accordingly
+    def self.movies(category = 'LATEST')
+      case category.upcase
+      when 'LATEST'
+        url = LATEST
+      when 'FIRST_ROUND'
+        url = FIRST_ROUND
+      when 'SECOND_ROUND'
+        url = SECOND_ROUND
+      end
+      result = movies_parser(url)
+      to_yaml(result)
+    end
+
+    # parse the movies acoordingly
+    def self.movies_parser(url)
+      document = open_html(url)
+      titles = get_titles(document)
+      stories = get_stories(document)
+      dates = get_dates(document)
+      trailers = get_trailer(document)
+      runtimes = get_runtime(document)
+      mix(titles, stories, dates, runtimes, trailers)
+    end
+
+    def self.encode_zh(text)
+      REFLECTION_SATITLE + URI.encode(Iconv.new('big5', 'utf-8').iconv(text)).to_s
+    end
+
+    def self.get_one_movie_name(doc)
+      name = doc.xpath(REFLECTION_NAME).text
+      name.gsub!(/[\t\r\n]/, '')
+    end
+
+    def self.get_reflection(doc)
+      doc.xpath(REFLECTION_CLASS).text.gsub!(/[\t\r\n]/, '').split
+    end
+
+    # get the details of movie
+    def self.movie_details(code)
+      open_html(MOVIE_BASE_URL + code + '/')
+    end
+
+    # open the destination url
+    def self.open_html(url)
+      Nokogiri::HTML(open(url))
+    end
+
+    # get the movie name
+    def self.get_titles(doc)
+      titles = doc.xpath(WHOLE_MOVIEWS_TITLES)
+      titles.map { |title| title.text.gsub(/[\t\n\r]/, '') }
+    end
+
+    # get the storyline of movie
+    def self.get_stories(doc)
+      storylines = doc.xpath(WHOLE_MOVIEWS_STORIES)
+      storylines.map(&:text) # { |story| story.text }
+    end
+
+    # get the runtime of movie
+    def self.get_runtime(doc)
+      days_times = split_day_and_time(doc)
+      days_times.map { |d_t| d_t[0].match(/\d+/).to_s }
+    end
+
+    # get the release date
+    def self.get_dates(doc)
+      days_times = split_day_and_time(doc)
+      days_times.map { |d_t| d_t[1].match(%r{\d+/\d+/\d+}).to_s } # mm/dd/yy
+    end
+
+    def self.split_day_and_time(doc)
+      gap = "\n\t\t\t\t\s\s\s\s\t"
+      days_times = doc.xpath(WHOLE_MOVIEWS_DATES)
+      days_times.map { |d_t| d_t.text.split(gap) }
+    end
+
+    # get the code of movies
+    def self.get_codes(doc)
+      codes = doc.xpath(WHOLE_MOVIEWS_CODES)
+      codes.map { |code| code.value.split('/')[2] }
+    end
+
+    # get the trailer link of the movies
+    def self.get_trailer(doc)
+      codes = get_codes(doc)
+      codes.map { |trailer| TRAILER_URL + trailer }
+    end
+
+    # build the hash for yaml output
+    def self.mix(t, s, d, ti, tr)
+      informations = t.each_with_index.map do |_, index|
+        { 'title' => t[index], 'story' => s[index], \
+          'date' => d[index], 'runtime(minutes)' => ti[index], \
+          'trailer' => tr[index] }
+      end
+      informations
+    end
+
+    # convert the schedules to yaml format
+    def self.to_yaml(mix)
+      mix.to_yaml
+    end
+  end
+end

data/lib/movie_crawler/version.rb

@@ -0,0 +1,3 @@
+module MovieCrawler
+  VERSION = '0.1.0'
+end

data/lib/movie_crawler.rb

@@ -0,0 +1,2 @@
+require 'movie_crawler/crawler.rb'
+require 'movie_crawler/version.rb'

data/movie_crawler.gemspec

@@ -0,0 +1,20 @@
+$:.push File.expand_path("../lib", __FILE__)
+require 'movie_crawler/version'
+Gem::Specification.new do |s|
+  s.name = 'movie_crawler'
+  s.version = MovieCrawler::VERSION
+  s.executables << 'app'
+  s.date = '2014-10-25'
+  s.summary = 'Grab the movies information from the atmovies.com'
+  s.description = 'Grab the movies information from the atmovies.com'
+  s.authors = ['Lee Chen', 'Chen Hung Tu', 'David Yang']
+  s.email = 'chung1350@hotmail.com'
+  s.files = `git ls-files`.split("\n")
+  s.test_files = `git ls-files -- {test,spec,features}/*`.split("\n")
+  s.homepage = 'https://github.com/ChenLiZhan/SOA-Crawler'
+  s.license = 'MIT'
+  s.add_development_dependency 'minitest'
+  s.add_development_dependency 'minitest-rg'
+  s.add_runtime_dependency 'nokogiri', '>= 1.6.2' # v.1.6.2 has documented problems
+  s.add_runtime_dependency 'iconv'
+end

data/spec/movies_spec.rb

@@ -0,0 +1,39 @@
+require 'minitest/autorun'
+require File.expand_path('../../lib/crawler', __FILE__)
+
+LATEST = 'http://www.atmovies.com.tw/movie/movie_new.html'
+SECOND_ROUND = 'http://www.atmovies.com.tw/movie/movie_now2-1.html'
+TRAILER_URL = 'http://app.atmovies.com.tw/movie/movie.cfm?action=trailer&film_id='
+
+# generate a random number to test either latest or second_round
+rand < 0.5 ? url = LATEST : url = SECOND_ROUND
+sample = MovieInfo.movies_parser(url)
+
+describe 'movies_parser should involve' do
+
+  it 'must be non-empty' do
+    sample.wont_be_empty
+  end
+
+  sample.each do |each_head|
+    it 'title must be string' do
+      each_head['title'].must_be_instance_of String
+    end
+
+    it 'story must be string' do
+      each_head['story'].must_be_instance_of String
+    end
+
+    it 'date must match the format' do
+      each_head['date'].must_match %r{\d+/\d+/\d+}
+    end
+
+    it 'runtime must be either empty or digit numbers' do
+      each_head['runtime(minutes)'].must_match(/\A|\d+/)
+    end
+
+    it 'trailer link must accessible' do
+      each_head['trailer'].must_include TRAILER_URL
+    end
+  end
+end

data/spec/rank_spec.rb

@@ -0,0 +1,35 @@
+require 'minitest/autorun'
+require 'minitest/rg'
+require File.expand_path('../../lib/crawler', __FILE__)
+
+# 1 to 3 means diffent test case related to us, taipei, dvd
+rand_rank = rand(1..3)
+sample = MovieInfo.get_table(rand_rank.to_s)
+
+describe 'table should involve' do
+
+  it 'must be non-empty' do
+    sample.wont_be_empty
+  end
+
+  it 'size must be ten' do
+    sample.size.must_equal 10
+  end
+
+  sample.each do |each_movie|
+
+    it 'each_movie must be non-empty' do
+      each_movie.wont_be_empty
+    end
+
+    it 'each_key must be string' do
+      each_movie.each_value { |key| key.must_be_instance_of String }
+    end
+
+    # Hash to_s would be like "{index=>\"movie_name\"}"
+    it 'each_key must match the format' do
+      each_movie.to_s.must_match %r{\{\d+=>\"\W+\"\}}
+    end
+  end
+
+end

metadata

@@ -0,0 +1,114 @@
+--- !ruby/object:Gem::Specification
+name: movie_crawler
+version: !ruby/object:Gem::Version
+  version: 0.1.0
+platform: ruby
+authors:
+- Lee Chen
+- Chen Hung Tu
+- David Yang
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2014-10-25 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: minitest
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: minitest-rg
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: nokogiri
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 1.6.2
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 1.6.2
+- !ruby/object:Gem::Dependency
+  name: iconv
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+description: Grab the movies information from the atmovies.com
+email: chung1350@hotmail.com
+executables:
+- app
+extensions: []
+extra_rdoc_files: []
+files:
+- Gemfile
+- README.md
+- Rakefile
+- bin/app
+- lib/movie_crawler.rb
+- lib/movie_crawler/crawler.rb
+- lib/movie_crawler/version.rb
+- movie_crawler.gemspec
+- spec/movies_spec.rb
+- spec/rank_spec.rb
+homepage: https://github.com/ChenLiZhan/SOA-Crawler
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.4.2
+signing_key: 
+specification_version: 4
+summary: Grab the movies information from the atmovies.com
+test_files:
+- spec/movies_spec.rb
+- spec/rank_spec.rb