bnext_robot 0.1.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 9c225a805567ccb4cb9db98087c3b6c09bd83f12
+  data.tar.gz: 52872627905196a8ea0b3d8683881c783f4a35e8
+SHA512:
+  metadata.gz: dcfe0e44c660d74c4213a5f9826ecd2cd02a5cc03509c966419a400123d3f7b8d09167badb70c9b571842b9bb92d8980d9fd64cd4dc305ccfb54371b2a996723
+  data.tar.gz: 634dd5d0f6d7e594e670e3b7c1295c2371e0539a95392a7ab091a0bfddf2546f3d24dc5ddd69e6cd0ee8a0b3ec1b79c6c1fbf4584396867eeb915f76fa4726dd

data/Gemfile

@@ -0,0 +1,9 @@
+# A sample Gemfile
+source 'https://rubygems.org'
+
+# gem "rails"
+gem 'oga'
+gem 'xpath'
+gem 'vcr'
+gem 'minitest'
+gem 'webmock'

data/Gemfile.lock

@@ -0,0 +1,40 @@
+GEM
+  remote: https://rubygems.org/
+  specs:
+    addressable (2.3.8)
+    ansi (1.5.0)
+    ast (2.1.0)
+    crack (0.4.2)
+      safe_yaml (~> 1.0.0)
+    hashdiff (0.2.2)
+    mini_portile (0.6.2)
+    minitest (5.8.1)
+    nokogiri (1.6.6.2)
+      mini_portile (~> 0.6.0)
+    oga (1.3.1)
+      ast
+      ruby-ll (~> 2.1)
+    ruby-ll (2.1.2)
+      ansi
+      ast
+    safe_yaml (1.0.4)
+    vcr (2.9.3)
+    webmock (1.22.1)
+      addressable (>= 2.3.6)
+      crack (>= 0.3.2)
+      hashdiff
+    xpath (2.0.0)
+      nokogiri (~> 1.3)
+
+PLATFORMS
+  ruby
+
+DEPENDENCIES
+  minitest
+  oga
+  vcr
+  webmock
+  xpath
+
+BUNDLED WITH
+   1.10.6

data/Rakefile

@@ -0,0 +1,8 @@
+require 'rake/testtask'
+
+task :default => [:spec]
+
+desc 'Run specs'
+Rake::TestTask.new(name=:spec) do |t|
+  t.pattern = 'spec/class_spec/*_spec.rb'
+end

data/bin/bnext_robot

@@ -0,0 +1,15 @@
+#!/usr/bin/env ruby
+require_relative '../lib/ext_class/bnext_robot'
+
+fail ArgumentError, "Usage:  bnext_robot [week/day]\n" if ARGV.count == 0
+
+bnext_robot = BNextRobot.new
+
+type = ARGV[0]
+if type == "week"
+  bnext_robot.show_week_rank
+elsif type == "day"
+  bnext_robot.show_day_rank
+else
+  puts "Please type week or day"
+end

data/bnext_robot.gemspec

@@ -0,0 +1,22 @@
+$LOAD_PATH.push File.expand_path('../robot', __FILE__)
+
+Gem::Specification.new do |s|
+  s.name        =  'bnext_robot'
+  s.version     =  '0.1.0'
+  s.date        =  '2015-10-18'
+  s.executables << 'bnext_robot'
+  s.summary     =  'Web scrapy for Business Next'
+  s.description =  'Web scrapy for Business Next, including showing day/week rank and feeds extraction'
+  s.authors     =  ['Jacky Pan', 'Angela Hung', 'Edison Lee', 'Tony Lee']
+  s.email       =  ['jackypan000@gmail.com', 'angela.hung@iss.nthu.edu.tw', 'dfg1021@hotmail.com.tw', 'tony123930@yahoo.com.tw']
+  s.files       =  `git ls-files`.split("\n")
+  s.test_files  =  `git ls-files spec/*`.split("\n")
+  s.homepage    =  'https://github.com/SOA-Upstart4/Team-HW-1-Ideate-and-Scrape'
+  s.license     =  'MIT'
+
+  s.add_development_dependency 'minitest'
+  s.add_development_dependency 'minitest-rg'
+  s.add_development_dependency 'vcr'
+  s.add_development_dependency 'webmock'
+  s.add_runtime_dependency 'oga'
+end

data/lib/ext_class/bnext_robot.rb

@@ -0,0 +1,96 @@
+require_relative '../int_module/crawl_runner'
+require_relative '../int_module/feeds_filter'
+require_relative '../int_class/feed'
+require_relative '../int_class/filter_condition'
+require 'oga'
+require 'open-uri'
+
+# BNextRobot Extract titles and links of daily/ weekly hot feeds.
+class BNextRobot
+  include Crawler
+  include FeedFilter
+
+  FEED_XPATH = "//a[contains(@class, 'item_title block_link')]/@href"
+  TITLE_XPATH = "//div[contains(@class, 'main_title')]"
+  TAG_XPATH = "//a[contains(@class, 'tag_link')]"
+  INFO_XPATH = "//span[contains(@class, 'info')]"
+  CONTENT_XPATH = "//div[contains(@class, 'content htmlview')]"
+  IMGS_XPATH = "//div[contains(@class, 'content htmlview')]/p/img/@src"
+
+  attr_accessor :day_rank_feeds, :week_rank_feeds
+
+  def initialize
+    load_page('http://www.bnext.com.tw/')
+    analyze
+    init_rank_feeds
+  end
+
+  def analyze
+    cat_tags = @web_data.scan(/<li>.*?<\/li>/)
+    atags = cat_tags.map { |x| x.match(/<a.*?<\/a>/).to_s }
+    hrefs = atags.map { |x| x.match(/href=\".*?\"/).to_s[7..-2] }
+    cat_names = atags.map { |x| x.match(/>.+?</).to_s[1..-2] }
+    cats_pair = cat_names.zip(hrefs).select { |n, ref| ref.start_with? 'categories' }
+
+    @cats = Hash.new(false)
+    cats_pair.map { |n, ref| @cats[n] = @domain + ref }
+    nil
+  end
+
+  def show_day_rank
+    @day_rank_feeds.map { |feed| puts "#{feed.title}: #{feed.link}" }
+    nil
+  end
+
+  def show_week_rank
+    @week_rank_feeds.map { |feed| puts "#{feed.title}: #{feed.link}" }
+    nil
+  end
+
+  def init_rank_feeds
+    token_gen = ["//div[@id = '", "_rank']//a[@class = 'content']"]
+    document = Oga.parse_html(@web_data)
+
+    day_rank_hrefs = document.xpath(token_gen.join('day') + '/@href').map(&:text)
+    week_rank_hrefs = document.xpath(token_gen.join('week') + '/@href').map(&:text)
+
+    day_rank_titles = document.xpath(token_gen.join('day')).map(&:text)
+    week_rank_titles = document.xpath(token_gen.join('week')).map(&:text)
+
+    day_rank = day_rank_titles.zip(day_rank_hrefs).select { |title, href| href.start_with? '/' }
+    day_rank = day_rank.map { |title, href| [title, @domain + href[1..-1]] }
+    week_rank = week_rank_titles.zip(week_rank_hrefs).select { |title, href| href.start_with? '/' }
+    week_rank = week_rank.map { |title, href| [title, @domain + href[1..-1]] }
+
+    @day_rank_feeds = day_rank.map { |title, href| Feed.new(title, "", "", [], href, "") }
+    @week_rank_feeds = week_rank.map { |title, href| Feed.new(title, "", "", [], href, "") }
+    nil
+  end
+
+  def get_feeds(cat, page_no)
+    # TODO: parse all feeds @ page: page_no
+    query_url = @domain + "categories/#{cat}/?p=#{page_no}"
+    document = Oga.parse_html(open(query_url))
+    path = document.xpath(FEED_XPATH).map(&:text)
+    # path.each do |feed_id|
+    #   feed = _extract_feed(feed_id)
+    #   puts "Title: #{feed.title}"
+    #   puts "Author: #{feed.author}"
+    #   puts "Date: #{feed.date}"
+    #   puts "Tags: " + feed.tags.join(", ")
+    # end
+    path.map { |feed_id| _extract_feed(feed_id) }
+  end
+
+  def _extract_feed(feed_id)
+    query_url = @domain[0..-2] + "#{feed_id}"
+    document = Oga.parse_html(open(query_url))
+    title = document.xpath(TITLE_XPATH).text
+    author = document.xpath(INFO_XPATH)[0].text.gsub('撰文者：'.force_encoding('ascii-8bit'), '')
+    date = document.xpath(INFO_XPATH)[1].text.gsub('發表日期：'.force_encoding('ascii-8bit'), '')
+    content = document.xpath(CONTENT_XPATH).text
+    tags = document.xpath(TAG_XPATH).map(&:text)
+    imgs = document.xpath(IMGS_XPATH).map(&:text)
+    Feed.new(title, author, date, tags, query_url, content, imgs)
+  end
+end

data/lib/int_class/feed.rb

@@ -0,0 +1,23 @@
+
+
+class Feed
+
+    attr_accessor :title, :author, :date, :tags, :link, :content, :imgs
+
+    def initialize( title='', author='', date='', tags=[], link='', content='', imgs=[] )
+        @title = title
+        @author = author
+        @date = date
+        @tags = tags
+        @link = link
+        @content = content
+        @imgs = imgs
+    end
+
+    def to_hash()
+    	symbs = %w(title author date tags link content imgs)
+    	values = [@title, @author, @date, @tags, @link, @content, @imgs]
+    	Hash[symbs.zip(values)]
+    end
+
+end

data/lib/int_class/filter_condition.rb

@@ -0,0 +1,41 @@
+
+
+class FilterCondition
+
+    attr_accessor :conds
+
+    def initialize()
+        @conds = Hash.new( false )
+    end
+
+    def date_must_before( date )
+        @conds[ "date_must_before" ] = date
+        self
+    end
+
+    def date_must_after( date )
+        @conds[ "date_must_after" ] = date
+        self
+    end
+
+    def tags_must_include( tags )
+        @conds[ "tags_must_include" ] = tags
+        self
+    end
+
+    def tags_must_exclude( tags )
+        @conds[ "tags_must_exclude" ] = tags
+        self
+    end
+
+    def title_must_include( terms )
+        @conds[ "title_must_include" ] = terms
+        self
+    end
+
+    def designated_authors( authors )
+        @conds[ "designated_authors" ] = authors
+        self
+    end
+
+end

data/lib/int_module/crawl_runner.rb

@@ -0,0 +1,26 @@
+require 'open-uri'
+
+module Crawler
+
+    attr_accessor :cats, :web_data, :domain
+
+    def load_page( url )
+        begin
+            @domain = url
+            @domain += "/" unless @domain.end_with? "/"
+            open( url ) { |f| @web_data = f.read }
+            1
+        rescue
+            0
+        end
+    end
+
+    def analyze
+        raise NotImplementedError.new("#{self.class.name}#analyze is an abstract method.")
+    end
+
+    def get_feeds( cat, max_num )
+        raise NotImplementedError.new("#{self.class.name}#get_feeds is an abstract method.")
+    end
+
+end

data/lib/int_module/feeds_filter.rb

@@ -0,0 +1,11 @@
+require_relative '../int_class/feed'
+require_relative '../int_class/filter_condition'
+
+module FeedFilter
+
+    def filter_feeds( feeds, condition )
+        # Implement filtering
+        feeds
+    end
+
+end

data/spec/class_spec/bnext_robot_spec.rb

@@ -0,0 +1,72 @@
+require 'minitest/autorun'
+require 'vcr'
+require 'webmock/minitest'
+require 'yaml'
+require_relative '../../lib/ext_class/bnext_robot'
+
+day_rank = [
+  "郭台銘投資製造的雲馬X1被爆抄很大，雲造科技回應：並無專利侵權行為: http://www.bnext.com.tw/article/view/id/37685",
+  "各位低持股的老闆皮緊一點！狼來了-談矽品案: http://www.bnext.com.tw/article/view/id/37666",
+  "Facebook開暗門，透過背景重新整理功能榨乾iPhone電力: http://www.bnext.com.tw/article/view/id/37684",
+  "高招！矽品提收購無效之訴，訴訟若拖2~3年不利日月光: http://www.bnext.com.tw/article/view/id/37672",
+  "台灣首次大型VR娛樂應用登場！中華職棒總冠軍賽，Lamigo桃猿熱鬧開打！: http://www.bnext.com.tw/article/view/id/37683",
+  "Gogoro創辦人陸學森：過去20年來，我學會的10件事。: http://www.bnext.com.tw/article/view/id/37688",
+  "被雷軍及郭台銘都看好的智慧電動車雲馬X1！90後創業家玩出新設計: http://www.bnext.com.tw/article/view/id/37663",
+  "台積電版 iPhone 秒殺三星版？別急，來看看這個測試: http://www.bnext.com.tw/ext_rss/view/id/1010354"
+]
+
+week_rank = [
+  "台積電勝三星？iPhone 6s 的 A9處理器事件總整理: http://www.bnext.com.tw/ext_rss/view/id/996449",
+  "Excel記帳雲端進化！Google表單比記帳App還好用: http://www.bnext.com.tw/ext_rss/view/id/955360",
+  "一台iPhone 6s竟有16種版本？性能有差異，消費者只能認了？: http://www.bnext.com.tw/ext_rss/view/id/1002363",
+  "傳華碩不滿？微軟自製筆電買氣旺、OEM廠或遭消滅？: http://www.bnext.com.tw/article/view/id/37652",
+  "蘋果穩居冠軍、Facebook強勢增長、Paypal首度進榜！解讀2015全球百大品牌: http://www.bnext.com.tw/article/view/id/37607",
+  "圖解行動支付兩大模式，你的錢未來這樣用！: http://www.bnext.com.tw/article/view/id/37609",
+  "韓流退燒？LG：韓企全球地位動搖、市佔全面敗退: http://www.bnext.com.tw/article/view/id/37624",
+  "消費者眼球都在哪？世界即時通訊及社群媒體使用情形分析: http://www.bnext.com.tw/article/view/id/37667"
+]
+
+VCR.configure do |config|
+    config.cassette_library_dir = './spec/testfiles/vcr_cassettes'
+    config.hook_into :webmock
+end
+
+bnext_robot = nil
+
+VCR.use_cassette('bnext_mainpage') do
+  bnext_robot = BNextRobot.new
+
+  describe "Get correct day rank articles" do
+
+    it 'has the right number of daily articles' do
+      bnext_robot.day_rank_feeds.size.must_equal day_rank.size
+    end
+
+    it 'has the right content' do
+      content = bnext_robot.day_rank_feeds.map { |feed| "#{feed.title.force_encoding("utf-8")}: #{feed.link.force_encoding("utf-8")}" }
+      content.must_equal day_rank
+    end
+  end
+
+  describe "Get correct week rank articles" do
+
+    it 'has the right number of daily articles' do
+      bnext_robot.week_rank_feeds.size.must_equal week_rank.size
+    end
+
+    it 'has the right content' do
+      content = bnext_robot.week_rank_feeds.map { |feed| "#{feed.title.force_encoding("utf-8")}: #{feed.link.force_encoding("utf-8")}" }
+      content.must_equal week_rank
+    end
+  end
+end
+
+VCR.use_cassette('bnext_techpage') do
+  bnext_tech = bnext_robot.get_feeds("tech", 1)
+  describe "Get correct list of each category" do
+
+    it 'get right number of feeds' do
+      bnext_tech.size.must_equal 20
+    end
+  end
+end