taiwanese_news_parser 0.0.1

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 067e4fb14a386429fc75be6ad0ffd3169677d9a7
+  data.tar.gz: b3c9af0188c2df29573c725c09919530a1a8099c
+SHA512:
+  metadata.gz: 5d6fde19818bfdf3a408dd985fec21b07dcf7a117a321a35e788bba3861cc8179f50203dea756cc8e841a1d3b3f7f1ece849f15a7084af987af9d63de54e1bf4
+  data.tar.gz: f32b846ff9bc138b611bb330bea80612f129ce62951f06d36a7fb953fa383f029bdbcea7007ff134eb6f613358cb773c84ca2e39d883e08df3dc445547e6e786

data/.gitignore

@@ -0,0 +1,17 @@
+*.gem
+*.rbc
+.bundle
+.config
+.yardoc
+Gemfile.lock
+InstalledFiles
+_yardoc
+coverage
+doc/
+lib/bundler/man
+pkg
+rdoc
+spec/reports
+test/tmp
+test/version_tmp
+tmp

data/Gemfile

@@ -0,0 +1,4 @@
+source 'https://rubygems.org'
+
+# Specify your gem's dependencies in taiwanese_news_parser.gemspec
+gemspec

data/LICENSE.txt

@@ -0,0 +1,22 @@
+Copyright (c) 2013 lulalala
+
+MIT License
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.md

@@ -0,0 +1,23 @@
+# TaiwaneseNewsParser
+
+台灣各新聞網站新聞解析器
+
+## Installation
+
+    gem 'taiwanese_news_parser'
+
+## Usage
+
+    TaiwaneseNewsParser.parse(url)
+
+會回傳新聞資訊的 hash
+
+## Contributing
+
+想要協助的朋友可以幫忙為其他新聞網站寫解析器。實作細節請參考個別解析器以及[wiki](https://github.com/lulalala/taiwanese_news_parser/wiki)。
+
+1. Fork it
+2. Create your feature branch (`git checkout -b my-new-feature`)
+3. Commit your changes (`git commit -am 'Add some feature'`)
+4. Push to the branch (`git push origin my-new-feature`)
+5. Create new Pull Request

data/Rakefile

@@ -0,0 +1,4 @@
+require "bundler/gem_tasks"
+require "rspec/core/rake_task"
+
+RSpec::Core::RakeTask.new

data/g0v.json

@@ -0,0 +1,37 @@
+{
+  "author": "lulalala",
+  "contributors": [],
+  "status": "alpha",
+  "name": "taiwanese_news_parser",
+  "name_zh": "台灣新聞網站解析器",
+  "description": "Parse Taiwanese based news agency website news articles",
+  "description_zh": "台灣新聞媒體網站的新聞解析庫",
+  "homepage": "https://github.com/lulalala/taiwanese_news_parser",
+  "document": "http://hack.g0v.tw/taiwanese_news_parser",
+  "repository": {
+    "type": "git",
+    "url": "https://github.com/lulalala/taiwanese_news_parser"
+  },
+  "licenses": [
+    {
+      "type": "MIT"
+    }
+  ],
+  "keywords": [
+    "ruby"
+  ],
+  "audience": [
+    "public"
+  ],
+  "products": [
+    "library",
+    "api"
+  ],
+  "projects": [
+    "taiwanese_news_parser"
+  ],
+  "thumbnail": "",
+  "needs": [
+    "programmer"
+  ]
+}

data/lib/taiwanese_news_parser/parser/apple_daily.rb

@@ -0,0 +1,69 @@
+class TaiwaneseNewsParser::Parser::AppleDaily < TaiwaneseNewsParser::Parser
+  def self.domain
+    'appledaily.com.tw'
+  end
+
+  def self.names
+    %w{蘋果日報}
+  end
+
+  def doc
+    @raw = open(url).read
+    @doc = Nokogiri::HTML(@raw)
+  end
+
+  #url = 'http://www.appledaily.com.tw/appledaily/article/headline/20130414/34951658'
+  def parse
+    @article[:title] = doc.at_css('#h1').text
+
+    @article[:company_name] = parse_company_name
+
+    @article[:content] = doc.css('.articulum').css('p,h2').text
+
+    @article[:reporter_name] = parse_reporter_name()
+
+    @article[:published_at] = self.class.parse_time(doc.css('.gggs time').text)
+
+    clean_up
+
+    @article
+  end
+
+  def parse_company_name
+    '蘋果日報'
+  end
+
+  def parse_reporter_name
+    text = doc.css('.articulum').css('p,h2').text.strip
+    if match = text.match(%r{◎記者(.+)$})
+      return reporter_name = match[1]
+    elsif match = text.match(%r{【(?:記者)?(.+?)[/／╱]})
+      reporter_name = match[1]
+    end
+    reporter_name
+  end
+
+  def clean_url
+    @article[:url].gsub!(%r{/([^/]*)$},'')
+  end
+
+  def self.parse_url_id(url)
+    # removes trailing slash
+    url[%r{http://www.appledaily\.com\.tw/\w+/article/\w+/((?:\d+/)+)},1][0..-2]
+  end
+
+  def self.parse_time(raw_time)
+    valid_formats = ['%Y年%m月%d日%H:%M', '%Y年%m月%d日']
+
+    date = nil
+    valid_formats.each do |format|
+      begin
+        date = DateTime.strptime(raw_time, format)
+      rescue
+      end
+      break if !date.nil?
+    end
+
+    return date
+  end
+end

data/lib/taiwanese_news_parser/parser/china_times.rb

@@ -0,0 +1,76 @@
+class TaiwaneseNewsParser::Parser::ChinaTimes < TaiwaneseNewsParser::Parser
+  def self.domain
+    'chinatimes.com'
+  end
+
+  def self.names
+    %w{中國時報 中時電子報 工商時報 旺報 時報週刊 中天 中視 中廣}
+  end
+
+  def doc
+    @raw = open(url).read
+    @doc = Nokogiri::HTML(@raw)
+  end
+
+  #url = 'http://news.chinatimes.com/mainland/11050505/112013041400325.html'
+  #url = 'http://www.chinatimes.com/realtimenews/%E6%AD%BB%E4%BA%A1%E9%9B%B2%E9%9C%84%E9%A3%9B%E8%BB%8A-%E7%BE%8E%E5%A9%A6%E5%A2%9C%E8%90%BD%E8%BA%AB%E4%BA%A1-20130720002354-260408'
+  def parse
+    @article[:title] = doc.at_css('.page_container header h1').text
+
+    @article[:company_name] = parse_company_name
+
+    @article[:content] = doc.css('.page_container article>p').text
+
+    #@article[:web_published_at] = Time.parse(doc.at_css('#story_update').text)
+
+    @article[:reporter_name] = parse_reporter_name()
+
+    t = doc.css('.reporter time').text.match(/(\d*)年(\d*)月(\d*)日 (\d*):(\d*)/)
+    @article[:published_at] = Time.new(t[1],t[2],t[3],t[4],t[5])
+
+    clean_up
+
+    @article
+  end
+
+  def parse_reporter_name
+    el = doc.at_css('.reporter a[rel=author]')
+    return el.text if el
+
+    text = doc.css('.reporter>text()').text
+    if match = text.match(%r{記者(.+?)[/／╱／]})
+      reporter_name = match[1]
+    elsif match = text.match(%r{【(.+?)[/／╱／]})
+      reporter_name = match[1]
+    else
+      reporter_name = text
+    end
+    reporter_name
+  end
+
+  def parse_company_name
+    n = doc.at_css('.reporter>a').text
+    if n == '時週精選'
+      n = '時報週刊'
+    elsif n == '新聞速報'
+      n = '中時電子報'
+    end
+    n
+  end
+
+  def clean_url
+    cleaner = TaiwaneseNewsParser::UrlCleaner.new('id')
+    @article[:url] = cleaner.clean(@article[:url])
+  end
+
+  def self.parse_url_id(url)
+    url_id = url[%r{http://news\.chinatimes\.com/\w+/(\d+/\d+)},1]
+    if url_id.nil?
+      url_id = url[%r{[^-]*+[^-]*+-(\d+)-\d+},1]
+    end
+    if url_id.nil?
+      url_id = url[%r{chinatimes\.com/(.+)},1]
+    end
+    url_id
+  end
+end

data/lib/taiwanese_news_parser/parser/cna.rb

@@ -0,0 +1,59 @@
+class TaiwaneseNewsParser::Parser::Cna < TaiwaneseNewsParser::Parser
+  def self.domain
+    'cna.com.tw'
+  end
+
+  def self.names
+    %{中央社}
+  end
+
+  def doc
+    @raw = open(url).read
+    @doc = Nokogiri::HTML(@raw)
+  end
+
+  #url = 'http://www.cna.com.tw/News/aSaM/201304120296-1.aspx'
+  def parse
+    @article[:title] = doc.at_css('.news_content h1').text
+
+    @article[:company_name] = '中央社'
+
+    @article[:content] = doc.css('.news_content .box_2').text
+
+    @article[:reporter_name] = parse_reporter_name()
+
+    match = doc.css('.news_content .box_2').text.strip.match( /(\d{3})(\d{2})(\d{2})/ )
+    date = []
+    date[0] = match[1].to_i + 1911
+    date[1] = match[2]
+    date[2] = match[3]
+    date_string = date.join('/') + ' ' + doc.css('.date').text
+    @article[:published_at] = Time.parse(date_string)
+
+    clean_up
+
+    @article
+  end
+
+  def parse_reporter_name
+    text = doc.css('.news_content .box_2').text
+    text = text[/（中央社(.*?)\d{1,2}日/,1]
+    cities = %w{台北 新北 台中 台南 高雄 基隆 新竹 嘉義 桃園 新竹 苗栗 彰化 南投 雲林 嘉義 屏東 宜蘭 花蓮 台東 澎湖 金門 連江}
+    cities.find do |city|
+      text.gsub!(/#{city}(?:縣市)?$/,'')
+    end
+    # TODO proper location name removal
+    if match = text.match(%r{記者(.+)})
+      reporter_name = match[1]
+    end
+    reporter_name
+  end
+
+  def reproduced?
+    false
+  end
+
+  def self.parse_url_id(url)
+    url[%r{/(\d+)(?:\-\d)?\.},1]
+  end
+end

data/lib/taiwanese_news_parser/parser/cts.rb

@@ -0,0 +1,52 @@
+class TaiwaneseNewsParser::Parser::Cts < TaiwaneseNewsParser::Parser
+  def self.domain
+    'cts.com.tw'
+  end
+
+  def self.names
+    ['華視']
+  end
+
+  def self.applicable?(url)
+    url.match(%r{cts\.com\.tw/})
+  end
+
+  def doc
+    @raw = open(url).read
+    @doc = Nokogiri::HTML(@raw)
+  end
+
+  #url = 'http://news.cts.com.tw/cts/politics/201403/201403191393958.html'
+  def parse
+    @article[:title] = doc.at_css('table h1').text
+    @article[:company_name] = parse_company_name
+    @article[:content] = doc.css('#ctscontent p').text
+
+    time = doc.at_css('td.style14 span.info').text[%r{^\d{4}/\d{1,2}/\d{1,2} \d{2}:\d{2}}]
+    @article[:published_at] = Time.parse("#{time}:00")
+
+    @article[:reporter_name] = parse_reporter_name()
+
+    clean_up
+
+    @article
+  end
+
+  def parse_reporter_name
+    text = doc.at_css('td.style14 span.info').text
+    text.gsub!(%r{^\d{4}/\d{1,2}/\d{1,2} \d{2}:\d{2}},'')
+    text.gsub!(%r{地區.+$},'')
+    if text.include?('綜合報導')
+      return nil
+    end
+    text[%r{(.+) 報導},1]
+  end
+
+  def parse_company_name
+    doc.at_css('table table div[align="right"] a img').attr(:alt)
+  end
+
+  def self.parse_url_id(url)
+    url[%r{/cts/.+/\d+/(\d+)\.html},1]
+  end
+end

data/lib/taiwanese_news_parser/parser/ettoday.rb

@@ -0,0 +1,53 @@
+class TaiwaneseNewsParser::Parser::Ettoday < TaiwaneseNewsParser::Parser
+  def self.domain
+    'ettoday.net'
+  end
+
+  def self.names
+    %w{東森}
+  end
+
+  def doc
+    @raw = open(url).read
+    @doc = Nokogiri::HTML(@raw)
+  end
+
+  #url = 'http://www.ettoday.net/news/20130128/158005.htm'
+  def parse
+    @article[:title] = doc.css('[itemprop=headline]').text
+
+    @article[:company_name] = '東森'
+
+    @article[:content] = doc.css('[itemprop=articleBody]>p').text
+
+    @article[:reporter_name] = parse_reporter_name()
+
+    t = doc.css('.news-time').text.match(/(\d*)年(\d*)月(\d*)日 (\d*):(\d*)/)
+    @article[:published_at] = Time.new(t[1],t[2],t[3],t[4],t[5])
+
+    clean_up
+
+    @article
+  end
+
+  def parse_reporter_name
+    text = doc.css('[itemprop=articleBody]').text
+    if match = text.match(%r{記者(.+?)[/／╱／]})
+      reporter_name = match[1]
+    end
+    reporter_name
+  end
+
+  def clean_url
+    cleaner = TaiwaneseNewsParser::UrlCleaner.new()
+    @article[:url] = cleaner.clean(@article[:url])
+  end
+
+  def self.parse_url_id(url)
+    url[%r{http://www\.ettoday\.net/\w+/(\d+/\d+)},1]
+  end
+
+  def reproduced?
+    false
+  end
+end

data/lib/taiwanese_news_parser/parser/liberty_times.rb

@@ -0,0 +1,66 @@
+class TaiwaneseNewsParser::Parser::LibertyTimes < TaiwaneseNewsParser::Parser
+  def self.domain
+    'libertytimes.com.tw'
+  end
+
+  def self.names
+    %{自由時報}
+  end
+
+  def self.applicable?(url)
+    url.match(%r{libertytimes\.com\.tw/liveNews/news\.php})
+  end
+
+  def doc
+    @raw = open(url).read
+    @doc = Nokogiri::HTML(@raw)
+  end
+
+  #url = 'http://www.libertytimes.com.tw/2013/new/apr/13/today-sp2.htm'
+  def parse
+    # new layout uses utf-8
+    @article[:title] = doc.at_css('#newsti text()').text
+    @article[:company_name] = parse_company_name
+    @article[:content] = doc.css('#newsc.news_content').text
+
+    time = doc.at_css('.conttime').text[%r{\d{4}/\d{1,2}/\d{1,2} \d{2}:\d{2}}]
+    if time.nil?
+      match = doc.at_css('.conttime').text.match(%r{(\d{2}):(\d{2})})
+      now = Time.now
+      today = Date.today
+      @article[:published_at] = Time.new(today.year, today.month, today.day, match[1].to_i, match[2].to_i)
+    else
+      @article[:published_at] = Time.parse("#{time}:00")
+    end
+
+    @article[:reporter_name] = parse_reporter_name()
+
+    clean_up
+
+    @article
+  end
+
+  def parse_reporter_name
+    if match = @article[:content].match(%r{〔(.*?)[/／╱](.*?)〕})
+      reporter_name = match[1][%r{記者(.+)},1]
+    elsif match = @article[:content].match(%r{記者(.+?)[/／╱]})
+      reporter_name = match[1]
+    elsif match = @article[:content].match(%r{（文／(.*?)）})
+      reporter_name = match[1]
+    end
+    reporter_name
+  end
+
+  def parse_company_name
+    '自由時報'
+  end
+
+  def clean_url
+    cleaner = TaiwaneseNewsParser::UrlCleaner.new('no')
+    @article[:url] = cleaner.clean(@article[:url])
+  end
+
+  def self.parse_url_id(url)
+    url[%r{news\.php\?no=(\d+)},1]
+  end
+end

data/lib/taiwanese_news_parser/parser/liberty_times_big5.rb

@@ -0,0 +1,51 @@
+class TaiwaneseNewsParser::Parser::LibertyTimesBig5 < TaiwaneseNewsParser::Parser
+  def self.domain
+    'libertytimes.com.tw'
+  end
+
+  def self.names
+    %{自由時報}
+  end
+
+  def self.applicable?(url)
+    url.match(%r{libertytimes\.com\.tw/\d+/\w+/\w+/\d+/.+\.htm})
+  end
+
+  #url = 'http://www.libertytimes.com.tw/2013/new/apr/13/today-sp2.htm'
+  def parse
+    @article[:title] = doc.at_css('#newtitle').text
+    @article[:company_name] = parse_company_name
+    @article[:content] = doc.css('#newsContent>span:not(#newtitle)>p:not(.picture)').text
+
+    @article[:reporter_name] = parse_reporter_name()
+    @article[:published_at] = Time.parse(doc.at_css('#date').text)
+
+    clean_up
+
+    @article
+  end
+
+  def parse_reporter_name
+    if match = @article[:content].match(%r{〔(.*?)[/／╱](.*?)〕})
+      reporter_name = match[1][%r{記者(.+)},1]
+    elsif match = @article[:content].match(%r{記者(.+?)[/／╱]})
+      reporter_name = match[1]
+    elsif match = @article[:content].match(%r{（文／(.*?)）})
+      reporter_name = match[1]
+    end
+    reporter_name
+  end
+
+  def parse_company_name
+    '自由時報'
+  end
+
+  def clean_url
+    cleaner = TaiwaneseNewsParser::UrlCleaner.new('')
+    @article[:url] = cleaner.clean(@article[:url])
+  end
+
+  def self.parse_url_id(url)
+    url[%r{http://www\.libertytimes\.com\.tw/(.*)\.htm},1]
+  end
+end

data/lib/taiwanese_news_parser/parser/now_news.rb

@@ -0,0 +1,53 @@
+class TaiwaneseNewsParser::Parser::NowNews < TaiwaneseNewsParser::Parser
+  def self.domain
+    'nownews.com'
+  end
+
+  def self.names
+    %w{NowNews 今日新聞}
+  end
+
+  def doc
+    @raw = open(url).read
+    @doc = Nokogiri::HTML(@raw)
+  end
+
+  #url = 'http://www.nownews.com/n/2014/03/21/1159861'
+  def parse
+    @article[:title] = doc.css('[itemprop=headline]').text
+
+    @article[:company_name] = self.class.names.first
+
+    @article[:content] = doc.css('[itemprop=articleBody]>p').text
+
+    @article[:reporter_name] = parse_reporter_name()
+
+    t = doc.css('#reporter_info p').text.match(/(\d*)年\s*(\d+)月\s*(\d+)日\D*(\d+):(\d+)/)
+    @article[:published_at] = Time.new(t[1],t[2],t[3],t[4],t[5])
+
+    clean_up
+
+    @article
+  end
+
+  def parse_reporter_name
+    text = doc.css('[itemprop=articleBody]').text
+    if match = text.match(%r{記者(.+?)[/／╱／]})
+      reporter_name = match[1]
+    end
+    reporter_name
+  end
+
+  def clean_url
+    cleaner = TaiwaneseNewsParser::UrlCleaner.new()
+    @article[:url] = cleaner.clean(@article[:url])
+  end
+
+  def self.parse_url_id(url)
+    url[%r{/\d+/\d+/\d+/(\d+)},1]
+  end
+
+  def reproduced?
+    false
+  end
+end

data/lib/taiwanese_news_parser/parser/tvbs.rb

@@ -0,0 +1,46 @@
+class TaiwaneseNewsParser::Parser::Tvbs < TaiwaneseNewsParser::Parser
+  def self.domain
+    'tvbs.com.tw'
+  end
+
+  def self.names
+    ['TVBS']
+  end
+
+  def self.applicable?(url)
+    url.match(%r{tvbs\.com\.tw/entry})
+  end
+
+  def doc
+    @raw = open(url).read
+    @doc = Nokogiri::HTML(@raw)
+  end
+
+  #url = 'http://news.tvbs.com.tw/entry/519673'
+  def parse
+    @article[:title] = doc.at_css('article h1').text
+    @article[:company_name] = parse_company_name
+    @article[:content] = doc.css('article .content').text
+
+    time = doc.at_css('article .meta-data .dateline').text[%r{時間：\d{4}/\d{1,2}/\d{1,2} \d{2}:\d{2}}]
+    @article[:published_at] = Time.parse("#{time}:00")
+
+    @article[:reporter_name] = parse_reporter_name()
+
+    clean_up
+
+    @article
+  end
+
+  def parse_reporter_name
+    doc.at_css('article .meta-data .reporter').text[%r{記者：(.+)},1]
+  end
+
+  def parse_company_name
+    self.class.names.first
+  end
+
+  def self.parse_url_id(url)
+    url[%r{/entry/(\d+)},1]
+  end
+end

data/lib/taiwanese_news_parser/parser/udn.rb

@@ -0,0 +1,43 @@
+class TaiwaneseNewsParser::Parser::Udn < TaiwaneseNewsParser::Parser
+  def self.domain
+    'udn.com'
+  end
+
+  def self.names
+    %{聯合報 聯合晚報}
+  end
+
+  #url = 'http://udn.com/NEWS/NATIONAL/NATS5/7807573.shtml'
+  def parse
+    @article[:title] = doc.at_css('#story_title').text
+    @article[:content] = doc.at_css('#story').text
+
+    #a.web_published_at = Time.parse(doc.at_css('#story_update').text)
+
+    @article[:company_name] = parse_company_name
+    @article[:reporter_name] = parse_reporter_name
+
+    @article[:published_at] = Time.parse(doc.at_css('#story_update').text)
+
+    clean_up
+
+    @article
+  end
+
+  def parse_company_name
+    get_company_name_and_reporter_name.match(%r{^(.*?)[/／╱]})[1]
+  end
+  def parse_reporter_name
+    get_company_name_and_reporter_name[%r{[/／╱]記者(.*)[/／╱]},1]
+  end
+
+  def self.parse_url_id(url)
+    url[%r{\w+/\w+/(\d+)},1]
+  end
+
+private
+
+  def get_company_name_and_reporter_name
+    doc.at_css('#story_author').text[%r{【(.*)】},1]
+  end
+end