web-page-parser 0.25 → 1.0.0

data/spec/parsers/independent_page_parser_spec.rb

@@ -0,0 +1,152 @@
+# -*- coding: utf-8 -*-
+require 'spec_helper'
+require 'net/http'
+include WebPageParser
+
+describe IndependentPageParserFactory do
+  before do
+    @valid_urls = [
+                   'http://www.independent.co.uk/news/world/saudi-authorities-stop-textmessage-tracking-of-women-for-now-9065486.html?somequery=string',
+                   'http://www.independent.co.uk/news/media/tv-radio/the-vex-factor-bbc-produces-almost-identical-programmes-to-those-by-itv-mps-are-told--by-itv-9059695.html',
+                  ]
+    @invalid_urls = [
+                     'http://www.independent.co.uk/sport/rugby/rugby-union/',
+                     'http://www.independent.co.uk/news/business/',
+                     'http://www.independent.co.uk/news/pictures/spencer-tunicks-nude-art-installations-9067645.html',
+                     'http://www.independent.co.uk/sport/rugby/rugby-union/international/chris-robshaw-flanker-answers-all-the-questions-about-red-rose-captaincy-9065632.html'
+                    ]
+  end
+
+  it "should detect independent articles from the url" do
+    @valid_urls.each do |url|
+      IndependentPageParserFactory.can_parse?(:url => url).should be_true
+    end
+  end
+
+  it "should ignore pages with the wrong url format" do
+    @invalid_urls.each do |url|
+      IndependentPageParserFactory.can_parse?(:url => url).should be_nil
+    end
+  end
+  
+end
+
+
+describe IndependentPageParserV1 do
+  describe "when parsing the Saudi authorities article" do
+    before do
+      @valid_options = { 
+        :url => 'www.independent.co.uk/news/world/saudi-authorities-stop-textmessage-tracking-of-women-for-now-9065486.html',
+        :page => File.read("spec/fixtures/independent/saudi-authorities-stop-textmessage-tracking-of-women-for-now-9065486.html"),
+        :valid_hash => 'cfc2994d68b1c59e10bf3225ae557d31'
+      }
+      @pa = IndependentPageParserV1.new(@valid_options)
+    end
+
+    it "should parse the title" do
+      @pa.title.should == "Saudi authorities stop text-message tracking of women… for now"
+    end
+
+    it "should parse the date" do
+      @pa.date.should == DateTime.parse("Jan 16 2014")
+    end
+
+    it "should calculate the hash correctly" do
+      @pa.hash.should == @valid_options[:valid_hash]
+    end
+
+    it "should parse the content" do
+      @pa.content[0].should == "A monitoring system which sends text alerts to Saudi women’s male 'guardians' when they cross the border has been temporarily suspended."
+      @pa.content[3].should == 'Many Saudi women have welcomed the freeze of the measure, including Sabria S. Jawhar, a Saudi columnist and assistant professor of applied linguistics at King Saud bin Abdulaziz University for Health Sciences.'
+      @pa.content.size.should == 11
+    end
+  end
+
+  describe "when parsing the syria article" do
+    before do
+      @valid_options = { 
+        :url => 'http://www.independent.co.uk/news/world/middle-east/innocent-starving-close-to-death-one-victim-of-the-siege-that-shames-syria-9065538.html',
+        :page => File.read('spec/fixtures/independent/innocent-starving-close-to-death-one-victim-of-the-siege-that-shames-syria-9065538.html'),
+        :valid_hash => 'bb552f44352d9807bc0299e5d12f6f0e'
+      }
+      @pa = IndependentPageParserV1.new(@valid_options)
+    end
+
+    it "should parse the title" do
+      @pa.title.should == "Innocent, starving, close to death: One victim of the siege that shames Syria"
+    end
+
+    it "should parse the date" do
+      @pa.date.should == DateTime.parse('Jan 16 2014')
+    end
+
+    it "should calculate the hash correctly" do
+      @pa.hash.should == @valid_options[:valid_hash]
+    end
+
+    it "should parse the content" do
+      @pa.content[0].should == "Israa al-Masri was still a toddler when she lost her battle to cling to life. But the image of her face, pictured just minutes before she finally succumbed to starvation, is becoming the symbol of a wider nightmare."
+      @pa.content.last.should == 'Given the weight of evidence of extreme hunger within Yarmouk, there is no particular reason to doubt the authenticity of other images such as the one above, which is also taken from footage supplied by an activist group, and featured by Al Jazeera. But despite rigorous checks, there is still no sure way of verifying them.'
+      @pa.content.size.should == 37
+    end
+  end
+
+  describe "when parsing the belgian-man article" do
+    before do
+      @valid_options = { 
+        :url => 'http://www.independent.co.uk/news/world/europe/belgian-man-who-skipped-100-restaurant-bills-is-killed-9081407.html',
+        :page => File.read('spec/fixtures/independent/belgian-man-who-skipped-100-restaurant-bills-is-killed-9081407.html'),
+        :valid_hash => '8e7ca0b6a3c3de210c743a36cea05990'
+      }
+      @pa = IndependentPageParserV1.new(@valid_options)
+    end
+
+    it "should parse the title" do
+      @pa.title.should == 'Belgian man who skipped 100 restaurant bills is killed'
+    end
+
+    it "should parse the date" do
+      @pa.date.should == DateTime.parse('Jan 23 2014')
+    end
+
+    it "should calculate the hash correctly" do
+      @pa.hash.should == @valid_options[:valid_hash]
+    end
+
+    it "should parse the content" do
+      @pa.content[0].should == 'He was a “happy-go-lucky” guy who was notorious for spicing up life on benefits in the medieval Belgian town of Ghent by strolling into a restaurant, calmly ordering lobster washed down with the finest brandy or some other gastronomic delight and then walking out without paying the bill.'
+      @pa.content[1].should == 'But, after 100 or so incidents spread over a five-year spree, Titus Clarysse has turned up dead, prompting police to launch an investigation of “murder or manslaughter”.'
+      @pa.content.size.should == 19
+    end
+  end
+
+  describe 'when parsing the uk-sanctuary article' do
+    before do
+      @valid_options = { 
+        :url => 'http://www.independent.co.uk/news/uk/politics/david-cameron-set-for-uturn-over-uk-sanctuary-for-most-vulnerable-syria-refugees-following-aid-agencies-plea-9077647.html',
+        :page => File.read('spec/fixtures/independent/david-cameron-set-for-uturn-over-uk-sanctuary-9077647.html'),
+        :valid_hash => '37a6522307cea4fbae37b8eb52191c1d'
+      }
+      @pa = IndependentPageParserV1.new(@valid_options)
+    end
+
+    it "should parse the title" do
+      @pa.title.should == 'David Cameron set for U-turn over UK sanctuary for most vulnerable Syria refugees following plea by aid agencies'
+    end
+
+    it "should parse the date" do
+      @pa.date.should == DateTime.parse('Jan 23 2014')
+    end
+
+    it "should calculate the hash correctly" do
+      @pa.hash.should == @valid_options[:valid_hash]
+    end
+
+    it "should parse the content" do
+      @pa.content[0].should == 'David Cameron opened the door yesterday for Britain to give sanctuary to some of the most vulnerable Syrian refugees trapped in appalling conditions in neighbouring countries.'
+      @pa.content.join(' ').should_not =~ /brightcove/
+      @pa.content.size.should == 16
+    end
+  end
+
+end

data/spec/parsers/new_york_times_page_parser_spec.rb

@@ -0,0 +1,190 @@
+# -*- coding: utf-8 -*-
+require 'spec_helper'
+require 'net/http'
+include WebPageParser
+
+describe NewYorkTimesPageParserFactory do
+  before do
+    @valid_urls = [
+                   "http://www.nytimes.com/2012/01/28/us/politics/no-more-nice-guys-fans-love-nuclear-newt.html?_r=1&ref=us",
+                   "http://www.nytimes.com/2012/01/29/business/global/greece-in-talks-with-creditors-on-debt-deal.html",
+                  ]
+    @invalid_urls = [
+                     "http://cityroom.blogs.nytimes.com/2012/01/27/the-week-in-pictures-for-jan-27/",
+                     "http://www.nytimes.com/pages/world/asia/index.html"
+                    ]
+  end
+
+  it "should detect new york times articles from the url" do
+    @valid_urls.each do |url|
+      NewYorkTimesPageParserFactory.can_parse?(:url => url).should be_true
+    end
+  end
+
+  it "should ignore pages with the wrong url format" do
+    @invalid_urls.each do |url|
+      NewYorkTimesPageParserFactory.can_parse?(:url => url).should be_nil
+    end
+  end
+  
+end
+
+
+describe NewYorkTimesPageParserV1 do
+  describe "when parsing the Gingrich article" do
+    before do
+      @valid_options = { 
+        :url => 'http://www.nytimes.com/2012/01/27/us/politics/the-long-run-gingrich-stuck-to-caustic-path-in-ethics-battles.html?src=me&ref=general',
+        :page => File.read("spec/fixtures/new_york_times/the-long-run-gingrich-stuck-to-caustic-path-in-ethics-battles.html"),
+        :valid_hash => '7562feadc3db5c9a4c474cc0e9db421a'
+      }
+      @pa = NewYorkTimesPageParserV1.new(@valid_options)
+    end
+
+    it "should parse the title" do
+      @pa.title.should == "Gingrich Stuck to Caustic Path in Ethics Battles"
+    end
+    
+    it "should parse the date" do
+      @pa.date.should == DateTime.parse("Sat Jan 26 2012")
+    end
+
+    it "should calculate the hash correctly" do
+      @pa.hash.should == @valid_options[:valid_hash]
+    end
+
+    it "should parse the content" do
+      @pa.content[0].should == "WASHINGTON — Newt Gingrich had an urgent warning for conservatives: Jim Wright, the Democratic speaker of the House, was out to destroy America."
+      @pa.content[4].should == "Mr. Gingrich, Democrats and Republicans here agree, emerged as one of Washington’s most aggressive practitioners of slash-and-burn politics; many fault him for erasing whatever civility once existed in the capital. He believed, and preached, that harsh language could win elections; in 1990, the political action committee he ran, Gopac, instructed Republican candidates to learn to “speak like Newt,” and offered a list of words to describe Democrats — like decay, traitors, radical, sick, destroy, pathetic, corrupt and shame."
+      @pa.content.size.should == 48
+    end
+  end
+
+  describe "when parsing the hamas-leader article" do
+    before do
+      @valid_options = { 
+        :url => 'http://www.nytimes.com/2012/01/28/world/middleeast/khaled-meshal-the-leader-of-hamas-vacates-damascus.html',
+        :page => File.read("spec/fixtures/new_york_times/khaled-meshal-the-leader-of-hamas-vacates-damascus.html"),
+        :valid_hash => '99ae48e19224402890b380019ca5fbda'
+      }
+      @pa = NewYorkTimesPageParserV1.new(@valid_options)
+    end
+
+    it "should parse the title" do
+      @pa.title.should == "Hamas Leader Abandons Longtime Base in Damascus"
+    end
+    
+    it "should parse the date" do
+      @pa.date.should == DateTime.parse("Fri Jan 27 2012")
+    end
+
+    it "should calculate the hash correctly" do
+      @pa.hash.should == @valid_options[:valid_hash]
+    end
+
+    it "should parse the content" do
+      @pa.content[0].should == "GAZA — Khaled Meshal, the leader of the Palestinian Islamist movement Hamas, has effectively abandoned his longtime base in Syria, where a popular uprising has left thousands dead, and has no plans to return, Hamas sources in Gaza said Friday."
+      @pa.content[4].should == %Q{On Sunday, Mr. Meshal is scheduled to make his first official visit to Jordan since he was deported in 1999. Qatar, one of Mr. Assad’s most vocal Arab critics, played mediator in arranging for Mr. Meshal’s visit to Jordan, which is expected to include a meeting with King Abdullah II. Jordan was the first Arab country to urge Mr. Assad to step down.}
+      @pa.content.last.should == "Ethan Bronner contributed reporting from Jerusalem."
+      @pa.content.size.should == 7
+      @pa.hash.should == @valid_options[:valid_hash]
+    end
+  end
+end
+
+describe NewYorkTimesPageParserV2 do
+  describe "when parsing the Gingrich article" do
+    before do
+      @valid_options = { 
+        :url => 'http://www.nytimes.com/2012/01/27/us/politics/the-long-run-gingrich-stuck-to-caustic-path-in-ethics-battles.html?src=me&ref=general',
+        :page => File.read("spec/fixtures/new_york_times/the-long-run-gingrich-stuck-to-caustic-path-in-ethics-battles.html"),
+        :valid_hash => '7562feadc3db5c9a4c474cc0e9db421a'
+      }
+      @pa = NewYorkTimesPageParserV2.new(@valid_options)
+    end
+
+    it "should parse the title" do
+      @pa.title.should == "Gingrich Stuck to Caustic Path in Ethics Battles"
+    end
+    
+    it "should parse the date" do
+      @pa.date.should == DateTime.parse("Sat Jan 26 2012")
+    end
+
+    it "should calculate the hash correctly" do
+      @pa.hash.should == @valid_options[:valid_hash]
+    end
+
+    it "should parse the content" do
+      @pa.content[0].should == "WASHINGTON — Newt Gingrich had an urgent warning for conservatives: Jim Wright, the Democratic speaker of the House, was out to destroy America."
+      @pa.content[4].should == "Mr. Gingrich, Democrats and Republicans here agree, emerged as one of Washington’s most aggressive practitioners of slash-and-burn politics; many fault him for erasing whatever civility once existed in the capital. He believed, and preached, that harsh language could win elections; in 1990, the political action committee he ran, Gopac, instructed Republican candidates to learn to “speak like Newt,” and offered a list of words to describe Democrats — like decay, traitors, radical, sick, destroy, pathetic, corrupt and shame."
+      @pa.content.size.should == 48
+    end
+  end
+
+  describe "when parsing the French comedian article" do
+    before do
+      @valid_options = { 
+        :url => 'http://www.nytimes.com/2014/01/12/world/europe/show-banned-french-comedian-has-new-one.html',
+        :page => File.read('spec/fixtures/new_york_times/show-banned-french-comedian-has-new-one.html'),
+        :valid_hash => 'ab9cafaac593c12b5b457a5bfdd3eda5'
+      }
+      @pa = NewYorkTimesPageParserV2.new(@valid_options)
+    end
+
+    it "should parse the title" do
+      @pa.title.should == 'Show Banned, French Comedian Has New One'
+    end
+    
+    it "should parse the date" do
+      @pa.date.should == DateTime.parse("Jan 12th 2014")
+    end
+
+    it "should calculate the hash correctly" do
+      @pa.hash.should == @valid_options[:valid_hash]
+    end
+
+    it "should parse the content" do
+      @pa.content[0].should == 'PARIS — A French comedian said Saturday that he had dropped a show banned for its anti-Semitic language and was planning one that would cause no objections.'
+      @pa.content[3].should == '“We live in a democratic country and I have to comply with the laws, despite the blatant political interference,” he said. “As a comedian, I have pushed the debate to the very edge of laughter.”'
+      @pa.content.size.should == 18
+    end
+  end
+
+
+  describe "retrieve_page" do
+    it "should retrieve the article from the nyt website" do
+      @pa = NewYorkTimesPageParserV1.new(:url => "http://www.nytimes.com/2012/08/22/us/politics/ignoring-calls-to-quit-akin-appeals-to-voters-in-ad.html?hp")
+      @pa.title.should =~ /ignoring/i
+    end
+
+    it "should retrieve the full article from the nyt website when given a first page url" do
+      @pa = NewYorkTimesPageParserV1.new(:url => "http://www.nytimes.com/2012/08/21/world/middleeast/syrian-rebels-coalesce-into-a-fighting-force.html?ref=world")
+      @pa.content.size.should > 40
+      @pa = NewYorkTimesPageParserV1.new(:url => "http://www.nytimes.com/2012/08/21/world/middleeast/syrian-rebels-coalesce-into-a-fighting-force.html")
+      @pa.content.size.should > 40
+    end
+
+    it "should retrieve more than the paywall url limit" do
+      urls = []
+      [
+       "http://feeds.nytimes.com/nyt/rss/HomePage",
+       "http://rss.nytimes.com/services/xml/rss/nyt/GlobalHome.xml",
+       "http://feeds.nytimes.com/nyt/rss/NYRegion",
+       "http://www.nytimes.com/services/xml/rss/nyt/World.xml"
+      ].each do |fu|
+        urls += Net::HTTP.get(URI(fu)).scan(/http:\/\/www.nytimes.com\/[0-9]{4}\/[^<"?]+/)
+      end
+
+      urls.uniq!
+      pending("Failing spec but works in practise. Needs a looksee.") { urls.size.should > 25 }
+      urls[0..24].each_with_index do |u,i|
+        @pa = NewYorkTimesPageParserV1.new(:url => u)
+        @pa.page.curl.header_str.to_s.scan(/^Location: .*/).grep(/myaccount.nytimes.com/).should be_empty
+        @pa.title.should_not =~ /^Log In/
+      end
+    end
+
+  end
+
+end

data/spec/parsers/washingtonpost_page_parser_spec.rb

@@ -0,0 +1,114 @@
+# -*- coding: utf-8 -*-
+require 'spec_helper'
+include WebPageParser
+
+describe WashingtonPostPageParserFactory do
+  before do
+    @valid_urls = [
+                   'http://www.washingtonpost.com/world/will-a-bust-follow-the-boom-in-britain/2014/01/18/3677a6ae-7f9d-11e3-97d3-b9925ce2c57b_story.html?tid=hpModule_04941f10-8a79-11e2-98d9-3012c1cd8d1e&hpid=z16',
+                   'http://www.washingtonpost.com/business/technology/nsa-program-defenders-question-snowdens-motives/2014/01/19/091fccaa-811d-11e3-bbe5-6a2a3141e3a9_story.html'
+                  ]
+    @invalid_urls = [
+                     'http://www.washingtonpost.com/politics/',
+                     'http://www.washingtonpost.com/local/',
+                     'http://www.washingtonpost.com/blogs/worldviews/',
+                     'http://www.washingtonpost.com/blogs/worldviews/wp/tag/iran/'
+                    ]
+  end
+
+  it "should detect washpo articles from the url" do
+    @valid_urls.each do |url|
+      WashingtonPostPageParserFactory.can_parse?(:url => url).should be_true
+    end
+  end
+
+  it "should ignore pages with the wrong url format" do
+    @invalid_urls.each do |url|
+      WashingtonPostPageParserFactory.can_parse?(:url => url).should be_nil
+    end
+  end
+  
+end
+
+describe WashingtonPostPageParserV1 do
+
+  describe 'when parsing the al-shabab article' do
+    before do
+      @valid_options = {
+                        :url => 'http://www.washingtonpost.com/world/national-security/pentagon-confirms-al-shabab-leader-killed-in-airstrike-in-somalia/2014/09/05/fc9fee06-3512-11e4-9e92-0899b306bbea_story.html',
+                        :page => File.read('spec/fixtures/washingtonpost/pentagon-confirms-al-shabab-leader-killed.html'),
+                        :valid_hash => 'FIXME'
+                       }
+      @pa = WashingtonPostPageParserV1.new(@valid_options)
+      
+    end
+
+    it 'should parse the title' do
+      @pa.title.should == 'White House confirms al-Shabab leader killed in airstrike in Somalia'
+    end
+
+    it 'should parse the content' do
+      @pa.content[0].should == 'In a major setback for al-Qaeda’s affiliate in East Africa, the Obama administration said Friday it had confirmed the death of a key Somali militant leader who had been targeted in an airstrike earlier in the week.'
+    end
+  end
+
+  describe 'when parsing the bust-boom article' do
+    before do
+      @valid_options = { 
+        :url => 'http://www.washingtonpost.com/world/will-a-bust-follow-the-boom-in-britain/2014/01/18/3677a6ae-7f9d-11e3-97d3-b9925ce2c57b_story.html?tid=hpModule_04941f10-8a79-11e2-98d9-3012c1cd8d1e&hpid=z16',
+        :page => File.read('spec/fixtures/washingtonpost/will-a-bust-follow-the-boom-in-britain.html'),
+        :valid_hash => '86020be298247aaecfc53e3d66f8c6ee'
+      }
+      @pa = WashingtonPostPageParserV1.new(@valid_options)
+    end
+
+    it "should parse the title" do
+      @pa.title.should == 'Will a bust follow the boom in Britain?'
+    end
+   
+    it 'should parse the date in UTC' do
+      @pa.date.should == DateTime.parse("January 18th 2014")
+      @pa.date.zone.should == '+00:00'
+    end
+
+    it "should parse the content" do
+      @pa.content[0].should == 'LONDON — For decades, the modest two-bedroom apartment off Abbey Road was home to some of London’s neediest, a small, leaky outpost in this city’s vast constellation of public housing.'
+      @pa.content[12].should == 'Crazy in the capital'
+      @pa.content.last.should == '“It’s about time the government did something to help,” he said. “I don’t come from a rich family, so I don’t have parents who will give 15,000 pounds for a deposit. That’s not available to me. I’m genuinely pleased Cameron has done something for the working man, which is me.”'
+      @pa.content.size.should == 25
+      @pa.hash.should == @valid_options[:valid_hash]
+    end
+  end
+
+  describe 'when parsing the sgt bowe article' do
+    before do
+      @valid_options = { 
+        :url => 'http://www.washingtonpost.com/world/national-security/sgt-bowe-bergdahls-capture-remains-amystery/2014/01/15/4f8ef686-7e28-11e3-9556-4a4bf7bcbd84_story.html?wprss=rss_national-security',
+        :page => File.read('spec/fixtures/washingtonpost/sgt-bowe-bergdahls-capture-remains-amystery.html'),
+        :valid_hash => '1fd07efe6bfbf5c4551e88d09a663e25'
+      }
+      @pa = WashingtonPostPageParserV1.new(@valid_options)
+    end
+
+    it "should parse the title" do
+      @pa.title.should == 'Sgt. Bowe Bergdahl’s capture remains a mystery'
+    end
+
+    it 'should parse the date in UTC' do
+      @pa.date.should == DateTime.parse("January 15th 2014")
+      @pa.date.zone.should == '+00:00'
+    end
+
+    it "should contain no javascript" do
+      @pa.content.join(' ').should_not =~ /function/
+    end
+
+    it "should parse the content" do
+      @pa.content[0].should == 'Correction: An earlier version of this article misspelled the name of Sgt. Bowe Bergdahl.'
+      @pa.content.size.should == 8 # The blockquote ends up as one big paragraph
+      @pa.hash.should == @valid_options[:valid_hash]
+    end
+
+  end
+
+end