web-page-parser 1.1.0 → 1.2.2

data/spec/parsers/guardian_page_spec.rb

@@ -248,4 +248,104 @@ describe GuardianPageParserV2 do
     end
   end
 
+  describe "when parsing the new format university extremist speakers article" do
+    before do
+      @valid_options = { 
+        :url => 'http://www.theguardian.com/uk-news/2015/mar/20/theresa-may-drops-rules-ordering-universities-ban-extremist-speakers',
+        :page => File.read("spec/fixtures/guardian/university-extremist-speakers.html"),
+        :valid_hash => 'aeeb32a7acd2de155be83fd7de6446cb'
+      }
+      @pa = GuardianPageParserV2.new(@valid_options)
+    end
+
+    it "should parse the title" do
+      @pa.title.should == "Theresa May drops rules on ordering universities to ban extremist speakers"
+    end
+
+    it "should parse the date in UTC" do
+      @pa.date.should == DateTime.parse("Friday 20 March 2015 17:38:30 GMT")
+      @pa.date.zone.should == '+00:00'
+    end
+
+    it "should parse the content" do
+      @pa.content[0].should == "The home secretary, Theresa May, has been forced to drop new statutory rules under which ministers could order universities and colleges to ban external extremist speakers."
+      @pa.content.size.should == 19
+      @pa.hash.should == @valid_options[:valid_hash]
+    end
+
+  end
+
+  describe "when parsing the Julian Assange article" do
+    before do
+      @valid_options = {
+        :url => 'https://www.theguardian.com/media/2016/dec/24/julian-assange-donald-trump-hillary-clinton-interview',
+        :page => File.read("spec/fixtures/guardian/julian-assange-donald-trump-hillary-clinton-interview.html"),
+        :valid_hash => '2757835e9e028a21b5e47c9199ade005'
+      }
+      @pa = GuardianPageParserV2.new(@valid_options)
+    end
+
+    it "should parse the title" do
+      @pa.title.should == "Julian Assange gives guarded praise of Trump and blasts Clinton in interview"
+    end
+
+    it "should parse the date in UTC" do
+      @pa.date.should == DateTime.parse("Saturday 24 December 2016 18:36:24 GMT")
+      @pa.date.zone.should == '+00:00'
+    end
+
+    it "should parse the content" do
+      @pa.content[0].should == "Julian Assange, the founder of WikiLeaks, has offered guarded praise of Donald Trump, arguing the president-elect “is not a DC insider” and could mean an opportunity for positive as well as negative change in the US."
+      @pa.content.last.should == "Dozens of journalists have been killed in Russia in the past two decades, and Freedom House considers the Russian press to be “not free” and notes: “The main national news agenda is firmly controlled by the Kremlin. The government sets editorial policy at state-owned television stations, which dominate the media landscape and generate propagandistic content.”"
+      @pa.content.size.should == 16
+      @pa.hash.should == @valid_options[:valid_hash]
+    end
+  end
+
+  describe "when parsing an article with duplicate headlines" do
+    before do
+      @valid_options = {
+        :url => 'https://www.theguardian.com/world/2016/dec/31/russia-syria-ceasefire-un-security-council-damascus-kazakhstan',
+        :page => File.read("spec/fixtures/guardian/duplicate-headline.html"),
+      }
+      @pa = GuardianPageParserV2.new(@valid_options)
+    end
+
+    it "should only return one of the titles" do
+      @pa.title.should == "Russia pushes for UN security council support for Syria ceasefire"
+    end
+
+  end
+
+  describe GuardianPageParserV3 do
+    describe "when parsing the Julian Assange article" do
+    before do
+      @valid_options = {
+        :url => 'https://www.theguardian.com/media/2016/dec/24/julian-assange-donald-trump-hillary-clinton-interview',
+        :page => File.read("spec/fixtures/guardian/julian-assange-donald-trump-hillary-clinton-interview.html"),
+        :valid_hash => 'a94b1cfb7abab286ab4e880e3c440d66'
+      }
+      @pa = GuardianPageParserV3.new(@valid_options)
+    end
+
+    it "should parse the title" do
+      @pa.title.should == "Julian Assange gives guarded praise of Trump and blasts Clinton in interview"
+    end
+
+    it "should parse the date in UTC" do
+      @pa.date.should == DateTime.parse("Saturday 24 December 2016 18:36:24 GMT")
+      @pa.date.zone.should == '+00:00'
+    end
+
+    it "should parse the content" do
+      @pa.content[0].should == "Julian Assange, the founder of WikiLeaks, has offered guarded praise of Donald Trump, arguing the president-elect “is not a DC insider” and could mean an opportunity for positive as well as negative change in the US."
+      @pa.content.last.should == "This article was amended on 29 December 2016 to remove a sentence in which it was asserted that Assange “has long had a close relationship with the Putin regime”. A sentence was also amended which paraphrased the interview, suggesting Assange said “there was no need for Wikileaks to undertake a whistleblowing role in Russia because of the open and competitive debate he claimed exists there”. It has been amended to more directly describe the question Assange was responding to when he spoke of Russia’s “many vibrant publications”."
+      @pa.content.size.should == 17
+      @pa.hash.should == @valid_options[:valid_hash]
+    end
+
+  end
+
+  end
+
 end

data/spec/parsers/independent_page_parser_spec.rb

@@ -153,4 +153,56 @@ describe IndependentPageParserV1 do
     end
   end
 
+  describe 'when parsing the lord-burns article' do
+    before do
+      @valid_options = { 
+        :url => 'http://www.independent.co.uk/news/media/lord-burns-channel-4-chairman-forced-to-step-down-by-ministers-amid-privatisation-fears-a6670691.html',
+        :page => File.read('spec/fixtures/independent/lord-burns.html'),
+        :valid_hash => 'f0efb3b2ea91266fe4a551867fcf6fb1'
+      }
+      @pa = IndependentPageParserV1.new(@valid_options)
+    end
+
+    it "should parse the title" do
+      @pa.title.should == 'Lord Burns: Channel 4 chairman forced to step down by ministers amid privatisation fears'
+    end
+
+    it "should parse the date" do
+      @pa.date.should == DateTime.parse('28 September 2015 18:03:52 BST')
+    end
+
+    it "should calculate the hash correctly" do
+      @pa.hash.should == @valid_options[:valid_hash]
+    end
+
+    it "should parse the content" do
+      @pa.content[0].should == 'Ministers have forced Lord Burns, the chairman of Channel 4, to step down, fuelling speculation that the not-for-profit broadcaster is being prepared for privatisation.'
+      @pa.content.size.should == 6
+    end
+  end
+
+  describe 'when parsing the boris-johnson article' do
+    before do
+      @valid_options = {
+        :url => 'http://www.independent.co.uk/news/uk/politics/boris-johnson-warns-low-immigration-could-stall-economic-growth-a6693486.html',
+        :page => File.read('spec/fixtures/independent/boris-johnson.html'),
+        :valid_hash => 'e9fc5ef9502d3b167c00d5cefb308495'
+      }
+      @pa = IndependentPageParserV1.new(@valid_options)
+    end
+
+    it "should parse the title" do
+      @pa.title.should == 'Boris Johnson warns low immigration could stall economic growth'
+    end
+
+    it "should calculate the hash correctly" do
+      @pa.hash.should == @valid_options[:valid_hash]
+    end
+
+    it "should exclude the 'read more' and image widget captions" do
+      @pa.content.to_s.should_not =~ /Farage says/
+      @pa.content.to_s.should_not =~ /A butcher/
+    end
+  end
+
 end

data/spec/parsers/new_york_times_page_parser_spec.rb

@@ -8,10 +8,14 @@ describe NewYorkTimesPageParserFactory do
     @valid_urls = [
                    "http://www.nytimes.com/2012/01/28/us/politics/no-more-nice-guys-fans-love-nuclear-newt.html?_r=1&ref=us",
                    "http://www.nytimes.com/2012/01/29/business/global/greece-in-talks-with-creditors-on-debt-deal.html",
+                   "https://www.nytimes.com/2014/01/12/world/europe/show-banned-french-comedian-has-new-one.html",
+                   "https://www.nytimes.com/2018/06/12/world/asia/trump-kim-policy.html?hp&action=click&pgtype=Homepage&clickSource=story-heading&module=span-ab-top-region&region=top-news&WT.nav=top-news"
                   ]
     @invalid_urls = [
                      "http://cityroom.blogs.nytimes.com/2012/01/27/the-week-in-pictures-for-jan-27/",
-                     "http://www.nytimes.com/pages/world/asia/index.html"
+                     "http://www.nytimes.com/pages/world/asia/index.html",
+                     "https://www.nytimes.com/section/business?module=SectionsNav&action=click&version=BrowseTree&region=TopBar&contentCollection=Business&pgtype=sectionfront",
+                     "https://www.nytimes.com/section/technology/personaltech"
                     ]
   end
 
@@ -151,35 +155,96 @@ describe NewYorkTimesPageParserV2 do
     end
   end
 
+  describe "when parsing the French comedian article with the 2018 formatting" do
+    before do
+      @valid_options = {
+        :url => 'https://www.nytimes.com/2014/01/12/world/europe/show-banned-french-comedian-has-new-one.html',
+        :page => File.read('spec/fixtures/new_york_times/show-banned-french-comedian-has-new-one-2018.html'),
+        :valid_hash => 'ab9cafaac593c12b5b457a5bfdd3eda5'
+      }
+      @pa = NewYorkTimesPageParserV2.new(@valid_options)
+    end
+
+    it "should parse the title" do
+      @pa.title.should == 'Show Banned, French Comedian Has New One'
+    end
+
+    it "should parse the date" do
+      @pa.date.should == DateTime.parse("2014-01-12T05:35:51+00:00")
+    end
+
+    it "should calculate the hash correctly" do
+      @pa.hash.should == @valid_options[:valid_hash]
+    end
+
+    it "should parse the content" do
+      @pa.content[0].should == 'PARIS — A French comedian said Saturday that he had dropped a show banned for its anti-Semitic language and was planning one that would cause no objections.'
+      @pa.content[3].should == '“We live in a democratic country and I have to comply with the laws, despite the blatant political interference,” he said. “As a comedian, I have pushed the debate to the very edge of laughter.”'
+      @pa.content.size.should == 18
+    end
+  end
+
+
+  describe "when parsing trump kim policy article" do
+    before do
+      @valid_options = {
+        :url => 'https://www.nytimes.com/2018/06/12/world/asia/trump-kim-policy.html',
+        :page => File.read('spec/fixtures/new_york_times/trump-kim-policy.html'),
+        :valid_hash => 'ab62998617a2fb91552122a9ac845e4c'
+      }
+      @pa = NewYorkTimesPageParserV2.new(@valid_options)
+    end
+
+    it "should parse the title" do
+      @pa.title.should == 'Vague on Details, Trump Is Betting on ‘Special Bond’ With Kim to Deliver Deal'
+    end
+
+    it "should parse the date" do
+      @pa.date.should == DateTime.parse("2018-06-12T16:35:00+00:00")
+    end
+
+    it "should calculate the hash correctly" do
+      @pa.hash.should == @valid_options[:valid_hash]
+    end
+
+    it "should parse the content" do
+      @pa.content[0].should == 'SINGAPORE — On paper, there is nothing President Trump extracted from North Korea’s leader, Kim Jong-un, in their summit meeting that Mr. Kim’s father and grandfather had not already given to past American presidents.'
+      @pa.content[8].should == '“I don’t know that I’ll ever admit that,” he added, “but I’ll find some kind of an excuse.”'
+      @pa.content.last.should == "Whatever he gets, it will be judged by one standard: whether he has “solved” the North Korea problem, as he vowed he would, rather than passing it on to his successor."
+      @pa.content.size.should == 28
+    end
+  end
+
 
   describe "retrieve_page" do
     it "should retrieve the article from the nyt website" do
-      @pa = NewYorkTimesPageParserV1.new(:url => "http://www.nytimes.com/2012/08/22/us/politics/ignoring-calls-to-quit-akin-appeals-to-voters-in-ad.html?hp")
+      @pa = NewYorkTimesPageParserV2.new(:url => "http://www.nytimes.com/2012/08/22/us/politics/ignoring-calls-to-quit-akin-appeals-to-voters-in-ad.html?hp")
       @pa.title.should =~ /ignoring/i
     end
 
     it "should retrieve the full article from the nyt website when given a first page url" do
-      @pa = NewYorkTimesPageParserV1.new(:url => "http://www.nytimes.com/2012/08/21/world/middleeast/syrian-rebels-coalesce-into-a-fighting-force.html?ref=world")
+      @pa = NewYorkTimesPageParserV2.new(:url => "http://www.nytimes.com/2012/08/21/world/middleeast/syrian-rebels-coalesce-into-a-fighting-force.html?ref=world")
       @pa.content.size.should > 40
-      @pa = NewYorkTimesPageParserV1.new(:url => "http://www.nytimes.com/2012/08/21/world/middleeast/syrian-rebels-coalesce-into-a-fighting-force.html")
+      @pa = NewYorkTimesPageParserV2.new(:url => "http://www.nytimes.com/2012/08/21/world/middleeast/syrian-rebels-coalesce-into-a-fighting-force.html")
       @pa.content.size.should > 40
     end
 
     it "should retrieve more than the paywall url limit" do
       urls = []
       [
-       "http://feeds.nytimes.com/nyt/rss/HomePage",
+       "http://rss.nytimes.com/services/xml/rss/nyt/HomePage.xml",
        "http://rss.nytimes.com/services/xml/rss/nyt/GlobalHome.xml",
-       "http://feeds.nytimes.com/nyt/rss/NYRegion",
+       "http://rss.nytimes.com/services/xml/rss/nyt/NYRegion.xml",
        "http://www.nytimes.com/services/xml/rss/nyt/World.xml"
       ].each do |fu|
-        urls += Net::HTTP.get(URI(fu)).scan(/http:\/\/www.nytimes.com\/[0-9]{4}\/[^<"?]+/)
+        next if urls.size > 25
+        urls += Net::HTTP.get(URI(fu)).scan(/https:\/\/www.nytimes.com\/[0-9]{4}\/[^<"?]+/)
+        urls.uniq!
       end
 
-      urls.uniq!
-      pending("Failing spec but works in practise. Needs a looksee.") { urls.size.should > 25 }
+      urls.size.should > 25
       urls[0..24].each_with_index do |u,i|
-        @pa = NewYorkTimesPageParserV1.new(:url => u)
+        @pa = NewYorkTimesPageParserV2.new(:url => u)
         @pa.page.curl.header_str.to_s.scan(/^Location: .*/).grep(/myaccount.nytimes.com/).should be_empty
         @pa.title.should_not =~ /^Log In/
       end

data/spec/parsers/rt_page_parser_spec.rb

@@ -0,0 +1,87 @@
+# -*- coding: utf-8 -*-
+require 'spec_helper'
+include WebPageParser
+
+describe RTPageParserFactory do
+  before do
+    @valid_urls = ["https://www.rt.com/uk/338045-privatization-aid-money-dfid/",
+                   "https://www.rt.com/business/338032-saudi-arabia-megafund-assets/",
+                   "https://www.rt.com/usa/338131-candy-thief-life-sentence/"]
+    @invalid_urls = ["https://www.rt.com/politics/",
+                     "https://www.rt.com/in-vision/337713-egyptair-hijacker-hostages-free/",
+                     "https://www.rt.com/in-motion/338243-anti-refugee-rally-uk/"]
+  end
+
+  it "should detect articles from the url" do
+    @valid_urls.each do |url|
+      RTPageParserFactory.can_parse?(:url => url).should be_true
+    end
+  end
+
+  it "should ignore pages with the wrong url format" do
+    @invalid_urls.each do |url|
+      RTPageParserFactory.can_parse?(:url => url).should be_nil
+    end
+  end
+
+end
+
+describe RTPageParserV1 do
+
+  describe "when parsing the privatised aid article" do
+    before do
+      @valid_options = { 
+        :url => 'https://www.rt.com/uk/338045-privatization-aid-money-dfid/',
+        :page => File.read("spec/fixtures/rt/338045.html"),
+        :valid_hash => '067d06dffae315daf6ab88026fdc7966'
+      }
+      @pa = RTPageParserV1.new(@valid_options)
+    end
+
+    it "should parse the title" do
+      @pa.title.should == "‘Scandal of privatized aid’: Free-market consultants cream off £450mn in UK govt funds"
+    end
+
+    it "should parse the date in UTC" do
+      @pa.date.should == DateTime.parse("2016-04-01 15:17")
+      @pa.date.zone.should == '+00:00'
+    end
+
+    it "should parse the content" do
+      @pa.content[0].should == "Free-market consultants in Britain are taking hundreds of millions of pounds ring-fenced to alleviate poverty in the developing world, as the government continues with its agenda of privatizing aid, a damning report has warned."
+      @pa.content[4].should == "The study examined how much of DfID’s work was geared towards supporting market-based development and the private sector in poor states. Recent projects included backing for a “business advocacy capacity development program” in Zimbabwe, and projects to increase private schooling in Kenya."
+      @pa.content[22].should == "ASI describes itself as a transparent, objective organization dedicated to making public services more robust. It also claims to support economic growth and civil society, while building “democratic and accountable institutions.”"
+      @pa.content.size.should == 23
+      @pa.hash.should == @valid_options[:valid_hash]
+    end
+  end
+
+  describe "when parsing the trump article" do
+    before do
+      @valid_options = { 
+        :url => 'https://www.rt.com/usa/338237-trump-fine-nato-breakup-obsolete/',
+        :page => File.read("spec/fixtures/rt/338237.html"),
+        :valid_hash => '633221af5d9b9a7161cd40eaaf22253d'
+      }
+      @pa = RTPageParserV1.new(@valid_options)
+    end
+
+    it "should parse the title" do
+      @pa.title.should == "Trump sparks NATO debate: ‘Obsolete’ or ‘tripwire that could lead to World War III’?"
+    end
+
+    it "should parse the date in UTC" do
+      @pa.date.should == DateTime.parse("2016-04-03 11:47")
+      @pa.date.zone.should == '+00:00'
+    end
+
+    it "should parse the content" do
+      @pa.content[0].should == "Republican presidential candidate Donald Trump slammed NATO on the campaign trail this week, saying he can live with breaking up the military alliance, which he calls “obsolete.”"
+      @pa.content[4].should == "NATO"
+      @pa.content[11].should == "The now-28 member-strong organization has defied its purported promise on a number of occasions."
+      @pa.content[24].should == "The cost of NATO"
+      @pa.content.size.should == 39
+      @pa.hash.should == @valid_options[:valid_hash]
+    end
+  end    
+end

data/spec/parsers/the_intercept_page_parser_spec.rb

@@ -64,4 +64,34 @@ describe TheInterceptPageParserV1 do
 
   end
 
+  describe 'when parsing the pentagon missionary article' do
+    before do
+      @valid_options = {
+                        :url => 'https://theintercept.com/2015/10/26/pentagon-missionary-spies-christian-ngo-front-for-north-korea-espionage/',
+                        :page => File.read('spec/fixtures/theintercept/pentagon-missionary.html'),
+                        :valid_hash => 'aa8a59955cc0c783f782c5c13701c71d'
+                       }
+      @pa = TheInterceptPageParserV1.new(@valid_options)
+    end
+
+    it 'should parse the title' do
+      @pa.title.should == "U.S. Military Used Christian NGO as Front for North Korea Espionage"
+    end
+
+    it 'should parse the content' do
+      @pa.content[0].should == 'ON MAY 10, 2007, in the East Room of the White House, President George W. Bush presided over a ceremony honoring the nation’s most accomplished community service leaders. Among those collecting a President’s Volunteer Service Award that afternoon was Kay Hiramine, the Colorado-based founder of a multimillion-dollar humanitarian organization.'
+       @pa.content[13].should == 'HISG WAS ESTABLISHED shortly after 9/11, when Hiramine led a group of three friends in creating a humanitarian organization that they hoped could provide disaster relief and sustainable development in poor and war-torn countries around the world, according to the organization’s incorporation documents.'
+       @pa.content[83].should == 'This report makes reference to a donation from Working Partners Foundation to Catholic Relief Services, based on Working Partners Foundation’s tax filings. Catholic Relief Services, which conducted a review after publication, said its own records contained no indication it received money from Working Partners Foundation or HISG.'
+       @pa.content.last.should == 'Top photo: U.S. President George W. Bush with Kay Hiramine prior to presenting him with a President’s Volunteer Service Award on May 10, 2007, in the East Room of the White House (photo flipped). '
+       @pa.content.size.should == 86
+       @pa.hash.should == @valid_options[:valid_hash]
+    end
+
+    it 'should parse the date in UTC' do
+      @pa.date.should == DateTime.parse('Oct. 26 2015 15:05:22')
+      @pa.date.zone.should == '+00:00'
+    end
+
+  end
+
 end

data/spec/parsers/washingtonpost_page_parser_spec.rb

@@ -6,7 +6,8 @@ describe WashingtonPostPageParserFactory do
   before do
     @valid_urls = [
                    'http://www.washingtonpost.com/world/will-a-bust-follow-the-boom-in-britain/2014/01/18/3677a6ae-7f9d-11e3-97d3-b9925ce2c57b_story.html?tid=hpModule_04941f10-8a79-11e2-98d9-3012c1cd8d1e&hpid=z16',
-                   'http://www.washingtonpost.com/business/technology/nsa-program-defenders-question-snowdens-motives/2014/01/19/091fccaa-811d-11e3-bbe5-6a2a3141e3a9_story.html'
+                   'http://www.washingtonpost.com/business/technology/nsa-program-defenders-question-snowdens-motives/2014/01/19/091fccaa-811d-11e3-bbe5-6a2a3141e3a9_story.html',
+                   'https://www.washingtonpost.com/world/middle_east/israel-ambassador-to-us-sends-anti-boycott-message-with-gift/2015/12/23/652d639c-a99b-11e5-b596-113f59ee069a_story.html'
                   ]
     @invalid_urls = [
                      'http://www.washingtonpost.com/politics/',
@@ -119,4 +120,95 @@ describe WashingtonPostPageParserV1 do
 
   end
 
+  describe 'when parsing the Israeli ambassador article' do
+    before do
+      @valid_options = {
+        :url => 'https://www.washingtonpost.com/world/middle_east/israel-ambassador-to-us-sends-anti-boycott-message-with-gift/2015/12/23/652d639c-a99b-11e5-b596-113f59ee069a_story.html',
+        :page => File.read('spec/fixtures/washingtonpost/israeli-ambassador.html'),
+        :valid_hash => 'c2e80bf1012949bf3a124576466b1b40'
+      }
+      @pa = WashingtonPostPageParserV1.new(@valid_options)
+    end
+
+    it "should parse the title" do
+      @pa.title.should eq 'Israel ambassador to US sends anti-boycott message with gift'
+    end
+
+    it 'should parse the date in UTC' do
+      @pa.date.should eq DateTime.parse("December 23 2015")
+      @pa.date.zone.should eq '+00:00'
+    end
+
+    it "should contain no javascript" do
+      @pa.content.join(' ').should_not =~ /function/
+    end
+
+    it "should parse the content" do
+      @pa.content[0].should eq 'JERUSALEM — Israel’s ambassador to the United States has dispatched a politically charged holiday gift.'
+      @pa.content.size.should eq 6
+      @pa.hash.should == @valid_options[:valid_hash]
+    end
+  end
+end
+
+describe WashingtonPostPageParserV2 do
+  describe 'when parsing the trump kim simmit article' do
+    before do
+      @valid_options = {
+        :url => 'https://www.washingtonpost.com/politics/trump-kim-summit-trump-says-we-have-developed-a-very-special-bond-at-end-of-historic-meeting/2018/06/12/ff43465a-6dba-11e8-bf86-a2351b5ece99_story.html?utm_term=.392b71a75a35',
+        :page => File.read('spec/fixtures/washingtonpost/trump-kim-summit.html'),
+        :valid_hash => '5b703096157e74b65fdf00fd9227ebbc'
+      }
+      @pa = WashingtonPostPageParserV2.new(@valid_options)
+    end
+
+    it "should parse the guid" do
+      @pa.guid.should eq "ff43465a-6dba-11e8-bf86-a2351b5ece99"
+    end
+
+    it "should parse the title" do
+      @pa.title.should eq 'Trump-Kim summit: Trump says after historic meeting, ‘We have developed a very special bond’'
+    end
+
+    it 'should parse the date in UTC' do
+      @pa.date.should eq DateTime.parse("2018-06-12T11:47:00-05:00")
+    end
+
+    it "should parse the content" do
+      @pa.content[0].should eq 'SINGAPORE —  President Trump concluded a historic summit with North Korean leader Kim Jong Un here Tuesday by sketching a path to prosperity for the isolated nation. But it remained highly uncertain whether the young dictator would embrace the offer by agreeing to eliminate his nuclear arsenal.'
+      @pa.content.last.should eq 'Carol Morello in Washington and Brian Murphy in Seoul contributed to this report.'
+      @pa.content[9].should eq 'Trump said that aides would begin additional talks soon and that he would potentially invite Kim to the White House and be open to visiting Pyongyang “at the appropriate time.” Yet he also acknowledged that disarmament would not come quickly.'
+      @pa.content.size.should eq 45
+      @pa.hash.should eq @valid_options[:valid_hash]
+    end
+  end
+
+    describe 'when parsing the bust-boom article from 2018' do
+    before do
+      @valid_options = {
+        :url => 'https://www.washingtonpost.com/world/will-a-bust-follow-the-boom-in-britain/2014/01/18/3677a6ae-7f9d-11e3-97d3-b9925ce2c57b_story.html?tid=hpModule_04941f10-8a79-11e2-98d9-3012c1cd8d1e&hpid=z16',
+        :page => File.read('spec/fixtures/washingtonpost/will-a-bust-follow-the-boom-in-britain-2018.html'),
+        :valid_hash => 'bbcdda8a8dffcabf71088039fb366e34'
+      }
+      @pa = WashingtonPostPageParserV2.new(@valid_options)
+    end
+
+    it "should parse the title" do
+      @pa.title.should == 'Will a bust follow the boom in Britain?'
+    end
+
+    it 'should parse the date in UTC' do
+      @pa.date.should eq DateTime.parse("2014-01-18T05:43:00-05:00")
+      @pa.date.zone.should eq '+00:00'
+    end
+
+    it "should parse the content" do
+      @pa.content[0].should eq 'LONDON — For decades, the modest two-bedroom apartment off Abbey Road was home to some of London’s neediest, a small, leaky outpost in this city’s vast constellation of public housing.'
+      @pa.content[12].should eq 'Crazy in the capital'
+      @pa.content.last.should eq '“It’s about time the government did something to help,” he said. “I don’t come from a rich family, so I don’t have parents who will give 15,000 pounds for a deposit. That’s not available to me. I’m genuinely pleased Cameron has done something for the working man, which is me.”'
+      @pa.content.size.should eq 25
+      @pa.hash.should eq @valid_options[:valid_hash]
+    end
+  end
+
 end