web-page-parser 0.10

data/spec/parser_factory_spec.rb

@@ -0,0 +1,18 @@
+$:.unshift File.join(File.dirname(__FILE__), '../lib')
+require 'web-page-parser'
+include WebPageParser
+
+describe ParserFactory do
+
+  it "should load parsers in the parsers directory" do
+    ParserFactory.factories.first.to_s.should == "TestPageParserFactory"
+  end
+
+  it "should provide the right PageParser for the given url" do
+    ParserFactory.parser_for(:url => "http://www.example.com").should be_a_kind_of TestPageParser
+  end
+
+  it "should return nil if no PageParser can be found for the given url" do
+    ParserFactory.parser_for(:url => "http://www.nowhere.nodomain").should be_nil
+  end
+end

data/spec/parsers/bbc_news_page_spec.rb

@@ -0,0 +1,144 @@
+$:.unshift File.join(File.dirname(__FILE__), '../../lib')
+require 'spec/base_parser_spec'
+require 'web-page-parser'
+include WebPageParser
+
+describe BbcNewsPageParserFactory do
+  before do
+    @valid_urls = [
+                   "http://news.bbc.co.uk/1/hi/entertainment/6984082.stm",
+                   "http://news.bbc.co.uk/1/hi/northern_ireland/7996478.stm",
+                   "http://news.bbc.co.uk/1/hi/uk/7995652.stm",
+                   "http://news.bbc.co.uk/1/hi/england/derbyshire/7996494.stm",
+                   "http://news.bbc.co.uk/2/low/uk_news/england/devon/7996447.stm"
+                  ]
+    @invalid_urls = [
+                     "http://news.bbc.co.uk/2/hi/health/default.stm",
+                     "http://news.bbc.co.uk/2/low/europe/default.stm",
+                     "http://news.bbc.co.uk/2/hi/in_pictures/default.stm",
+                     "http://news.bbc.co.uk/sport",
+                     "http://newsforums.bbc.co.uk/nol/thread.jspa?forumID=6422&edition=1&ttl=20090509133749",
+                     "http://www.bbc.co.uk/blogs/nickrobinson/",
+                     "http://news.bbc.co.uk/hi/english/static/in_depth/health/2000/heart_disease/default.stm",
+                     "http://news.bbc.co.uk/1/shared/spl/hi/pop_ups/08/middle_east_views_on_netanyahu0s_us_visit/html/1.stm"
+                    ]
+  end
+
+  it "should detect bbc news articles from the url" do
+    @valid_urls.each do |url|
+      BbcNewsPageParserFactory.can_parse?(:url => url).should be_true
+    end
+  end
+
+  it "should ignore pages with the wrong url format" do
+    @invalid_urls.each do |url|
+      BbcNewsPageParserFactory.can_parse?(:url => url).should be_nil
+    end
+  end
+  
+  it "should ignore 'in pictures' articles" do
+    BbcNewsPageParserFactory.can_parse?(:url => 'http://news.bbc.co.uk/1/hi/in_pictures/8039882.stm').should be_nil
+  end
+end
+
+describe BbcNewsPageParserV2 do
+  it_should_behave_like AllPageParsers
+  before do
+    @valid_options = { 
+      :url => 'http://news.bbc.co.uk/1/hi/world/middle_east/8011268.stm',
+      :page => File.read("spec/fixtures/bbc_news/8011268.stm.html"),
+      :valid_hash => 'd9e201abec3f4b9e38865b5135281978'
+    }
+    @pa = BbcNewsPageParserV2.new(@valid_options)
+  end
+
+  it "should parse the title" do
+    @pa.title.should == "Obama invites Middle East heads"
+  end
+  
+  it "should convert iso-8859-1 in the title to utf8" do
+    page = BbcNewsPageParserV2.new(:page => '<meta name="Headline" content="'+"\243"+'100K mortgage claim by Woodward"')
+    page.title.should == "£100K mortgage claim by Woodward"
+  end
+  
+  it "should convert iso-8859-1 in the content to utf8" do
+    page = BbcNewsPageParserV2.new(:page => "S BO -->\243100K mortgage claim by Woodward<!-- E BO")
+    page.content.first.should == "£100K mortgage claim by Woodward"
+  end
+  
+
+  it "should parse the date in UTC" do
+    # 2009/04/21 19:50:44
+    @pa.date.should == DateTime.parse("Apr 21 19:50:44 +0000 2009")
+    @pa.date.zone.should == '+00:00'
+  end
+  
+  it "should parse the content" do
+    @pa.content[0].should == "US officials say the leaders of Israel, Egypt and the Palestinians have been invited for talks in Washington in a new push for Middle East peace."
+    @pa.content.last.should == "The US supports a two-state solution, with Israel existing peacefully alongside a Palestinian state."
+    @pa.content.size.should == 15
+  end
+
+  it "should decode html entities" do
+    @pa.content[8].should == 'He added: "We are actively working to finalise dates for the visits."'
+  end
+
+  it "should calculate a valid hash of the content" do
+    @pa.hash.should == @valid_options[:valid_hash]
+  end
+  
+  it "should parse 'from our own correspondent' pages" do
+    page = BbcNewsPageParserV2.new(:url => "http://news.bbc.co.uk/1/hi/programmes/from_our_own_correspondent/8029015.stm",
+                                   :page => File.read("spec/fixtures/bbc_news/8029015.stm.html"))
+    page.title.should == "Cairo's terrifying traffic chaos"
+    page.content.first.should == "Christian Fraser discovers that a brush with death on Cairo's congested roads leaves no appetite for life in the fast lane."
+  end
+  
+  it "should parse 'magazine' pages" do
+    page = BbcNewsPageParserV2.new(:url => "http://news.bbc.co.uk/1/hi/magazine/8063681.stm",
+                                   :page => File.read("spec/fixtures/bbc_news/8063681.stm.html"))
+    page.title.should == "My night with Parisien prostitutes"
+    page.content.first.should == "Wandering around the red light district of Paris as a teenager taught me all I need to know - about teenagers, not women, says Laurie Taylor in his weekly column."
+  end
+
+end
+
+describe BbcNewsPageParserV1 do
+  before do
+    @valid_options = { 
+      :url => 'http://news.bbc.co.uk/1/hi/england/bradford/6072486.stm',
+      :page => File.read("spec/fixtures/bbc_news/6072486.stm.html"),
+      :valid_hash => 'aaf7ed1219eb69c3126ea5d0774fbe7d'
+    }
+    @pa = BbcNewsPageParserV1.new(@valid_options)
+  end
+
+  it "should parse the title" do
+    @pa.title.should == "Son-in-law remanded over killing"
+  end
+
+  it "should parse the date in UTC" do
+    @pa.date.should == DateTime.parse("Sat Oct 21 14:41:10 +0000 2006")
+    @pa.date.zone.should == '+00:00'
+  end
+
+  it "should parse the content exactly like the old News Sniffer library" do
+    @pa.content.first.should == "<B>The son-in-law of a 73-year-old Castleford widow has been charged with her murder.</B>"
+    @pa.content.last.should == 'He denied the charges against him through his solicitor and is due to appear at Leeds Crown Court on Friday.'
+    @pa.content.size.should == 5
+    @pa.hash.should == @valid_options[:valid_hash]
+  end
+
+  it "should convert apostrophe and pound sign html entities in content" do
+    @pa = BbcNewsPageParserV1.new :page => 'S SF -->John&apos;s code sucks &amp; blows<!-- E BO'
+    @pa.content.to_s.should match Regexp.new("John's")
+    @pa.content.to_s.should match /sucks & blows/
+  end
+
+  it "should convert apostrophe and pound sign html entities in page titles" do
+    @pa = BbcNewsPageParserV1.new :page => '<meta name="Headline" content="John&apos;s code sucks &amp; blows!"/>'
+    @pa.title.should match Regexp.new("John's")
+    @pa.title.should match /sucks & blows/
+  end
+
+end

data/spec/spec.opts

@@ -0,0 +1,4 @@
+--colour
+--format s
+--loadby mtime
+--reverse

metadata

@@ -0,0 +1,92 @@
+--- !ruby/object:Gem::Specification 
+name: web-page-parser
+version: !ruby/object:Gem::Version 
+  version: "0.10"
+platform: ruby
+authors: 
+- John Leach
+autorequire: 
+bindir: bin
+cert_chain: []
+
+date: 2009-06-20 00:00:00 +01:00
+default_executable: 
+dependencies: 
+- !ruby/object:Gem::Dependency 
+  name: oniguruma
+  type: :runtime
+  version_requirement: 
+  version_requirements: !ruby/object:Gem::Requirement 
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        version: 1.1.0
+    version: 
+- !ruby/object:Gem::Dependency 
+  name: htmlentities
+  type: :runtime
+  version_requirement: 
+  version_requirements: !ruby/object:Gem::Requirement 
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        version: 4.0.0
+    version: 
+description: A Ruby library to parse the content out of web pages, such as BBC News pages.  Used by the News Sniffer project.
+email: john@johnleach.co.uk
+executables: []
+
+extensions: []
+
+extra_rdoc_files: 
+- README.rdoc
+- LICENSE
+files: 
+- lib/web-page-parser
+- lib/web-page-parser/base_parser.rb
+- lib/web-page-parser/parsers
+- lib/web-page-parser/parsers/test_page_parser.rb
+- lib/web-page-parser/parsers/bbc_news_page_parser.rb
+- lib/web-page-parser/parser_factory.rb
+- lib/web-page-parser.rb
+- README.rdoc
+- LICENSE
+has_rdoc: true
+homepage: http://github.com/johnl/web-page-parser/tree/master
+post_install_message: 
+rdoc_options: []
+
+require_paths: 
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement 
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      version: "0"
+  version: 
+required_rubygems_version: !ruby/object:Gem::Requirement 
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      version: "0"
+  version: 
+requirements: []
+
+rubyforge_project: web-page-parser
+rubygems_version: 1.3.1
+signing_key: 
+specification_version: 2
+summary: A parser for web pages
+test_files: 
+- spec/parser_factory_spec.rb
+- spec/base_parser_spec.rb
+- spec/fixtures
+- spec/fixtures/bbc_news
+- spec/fixtures/bbc_news/8063681.stm.html
+- spec/fixtures/bbc_news/8011268.stm.html
+- spec/fixtures/bbc_news/6072486.stm.html
+- spec/fixtures/bbc_news/8029015.stm.html
+- spec/fixtures/bbc_news/7745137.stm.html
+- spec/parsers
+- spec/parsers/bbc_news_page_spec.rb
+- spec/spec.opts