web-page-parser 0.21 → 0.22

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (6) hide show
  1. data/lib/web-page-parser/parsers/bbc_news_page_parser.rb +4 -0
  2. data/spec/fixtures/bbc_news/12921632.html +1621 -0
  3. data/spec/fixtures/bbc_news/13293006.html +1357 -0
  4. data/spec/parsers/bbc_news_page_spec.rb +13 -2
  5. data/spec/spec.opts +0 -2
  6. metadata +8 -4
data/lib/web-page-parser/parsers/bbc_news_page_parser.rb CHANGED
@@ -115,6 +115,8 @@ module WebPageParser
115
115
  STRIP_HIDDEN_A = ORegexp.new('<a class=.hidden.+?</a>','m')
116
116
  STRIP_STORY_FEATURE = ORegexp.new('<div class=.story\-feature.+?</div>', 'm')
117
117
  STRIP_HYPERPUFF_RE = ORegexp.new('<div class=.embedded-hyper.+?<div class=.hyperpuff.+?</div>.+?</div>', 'm')
118
+ STRIP_MARKETDATA_RE = ORegexp.new('<div class=.market\-data.+?</div>', 'm')
119
+ STRIP_EMBEDDEDHYPER_RE = ORegexp.new('<div class=.embedded\-hyper.+?</div>', 'm')
118
120
 
119
121
  def content_processor
120
122
  @content = STRIP_PAGE_BOOKMARKS.gsub(@content, '')
@@ -126,6 +128,8 @@ module WebPageParser
126
128
  @content = STRIP_HIDDEN_A.gsub(@content, '')
127
129
  @content = STRIP_STORY_FEATURE.gsub(@content, '')
128
130
  @content = STRIP_HYPERPUFF_RE.gsub(@content, '')
131
+ @content = STRIP_MARKETDATA_RE.gsub(@content, '')
132
+ @content = STRIP_EMBEDDEDHYPER_RE.gsub(@content, '')
129
133
  super
130
134
  end
131
135
  end