newscrapi 0.0.11

data/lib/newscrapi.rb

@@ -0,0 +1,2 @@
+
+require 'newscrapi/scrapper'

data/lib/newscrapi/encoding.rb

@@ -0,0 +1,44 @@
+
+require 'rchardet'
+require 'nokogiri'
+require 'iconv'
+
+module Newscrapi
+  module Encoding
+
+    def self.guess_html_encoding(obj)
+      doc, page = parse_parameters_doc_page(obj)
+
+      meta_encoding = doc.meta_encoding
+      return meta_encoding unless meta_encoding.nil?
+      CharDet.detect(page)['encoding']
+    end
+
+    def self.get_html_doc_with_changed_encoding(obj, encode_to)
+      doc, page = parse_parameters_doc_page(obj)
+
+      if encode_to
+        guessed_encoding = guess_html_encoding(page)
+        if guessed_encoding != encode_to
+          doc = doc.serialize(:encoding => encode_to)
+          page = doc.to_s
+          return Nokogiri::HTML(page)
+        end
+      end
+      doc
+    end
+
+    private
+
+    def self.parse_parameters_doc_page(obj)
+      if (obj.class == String)
+        page = obj
+        doc = Nokogiri::HTML(page)
+      elsif (obj.class == Nokogiri::HTML::Document)
+        doc = obj
+        page = doc.to_s
+      else raise Exception.new("Not supported type #{obj.class.to_s}") end
+      return doc, page
+    end
+  end
+end

data/lib/newscrapi/feedzirra.rb

@@ -0,0 +1,17 @@
+# feedzirra entries are extended by methods for scrapping content
+require 'feedzirra/feed_entry_utilities'
+
+module Feedzirra
+  module FeedEntryUtilities
+
+    # Scrap the content based on the URL and the existing content and return it
+    def scrap_content(scrapper = Newscrapi::Scrapper.default, options = {})
+      scrapper.scrap_content(self.url, options) || self.content.to_s
+    end
+
+    # Scrap the content or use the existing one and change the feed entry
+    def scrap_content!(scrapper = Newscrapi::Scrapper.default, options = {})
+      self.content = scrap_content(scrapper, options)
+    end
+  end
+end

data/lib/newscrapi/mapping.rb

@@ -0,0 +1,50 @@
+require 'newscrapi/scrapper'
+require 'iconv'
+
+class Newscrapi::Mapping
+
+  attr_reader :content_xpaths_list, :url_pattern_regexp
+
+  def initialize
+    @content_xpaths_list = []
+  end
+
+  def url_pattern(pattern)
+    @url_pattern_regexp = pattern.class == String ?
+      Regexp.compile("^#{Regexp.escape(pattern).gsub('\*','.*')}$") : pattern
+  end
+
+  def content_at(content_xpath)
+    @content_xpaths_list << content_xpath
+  end
+
+  def iconv(args)
+    suppose_encoding(args[:from])
+    convert_to(args[:to])
+  end
+
+=begin
+  def suppose_encoding(encoding = nil)
+    return @supposed_encoding if encoding.nil?
+    @supposed_encoding = encoding
+  end
+=end
+
+  def matches_url?(url)
+    url =~ @url_pattern_regexp
+  end
+
+  def scrap_content(obj, content_scrapper = nil)
+    doc = Newscrapi::Scrapper.parse_page(obj)
+    @content_xpaths_list.each do |content_xpath|
+      content_section = doc.xpath(content_xpath)
+      if content_section.count > 0
+        content = content_section.to_a.join("\n")
+        content = content_scrapper.clean_content(content) unless content_scrapper.nil?
+        return content
+      end
+    end
+    nil
+  end
+end
+

data/lib/newscrapi/scrapper.rb

@@ -0,0 +1,129 @@
+
+require 'open-uri'
+require 'nokogiri'
+
+require 'newscrapi/encoding'
+require 'newscrapi/mapping'
+
+module Newscrapi
+
+  class Scrapper
+
+    class << self
+      attr_accessor :default_config_file, :default
+      default_config_file = "#{File.dirname(__FILE__)}/../config/content_scrapper.rb"
+
+      def create_new_default(*args)
+        self.default = self.new(*args)
+      end
+    end
+
+    def set_as_default
+      Newscrapi::Scrapper.default = self
+    end
+
+    attr_reader :content_mappings, :scrapping_exception_handler_block,
+      :missing_url_matcher_handler_block, :missing_content_handler_block
+
+    def self.parse_page(obj)
+      return obj if obj.class == Nokogiri::HTML::Document
+      Nokogiri::HTML(obj)
+    end
+
+    def initialize(scrapper_config_file = nil)
+      @content_mappings = []
+      config_file = scrapper_config_file || Newscrapi::Scrapper.default_config_file
+      self.instance_eval(File.read(config_file), config_file) unless config_file.nil?
+    end
+
+    def encode_to(encoding = nil)
+      @encode_to = encoding unless encoding.nil?
+      @encode_to
+    end
+
+    def content_mapping(&block)
+      new_mapping = Newscrapi::Mapping.new
+      new_mapping.instance_eval(&block)
+      @content_mappings << new_mapping
+    end
+
+    def clean_content(content)
+      @content_cleaner_block.nil? ? content : @content_cleaner_block.call(content)
+    end
+
+    def sanitize_tags(&sanitize_settings)
+      @content_cleaner_block = lambda do |content|
+        require 'sanitize'
+        Sanitize.clean(content, sanitize_settings.call())
+      end
+    end
+
+    def loofah_tags(scrap_type)
+      @content_scrapper_block = lambda do |content|
+        require 'loofah'
+        Loofah.document(content).scrub!(scrap_type).to_s
+      end
+    end
+
+    def matching_content_mapper(url)
+      content_mappings.each { | content_mapping | return content_mapping if content_mapping.matches_url?(url) }
+      nil
+    end
+
+    def scrap_content(url, options = {})
+      if (content_mapping = matching_content_mapper(url)).nil?
+        @missing_url_matcher_handler_block.call(url) unless @missing_url_matcher_handler_block.nil?
+        return nil
+      end
+      return nil if content_mapping.content_xpaths_list.empty?
+      begin
+        use_page = ensure_encoding(options[:use_page] || Kernel.open(url).read)
+
+        doc = Newscrapi::Scrapper.parse_page(use_page)
+        scrapped_content = content_mapping.scrap_content(doc, content_scrapper = self)
+
+        @missing_content_handler_block.call(url) if !@missing_content_handler_block.nil? and scrapped_content.nil?
+        return scrapped_content
+      rescue Exception
+        @scrapping_exception_handler_block.call($!, url) unless @scrapping_exception_handler_block.nil?
+        return nil
+      end
+      nil
+    end
+
+    def rescue_scrapping(&block)
+      @scrapping_exception_handler_block = block
+    end
+
+    def missing_url_matcher(&block)
+      @missing_url_matcher_handler_block = block
+    end
+
+    def missing_content(&block)
+      @missing_content_handler_block = block
+    end
+
+    def report_to_stderr
+      rescue_scrapping do |exception, url|
+        STDERR << "error occured during scrapping page #{url}\n"
+        STDERR << "#{exception.message}\n"
+        STDERR << exception.backtrace.join("\n")
+      end
+
+      missing_url_matcher do |url|
+        STDERR << "missing matcher for #{url}\n"
+      end
+
+      missing_content do |url|
+        STDERR << "empty content for #{url}\n"
+      end
+    end
+
+    private
+
+    def ensure_encoding(str)
+      Newscrapi::Encoding::get_html_doc_with_changed_encoding(str, self.encode_to)
+    end
+
+  end
+end

data/lib/newscrapi/testing.rb

@@ -0,0 +1,19 @@
+require 'newscrapi/scrapper'
+
+class Newscrapi::Scrapper
+
+  alias :old_initialize :initialize
+
+  def initialize
+    old_initialize
+    testing_report_to_stderr
+  end
+
+  def testing_report_to_stderr
+    rescue_scrapping do |exception, url|
+#      extended_exception = Exception.new("error occured during scrapping page #{url}: #{exception.message}")
+#      extended_exception.set_backtrace(exception.backtrace)
+      raise exception #extended_exception
+    end
+  end
+end

data/rails/init.rb

@@ -0,0 +1,3 @@
+require 'newscrapi'
+
+Newscrapi::Scrapper.default_config_file = "#{RAILS_ROOT}/config/content_scrapper.rb"

data/test/helper.rb

@@ -0,0 +1,9 @@
+require 'rubygems'
+require 'test/unit'
+require 'shoulda'
+
+$LOAD_PATH.unshift(File.join(File.dirname(__FILE__), '..', 'lib'))
+$LOAD_PATH.unshift(File.dirname(__FILE__))
+
+require 'newscrapi/scrapper'
+require 'newscrapi/testing'

data/test/test_encoding.rb

@@ -0,0 +1,43 @@
+
+require 'helper'
+require 'nokogiri'
+require 'newscrapi/encoding'
+
+class TestEncoding < Test::Unit::TestCase
+
+  context "on guessing the encoding of a page with a metatag defined" do
+    setup do
+      @page = File.open("#{File.dirname(__FILE__)}/test_pages/windows-1250_page.html").read
+      @doc = Nokogiri::HTML(@page)
+    end
+    should "detect the page encoding correctly for string input" do
+      assert_equal 'windows-1250', Newscrapi::Encoding.guess_html_encoding(@page)
+    end
+    should "detect the page encoding correctly for parsed document input" do
+      assert_equal 'windows-1250', Newscrapi::Encoding.guess_html_encoding(@doc)
+    end
+  end
+
+  context "on guessing the encoding of a page without the encoding metatag defined" do
+    setup do
+      @page = File.open("#{File.dirname(__FILE__)}/test_pages/utf-8_page.html").read
+      @doc = Nokogiri::HTML(@page)
+    end
+    should "detect the page encoding correctly for string input" do
+      assert_equal 'utf-8', Newscrapi::Encoding.guess_html_encoding(@page)
+    end
+    should "detect the page encoding correctly for parsed document input" do
+      assert_equal 'utf-8', Newscrapi::Encoding.guess_html_encoding(@doc)
+    end
+  end
+
+  context "on not supported class type encoding guessing" do
+    should "raise exception" do
+      assert_raise Exception do
+        Newscrapi::Encoding.guess_html_encoding(5)
+      end
+    end
+  end
+
+end
+

data/test/test_mapping.rb

@@ -0,0 +1,58 @@
+
+require 'helper'
+require 'mocha'
+
+class TestMapping < Test::Unit::TestCase
+
+  context "on empty content mapping creation" do
+    setup do
+      @mapping = Newscrapi::Mapping.new
+      @mapping.instance_eval do
+        url_pattern /^http:\/\/www\.matchme\.com\//
+          content_at '//div[@id="failing_content"]'
+        content_at '//div[@id="itext_content"]'
+        content_at '//div[@id="itext_second_content"]'
+      end
+    end
+    should "match the right urls" do
+      assert @mapping.matches_url?('http://www.matchme.com/')
+    end
+    should "not match the wrong urls" do
+      assert !@mapping.matches_url?('https://www.somethingelse.org/hfas')
+    end
+    context "scrapping content for a specific site" do
+      setup do
+        pretty_content = File.open("#{File.dirname(__FILE__)}/test_pages/pretty.html").read 
+        @document = Nokogiri::HTML(pretty_content)
+      end
+      should "extract the content" do
+        assert_match(%r{<p><strong>This is a strong text</strong></p>},
+                     @mapping.scrap_content(@document))
+      end
+    end
+    context "on document with two content parts" do
+      setup do
+        two_content = File.open("#{File.dirname(__FILE__)}/test_pages/twocontent.html").read 
+        @document = Nokogiri::HTML(two_content)
+      end
+      should "evaluate the contents in the order as they were added" do
+        assert_match(%r{The first one is matched}, @mapping.scrap_content(@document))
+      end
+    end
+  end
+
+  context "on url matcher definition using wildcards" do
+    setup do
+      @mapping = Newscrapi::Mapping.new
+      @mapping.instance_eval do
+        url_pattern 'http://*.example.com/*'
+      end
+    end
+    should "match urls with matching wildcards" do
+      assert @mapping.matches_url?('http://test.example.com/path/to/doc.html')
+    end
+    should "not match urls with not matching wildcards" do
+      assert !@mapping.matches_url?('http://test.example2.com/path/to/doc.html')
+    end
+  end
+end

data/test/test_pages.rb

@@ -0,0 +1,69 @@
+
+require 'helper'
+require 'mocha'
+
+class TestPages < Test::Unit::TestCase
+
+  context "on page containing CDATA" do
+    setup do
+      @scrapper = Newscrapi::Scrapper.new
+      @scrapper.instance_eval do
+        content_mapping do
+          url_pattern /.*/
+          content_at '//div[@class="art-full adwords-text"]'
+        end
+        loofah_tags(:strip)
+      end
+      @scrapper.rescue_scrapping do |exception,url|
+        puts exception
+      end
+      cdata_content = File.open("#{File.dirname(__FILE__)}/test_pages/cdata.html").read
+      Kernel.expects(:open).returns(StringIO.new(cdata_content))
+    end
+    should "not escape the cdata entries, should leave cdata unvisible" do
+      assert_match /<!--</, @scrapper.scrap_content('http://www.cdata.url/hsdae')
+    end
+  end
+
+  context "on page encoding conversion" do
+    setup do
+      @scrapper = Newscrapi::Scrapper.new
+      @scrapper.instance_eval do
+        encode_to 'utf-8'
+        content_mapping do
+          url_pattern /.*/
+            content_at '//div[@id="itext_content"]'
+        end
+      end
+      content = File.open("#{File.dirname(__FILE__)}/test_pages/windows-1250_page.html").read
+      Kernel.expects(:open).returns(StringIO.new(content))
+    end
+    should "convert the document to utf-8 encoding" do
+      require 'rchardet'
+      scrapped_page = @scrapper.scrap_content('http://hop.kop')
+      assert_equal 'utf-8', CharDet.detect(scrapped_page)['encoding']
+    end
+  end
+
+=begin
+  context "on pattern for page for encoding" do
+    setup do
+      @scrapper = Newscrapi::Scrapper.new
+      @scrapper.instance_eval do
+        encode_to 'windows-1250'
+        content_mapping do
+          url_pattern /^http:\/\/www\.matchme\.com\//
+            content_at '//div[@class="node node-story"]/div[@class="content"]/p'
+          suppose_encoding 'utf-8'
+        end
+      end
+      CharDet.expects(:detect).never
+      @page = File.open("#{File.dirname(__FILE__)}/test_pages/page_without_encoding_meta_tag.html").read
+    end
+    should "scrap a page with converted content" do
+      assert_not_nil @scrapper.scrap_content('http://www.matchme.com', @page)
+    end
+  end
+=end
+end
+