feeds-crawler 0.2.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: ae6d80efb92d85a3e18719e4c23a93a74edb888b
+  data.tar.gz: 4f2d7eb77ba67427f932f1779d90299ab2b3f35a
+SHA512:
+  metadata.gz: 72c2ff85052f577433507bffb91fce51eeb14518acfc5b194c222c0e00576ca156329caa77b51a887128835c883a7b9923248b187c9b11d1a3a199d8e147ac35
+  data.tar.gz: 169ce845207dd426522651276642cab9d89c261a393a0e9c1fbcc3ee9d014b0c5778ea5ebc5f46b5628beafc40554818c926eb8398f662e7ac2646f681e44d4e

data/.gitignore

@@ -0,0 +1,56 @@
+*.gem
+*.rbc
+/.config
+/coverage/
+/InstalledFiles
+/pkg/
+/spec/reports/
+/spec/examples.txt
+/test/tmp/
+/test/version_tmp/
+/tmp/
+
+# Used by dotenv library to load environment variables.
+# .env
+
+## Specific to RubyMotion:
+.dat*
+.repl_history
+build/
+*.bridgesupport
+build-iPhoneOS/
+build-iPhoneSimulator/
+
+## Specific to RubyMotion (use of CocoaPods):
+#
+# We recommend against adding the Pods directory to your .gitignore. However
+# you should judge for yourself, the pros and cons are mentioned at:
+# https://guides.cocoapods.org/using/using-cocoapods.html#should-i-check-the-pods-directory-into-source-control
+#
+# vendor/Pods/
+
+## Documentation cache and generated files:
+/.yardoc/
+/_yardoc/
+/doc/
+/rdoc/
+
+## Environment normalization:
+/.bundle/
+/vendor/bundle
+/lib/bundler/man/
+
+# for a library or gem, you might want to ignore these files since the code is
+# intended to run in multiple environments; otherwise, check them in:
+# Gemfile.lock
+# .ruby-version
+# .ruby-gemset
+
+# unless supporting rvm < 1.11.0 or doing something fancy, ignore this:
+.rvmrc
+
+# RubyMine files
+.idea/
+
+# RSpec files
+.rspec_status

data/.rspec

@@ -0,0 +1,2 @@
+--format documentation
+--color

data/Gemfile

@@ -0,0 +1,3 @@
+source 'https://rubygems.org'
+
+gemspec

data/Gemfile.lock

@@ -0,0 +1,77 @@
+PATH
+  remote: .
+  specs:
+    feeds-crawler (0.2.0)
+      parallel (~> 1.12, >= 1.12.0)
+      ruby-readability (~> 0.7, >= 0.7.0)
+      sanitize (~> 4.5, >= 4.5.0)
+
+GEM
+  remote: https://rubygems.org/
+  specs:
+    ast (2.3.0)
+    coderay (1.1.1)
+    crass (1.0.2)
+    diff-lcs (1.3)
+    guess_html_encoding (0.0.11)
+    method_source (0.8.2)
+    mini_portile2 (2.2.0)
+    nokogiri (1.8.0)
+      mini_portile2 (~> 2.2.0)
+    nokogumbo (1.4.13)
+      nokogiri
+    parallel (1.12.0)
+    parser (2.4.0.0)
+      ast (~> 2.2)
+    powerpack (0.1.1)
+    pry (0.10.4)
+      coderay (~> 1.1.0)
+      method_source (~> 0.8.1)
+      slop (~> 3.4)
+    rainbow (2.2.2)
+      rake
+    rake (10.5.0)
+    rspec (3.6.0)
+      rspec-core (~> 3.6.0)
+      rspec-expectations (~> 3.6.0)
+      rspec-mocks (~> 3.6.0)
+    rspec-core (3.6.0)
+      rspec-support (~> 3.6.0)
+    rspec-expectations (3.6.0)
+      diff-lcs (>= 1.2.0, < 2.0)
+      rspec-support (~> 3.6.0)
+    rspec-mocks (3.6.0)
+      diff-lcs (>= 1.2.0, < 2.0)
+      rspec-support (~> 3.6.0)
+    rspec-support (3.6.0)
+    rubocop (0.50.0)
+      parallel (~> 1.10)
+      parser (>= 2.3.3.1, < 3.0)
+      powerpack (~> 0.1)
+      rainbow (>= 2.2.2, < 3.0)
+      ruby-progressbar (~> 1.7)
+      unicode-display_width (~> 1.0, >= 1.0.1)
+    ruby-progressbar (1.8.3)
+    ruby-readability (0.7.0)
+      guess_html_encoding (>= 0.0.4)
+      nokogiri (>= 1.6.0)
+    sanitize (4.5.0)
+      crass (~> 1.0.2)
+      nokogiri (>= 1.4.4)
+      nokogumbo (~> 1.4.1)
+    slop (3.6.0)
+    unicode-display_width (1.3.0)
+
+PLATFORMS
+  ruby
+
+DEPENDENCIES
+  bundler (~> 1.15)
+  feeds-crawler!
+  pry
+  rake (~> 10.0)
+  rspec (~> 3.0)
+  rubocop
+
+BUNDLED WITH
+   1.15.3

data/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2017 Andrey Tatarenko
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

data/README.md

@@ -0,0 +1,53 @@
+# Feeds Crawler
+
+This gem allows to crawl news articles from RSS feeds. It realizes parallel execution for better performance and sanitize collected text from unnecessary HTML tags.
+
+## Installation
+
+Add this line to your application's Gemfile:
+
+```ruby
+gem 'feeds-crawler'
+```
+
+And then execute:
+
+    $ bundle
+
+Or install it yourself as:
+
+    $ gem install feeds-crawler
+
+## Usage
+
+```ruby
+require 'feeds_crawler'
+
+FeedsCrawler.crawl('rss_feed_url')
+# => ["Article 1", "Article 2", "Article 3"]
+```
+
+It accepts as many RSS feeds as you want:
+```ruby
+my_favorite_rss_feeds = %w[
+  first_rss_feed
+  second_rss_feed
+]
+
+FeedsCrawler.crawl(my_favorite_rss_feeds)
+# => ["Article from feed #1", "Article from feed #2"]
+```
+or:
+```ruby
+FeedsCrawler.crawl('first_rss_feed_url', 'second_rss_feed_url')
+# => ["Article from feed #1", "Article from feed #2"]
+```
+
+
+## Contributing
+
+Bug reports and pull requests are welcome on GitHub at https://github.com/andrey17076/feeds-crawler.
+
+## License
+
+The gem is available as open source under the terms of the [MIT License](http://opensource.org/licenses/MIT).

data/Rakefile

@@ -0,0 +1,8 @@
+require 'bundler/gem_tasks'
+require 'rspec/core/rake_task'
+
+RSpec::Core::RakeTask.new(:spec)
+
+task default: :spec
+task c: :console
+task(:console) { sh('./bin/console') }

data/bin/console

@@ -0,0 +1,7 @@
+#!/usr/bin/env ruby
+
+require 'bundler/setup'
+require 'feeds_crawler'
+require 'pry'
+
+Pry.start

data/bin/setup

@@ -0,0 +1,6 @@
+#!/usr/bin/env bash
+set -euo pipefail
+IFS=$'\n\t'
+set -vx
+
+bundle install

data/feeds-crawler.gemspec

@@ -0,0 +1,28 @@
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+
+Gem::Specification.new do |spec|
+  spec.name          = 'feeds-crawler'
+  spec.version       = '0.2.0'
+  spec.authors       = ['Andrey Tatarenko']
+  spec.email         = ['andrey17076@gmail.com']
+
+  spec.summary       = 'This gem allows to crawl news articles from RSS feeds.'
+  spec.homepage      = 'https://github.com/andrey17076/feed-crawler'
+  spec.license       = 'MIT'
+  spec.files         = `git ls-files -z`.split("\x0")
+  spec.executables   = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
+  spec.test_files    = spec.files.grep(%r{^spec/})
+  spec.require_paths = ['lib']
+
+  spec.add_development_dependency 'bundler', '~> 1.15'
+  spec.add_development_dependency 'rake', '~> 10.0'
+  spec.add_development_dependency 'rspec', '~> 3.0'
+  spec.add_development_dependency 'pry'
+  spec.add_development_dependency 'rubocop'
+
+  spec.add_runtime_dependency 'parallel',  '~> 1.12', '>= 1.12.0'
+  spec.add_runtime_dependency 'ruby-readability', '~> 0.7', '>= 0.7.0'
+  spec.add_runtime_dependency 'sanitize', '~> 4.5', '>= 4.5.0'
+end
+

data/lib/crawlers/helpers/content.rb

@@ -0,0 +1,20 @@
+require 'ruby-readability'
+require 'sanitize'
+
+module Crawlers
+  module Helpers
+    module Content
+      def extract_primary_content(html_text)
+        content = Readability::Document.new(html_text).content
+        sanitized_content = Sanitize.clean(content)
+        remove_trailing_spaces(sanitized_content)
+      end
+
+      private
+
+      def remove_trailing_spaces(text)
+        text.strip.gsub(/(?<=\n)\s+/, '')
+      end
+    end
+  end
+end

data/lib/crawlers/rss.rb

@@ -0,0 +1,45 @@
+require 'rss'
+require 'open-uri'
+require 'parallel'
+require 'crawlers/helpers/content'
+
+module Crawlers
+  class Rss
+    include Helpers::Content
+
+    def initialize(rss_url)
+      @rss_url = rss_url
+    end
+
+    def articles
+      articles = Parallel.map(rss_feed_items) do |feed_item|
+        crawl_article(feed_item)
+      end
+      articles.reject(&:empty?)
+    end
+
+    private
+
+    def rss_feed_items
+      rss_feed = page_content(@rss_url)
+      parse_feed(rss_feed)
+    end
+
+    def parse_feed(rss_feed)
+      RSS::Parser.parse(rss_feed)&.items
+    rescue RSS::Error
+      []
+    end
+
+    def crawl_article(feed_item)
+      page_with_article = page_content(feed_item.link)
+      extract_primary_content(page_with_article)
+    end
+
+    def page_content(page_url)
+      open(page_url).read
+    rescue StandardError
+      ''
+    end
+  end
+end

data/lib/feeds_crawler.rb

@@ -0,0 +1,16 @@
+require 'parallel'
+require 'crawlers/rss'
+
+class FeedsCrawler
+  def self.crawl(*rss_urls)
+    urls = Array(rss_urls)
+    crawl_feeds_articles(urls).flatten
+  end
+
+  def self.crawl_feeds_articles(rss_urls)
+    Parallel.map(rss_urls) do |url|
+      Crawlers::Rss.new(url).articles
+    end
+  end
+  private_class_method :crawl_feeds_articles
+end

data/spec/crawlers/helpers/content_spec.rb

@@ -0,0 +1,31 @@
+require 'spec_helper'
+
+RSpec.describe Crawlers::Helpers::Content do
+  let(:dummy_class) { Class.new { include Crawlers::Helpers::Content } }
+  describe '.extract_primary_content' do
+    let(:html_text) { open('spec/fixtures/page_with_content.html').read }
+    subject { dummy_class.new.extract_primary_content(html_text) }
+
+    it 'extracts real content of a document' do
+      is_expected.to eq('Real Content')
+    end
+
+    context 'when real content still has markup element' do
+      let(:markup) { '<div><p>Content</p></div>' }
+      before { allow_any_instance_of(Readability::Document).to receive(:content).and_return(markup) }
+
+      it 'removes any markup' do
+        is_expected.to eq('Content')
+      end
+    end
+
+    context 'when real content has trailing spaces' do
+      let(:content) { "First line\n Second line" }
+      before { allow(Sanitize).to receive(:clean).and_return(content) }
+
+      it 'removes this spaces' do
+        is_expected.to eq("First line\nSecond line")
+      end
+    end
+  end
+end

data/spec/crawlers/rss_spec.rb

@@ -0,0 +1,72 @@
+require 'spec_helper'
+
+RSpec.describe Crawlers::Rss do
+  describe '#articles' do
+    subject { described_class.new('url').articles }
+
+    context 'when rss url is invalid' do
+      before { allow(Kernel).to receive(:open).and_raise(Errno::ENOENT) }
+
+      it { is_expected.to be_empty }
+    end
+
+    context 'when rss url leads to error page' do
+      before { allow(Kernel).to receive(:open).and_raise(OpenURI::HTTPError) }
+
+      it { is_expected.to be_empty }
+    end
+
+    context 'when rss url leads to page with redirect' do
+      before { allow(Kernel).to receive(:open).and_raise(RuntimeError) }
+
+      it { is_expected.to be_empty }
+    end
+
+    context 'when something goes wrong even when page loads' do
+      before do
+        allow(Kernel).to receive_message_chain(:open, :read).and_raise(StandardError)
+      end
+
+      it { is_expected.to be_empty }
+    end
+
+    context 'when rss url is valid' do
+      before do
+        allow(Kernel).to receive_message_chain(:open, :read).and_return('page content')
+      end
+
+      context 'when page content is bad formatted markup' do
+        before { allow(RSS::Parser).to receive(:parse).and_raise(RSS::Error) }
+
+        it { is_expected.to be_empty }
+      end
+
+      context 'when page content is not rss feed' do
+        before { allow(RSS::Parser).to receive(:parse).and_return(nil) }
+
+        it { is_expected.to be_empty }
+      end
+
+      context 'when page content is real rss feed' do
+        let(:feed_items) { [instance_double(RSS::Rss::Channel::Item, link: '')] }
+
+        before do
+          allow(RSS::Parser).to receive_message_chain(:parse, :items).and_return(feed_items)
+          allow_any_instance_of(described_class).to receive(:extract_primary_content).and_return(feed_item_article)
+        end
+
+        context 'when link does not leads to page with content' do
+          let(:feed_item_article) { '' }
+
+          it { is_expected.to be_empty }
+        end
+
+        context 'when link leads to page with content' do
+          let(:feed_item_article) { 'Content' }
+
+          it { is_expected.to match_array([feed_item_article]) }
+        end
+      end
+    end
+  end
+end

data/spec/feeds_crawler_spec.rb

@@ -0,0 +1,28 @@
+require 'spec_helper'
+
+RSpec.describe FeedsCrawler do
+  describe '.crawl' do
+    let(:rss_url) { 'spec/fixtures/rss_feed.xml' }
+
+    subject { described_class.crawl(rss_url) }
+
+    it 'crawls crawlers feeds articles' do
+      is_expected.to match_array(['Real Content'])
+    end
+
+    context 'when passing an argument list' do
+      let(:article) { 'article' }
+      let(:rss_urls) { %w[first_url second_url] }
+
+      before do
+        allow_any_instance_of(Crawlers::Rss).to receive(:articles).and_return([article])
+      end
+
+      subject { described_class.crawl(*rss_urls).count }
+
+      it 'crawls articles for each argument' do
+        is_expected.to eq(rss_urls.length)
+      end
+    end
+  end
+end

data/spec/fixtures/page_with_content.html

@@ -0,0 +1,8 @@
+<html>
+<head>
+    <title>title!</title>
+</head>
+<body class='foo'>
+    <div class='comment'>Real Content</div>
+</body>
+</html>

data/spec/fixtures/rss_feed.xml

@@ -0,0 +1,10 @@
+<rss version="2.0">
+  <channel>
+    <title>Feed</title>
+    <link>link.com</link>
+    <description>Description</description>
+    <item>
+      <link>spec/fixtures/page_with_content.html</link>
+    </item>
+  </channel>
+</rss>

data/spec/spec_helper.rb

@@ -0,0 +1,13 @@
+require 'bundler/setup'
+require 'pry'
+require 'feeds_crawler'
+
+RSpec.configure do |config|
+  config.example_status_persistence_file_path = '.rspec_status'
+
+  config.disable_monkey_patching!
+
+  config.expect_with :rspec do |c|
+    c.syntax = :expect
+  end
+end

metadata

@@ -0,0 +1,201 @@
+--- !ruby/object:Gem::Specification
+name: feeds-crawler
+version: !ruby/object:Gem::Version
+  version: 0.2.0
+platform: ruby
+authors:
+- Andrey Tatarenko
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2017-09-16 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.15'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.15'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '10.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '10.0'
+- !ruby/object:Gem::Dependency
+  name: rspec
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.0'
+- !ruby/object:Gem::Dependency
+  name: pry
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: rubocop
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: parallel
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.12'
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 1.12.0
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.12'
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 1.12.0
+- !ruby/object:Gem::Dependency
+  name: ruby-readability
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.7'
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.7.0
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.7'
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.7.0
+- !ruby/object:Gem::Dependency
+  name: sanitize
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '4.5'
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 4.5.0
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '4.5'
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 4.5.0
+description: 
+email:
+- andrey17076@gmail.com
+executables:
+- console
+- setup
+extensions: []
+extra_rdoc_files: []
+files:
+- ".gitignore"
+- ".rspec"
+- Gemfile
+- Gemfile.lock
+- LICENSE
+- README.md
+- Rakefile
+- bin/console
+- bin/setup
+- feeds-crawler.gemspec
+- lib/crawlers/helpers/content.rb
+- lib/crawlers/rss.rb
+- lib/feeds_crawler.rb
+- spec/crawlers/helpers/content_spec.rb
+- spec/crawlers/rss_spec.rb
+- spec/feeds_crawler_spec.rb
+- spec/fixtures/page_with_content.html
+- spec/fixtures/rss_feed.xml
+- spec/spec_helper.rb
+homepage: https://github.com/andrey17076/feed-crawler
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.6.12
+signing_key: 
+specification_version: 4
+summary: This gem allows to crawl news articles from RSS feeds.
+test_files:
+- spec/crawlers/helpers/content_spec.rb
+- spec/crawlers/rss_spec.rb
+- spec/feeds_crawler_spec.rb
+- spec/fixtures/page_with_content.html
+- spec/fixtures/rss_feed.xml
+- spec/spec_helper.rb