bf_multi_rss 0.3.5 → 0.3.6

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: ee6ef9c51e1c46557598c01e7c86409deb1570fa
-  data.tar.gz: 5978d5e0ae579add6e9b67834250f31ac632038c
+  metadata.gz: 52417ce78b48cad3fceead305a2fcacff48a9004
+  data.tar.gz: a86c24913778cd6ab30bc5f0ee63273b0bb40161
 SHA512:
-  metadata.gz: 9b562dc4a5c91a6e84c1fe8f7c6d814f1e6f43dde5c1ea1f28a1b073b92dc3d9ac410b5440b27cd79abad06316a4185e62ac21588b246ec654e14a4950fd8a69
-  data.tar.gz: d187969800ceaf6ca23e02b7509ad5e1e75a81680b6a9e36ce77b0531a5b61a84f0ef3277a64923202723d96e24e836dcff7d9cf4a89166670ee00923b22b051
+  metadata.gz: 6ffec392ce382bdde93b3057f2b9715b48e0a06bcfe5e37ddb1cee34be6b88ebdf63494a2bf4e5f3626456db778dff4c1497119be7d7b334be0fad063c5676f6
+  data.tar.gz: 3c516378eb75b04235d1d8f9afc540b814212e9528bb61bce0e498f6f69db6005af3441966ddb8a382603f95d0378deb9d1c3bd2027c0a2f194ac0f68cb1016d

data/.gitignore

@@ -0,0 +1,12 @@
+/.bundle/
+/.yardoc
+/Gemfile.lock
+/_yardoc/
+/coverage/
+/doc/
+/pkg/
+/spec/reports/
+/tmp/
+
+# rspec failure tracking
+.rspec_status

data/.rspec

@@ -0,0 +1,2 @@
+--format documentation
+--color

data/.rubocop.yml

@@ -0,0 +1,86 @@
+Metrics/BlockLength:
+  Enabled: false
+
+# Disable global vars
+Style/SpecialGlobalVars:
+  Enabled: false
+
+# Disable class length restriction
+Metrics/ClassLength:
+  Enabled: false
+
+# Disable module length restriction
+Metrics/ModuleLength:
+  Enabled: false
+
+# Disable method length restriction
+Metrics/MethodLength:
+  Enabled: false
+
+# Disable checking of ABC metrics (assignment, branch, condition)
+Metrics/AbcSize:
+  Enabled: false
+
+# Disable method perceived complexity (how it is hard to read a method)
+Metrics/PerceivedComplexity:
+  Enabled: false
+
+# Disable cyclomatic complexity which is a number of paths through a method
+Metrics/CyclomaticComplexity:
+  Enabled: false
+
+# Block nesting is 5
+Metrics/BlockNesting:
+  Max: 5
+
+# Set max line length to 120
+Metrics/LineLength:
+  Max: 120
+
+Style/Documentation:
+  Enabled: false
+
+# Trailing comma helps not to forget to put it
+# if you want to add new value below
+Style/TrailingCommaInLiteral:
+  Enabled: false
+
+# It is totally ok to use block chain by my preference
+# It promotes functional programming style
+Style/MultilineBlockChain:
+  Enabled: false
+
+# Empty lines are enabled to simplify vim paragraph selection
+Style/EmptyLinesAroundBlockBody:
+  Enabled: false
+
+Style/EmptyLinesAroundClassBody:
+  Enabled: false
+
+Style/EmptyLinesAroundMethodBody:
+  Enabled: false
+
+# Disable multiline method indentation, not comfortable with it
+Style/MultilineMethodCallIndentation:
+  Enabled: false
+
+# extend self uses ruby inheritance instead of copying methods
+Style/ModuleFunction:
+  Enabled: false
+
+Style/Lambda:
+  Enabled: false
+
+AllCops:
+  DisplayCopNames: true
+  Exclude:
+    - db/**/*
+    - bin/**
+    - config/**/**
+    - spec/dummy/bin/**
+    - spec/dummy/db/**
+    - Guardfile
+
+Rails:
+  Enabled: false
+  Exclude:

data/.travis.yml

@@ -0,0 +1,5 @@
+sudo: false
+language: ruby
+rvm:
+  - 2.3.1
+before_install: gem install bundler -v 1.14.5

data/bf_multi_rss.gemspec

@@ -0,0 +1,27 @@
+# coding: utf-8
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+
+Gem::Specification.new do |spec|
+  spec.name = 'bf_multi_rss'
+  spec.version = '0.3.6'
+  spec.authors = ['Sebastian Schürmann']
+  spec.email = ['sebs@2xs.org']
+
+  spec.summary = 'Parallel rss fetching'
+  spec.homepage = 'https://github.com/bundesfeeds'
+  spec.license = 'MIT'
+
+  spec.files = `git ls-files -z`.split("\x0").reject do |f|
+    f.match(%r{^(test|spec|features)/})
+  end
+  spec.bindir        = 'exe'
+  spec.executables   = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
+  spec.require_paths = ['lib']
+
+  spec.add_development_dependency 'bundler', '~> 1.14'
+  spec.add_development_dependency 'rake', '~> 10.0'
+  spec.add_development_dependency 'rspec', '~> 3.0'
+  spec.add_runtime_dependency 'http', '~> 2.2'
+  spec.add_runtime_dependency 'parallel', '~> 1.10'
+end

data/lib/bf_multi_rss.rb

@@ -0,0 +1,2 @@
+module BfMultiRss
+end

data/lib/bf_multi_rss/fetcher.rb

@@ -0,0 +1,80 @@
+require 'rss'
+require 'http'
+require 'parallel'
+require 'bf_multi_rss/rss_result'
+require 'bf_multi_rss/rss_error'
+
+module BfMultiRss
+  class NotInvertibleError < StandardError
+  end
+
+  class Fetcher
+    attr_reader :concurrency, :errors
+    def initialize(concurrency = 4)
+      @concurrency = concurrency
+      @errors = []
+    end
+
+    def fetch_uri(uri)
+      response = HTTP.get(uri)
+      raise_errors(response, uri)
+      response.to_s
+    end
+
+    def fetch_rss(uri)
+      response = fetch_uri(uri)
+      rss = RSS::Parser.parse(response, false)
+      if rss.nil?
+        err = 'ParseErr ' + uri
+        raise NotInvertibleError, err
+      end
+      rss.items
+    end
+
+    def raise_errors(response, uri)
+      case response.status
+      when 500
+        err = 'Http500 ' + uri
+        raise NotInvertibleError, err
+      when 404
+        err = 'Http404 ' + uri
+        raise NotInvertibleError, err
+      when 301
+        err = 'Http301 ' + uri
+        raise NotInvertibleError, err
+      end
+    end
+
+    def fetch_all(uris)
+      @errors = []
+      results = Parallel.map(
+        uris,
+        in_processes: @concurrency
+      ) do |uri|
+        begin
+          posts = fetch_rss(uri)
+          BfMultiRss::RssResult.new(uri, posts)
+
+        rescue  REXML::ParseException,
+                OpenURI::HTTPError,
+                Errno::EHOSTUNREACH,
+                Errno::ETIMEDOUT,
+                RSS::NotWellFormedError,
+                Net::OpenTimeout,
+                Net::ReadTimeout,
+                Errno::ECONNREFUSED,
+                Errno::ECONNRESET,
+                NotInvertibleError => e
+          BfMultiRss::RssError.new(uri, e)
+        end
+      end
+      errors = results.select do |result|
+        result.is_a? BfMultiRss::RssError
+      end
+      @errors = errors
+      results.select do |result|
+        !result.is_a? BfMultiRss::RssError
+      end
+    end
+  end
+end

data/lib/bf_multi_rss/rss_error.rb

@@ -0,0 +1,9 @@
+module BfMultiRss
+  class RssError
+    attr_reader :uri, :e
+    def initialize(uri, e)
+      @uri = uri
+      @e = e
+    end
+  end
+end

data/lib/bf_multi_rss/rss_result.rb

@@ -0,0 +1,11 @@
+module BfMultiRss
+  class RssResult
+    # remove the public setter interface
+    attr_reader :posts, :src
+
+    def initialize(src, posts = [])
+      @posts = posts
+      @src = src
+    end
+  end
+end

metadata

@@ -1,12 +1,12 @@
 --- !ruby/object:Gem::Specification
 name: bf_multi_rss
 version: !ruby/object:Gem::Version
-  version: 0.3.5
+  version: 0.3.6
 platform: ruby
 authors:
 - Sebastian Schürmann
 autorequire: 
-bindir: bin
+bindir: exe
 cert_chain: []
 date: 2017-03-13 00:00:00.000000000 Z
 dependencies:
@@ -81,20 +81,29 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '1.10'
 description: 
-email: rubycoder@example.com
+email:
+- sebs@2xs.org
 executables: []
 extensions: []
 extra_rdoc_files: []
 files:
+- ".gitignore"
+- ".rspec"
+- ".rubocop.yml"
+- ".travis.yml"
 - CODE_OF_CONDUCT.md
 - Gemfile
-- Gemfile.lock
 - Guardfile
 - LICENSE.txt
 - README.md
 - Rakefile
+- bf_multi_rss.gemspec
 - bin/console
 - bin/setup
+- lib/bf_multi_rss.rb
+- lib/bf_multi_rss/fetcher.rb
+- lib/bf_multi_rss/rss_error.rb
+- lib/bf_multi_rss/rss_result.rb
 homepage: https://github.com/bundesfeeds
 licenses:
 - MIT

data/Gemfile.lock

@@ -1,122 +0,0 @@
-PATH
-  remote: .
-  specs:
-    bf_multi_rss (0.3.5)
-      http (~> 2.2)
-      parallel (~> 1.10)
-
-GEM
-  remote: https://rubygems.org/
-  specs:
-    addressable (2.5.0)
-      public_suffix (~> 2.0, >= 2.0.2)
-    ast (2.3.0)
-    coderay (1.1.1)
-    crack (0.4.3)
-      safe_yaml (~> 1.0.0)
-    diff-lcs (1.3)
-    domain_name (0.5.20170223)
-      unf (>= 0.0.5, < 1.0.0)
-    ffi (1.9.18)
-    formatador (0.2.5)
-    guard (2.14.1)
-      formatador (>= 0.2.4)
-      listen (>= 2.7, < 4.0)
-      lumberjack (~> 1.0)
-      nenv (~> 0.1)
-      notiffany (~> 0.0)
-      pry (>= 0.9.12)
-      shellany (~> 0.0)
-      thor (>= 0.18.1)
-    guard-compat (1.2.1)
-    guard-rspec (4.7.3)
-      guard (~> 2.1)
-      guard-compat (~> 1.1)
-      rspec (>= 2.99.0, < 4.0)
-    guard-rubocop (1.2.0)
-      guard (~> 2.0)
-      rubocop (~> 0.20)
-    hashdiff (0.3.2)
-    http (2.2.1)
-      addressable (~> 2.3)
-      http-cookie (~> 1.0)
-      http-form_data (~> 1.0.1)
-      http_parser.rb (~> 0.6.0)
-    http-cookie (1.0.3)
-      domain_name (~> 0.5)
-    http-form_data (1.0.1)
-    http_parser.rb (0.6.0)
-    listen (3.1.5)
-      rb-fsevent (~> 0.9, >= 0.9.4)
-      rb-inotify (~> 0.9, >= 0.9.7)
-      ruby_dep (~> 1.2)
-    lumberjack (1.0.11)
-    method_source (0.8.2)
-    nenv (0.3.0)
-    notiffany (0.1.1)
-      nenv (~> 0.1)
-      shellany (~> 0.0)
-    parallel (1.11.1)
-    parser (2.4.0.0)
-      ast (~> 2.2)
-    powerpack (0.1.1)
-    pry (0.10.4)
-      coderay (~> 1.1.0)
-      method_source (~> 0.8.1)
-      slop (~> 3.4)
-    public_suffix (2.0.5)
-    rainbow (2.2.1)
-    rake (10.5.0)
-    rb-fsevent (0.9.8)
-    rb-inotify (0.9.8)
-      ffi (>= 0.5.0)
-    rspec (3.5.0)
-      rspec-core (~> 3.5.0)
-      rspec-expectations (~> 3.5.0)
-      rspec-mocks (~> 3.5.0)
-    rspec-core (3.5.4)
-      rspec-support (~> 3.5.0)
-    rspec-expectations (3.5.0)
-      diff-lcs (>= 1.2.0, < 2.0)
-      rspec-support (~> 3.5.0)
-    rspec-mocks (3.5.0)
-      diff-lcs (>= 1.2.0, < 2.0)
-      rspec-support (~> 3.5.0)
-    rspec-support (3.5.0)
-    rubocop (0.47.1)
-      parser (>= 2.3.3.1, < 3.0)
-      powerpack (~> 0.1)
-      rainbow (>= 1.99.1, < 3.0)
-      ruby-progressbar (~> 1.7)
-      unicode-display_width (~> 1.0, >= 1.0.1)
-    ruby-progressbar (1.8.1)
-    ruby_dep (1.5.0)
-    safe_yaml (1.0.4)
-    shellany (0.0.1)
-    slop (3.6.0)
-    thor (0.19.4)
-    unf (0.1.4)
-      unf_ext
-    unf_ext (0.0.7.2)
-    unicode-display_width (1.1.3)
-    webmock (2.3.2)
-      addressable (>= 2.3.6)
-      crack (>= 0.3.2)
-      hashdiff
-
-PLATFORMS
-  ruby
-
-DEPENDENCIES
-  bf_multi_rss!
-  bundler (~> 1.14)
-  guard
-  guard-rspec
-  guard-rubocop
-  rake (~> 10.0)
-  rspec
-  rubocop
-  webmock
-
-BUNDLED WITH
-   1.14.6