rssdump 0.1.0 → 1.0.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 95a5bf76f09fe0626531a62ffbabc791fd1ca4ac
-  data.tar.gz: 8ba6115c49373a2281dcb120fa29b334d28abbe9
+  metadata.gz: 4a064db5574bf273163d1194ea130f14f9124f48
+  data.tar.gz: 3aec73daf1261d65a697fbfdf113d33c22d21a33
 SHA512:
-  metadata.gz: 1ffe505a06cf39cc23cd5dc40818163f32f00e75c1dc93d822f3fd797b198c6c96f1b3e06f0e6fb0d4295035a3a13d826ecf42f0844bcc26ca229123d8e1293e
-  data.tar.gz: 103b7df56f448c4e15c50b923b601716c15d98c54cc755b004723c96d0b8c4a7b8b4ff4d05816f51ea9e7041c61070b628f4df2e95ac31cd4a559c297eee2866
+  metadata.gz: 0b27cb123ebd759ff853f35d4ae763a056fc2696750284039c862bea6de4a5632871d822718c98dc666cedca93de069a18678881206dba410ec43e6cd5579bc2
+  data.tar.gz: c54ab35de1b51eaad4bd3baf42507f83a52cd7ba68b7af77e9c5eea13f4766c706223afcbafcb392ae0a34f5aef063d2225ae9e41456ddb6de62736a029e4975

data/lib/rssdump/item.rb

@@ -1,6 +1,6 @@
 
 module Rssdump
   class Item
-    attr_accessor :link, :title, :description, :category, :pub_date, :feed, :feed_name
+    attr_accessor :link, :title, :description, :category, :pub_date, :feed
   end
 end

data/lib/rssdump/scraper.rb

@@ -7,12 +7,11 @@ module Rssdump
     include Logging
     include Cleaning
 
-    attr_reader :errors
-
-    def scrap feed, feed_name = "_"
-      @errors = []
-    	rss = SimpleRSS.parse ensure_valid(open(feed).read)
-      rss.items.map do |item|
+    def scrap feed_url
+    	rss = SimpleRSS.parse ensure_valid(open(feed_url).read)
+      status = :ok
+      errors = []
+      items = rss.items.map do |item|
         begin
           ritem = Item.new
           ritem.title = clean_html(item.title)
@@ -20,10 +19,11 @@ module Rssdump
           ritem.description = clean_html(item.description)
           ritem.pub_date = item.pubDate || item.updated
           ritem.link = clean_link(item.link)
-          ritem.feed = feed
-          ritem.feed_name = feed_name
+          ritem.feed = feed_url
           ritem
         rescue => e
+          status = :ko
+          errors << e
           logger.error "An error occurred during cleaning with item #{item.link}."
           logger.error "#{e}\n#{e.backtrace.join("\n")}"
           logger.warn "Ignoring item #{item.link}."
@@ -32,6 +32,11 @@ module Rssdump
       end.select do |item|
         !item.nil?
       end
+      {
+        status: status,
+        errors: errors,
+        items: items
+      }
     end
   end
 end

data/lib/rssdump/version.rb

@@ -1,3 +1,3 @@
 module Rssdump
-  VERSION = "0.1.0"
+  VERSION = "1.0.0"
 end

data/lib/rssdump.rb

@@ -1,7 +1,6 @@
 require "logging"
 
 require_relative "./rssdump/cleaning"
-require_relative "./rssdump/mongo_store"
 require_relative "./rssdump/item"
 require_relative "./rssdump/dumper"
 require_relative "./rssdump/scraper"

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: rssdump
 version: !ruby/object:Gem::Version
-  version: 0.1.0
+  version: 1.0.0
 platform: ruby
 authors:
 - Damien Cram
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2016-05-04 00:00:00.000000000 Z
+date: 2016-07-06 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: nokogiri
@@ -66,20 +66,6 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
-- !ruby/object:Gem::Dependency
-  name: mongo
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '2.1'
-  type: :runtime
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '2.1'
 - !ruby/object:Gem::Dependency
   name: awesome_print
   requirement: !ruby/object:Gem::Requirement
@@ -122,8 +108,7 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
-description: Retrieves all items from an RSS feed and stores them to a MongoDB collection.
-  Rssdump is based on simple-rss.
+description: Retrieves all items from an RSS feed and clean them.
 email:
 - damien.cram@laposte.net
 executables: []
@@ -134,7 +119,6 @@ files:
 - lib/rssdump/cleaning.rb
 - lib/rssdump/dumper.rb
 - lib/rssdump/item.rb
-- lib/rssdump/mongo_store.rb
 - lib/rssdump/scraper.rb
 - lib/rssdump/tasks.rb
 - lib/rssdump/version.rb
@@ -160,6 +144,5 @@ rubyforge_project:
 rubygems_version: 2.5.1
 signing_key: 
 specification_version: 4
-summary: Retrieves all items from an RSS feed and stores them to a MongoDB collection.
-  Rssdump is based on simple-rss.
+summary: Retrieves all items from an RSS feed and clean them.
 test_files: []

data/lib/rssdump/mongo_store.rb

@@ -1,48 +0,0 @@
-require 'mongo'
-
-module Rssdump
-  class MongoStore
-    DEFAULT_URL = "mongodb://127.0.0.1:27017/rssdump"
-    COLL_ITEMS = "items"
-
-    def initialize url
-      @url = url || DEFAULT_URL
-      client[COLL_ITEMS].indexes.create_one({ :link => 1 }, :unique => true)
-      client[COLL_ITEMS].indexes.create_one({ :pub_date => 1 }, :unique => false)
-      @logger = Logging.logger[self]
-    end
-
-    def upsert item
-      if client[COLL_ITEMS].find({link: item.link}).count == 0
-        @logger.debug "Inserting new item #{item.link} to store"
-        client[COLL_ITEMS].insert_one({
-          v: Rssdump::VERSION,
-          title: item.title,
-          link: item.link,
-          feed: item.feed,
-          feed_name: item.feed_name,
-          category: item.category,
-          description: item.description,
-          pub_date: item.pub_date,
-        })
-        true
-      else
-        false
-      end
-    end
-
-    def nb_items
-      client[COLL_ITEMS].count
-    end
-
-    def disk_usage_mb
-      client.database.command({dbStats: 1, scale: 1024**2}).first["dataSize"]
-    end
-
-    private
-
-    def client
-      @client ||= Mongo::Client.new(@url)
-    end
-  end
-end