rssdump 1.0.0 → 1.1.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 4a064db5574bf273163d1194ea130f14f9124f48
-  data.tar.gz: 3aec73daf1261d65a697fbfdf113d33c22d21a33
+  metadata.gz: b7d6af91750470ce86258f4894c30b203321a327
+  data.tar.gz: a4ee133486a968fef2529416135e3c41269bfe59
 SHA512:
-  metadata.gz: 0b27cb123ebd759ff853f35d4ae763a056fc2696750284039c862bea6de4a5632871d822718c98dc666cedca93de069a18678881206dba410ec43e6cd5579bc2
-  data.tar.gz: c54ab35de1b51eaad4bd3baf42507f83a52cd7ba68b7af77e9c5eea13f4766c706223afcbafcb392ae0a34f5aef063d2225ae9e41456ddb6de62736a029e4975
+  metadata.gz: 04fd37d7ac11f96b795e4ded5517e85cae6759cabdd99fe4e75156de740da3ce67b483f5a928914a159c23835ad711cbad3ceaad72cbe4221987bcff3b8edd06
+  data.tar.gz: 0301743aefbe3176381d1b2f79053d5f7fc3389eb74fa40a11384e10ca87c1e932fa0a3f9f03c63b38889aa71c480065c8339b960cc71ae0b9b922b081cd53dc

data/lib/rssdump.rb

@@ -2,6 +2,5 @@ require "logging"
 
 require_relative "./rssdump/cleaning"
 require_relative "./rssdump/item"
-require_relative "./rssdump/dumper"
 require_relative "./rssdump/scraper"
 require_relative "./rssdump/version"

data/lib/rssdump/scraper.rb

@@ -7,8 +7,13 @@ module Rssdump
     include Logging
     include Cleaning
 
-    def scrap feed_url
-    	rss = SimpleRSS.parse ensure_valid(open(feed_url).read)
+    def scrap feed
+      scrap_from_body(open(feed).read, feed)
+    end
+
+    def scrap_from_body body, feed_url
+      @errors = []
+    	rss = SimpleRSS.parse ensure_valid(body)
       status = :ok
       errors = []
       items = rss.items.map do |item|

data/lib/rssdump/version.rb

@@ -1,3 +1,3 @@
 module Rssdump
-  VERSION = "1.0.0"
+  VERSION = "1.1.0"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: rssdump
 version: !ruby/object:Gem::Version
-  version: 1.0.0
+  version: 1.1.0
 platform: ruby
 authors:
 - Damien Cram
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2016-07-06 00:00:00.000000000 Z
+date: 2016-07-09 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: nokogiri
@@ -117,7 +117,6 @@ extra_rdoc_files: []
 files:
 - lib/rssdump.rb
 - lib/rssdump/cleaning.rb
-- lib/rssdump/dumper.rb
 - lib/rssdump/item.rb
 - lib/rssdump/scraper.rb
 - lib/rssdump/tasks.rb

data/lib/rssdump/dumper.rb

@@ -1,69 +0,0 @@
-require 'thread/pool'
-
-module Rssdump
-  class Dumper
-    def initialize opt
-      @mongo_url = opt[:mongo_url] || raise("Missing opt :mongo_url")
-      @store = MongoStore.new @mongo_url
-      @logger = Logging.logger[self]
-    end
-
-    def dump_feed feed_name, feed_url
-      begin
-        scraper = Scraper.new
-        @logger.info "Scraping feed #{feed_name} - #{feed_url}"
-        items = scraper.scrap feed_url, feed_name
-        nb = 0
-        for item in items
-          inserted = @store.upsert item
-          nb+=1 if inserted
-        end
-        if nb > 0
-          @logger.info "#{nb} new items inserted for feed #{feed_name} - #{feed_url}"
-        else
-          @logger.debug "No new item inserted for feed #{feed_name} - #{feed_url}"
-        end
-        if(nb == items.count)
-          @logger.warn "All #{nb} items were new for feed #{feed_name}. Consider increasing the scraping frequency"
-        end
-        {status: 0, nb_inserted: nb}
-      rescue => e
-        @logger.error "An error occurred while scraping feed #{feed_name} - #{feed_url}"
-        @logger.error "#{e}\n#{e.backtrace.join("\n")}"
-        {status: 1, error: e}
-      end
-    end
-
-    def dump file, opts = {threads: 1}
-      @logger.info "Starting a batch scrap from feed file #{file} [opts: #{opts}]"
-      pool = Thread.pool(opts[:threads])
-      mutex = Mutex.new
-
-      @start = Time.now
-      nb_inserted = 0
-      IO.readlines(file, :encoding => 'UTF-8').select do |l|
-        !l.strip.empty? && !l.strip.start_with?('#')
-      end.map do |l|
-        l.split("\t").map(&:strip)
-      end.each do |feed_name, feed_url|
-        pool.process do
-          res = dump_feed(feed_name, feed_url)
-          mutex.synchronize do
-            nb_inserted += res[:nb_inserted] if res[:status] == 0
-          end
-        end
-      end
-
-      # wait until all tasks are finished
-
-      @logger.info "Waiting for all pools to process."
-      pool.shutdown
-      @logger.info "All pools processed !"
-
-      @logger.info "Finished dumping file #{file} in #{"%.3f" % (Time.now - @start)} seconds."
-      @logger.info "Num of new items inserted #{nb_inserted}."
-      @logger.info "Total number of items in db: #{@store.nb_items}."
-      @logger.info "Data size : #{ '%.2f' % @store.disk_usage_mb}Mb."
-    end
-  end
-end