rssdump 0.1.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 95a5bf76f09fe0626531a62ffbabc791fd1ca4ac
+  data.tar.gz: 8ba6115c49373a2281dcb120fa29b334d28abbe9
+SHA512:
+  metadata.gz: 1ffe505a06cf39cc23cd5dc40818163f32f00e75c1dc93d822f3fd797b198c6c96f1b3e06f0e6fb0d4295035a3a13d826ecf42f0844bcc26ca229123d8e1293e
+  data.tar.gz: 103b7df56f448c4e15c50b923b601716c15d98c54cc755b004723c96d0b8c4a7b8b4ff4d05816f51ea9e7041c61070b628f4df2e95ac31cd4a559c297eee2866

data/lib/rssdump/cleaning.rb

@@ -0,0 +1,36 @@
+require 'nokogiri'
+require 'time'
+
+module Rssdump
+  module Cleaning
+    URLS = /https?:\/\/[\S]+/
+
+    def clean_link url
+      (url || "").strip
+    end
+
+    def clean_html txt
+      if txt
+        ensure_valid txt
+        c = txt ? Nokogiri::HTML(CGI.unescapeHTML(txt)).text : ""
+        c = c.gsub(URLS, "").split("LIRE AUSSI")[0] || ""
+        c = c.gsub(/[\n\r\t ]+/, " ").strip
+        c
+      else
+        ""
+      end
+    end
+
+    def parse_pub_date str
+      Time.rfc2822 str
+    end
+
+    def ensure_valid txt
+      if !txt.valid_encoding?
+        txt.force_encoding("ISO-8859-1").encode("UTF-8")
+      else
+        txt
+      end
+    end
+  end
+end

data/lib/rssdump/dumper.rb

@@ -0,0 +1,69 @@
+require 'thread/pool'
+
+module Rssdump
+  class Dumper
+    def initialize opt
+      @mongo_url = opt[:mongo_url] || raise("Missing opt :mongo_url")
+      @store = MongoStore.new @mongo_url
+      @logger = Logging.logger[self]
+    end
+
+    def dump_feed feed_name, feed_url
+      begin
+        scraper = Scraper.new
+        @logger.info "Scraping feed #{feed_name} - #{feed_url}"
+        items = scraper.scrap feed_url, feed_name
+        nb = 0
+        for item in items
+          inserted = @store.upsert item
+          nb+=1 if inserted
+        end
+        if nb > 0
+          @logger.info "#{nb} new items inserted for feed #{feed_name} - #{feed_url}"
+        else
+          @logger.debug "No new item inserted for feed #{feed_name} - #{feed_url}"
+        end
+        if(nb == items.count)
+          @logger.warn "All #{nb} items were new for feed #{feed_name}. Consider increasing the scraping frequency"
+        end
+        {status: 0, nb_inserted: nb}
+      rescue => e
+        @logger.error "An error occurred while scraping feed #{feed_name} - #{feed_url}"
+        @logger.error "#{e}\n#{e.backtrace.join("\n")}"
+        {status: 1, error: e}
+      end
+    end
+
+    def dump file, opts = {threads: 1}
+      @logger.info "Starting a batch scrap from feed file #{file} [opts: #{opts}]"
+      pool = Thread.pool(opts[:threads])
+      mutex = Mutex.new
+
+      @start = Time.now
+      nb_inserted = 0
+      IO.readlines(file, :encoding => 'UTF-8').select do |l|
+        !l.strip.empty? && !l.strip.start_with?('#')
+      end.map do |l|
+        l.split("\t").map(&:strip)
+      end.each do |feed_name, feed_url|
+        pool.process do
+          res = dump_feed(feed_name, feed_url)
+          mutex.synchronize do
+            nb_inserted += res[:nb_inserted] if res[:status] == 0
+          end
+        end
+      end
+
+      # wait until all tasks are finished
+
+      @logger.info "Waiting for all pools to process."
+      pool.shutdown
+      @logger.info "All pools processed !"
+
+      @logger.info "Finished dumping file #{file} in #{"%.3f" % (Time.now - @start)} seconds."
+      @logger.info "Num of new items inserted #{nb_inserted}."
+      @logger.info "Total number of items in db: #{@store.nb_items}."
+      @logger.info "Data size : #{ '%.2f' % @store.disk_usage_mb}Mb."
+    end
+  end
+end

data/lib/rssdump/item.rb

@@ -0,0 +1,6 @@
+
+module Rssdump
+  class Item
+    attr_accessor :link, :title, :description, :category, :pub_date, :feed, :feed_name
+  end
+end

data/lib/rssdump/mongo_store.rb

@@ -0,0 +1,48 @@
+require 'mongo'
+
+module Rssdump
+  class MongoStore
+    DEFAULT_URL = "mongodb://127.0.0.1:27017/rssdump"
+    COLL_ITEMS = "items"
+
+    def initialize url
+      @url = url || DEFAULT_URL
+      client[COLL_ITEMS].indexes.create_one({ :link => 1 }, :unique => true)
+      client[COLL_ITEMS].indexes.create_one({ :pub_date => 1 }, :unique => false)
+      @logger = Logging.logger[self]
+    end
+
+    def upsert item
+      if client[COLL_ITEMS].find({link: item.link}).count == 0
+        @logger.debug "Inserting new item #{item.link} to store"
+        client[COLL_ITEMS].insert_one({
+          v: Rssdump::VERSION,
+          title: item.title,
+          link: item.link,
+          feed: item.feed,
+          feed_name: item.feed_name,
+          category: item.category,
+          description: item.description,
+          pub_date: item.pub_date,
+        })
+        true
+      else
+        false
+      end
+    end
+
+    def nb_items
+      client[COLL_ITEMS].count
+    end
+
+    def disk_usage_mb
+      client.database.command({dbStats: 1, scale: 1024**2}).first["dataSize"]
+    end
+
+    private
+
+    def client
+      @client ||= Mongo::Client.new(@url)
+    end
+  end
+end

data/lib/rssdump/scraper.rb

@@ -0,0 +1,37 @@
+# encoding: utf-8
+require 'simple-rss'
+require 'open-uri'
+
+module Rssdump
+  class Scraper
+    include Logging
+    include Cleaning
+
+    attr_reader :errors
+
+    def scrap feed, feed_name = "_"
+      @errors = []
+    	rss = SimpleRSS.parse ensure_valid(open(feed).read)
+      rss.items.map do |item|
+        begin
+          ritem = Item.new
+          ritem.title = clean_html(item.title)
+          ritem.category = clean_html(item.category)
+          ritem.description = clean_html(item.description)
+          ritem.pub_date = item.pubDate || item.updated
+          ritem.link = clean_link(item.link)
+          ritem.feed = feed
+          ritem.feed_name = feed_name
+          ritem
+        rescue => e
+          logger.error "An error occurred during cleaning with item #{item.link}."
+          logger.error "#{e}\n#{e.backtrace.join("\n")}"
+          logger.warn "Ignoring item #{item.link}."
+          nil
+        end
+      end.select do |item|
+        !item.nil?
+      end
+    end
+  end
+end

data/lib/rssdump/tasks.rb

@@ -0,0 +1,37 @@
+require_relative "../rssdump"
+require 'awesome_print'
+
+def obj_to_hash obj
+  obj.instance_variables.each_with_object({}) do |var, hash|
+    hash[var.to_s.delete("@")] = obj.instance_variable_get(var)
+  end
+end
+
+namespace :collect do
+  desc "Retrieves a feed given a URL and prints the parsed items to the console"
+  task :feed do
+     url = ENV["URL"] || raise("Missing URL variable")
+     items = Rssdump::Scraper.new.scrap url
+     items.each do |item|
+       ap obj_to_hash(item)
+     end
+  end
+  desc "Retrieves a feed given a URL, parses it using SimpleRSS and prints items to the console"
+  task :feed_simplerss do
+    url = ENV["URL"] || raise("Missing URL variable")
+    rss = SimpleRSS.parse open(url)
+    rss.items.map do |item|
+      ap '-'*80
+      ap "title: [#{item.title ? item.title.encoding : ""}]#{item.title}"
+      ap "description: [#{item.description ? item.description.encoding : ""}]#{item.description}"
+      ap "pubDate: #{item.pubDate}"
+      ap "category: [#{item.category ? item.category.encoding : ""}]#{item.category}"
+      ap "link: [#{item.link ? item.link.encoding : ""}]#{item.link}"
+    end
+  end
+  desc "Retrieves a feed given a URL and prints raw body to the console"
+  task :feed_raw do
+    url = ENV["URL"] || raise("Missing URL variable")
+    ap open(url).read
+  end
+end

data/lib/rssdump/version.rb

@@ -0,0 +1,3 @@
+module Rssdump
+  VERSION = "0.1.0"
+end

data/lib/rssdump.rb

@@ -0,0 +1,8 @@
+require "logging"
+
+require_relative "./rssdump/cleaning"
+require_relative "./rssdump/mongo_store"
+require_relative "./rssdump/item"
+require_relative "./rssdump/dumper"
+require_relative "./rssdump/scraper"
+require_relative "./rssdump/version"

metadata

@@ -0,0 +1,165 @@
+--- !ruby/object:Gem::Specification
+name: rssdump
+version: !ruby/object:Gem::Version
+  version: 0.1.0
+platform: ruby
+authors:
+- Damien Cram
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2016-05-04 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: nokogiri
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: logging
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: simple-rss
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: thread
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: mongo
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '2.1'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '2.1'
+- !ruby/object:Gem::Dependency
+  name: awesome_print
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: rspec
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+description: Retrieves all items from an RSS feed and stores them to a MongoDB collection.
+  Rssdump is based on simple-rss.
+email:
+- damien.cram@laposte.net
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- lib/rssdump.rb
+- lib/rssdump/cleaning.rb
+- lib/rssdump/dumper.rb
+- lib/rssdump/item.rb
+- lib/rssdump/mongo_store.rb
+- lib/rssdump/scraper.rb
+- lib/rssdump/tasks.rb
+- lib/rssdump/version.rb
+homepage: http://github.com/pompadour/rssdump.git
+licenses: []
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.5.1
+signing_key: 
+specification_version: 4
+summary: Retrieves all items from an RSS feed and stores them to a MongoDB collection.
+  Rssdump is based on simple-rss.
+test_files: []