opml_janitor 0.0.1

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 739ca7e9b2c084b9bcbb7d141f4d852ff908eb12
+  data.tar.gz: 12fea1d3defa5bb56ae7c37ee5d2240ee68e8020
+SHA512:
+  metadata.gz: 9d9d7b67ea243c90f896ec75ee442c395c02ff6d26a6615f912a7ce581a2b2f3a4f761337f7a39233c7c6816cc576f2892ad46ab8d25fb27a545f9b0e70369e0
+  data.tar.gz: 201af0d908a7375fcf390da31b519c82434147ade3f4e16ea331a7c5305d0978819e7df168a801b9c2813b77c30cd4b015b1197f6aed9aa1bca8c0c1161f39ce

data/.gitignore

@@ -0,0 +1,14 @@
+/.bundle/
+/.yardoc
+/Gemfile.lock
+/_yardoc/
+/coverage/
+/doc/
+/pkg/
+/spec/reports/
+/tmp/
+*.bundle
+*.so
+*.o
+*.a
+mkmf.log

data/Gemfile

@@ -0,0 +1,4 @@
+source 'https://rubygems.org'
+gem 'opml_saw', :git => "git://github.com/feedbin/opml_saw.git", :branch => "master"
+# Specify your gem's dependencies in opml_janitor.gemspec
+gemspec

data/LICENSE.txt

@@ -0,0 +1,22 @@
+Copyright (c) 2015 chrislee35
+
+MIT License
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.md

@@ -0,0 +1,62 @@
+# OpmlJanitor
+
+Tool to clean up broken and stale RSS feeds from an OPML file.
+It parses the XML, and for each feed, it downloads the RSS/Atom/etc., validates that the feed has been active within the given time frame, and writes the result to a new OPML XML document containing only the good feeds.
+
+## Installation
+
+Add this line to your application's Gemfile:
+
+    gem 'opml_janitor'
+
+And then execute:
+
+    $ bundle
+
+Or install it yourself as:
+
+    $ gem install opml_janitor
+
+## Usage
+
+	require 'opml_janitor'
+	
+	opml_janitor = OpmlJanitor::Parser.from_filehandle("example.opml")
+	opml_janitor.debug = true
+	opml_janitor.threads = 20
+	opml_janitor.validate!(Time.now - (30*24*60*60))
+	result = opml_janitor.to_xml
+	
+
+Or you could use the bundled tool
+
+	$ opml_janitor -h
+	Usage: opml_janitor [-hv] [-t <# threads>] [-s <date>] [-i <input file>] [-o <output file>]
+	  -h    displays this help text
+	  -v    turns on debugging messages (encouraged)
+	  -t #  set the number of threads (default 1) (highly encouraged)
+	  -i    specify a file for input (default is standard input)
+	  -o    specify a file for output (default is standard output)
+	  
+
+Example: 
+
+	$ opml_janitor -i subscriptions.xml -o sub.xml -s "2015-01-01" -v -t 20
+
+* Reads subscriptions from subscriptions.xml
+* Writes results to sub.xml
+* Must have posts newer than 2015-01-01 00:00:00 (local time zone)
+* Verbose messages
+* 20 threads
+
+## Related
+
+Related work: <a href='https://github.com/feedbin/opml_saw'>opml_saw</a>
+
+## Contributing
+
+1. Fork it ( https://github.com/chrislee35/opml_janitor/fork )
+2. Create your feature branch (`git checkout -b my-new-feature`)
+3. Commit your changes (`git commit -am 'Add some feature'`)
+4. Push to the branch (`git push origin my-new-feature`)
+5. Create a new Pull Request

data/Rakefile

@@ -0,0 +1,20 @@
+#!/usr/bin/env rake
+require "bundler/gem_tasks"
+
+require 'rake/testtask'
+require 'rdoc/task'
+
+Rake::TestTask.new do |t|
+	t.libs << 'lib'
+	t.test_files = FileList['test/test_*.rb']
+	t.verbose = true
+end
+
+RDoc::Task.new do |rd|
+  rd.main = "README.doc"
+  rd.rdoc_files.include("README.md", "lib/**/*.rb")
+  rd.options << "--all"
+  rd.options << "--verbose"
+end
+
+task :default => :test

data/bin/opml_janitor

@@ -0,0 +1,59 @@
+#!/usr/bin/env ruby
+require 'opml_janitor'
+require 'getoptlong'
+require 'time'
+
+opts = GetoptLong.new(
+  [ '--help', '-h', GetoptLong::NO_ARGUMENT ],
+  [ '--debug', '-v', GetoptLong::NO_ARGUMENT ],
+  [ '--since', '-s', GetoptLong::REQUIRED_ARGUMENT ],
+  [ '--input', '-i', GetoptLong::REQUIRED_ARGUMENT ],
+  [ '--output', '-o', GetoptLong::REQUIRED_ARGUMENT ],
+  [ '--threads', '-t', GetoptLong::REQUIRED_ARGUMENT ]
+)
+
+options = {
+  :help => false,
+  :debug => false,
+  :threads => 1,
+  :since => nil,
+  :input => $stdin,
+  :output => $stdout
+}
+
+opts.each do |opt, arg|
+	case opt
+	when '--help'
+		options[:help] = true
+	when '--debug'
+		options[:debug] = true
+  when '--since'
+    options[:since] = Time.parse(arg)
+  when '--input'
+    options[:input] = File.open(arg, 'r')
+  when '--output'
+    options[:output] = File.open(arg, 'w')
+  when '--threads'
+    options[:threads] = arg.to_i
+  else
+    options[:help] = true
+  end
+end
+
+if options[:help]
+  puts "Usage: #{$0} [-hv] [-t <# threads>] [-s <date>] [-i <input file>] [-o <output file>]"
+  puts "  -h    displays this help text"
+  puts "  -v    turns on debugging messages (encouraged)"
+  puts "  -t #  set the number of threads (default 1) (highly encouraged)"
+  puts "  -i    specify a file for input (default is standard input)"
+  puts "  -o    specify a file for output (default is standard output)"
+  exit
+end
+
+opml_janitor = OpmlJanitor::Parser.from_filehandle(options[:input])
+opml_janitor.debug = options[:debug]
+opml_janitor.threads = options[:threads]
+opml_janitor.validate!(options[:since])
+result = opml_janitor.to_xml
+options[:output].puts(result)
+

data/lib/opml_janitor.rb

@@ -0,0 +1,177 @@
+require "opml_janitor/version"
+require "opml_janitor/outline"
+require 'nokogiri'
+require 'open-uri'
+require 'rss'
+require 'timeout'
+require 'thread'
+require 'pp'
+
+module OpmlJanitor # :nodoc:
+  # The Parser class takes in the contents of an OPML XML document and
+  # can filter and save the results
+  class Parser
+    # initialize takes the contents of an OPML XML document and a flag
+    # for debug messages (default false)
+    def initialize(contents, debug = false)
+      @xml = contents
+      @opml = Nokogiri::XML.parse(@xml)
+      @debug = debug
+      @threads = 1
+      @timeout = 20
+    end
+    
+    # debug= sets the debug flag
+    def debug=(debug)
+      @debug = debug
+    end
+    
+    # timeout= sets the timeout for downloading and processing each feed
+    # the default is 20 seconds
+    def timeout=(timeout)
+      @timeout = timeout
+    end
+    
+    ##
+    # threads= sets the number of threads for running the validation process
+    def threads=(threads)
+      @threads = threads
+    end
+    
+    ##
+    # from_filehandle allows the OPML XML document to be read from a given
+    # filehandle and returns an initialized Parser instance
+    def self.from_filehandle(filehandle)
+      contents = filehandle.read
+      Parser.new(contents)
+    end
+    
+    ##
+    # from_file takes in a filename and returns an initialized Parser instance
+    def self.from_file(filename)
+      from_filehandle(File.open(filename, 'r'))
+    end
+    
+    ##
+    # from_url takes in a URL as a string and returns an initialized Parser instance
+    def self.from_url(url)
+      from_filehandle(open(url).read)
+    end
+    
+    ##
+    # validate! takes in one argument, +since+, specifing a Time object. Since is used to check if any posts have been posted since that time, thus detecting "stale" blogs/rss feeds 
+    def validate!(since = nil)
+      # this threading methodology is highly expensive for simple blocks, but a life-saver for IO-bound blocks
+      @work_queue = Queue.new
+      data = @opml.css("body").children
+      boss = Thread.new do
+        filter!(data)
+      end
+      
+      workers = (0...@threads).map do
+        @work_queue.push(false) # this will end each thread
+        Thread.new do
+          begin
+            while work = @work_queue.pop()
+              val = validate_callback(work[:outline], since)
+              spaces = 80 - work[:outline][:xml_url].length
+              spaces = 1 if spaces < 1
+              puts "#{work[:outline][:xml_url]}#{' ' * spaces}#{val}" if @debug
+              unless val == "PASS"
+                work[:node].unlink
+              end
+            end
+          rescue ThreadError
+          end
+        end
+      end
+      boss.join
+      workers.map(&:join)
+    end
+    
+    ##
+    # to_xml outputs the current OPML XML structure as a String containing all the XML markup
+    def to_xml
+      @opml.to_xml
+    end
+
+    private
+
+    ##
+    # filter! recurses down the OPML body, looking for outline tags, and pushes each
+    # leaf node onto a work queue
+    def filter!(data)
+      data.each do |node|
+        if node.name == 'outline'
+          outline = Outline.new(node).to_hash
+          if node.children.length > 0
+            title = outline[:title] || outline[:text]
+            filter!(node.children)
+          else
+            @work_queue.push({ :outline => outline, :node => node})
+          end
+        end
+      end      
+    end
+    
+    ##
+    # validate_callback tries to download a feed and verify that it has been updated
+    # since the +since+ time
+    def validate_callback(feed, since=nil)
+      val = "FAIL"
+      begin
+        Timeout::timeout(@timeout) {
+          open(feed[:xml_url]) do |rss|
+            feed = RSS::Parser.parse(rss)
+            if feed
+              last_updated = Time.at(0)
+              feed.items.each do |item|
+                #p item.class
+                updated = nil
+                if item.respond_to?(:updated)
+                  updated = item.updated.content
+                elsif item.respond_to?(:date)
+                  updated = item.date
+                end
+                next unless updated
+                if updated and updated > last_updated
+                  last_updated = updated
+                end
+              end
+              if since
+                #p last_updated
+                if last_updated and last_updated > since
+                  val = "PASS"
+                else
+                  val = "STALE"
+                end
+              else
+                val = "PASS"
+              end
+            else
+              val = "NOFEED"
+            end
+          end
+        }
+      rescue EOFError => e
+        val = "EOFError"
+      rescue OpenURI::HTTPError => e
+        val = "HTTPError"
+      rescue RSS::Error => e
+        val = "RSSError"
+      rescue Timeout::Error => e
+        val = "Timedout"
+      rescue SocketError => e
+        val = "SocketError"
+      rescue RuntimeError => e
+        val = "Redirect Loop"
+      rescue Errno::ECONNREFUSED => e
+        val = "Connection Refused"
+      rescue Exception => e
+        val = "Unexpected error: #{e}"
+      end
+      val      
+    end
+  end
+  
+end

data/lib/opml_janitor/outline.rb

@@ -0,0 +1,35 @@
+module OpmlJanitor
+  
+  class Outline
+    
+    attr_reader :hash
+    
+    def initialize(node)
+      @node = node
+      @hash = {}
+    end
+    
+    def to_hash
+      @node.attributes.each do |attribute|
+        key = underscore(attribute[0]).to_sym
+        @hash[key] = @node.attr(attribute[0])
+      end
+      @hash
+    end
+    
+    private 
+    
+    # from ActiveSupport
+    def underscore(camel_cased_word)
+      word = camel_cased_word.to_s.dup
+      word.gsub!(/::/, '/')
+      word.gsub!(/([A-Z\d]+)([A-Z][a-z])/,'\1_\2')
+      word.gsub!(/([a-z\d])([A-Z])/,'\1_\2')
+      word.tr!("-", "_")
+      word.downcase!
+      word
+    end
+    
+  end
+  
+end

data/lib/opml_janitor/version.rb

@@ -0,0 +1,3 @@
+module OpmlJanitor
+  VERSION = "0.0.1"
+end

data/opml_janitor.gemspec

@@ -0,0 +1,25 @@
+# coding: utf-8
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require 'opml_janitor/version'
+
+Gem::Specification.new do |spec|
+  spec.name          = "opml_janitor"
+  spec.version       = OpmlJanitor::VERSION
+  spec.authors       = ["chrislee35"]
+  spec.email         = ["rubygems@chrislee.dhs.org"]
+  spec.summary       = %q{Parses an OPML file, verifies the feeds, and writes the resulting OPML}
+  spec.description   = %q{This gem provides a tool for cleaning up OPML feeds.}
+  spec.homepage      = "https://github.com/chrislee35/opml_janitor"
+  spec.license       = "MIT"
+
+  spec.files         = `git ls-files -z`.split("\x0")
+  spec.executables   = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
+  spec.test_files    = spec.files.grep(%r{^(test|spec|features)/})
+  spec.require_paths = ["lib"]
+
+  spec.add_runtime_dependency "nokogiri", "~> 1.6"
+  spec.add_development_dependency "minitest", "~> 5.5"
+  spec.add_development_dependency "bundler", "~> 1.7"
+  spec.add_development_dependency "rake", "~> 10.0"
+end

data/test/helper.rb

@@ -0,0 +1,6 @@
+require 'minitest/autorun'
+require 'minitest/test'
+require 'minitest/unit'
+require 'minitest/pride'
+include MiniTest::Assertions
+require File.expand_path('../../lib/opml_janitor.rb', __FILE__)

data/test/test_opml_janitor.rb

@@ -0,0 +1,27 @@
+unless Kernel.respond_to?(:require_relative)
+	module Kernel
+		def require_relative(path)
+			require File.join(File.dirname(caller[0]), path.to_str)
+		end
+	end
+end
+
+require_relative 'helper'
+require 'pp'
+
+class TestOPMLJanitor < Minitest::Test
+  def test_opml_parse
+    #opmljanitor = OpmlJanitor::Parser.from_file("test/test.opml")
+  end
+  
+  def test_opml_validation
+    #opmljanitor = OpmlJanitor::Parser.from_file("test/test2.opml")
+    #pp opmljanitor.validate
+  end
+  
+  def test_opml_validation_with_time
+    opmljanitor = OpmlJanitor::Parser.from_file("test/test2.opml")
+    opmljanitor.validate!(Time.now - (265*24*60*60))
+    xml = opmljanitor.to_xml
+  end
+end

metadata

@@ -0,0 +1,115 @@
+--- !ruby/object:Gem::Specification
+name: opml_janitor
+version: !ruby/object:Gem::Version
+  version: 0.0.1
+platform: ruby
+authors:
+- chrislee35
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2015-04-13 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: nokogiri
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.6'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.6'
+- !ruby/object:Gem::Dependency
+  name: minitest
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '5.5'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '5.5'
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.7'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.7'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '10.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '10.0'
+description: This gem provides a tool for cleaning up OPML feeds.
+email:
+- rubygems@chrislee.dhs.org
+executables:
+- opml_janitor
+extensions: []
+extra_rdoc_files: []
+files:
+- ".gitignore"
+- Gemfile
+- LICENSE.txt
+- README.md
+- Rakefile
+- bin/opml_janitor
+- lib/opml_janitor.rb
+- lib/opml_janitor/outline.rb
+- lib/opml_janitor/version.rb
+- opml_janitor.gemspec
+- test/helper.rb
+- test/test_opml_janitor.rb
+homepage: https://github.com/chrislee35/opml_janitor
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.2.2
+signing_key: 
+specification_version: 4
+summary: Parses an OPML file, verifies the feeds, and writes the resulting OPML
+test_files:
+- test/helper.rb
+- test/test_opml_janitor.rb