jm3-rubunkulous 0.0.2

data/bin/rubunkulous

@@ -0,0 +1,156 @@
+#!/usr/bin/env ruby
+
+# Ru-BUNK-u-lous.
+# n. A reentrant link-checker for delicious power-users, stress-tested with stores of 12,000+ links.
+# by jm3 (John Manoogian III)
+
+require 'rubygems'
+require 'curb'
+require 'moneta'
+require 'moneta/basic_file'
+require 'moneta/xattr'
+require 'net/https'
+require 'progressbar'
+require 'rexml/document'
+include REXML
+
+Pointer_Key = 'index_pointer'
+Max_XAttr_Key_Length = 126 # i swear to christ...
+
+@verbose = false
+@start_override = 0
+
+ARGV.each do|arg|
+  if arg =~ /--start=(\d+)/ or arg =~ /^(\d+)$/
+    @start_override = $1.to_i
+  end
+end
+
+def link_cache
+  f = '.cached_links.xml'
+  File.writable?('.') ? File.join('.', f) : File.join(File.expand_path('~'), f)
+end
+
+def credentials
+  creds = File.join( '.', 'credentials.yml')
+  return [] unless File.exist?(creds)
+  creds = YAML.load(File.read(creds))[:delicious]
+  [creds[:user], creds[:password]]
+end
+
+def fetch_links
+  @response = ''
+  begin    
+    url = URI.parse( 'https://api.del.icio.us/v1/posts/all')
+    http = Net::HTTP.new(url.host, url.port)
+    http.use_ssl = true
+    http.verify_mode = OpenSSL::SSL::VERIFY_NONE
+    http.start do |http|
+      if !url.query.nil?
+        path = url.path + "?" + url.query
+      else
+        path = url.path
+      end
+      req = Net::HTTP::Get.new(path)
+      user, password = credentials
+      req.basic_auth user, password
+      res = http.request(req)
+      @response = res.body
+    end
+  rescue Net::HTTPError
+    raise "Can't connect to the server, please check the username/password.\nError: #{$!} \n"
+  end
+  
+  begin
+    @doc = Document.new @response
+  rescue REXML::ParseException
+    raise "\nPosts#find XML parsing failed\nError: #{$!}\n"
+  end
+end
+
+def links
+  # Check if we've got cached XML from a previous run
+  unless File.exists?(link_cache)
+    puts "Caching Delicious locally to avoid making slow API calls more than once."
+    xml = fetch_links
+    File.open(link_cache, 'w') {|f| f.write(xml) }
+
+    # by default, the Delicious API returns reponses with encoding errors; this fixes them:
+    if `which tidy`.empty?
+      puts "WARNING: you don't have tidy installed, which means that occasional errors in Delicious's API xml may hang up Rubunkulous. Try: sudo port install tidy  to get it."
+    else
+      `tidy -xml .cached_links.xml &> /dev/null`
+    end
+  end
+
+  puts "Loading link data saved from previous run (this could take a second...)"
+  xmldoc = Document.new( File.new(link_cache))
+  xmldoc.elements.to_a('posts/post')
+end
+
+def check(links)
+  return unless links.size > 0
+  interrupted = false
+  trap("INT") { interrupted = true }
+
+  @links_checked, @num_fails, @total_links = 0,0,0
+
+  def print_report
+    puts "\n#{@links_checked} links checked (#{@num_fails} failures) - #{@total_links - @links_checked} links to go."
+    puts "\nTo clear last-checked counter and re-check all links, type: \nxattr .moneta_cache/xattr_cache -index_pointer"
+  end
+
+  def log_failed(url, desc, error)
+    # truncate xattr key names or suffer the consequences
+    @cache.store(url[0..Max_XAttr_Key_Length], "#{desc} (#{url}) failed with #{error} at #{Time.now.to_s}") 
+  end
+
+  @cache = Moneta::Xattr.new(:file => File.join(File.dirname(__FILE__), ".moneta_cache", "xattr_cache"))
+  last_index = (@start_override > 0 ? @start_override : nil) || @cache[Pointer_Key] || 0
+
+  puts "Left off at link ##{last_index} (of #{links.size} total links)." if last_index > 0
+
+  # skip previously checked links
+  links = links[last_index..links.size]
+  @total_links = links.size
+  puts "#{@total_links} links to check."
+  @pbar = ProgressBar.new("  Checking", links.size)
+
+  links.each do |link|
+
+    if interrupted
+      print_report
+      exit
+    end
+
+    url  = link.attributes['href']
+    desc = link.attributes['description']
+
+    begin
+      response = Curl::Easy.perform(url) do |curl|
+        curl.follow_location  = true
+        curl.max_redirects    = 2
+        curl.connect_timeout  = 3
+        curl.timeout          = 5 # total request timeout; saves us from having to wait for the request to complete if you bookmarked a 500mb file or something...
+      end
+    rescue
+      log_failed(url, desc, response ? response.response_code : 666)
+    end
+
+    @links_checked += 1
+    @pbar.inc
+    @cache.store(Pointer_Key, last_index + @links_checked)
+    next unless response
+
+    if response.response_code != 200
+      puts "x FAIL #{response.response_code} #{url} (#{@links_checked})" if @verbose
+      @num_fails += 1
+      log_failed(url, desc, response.response_code)
+    else
+      puts "> OK #{url} (#{@links_checked})" if @verbose
+    end
+  end
+  print_report
+end
+
+check(links)

metadata

@@ -0,0 +1,82 @@
+--- !ruby/object:Gem::Specification 
+name: jm3-rubunkulous
+version: !ruby/object:Gem::Version 
+  version: 0.0.2
+platform: ruby
+authors: 
+- John Manoogian III
+autorequire: 
+bindir: bin
+cert_chain: []
+
+date: 2009-05-16 00:00:00 -07:00
+default_executable: 
+dependencies: 
+- !ruby/object:Gem::Dependency 
+  name: curb
+  type: :runtime
+  version_requirement: 
+  version_requirements: !ruby/object:Gem::Requirement 
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        version: "0"
+    version: 
+- !ruby/object:Gem::Dependency 
+  name: moneta
+  type: :runtime
+  version_requirement: 
+  version_requirements: !ruby/object:Gem::Requirement 
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        version: "0"
+    version: 
+- !ruby/object:Gem::Dependency 
+  name: progressbar
+  type: :runtime
+  version_requirement: 
+  version_requirements: !ruby/object:Gem::Requirement 
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        version: "0"
+    version: 
+description: reentrant link-checker for del.icio.us power-users
+email: jm3@jm3.net
+executables: []
+
+extensions: []
+
+extra_rdoc_files: []
+
+files: 
+- bin/rubunkulous
+has_rdoc: true
+homepage: http://jm3.net/
+post_install_message: 
+rdoc_options: []
+
+require_paths: 
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement 
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      version: "0"
+  version: 
+required_rubygems_version: !ruby/object:Gem::Requirement 
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      version: "0"
+  version: 
+requirements: []
+
+rubyforge_project: rubunkulous
+rubygems_version: 1.2.0
+signing_key: 
+specification_version: 2
+summary: reentrant link-checker for del.icio.us power-users
+test_files: []
+