url_processor 0.1.0 → 0.5.0

checksums.yaml

@@ -0,0 +1,15 @@
+---
+!binary "U0hBMQ==":
+  metadata.gz: !binary |-
+    ODM4YTQ5NWIzYzgzZWU3NDg4Mzk2Y2M3NmQyM2I1YzA5Nzk1NzIzNw==
+  data.tar.gz: !binary |-
+    NWJlNDU2YTdjZTM3MTVlNWE4ZmUxZjQ2ZmI2MjVkYWM0NDU0ZTg1Ng==
+SHA512:
+  metadata.gz: !binary |-
+    ZWYyOTU3ZTU2YTZmZDgxNTljYjAzMmMxMzdiMWFiMThmZmZhM2UzZjcyMDlj
+    MjVkMWIxMWE4OWUyYTMzOTBhYzgxZTZhNzJlNGVmYjk3YTU5NzAyNzI5ZDg5
+    OWUzODI5NTFhODI3ZGFhYTEwNjRjOTZjMGFkNmZlYjhkNDJlYzA=
+  data.tar.gz: !binary |-
+    YTkwNTgyNTA4YTcxOWNlZWNiMmE3YTJhNDc3NmVkZTE0YzYwMDljMWUzMWM0
+    NjhhYWJiMjJkNmExNmRkZDc2NTExZWE2YzgwYTFkODMwZGM3NDQ4ZjU1ODA4
+    MTk4N2ZiODg2NTA4Y2MyZTM5NGM2NDI5ZGY4OGMxNmZkNzZiMmQ=

data/.gitignore

@@ -0,0 +1,17 @@
+*.gem
+*.rbc
+.bundle
+.config
+.yardoc
+Gemfile.lock
+InstalledFiles
+_yardoc
+coverage
+doc/
+lib/bundler/man
+pkg
+rdoc
+spec/reports
+test/tmp
+test/version_tmp
+tmp

data/.rspec

@@ -1 +1,2 @@
+--format documentation
 --color

data/.travis.yml

@@ -0,0 +1,9 @@
+script: bundle exec rake
+rvm:
+- 1.9.3
+deploy:
+  provider: rubygems
+  api_key:
+    secure: lHnBn15z5MLSKiCqcqG0Bgj7r089C0uzI9zxzNtEGJ4fScIrRREJkYEb0l0ndCizek5++ChVOVM1/qguVEcFlGyUJ4DEXzWynRzsWui2sN+Xt9AaEUXGc23L/ljSg6oiH8G1TgIfeTODxn95WIUfFI6ydu5rZddsDeAWbXBxp4A=
+  on:
+    tags: true

data/Gemfile

@@ -1,14 +1,4 @@
-source "http://rubygems.org"
-# Add dependencies required to use your gem here.
-# Example:
-#   gem "activesupport", ">= 2.3.5"
+source 'https://rubygems.org'
 
-# Add dependencies to develop your gem here.
-# Include everything needed to run rake, tests, features, etc.
-group :development do
-  gem "rspec", "~> 2.8.0"
-  gem "rdoc", "~> 3.12"
-  gem "bundler", "~> 1.0"
-  gem "jeweler", "~> 2.0.1"
-  gem "simplecov", ">= 0"
-end
+# Specify your gem's dependencies in url_processor.gemspec
+gemspec

data/LICENSE.txt

@@ -1,5 +1,7 @@
 Copyright (c) 2014 Eric Hayes
 
+MIT License
+
 Permission is hereby granted, free of charge, to any person obtaining
 a copy of this software and associated documentation files (the
 "Software"), to deal in the Software without restriction, including

data/README.md

@@ -0,0 +1,108 @@
+# url_processor [![Build Status](https://travis-ci.org/ejhayes/url_processor.png?branch=master)](https://travis-ci.org/ejhayes/url_processor) [![Gem Version](https://badge.fury.io/rb/url_processor.png)](http://badge.fury.io/rb/url_processor) [![Code Climate](https://codeclimate.com/github/ejhayes/url_processor.png)](https://codeclimate.com/github/ejhayes/url_processor)
+
+Fast and easy way to process urls.
+
+## Installation
+
+Add this line to your application's Gemfile:
+
+    gem 'url_processor'
+
+And then execute:
+
+    $ bundle
+
+Or install it yourself as:
+
+    $ gem install url_processor
+
+## Usage
+
+Example use case:
+
+    require 'url_processor'
+    require 'trollop'
+
+    opts = Trollop::options do
+      banner <<-EOS
+    This utility validates the quality of links used by courtreference.com.
+    EOS
+      opt :log, "output to logfile (default is STDOUT)", :type => :string
+      opt :debug, "enable debugging mode"
+      opt :batch_size, "maximum batch size (records to retrieve at a time before processing urls)", :default => 100
+      opt :max_concurrency, "maximum number of threads to spawn", :type => :int, :default => 20
+      opt :max_retries, "maximum number of times to try a url before failing", :type => :int, :default => 3
+      opt :max_timeout, "maximum duration in seconds to wait for url to load", :type => :int, :default => 5
+      opt :cookies_file, "file to store cookie information", :type => :string, :default => '/tmp/whatever_cookies'
+      opt :max_total_connections, "maximum number of connections to keep open at a time", :type => :int, :default => 100
+    end
+
+    # Additional validations
+    opts[:log] = STDOUT if opts[:log] == nil
+
+    link_check = UrlProcessor.create do |config|
+      config.log = opts[:log]
+      config.debug = opts[:debug]
+      config.batch_size = opts[:batch_size]
+      config.max_concurrency = opts[:max_concurrency]
+      config.max_retries = opts[:max_retries]
+      config.cookies_file = opts[:cookies_file]
+      config.max_total_connections = opts[:max_total_connections]
+
+      config.retrieves_links_by_id_with do |link_id|
+        puts "I was called with #{link_id}"
+      end
+
+      config.retrieves_all_links_with do
+        links = []
+        (1..3).each do |i|
+          links << OpenStruct.new(:id => i, :urls => [{:url => 'http://www.example.com'}])
+        end
+        links
+      end
+
+      config.creates_new_link_request_with do |url, params|
+        UrlProcessor::LinkRequest.new(url, params)
+      end
+
+      config.processes_response_with do |response|
+        if response.return_code == :ok
+          config.logger.info "#{response.return_code} - #{response.effective_url}".green
+          destroyed_broken_link = OpenStruct.new(:id => 1, :serializable_hash => { :about => 'Not a real record' })
+          config.logger.debug "broken link destroyed (#{destroyed_broken_link.id}): #{destroyed_broken_link.serializable_hash}".green
+        else
+          config.logger.info "#{response.return_code} - #{response.effective_url}".red
+        end
+      end
+    end
+
+    puts "Running this thing..."
+    link_check.run
+
+## Updating this gem
+
+If you are making changes to this gem, here's some stuff you will need to know:
+
+## Running the executables
+
+If you have an exetable file called "my_executable" in the bin folder, you can run it by doing:
+
+    bundle exec my_executable
+
+If you want to play around in the interactive console with the gem already loaded, you can do this:
+
+    bundle exec rake console
+
+To run the tests
+
+    bundle exec rake spec
+
+Please note that we are using simplecov for code coverage.
+
+## Contributing
+
+1. Fork it ( http://github.com/ejhayes/url_processor/fork )
+2. Create your feature branch (`git checkout -b my-new-feature`)
+3. Commit your changes (`git commit -am 'Add some feature'`)
+4. Push to the branch (`git push origin my-new-feature`)
+5. Create new Pull Request

data/Rakefile

@@ -1,50 +1,18 @@
-# encoding: utf-8
+require "bundler/gem_tasks"
+require "rspec/core/rake_task"
 
-require 'rubygems'
-require 'bundler'
-begin
-  Bundler.setup(:default, :development)
-rescue Bundler::BundlerError => e
-  $stderr.puts e.message
-  $stderr.puts "Run `bundle install` to install missing gems"
-  exit e.status_code
-end
-require 'rake'
-
-require 'jeweler'
-Jeweler::Tasks.new do |gem|
-  # gem is a Gem::Specification... see http://guides.rubygems.org/specification-reference/ for more options
-  gem.name = "url_processor"
-  gem.homepage = "http://github.com/ejhayes/url_processor"
-  gem.license = "MIT"
-  gem.summary = %Q{Fast and reliable link checker.}
-  gem.description = %Q{Fast and easy way to validate tons of urls without locking up your system or eating up too much memory.}
-  gem.email = "eric@deployfx.com"
-  gem.authors = ["Eric Hayes"]
-  # dependencies defined in Gemfile
-end
-Jeweler::RubygemsDotOrgTasks.new
-
-require 'rspec/core'
-require 'rspec/core/rake_task'
-RSpec::Core::RakeTask.new(:spec) do |spec|
-  spec.pattern = FileList['spec/**/*_spec.rb']
-end
-
-desc "Code coverage detail"
-task :simplecov do
-  ENV['COVERAGE'] = "true"
-  Rake::Task['spec'].execute
-end
+RSpec::Core::RakeTask.new(:spec)
 
 task :default => :spec
+task :test => :spec
 
-require 'rdoc/task'
-Rake::RDocTask.new do |rdoc|
-  version = File.exist?('VERSION') ? File.read('VERSION') : ""
-
-  rdoc.rdoc_dir = 'rdoc'
-  rdoc.title = "url_processor #{version}"
-  rdoc.rdoc_files.include('README*')
-  rdoc.rdoc_files.include('lib/**/*.rb')
+desc 'Interactive console with gem pre-loaded'
+task :console do
+  require 'irb'
+  require 'irb/completion'
+  require 'ruby-debug'
+  $LOAD_PATH.unshift File.expand_path('../../lib', __FILE__)
+  require 'url_processor'
+  ARGV.clear
+  IRB.start
 end

data/TODO

@@ -0,0 +1,12 @@
+Things to be done, eventually
+=============================
+
+- rdoc documentation in code
+- simplecov code coverage
+- rspec tests
+- integration with travisci
+- integration with code climate
+- published gem to rubygems
+- ppa setup for libcurl
+- post-install message to install correct libcurl message
+  in the event that asyncdns is not properly setup

data/bin/url_processor

@@ -0,0 +1,5 @@
+#!/usr/bin/env ruby
+
+require 'url_processor'
+
+puts "THIS IS A TEST OF THE TOOL, version: #{UrlProcessor::VERSION}"

data/lib/url_processor/base.rb

@@ -0,0 +1,129 @@
+module UrlProcessor
+  class Base
+    attr_reader :config
+
+    def initialize(c)
+      raise ArgumentError unless c.is_a? UrlProcessor::Config
+      @config = c
+
+      # connect to the db
+      #OnlinesearchesModels::connect
+    end
+
+    def new_broken_link(params={})
+      raise NotImplementedError.new "new_broken_link not implemented"
+    end
+
+    def report_broken_link(link_id, params={})
+      url_type_code = params[:url_type_code]
+      response_code = params[:response_code]
+      begin
+        link = config.get_link_by_id.call(link_id)
+        broken_link = new_broken_link(
+          :link_id => link.id, 
+          :fips_code => link.fips_code, 
+          :url_type_code => url_type_code, 
+          :response_code => response_code,
+          :reported_by => 'QC Report'
+        )
+        broken_link.save
+        config.logger.debug "broken link created (#{broken_link.id}): #{broken_link.serializable_hash}".red
+      rescue ActiveRecord::RecordNotFound => e
+        config.logger.warn "#{e}".red
+      end
+    end
+
+    def pre_process_link(link)
+      # do nothing
+    end
+
+    def process_response(response)
+      raise NotImplementedError.new "process_reponse is not implemented"
+    end
+
+    def new_link_request(url, params={})
+      raise NotImplementedError.new "link_request is not implemented"
+    end
+
+    def find_in_batches(collection, batch_size)
+      if collection.is_a? Array
+        collection.each do |element|
+          yield element
+        end
+      else
+        collection.find_in_batches(batch_size: batch_size) do |group|
+          # Output progress information
+          config.logger.info "PROCESSED: #{processed_links}, NEXT GROUP SIZE: #{group.size}".yellow
+
+          # for debuggin purposes we do not want to process everything
+          if config.debug && processed_links >= config.batch_size
+            config.logger.debug "FINISHED first batch (#{@batch_size} records), exiting".yellow
+            return
+          end
+
+          group.each do |element|
+            yield element
+          end
+        end
+      end
+    end
+
+    def run
+      processed_links = 0
+      hydra = Typhoeus::Hydra.new(max_concurrency: config.max_concurrency, max_total_connections: config.max_total_connections)
+
+      find_in_batches(config.links.call, config.batch_size) do |link|
+        # any custom pre-processing
+        pre_process_link(link)
+
+        if link.urls.empty?
+          # In the event that we have a link that actually has no urls associated with it
+          report_broken_link link.id, :response_code => :has_no_urls if config.report_records_without_urls
+        else
+          # Each record has 2 urls associated with it, process each separately
+          link.urls.each do |url|
+            config.logger.debug "link: #{link.serializable_hash}, url: #{url}".yellow
+
+            link_request = config.new_link_request.call(
+              url[:url], 
+              followlocation: true, 
+              method: :head, 
+              ssl_verifypeer: false, 
+              ssl_verifyhost: 2, 
+              cookiefile: config.cookies_file, 
+              cookiejar: config.cookies_file, 
+              link_id: link.id,
+              url_type_code: url[:url_type_code],
+              timeout: config.max_timeout,
+              connecttimeout: config.max_timeout,
+              max_retries: config.max_retries,
+              forbid_reuse: 1,
+              nosignal: 1
+            )
+
+            link_request.on_complete do |response|
+              processed_links += 1
+
+              if ([:operation_timedout, :couldnt_resolve_host].include? response.return_code) && response.request.retry_request?
+                config.logger.info "#{response.return_code} - #{response.effective_url} timed out, retrying".yellow
+                hydra.queue response.request
+              elsif response.return_code == :got_nothing && response.request.options[:method] != :get
+                config.logger.info "#{response.return_code} - #{response.effective_url} empty response, attempting GET request instead".yellow
+                
+                # set to GET request since HEAD may fail in some cases
+                response.request.options[:method] = :get
+                hydra.queue response.request
+              else
+                config.process_response.call response
+              end
+            end
+
+            hydra.queue link_request
+          end
+        end
+
+        hydra.run
+      end
+    end
+  end
+end

data/lib/url_processor/config.rb

@@ -0,0 +1,72 @@
+module UrlProcessor
+
+  class Config
+    # configuration parameters
+    attr_accessor :batch_size
+    attr_accessor :max_concurrency
+    attr_accessor :max_retries
+    attr_accessor :cookies_file
+    attr_accessor :max_total_connections
+    attr_accessor :max_timeout
+
+    # debugging
+    attr_reader :debug
+    def debug=(val)
+      @debug = val
+      update_logging_level
+    end
+
+    # logging
+    attr_reader :logger
+
+    def log=(logging_location)
+      @logger = Logger.new(logging_location)
+      update_logging_level
+    end
+
+    # get individual link
+    attr_reader :get_link_by_id
+    def retrieves_links_by_id_with(&block)
+      @get_link_by_id = block
+    end
+
+    # get all links
+    attr_reader :links
+    def retrieves_all_links_with(&block)
+      @links = block
+    end
+
+    # create new link request
+    attr_reader :new_link_request
+    def creates_new_link_request_with(&block)
+      @new_link_request = block
+    end
+
+    # process responses
+    attr_reader :process_response
+    def processes_response_with(&block)
+      @process_response = block
+    end
+
+    def validate!
+      raise NotImplementedError.new("retrieves_links_by_id_with not set") if get_link_by_id.nil?
+      raise NotImplementedError.new("retrieves_all_links_with not set") if links.nil?
+      raise NotImplementedError.new("creates_new_link_request_with not set") if new_link_request.nil?
+      raise NotImplementedError.new("processes_response_with not set") if process_response.nil?
+    end
+
+    private
+
+    def update_logging_level
+      unless logger.nil?
+        if debug
+          logger.level = Logger::DEBUG
+        else
+          logger.level = Logger::WARN
+        end
+      end
+    end
+
+  end
+
+end

data/lib/url_processor/error.rb

@@ -0,0 +1,9 @@
+module UrlProcessor
+
+  class Error < StandardError
+  end
+
+  class NotConfiguredError < Error
+  end
+
+end

data/lib/url_processor/link_request.rb

@@ -0,0 +1,12 @@
+module UrlProcessor
+  class LinkRequest < RetryableRequest
+    attr_accessor :link_id
+    attr_accessor :url_type_code
+
+    def initialize(base_url, options={})
+      @link_id = options.delete(:link_id)
+      @url_type_code = options.delete(:url_type_code)
+      super(base_url, options)
+    end
+  end
+end

data/lib/url_processor/retryable_request.rb

@@ -0,0 +1,27 @@
+module UrlProcessor
+  class RetryableRequest < Typhoeus::Request
+    def initialize(base_url, options={})
+      @attempts = 0
+      @max_retries = options.delete(:max_retries) || 3
+      super(base_url, options)
+    end
+
+    def on_complete(&block)
+      @attempts += 1
+      super(&block)
+    end
+
+    def retry_request
+      if retry_request?
+        options[:method] = :get
+        return self
+      else
+        return nil
+      end
+    end
+
+    def retry_request?
+      @attempts <= @max_retries
+    end
+  end
+end

data/lib/url_processor/runner.rb

@@ -0,0 +1,16 @@
+module UrlProcessor
+  class Runner
+
+    #attr_reader :callbacks
+
+    def initialize(config)
+      raise ArgumentError.new("invalid config '#{config}', expected ::Config") unless config.is_a? UrlProcessor::Config
+      @runner = Base.new(config)
+    end
+
+    def run
+      @runner.run
+    end
+
+  end
+end

data/lib/url_processor/version.rb

@@ -0,0 +1,3 @@
+module UrlProcessor
+  VERSION = "0.5.0"
+end

data/lib/url_processor.rb

@@ -0,0 +1,24 @@
+require 'typhoeus'
+require 'colorize'
+require 'ostruct'
+require 'url_processor/version'
+require 'url_processor/error'
+require 'url_processor/runner'
+require 'url_processor/config'
+require 'url_processor/base'
+require 'url_processor/retryable_request'
+require 'url_processor/link_request'
+
+module UrlProcessor
+
+  def self.create(&block)
+    config = Config.new
+    block.call config
+
+    # validate before returning
+    config.validate!
+
+    Runner.new(config)
+  end
+
+end

data/spec/spec_helper.rb

@@ -1,29 +1,2 @@
-require 'simplecov'
-
-module SimpleCov::Configuration
-  def clean_filters
-    @filters = []
-  end
-end
-
-SimpleCov.configure do
-  clean_filters
-  load_adapter 'test_frameworks'
-end
-
-ENV["COVERAGE"] && SimpleCov.start do
-  add_filter "/.rvm/"
-end
-$LOAD_PATH.unshift(File.join(File.dirname(__FILE__), '..', 'lib'))
-$LOAD_PATH.unshift(File.dirname(__FILE__))
-
-require 'rspec'
+$LOAD_PATH.unshift File.expand_path('../../lib', __FILE__)
 require 'url_processor'
-
-# Requires supporting files with custom matchers and macros, etc,
-# in ./support/ and its subdirectories.
-Dir["#{File.dirname(__FILE__)}/support/**/*.rb"].each {|f| require f}
-
-RSpec.configure do |config|
-
-end

data/spec/url_processor/base_spec.rb

@@ -0,0 +1,26 @@
+require 'spec_helper'
+
+describe UrlProcessor::Base do
+  let(:config) { UrlProcessor::Config.new }
+
+  describe '.new' do
+    it 'can be created' do
+      UrlProcessor::Base.new(config)
+    end
+
+    it 'throws an error if config is not set' do
+      expect { UrlProcessor::Base.new }.to raise_error ArgumentError
+    end
+
+    it 'throws an error if config is not of type UrlProcessor::Config' do
+      expect { UrlProcessor::Base.new( double('not a valid type') ) }.to raise_error ArgumentError
+    end
+  end
+
+  describe :attributes do
+    it 'sets config' do
+      url_processor = UrlProcessor::Base.new(config)
+      url_processor.config.should eq config
+    end
+  end
+end

data/spec/url_processor/config_spec.rb

@@ -0,0 +1,146 @@
+require 'spec_helper'
+
+describe UrlProcessor::Config do
+  let(:config) { UrlProcessor::Config.new }
+
+  describe :validation do
+    before(:each) do
+      config.debug = true
+      config.batch_size = 1
+      config.max_concurrency = 1
+      config.max_retries = 1
+      config.cookies_file = 'test'
+      config.max_total_connections = 1
+      config.max_timeout = 1
+      config.log = STDOUT
+
+      config.retrieves_links_by_id_with do |val|
+      end
+
+      config.retrieves_all_links_with do
+      end
+
+      config.creates_new_link_request_with do |url, params|
+      end
+
+      config.processes_response_with do |response|
+      end
+    end
+
+    it 'is valid' do
+      config.validate!
+    end
+
+    it 'throws an error if get_link_by_id is not set' do
+      config.stub(:get_link_by_id).and_return(nil)
+      expect { config.validate! }.to raise_error NotImplementedError
+    end
+
+    it 'throws an error if links is not set' do
+      config.stub(:links).and_return(nil)
+      expect { config.validate! }.to raise_error NotImplementedError
+    end
+
+    it 'throws an error if new_link_request is not set' do
+      config.stub(:new_link_request).and_return(nil)
+      expect { config.validate! }.to raise_error NotImplementedError
+    end
+
+    it 'throws an error if process_response is not set' do
+      config.stub(:process_response).and_return(nil)
+      expect { config.validate! }.to raise_error NotImplementedError
+    end
+  end
+
+  describe :attributes do
+    it 'sets attributes' do
+      [
+        :debug,
+        :batch_size,
+        :max_concurrency,
+        :max_retries,
+        :cookies_file,
+        :max_total_connections,
+        :max_timeout
+      ].each do |param|
+        config.should respond_to(param)
+        config.should respond_to("#{param}=")
+      end
+    end
+  end
+
+  describe :logging do
+    it 'is nil by default' do
+      config.logger.should be_nil
+    end
+
+    it 'creates a valid logger object when log= is set' do
+      config.log = STDOUT
+      config.logger.should be_a Logger
+    end
+
+    # Debug
+    it 'sets logging level to DEBUG if debug is set first' do
+      config.debug = true
+      config.log = STDOUT
+      config.logger.level.should eq Logger::DEBUG
+    end
+
+    it 'sets logging level to DEBUG if debug is set last' do
+      config.log = STDOUT
+      config.debug = true
+      config.logger.level.should eq Logger::DEBUG
+    end
+
+    it 'sets logging level to WARN if debug level is not specified' do
+      config.log = STDOUT
+      config.logger.level.should eq Logger::WARN
+    end
+
+    it 'sets logging level to WARN if debug is changed from true -> false' do
+      config.debug = true
+      config.log = STDOUT
+      config.debug = false
+      config.logger.level.should eq Logger::WARN
+    end
+  end
+
+  describe :callbacks do
+    
+    it 'sets get_link_by_id' do
+      config.retrieves_links_by_id_with do |link_id|
+        link_id.should eq 456
+      end
+
+      config.get_link_by_id.call(456)
+    end
+
+    it 'sets links' do
+      expected_response = double('expected_response')
+      config.retrieves_all_links_with do
+        expected_response
+      end
+
+      config.links.call.should eq expected_response
+    end
+
+    it 'sets new_link_request' do
+      expected_response = double('expected_response')
+      config.creates_new_link_request_with do |url, params|
+        expected_response
+      end
+
+      config.new_link_request.call.should eq expected_response
+    end
+
+    it 'sets process_response' do
+      expected_response = double('expected_response')
+      config.processes_response_with do |response|
+        expected_response
+      end
+
+      config.process_response.call.should eq expected_response
+    end
+  end
+  
+end

data/spec/url_processor/runner_spec.rb

@@ -0,0 +1,47 @@
+require 'spec_helper'
+
+describe UrlProcessor::Runner do
+  let(:config) { UrlProcessor::Config.new }
+
+  describe ".new" do
+    describe :happy do
+      it 'can be created' do
+        UrlProcessor::Runner.new(config)
+      end
+
+      it 'creates a UrlProcessor::Base with the configuration object' do
+        UrlProcessor::Base.should_receive(:new).with(config)
+        runner = UrlProcessor::Runner.new(config)
+      end
+    end
+
+    describe :sad do
+      it 'raises an error if configuration not provided' do
+        expect { UrlProcessor::Runner.new }.to raise_error ArgumentError
+      end
+
+      it 'raises an error if configuration is not a UrlProcessor::Config type' do
+        invalid_config = double('invalid_config')
+
+        expect { UrlProcessor::Runner.new(invalid_config) }.to raise_error ArgumentError
+      end
+    end
+  end
+
+  describe ".run" do
+
+    it 'can be called' do
+      runner = UrlProcessor::Runner.new(config)
+      runner.should respond_to :run
+    end
+
+    it 'calls the run on the base runner' do
+      UrlProcessor::Base.any_instance.should_receive(:run)
+
+      runner = UrlProcessor::Runner.new(config)
+      runner.run
+    end
+
+  end
+  
+end

data/spec/url_processor_spec.rb

@@ -1,7 +1,36 @@
-require File.expand_path(File.dirname(__FILE__) + '/spec_helper')
+require 'spec_helper'
+
+describe UrlProcessor do
+
+  describe :gem_attributes do
+    it 'should have a version number' do
+      UrlProcessor::VERSION.should_not be_nil
+    end
+  end
+
+  describe '.create' do
+
+    it 'requires a block with a config argument' do
+      UrlProcessor::Config.any_instance.stub(:validate!)
+      UrlProcessor::Config.any_instance.should_receive(:some_random_call)
+
+      UrlProcessor.create do |config|
+        config.some_random_call
+      end
+    end
+
+    it 'throws an error if a block is not passed' do
+      expect { UrlProcessor.create }.to raise_error NoMethodError
+    end
+
+    it 'returns a runner' do
+      processor = UrlProcessor.create do |config|
+        config.stub(:validate!)
+      end
+
+      processor.should be_a UrlProcessor::Runner
+    end
 
-describe "UrlProcessor" do
-  it "fails" do
-    fail "hey buddy, you should probably rename this file and start specing for real"
   end
+
 end

data/url_processor.gemspec

@@ -1,63 +1,31 @@
-# Generated by jeweler
-# DO NOT EDIT THIS FILE DIRECTLY
-# Instead, edit Jeweler::Tasks in Rakefile, and run 'rake gemspec'
-# -*- encoding: utf-8 -*-
+# coding: utf-8
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require 'url_processor/version'
 
-Gem::Specification.new do |s|
-  s.name = "url_processor"
-  s.version = "0.1.0"
+Gem::Specification.new do |spec|
+  spec.name          = "url_processor"
+  spec.version       = UrlProcessor::VERSION
+  spec.authors       = ["Eric Hayes"]
+  spec.email         = ["eric@deployfx.com"]
+  spec.summary       = %q{Fast and reliable link checker.}
+  spec.description   = %q{Fast and easy way to validate tons of urls without locking up your system or eating up too much memory.}
+  spec.homepage      = "https://github.com/ejhayes/url_processor"
+  spec.license       = "MIT"
 
-  s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
-  s.authors = ["Eric Hayes"]
-  s.date = "2014-02-20"
-  s.description = "Fast and easy way to validate tons of urls without locking up your system or eating up too much memory."
-  s.email = "eric@deployfx.com"
-  s.extra_rdoc_files = [
-    "LICENSE.txt",
-    "README.rdoc"
-  ]
-  s.files = [
-    ".document",
-    ".rspec",
-    "Gemfile",
-    "Gemfile.lock",
-    "LICENSE.txt",
-    "README.rdoc",
-    "Rakefile",
-    "VERSION",
-    "lib/url_processor.rb",
-    "spec/spec_helper.rb",
-    "spec/url_processor_spec.rb",
-    "url_processor.gemspec"
-  ]
-  s.homepage = "http://github.com/ejhayes/url_processor"
-  s.licenses = ["MIT"]
-  s.require_paths = ["lib"]
-  s.rubygems_version = "1.8.23"
-  s.summary = "Fast and reliable link checker."
+  spec.files         = `git ls-files -z`.split("\x0")
+  spec.executables   = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
+  spec.test_files    = spec.files.grep(%r{^(test|spec|features)/})
+  spec.require_paths = ["lib"]
 
-  if s.respond_to? :specification_version then
-    s.specification_version = 3
+  # gem dependencies
+  spec.add_runtime_dependency "typhoeus", "~> 0.6.7"
+  spec.add_runtime_dependency "colorize", "~> 0.6.0"
 
-    if Gem::Version.new(Gem::VERSION) >= Gem::Version.new('1.2.0') then
-      s.add_development_dependency(%q<rspec>, ["~> 2.8.0"])
-      s.add_development_dependency(%q<rdoc>, ["~> 3.12"])
-      s.add_development_dependency(%q<bundler>, ["~> 1.0"])
-      s.add_development_dependency(%q<jeweler>, ["~> 2.0.1"])
-      s.add_development_dependency(%q<simplecov>, [">= 0"])
-    else
-      s.add_dependency(%q<rspec>, ["~> 2.8.0"])
-      s.add_dependency(%q<rdoc>, ["~> 3.12"])
-      s.add_dependency(%q<bundler>, ["~> 1.0"])
-      s.add_dependency(%q<jeweler>, ["~> 2.0.1"])
-      s.add_dependency(%q<simplecov>, [">= 0"])
-    end
-  else
-    s.add_dependency(%q<rspec>, ["~> 2.8.0"])
-    s.add_dependency(%q<rdoc>, ["~> 3.12"])
-    s.add_dependency(%q<bundler>, ["~> 1.0"])
-    s.add_dependency(%q<jeweler>, ["~> 2.0.1"])
-    s.add_dependency(%q<simplecov>, [">= 0"])
-  end
+  # not required to run the gem
+  spec.add_development_dependency "bundler", "~> 1.5"
+  spec.add_development_dependency "rake"
+  spec.add_development_dependency "debugger"
+  spec.add_development_dependency "rspec"
+  spec.add_development_dependency "trollop"
 end
-

metadata

@@ -1,84 +1,102 @@
 --- !ruby/object:Gem::Specification
 name: url_processor
 version: !ruby/object:Gem::Version
-  version: 0.1.0
-  prerelease: 
+  version: 0.5.0
 platform: ruby
 authors:
 - Eric Hayes
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2014-02-20 00:00:00.000000000 Z
+date: 2014-04-23 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
-  name: rspec
+  name: typhoeus
   requirement: !ruby/object:Gem::Requirement
-    none: false
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
-        version: 2.8.0
-  type: :development
+        version: 0.6.7
+  type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
-    none: false
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
-        version: 2.8.0
+        version: 0.6.7
 - !ruby/object:Gem::Dependency
-  name: rdoc
+  name: colorize
   requirement: !ruby/object:Gem::Requirement
-    none: false
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
-        version: '3.12'
-  type: :development
+        version: 0.6.0
+  type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
-    none: false
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
-        version: '3.12'
+        version: 0.6.0
 - !ruby/object:Gem::Dependency
   name: bundler
   requirement: !ruby/object:Gem::Requirement
-    none: false
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
-        version: '1.0'
+        version: '1.5'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
-    none: false
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
-        version: '1.0'
+        version: '1.5'
 - !ruby/object:Gem::Dependency
-  name: jeweler
+  name: rake
   requirement: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ~>
+    - - ! '>='
       - !ruby/object:Gem::Version
-        version: 2.0.1
+        version: '0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ~>
+    - - ! '>='
       - !ruby/object:Gem::Version
-        version: 2.0.1
+        version: '0'
 - !ruby/object:Gem::Dependency
-  name: simplecov
+  name: debugger
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: rspec
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: trollop
   requirement: !ruby/object:Gem::Requirement
-    none: false
     requirements:
     - - ! '>='
       - !ruby/object:Gem::Version
@@ -86,58 +104,69 @@ dependencies:
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
-    none: false
     requirements:
     - - ! '>='
       - !ruby/object:Gem::Version
         version: '0'
 description: Fast and easy way to validate tons of urls without locking up your system
   or eating up too much memory.
-email: eric@deployfx.com
-executables: []
+email:
+- eric@deployfx.com
+executables:
+- url_processor
 extensions: []
-extra_rdoc_files:
-- LICENSE.txt
-- README.rdoc
+extra_rdoc_files: []
 files:
-- .document
+- .gitignore
 - .rspec
+- .travis.yml
 - Gemfile
-- Gemfile.lock
 - LICENSE.txt
-- README.rdoc
+- README.md
 - Rakefile
-- VERSION
+- TODO
+- bin/url_processor
 - lib/url_processor.rb
+- lib/url_processor/base.rb
+- lib/url_processor/config.rb
+- lib/url_processor/error.rb
+- lib/url_processor/link_request.rb
+- lib/url_processor/retryable_request.rb
+- lib/url_processor/runner.rb
+- lib/url_processor/version.rb
 - spec/spec_helper.rb
+- spec/url_processor/base_spec.rb
+- spec/url_processor/config_spec.rb
+- spec/url_processor/runner_spec.rb
 - spec/url_processor_spec.rb
 - url_processor.gemspec
-homepage: http://github.com/ejhayes/url_processor
+homepage: https://github.com/ejhayes/url_processor
 licenses:
 - MIT
+metadata: {}
 post_install_message: 
 rdoc_options: []
 require_paths:
 - lib
 required_ruby_version: !ruby/object:Gem::Requirement
-  none: false
   requirements:
   - - ! '>='
     - !ruby/object:Gem::Version
       version: '0'
-      segments:
-      - 0
-      hash: 4095729639175042089
 required_rubygems_version: !ruby/object:Gem::Requirement
-  none: false
   requirements:
   - - ! '>='
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 1.8.23
+rubygems_version: 2.2.2
 signing_key: 
-specification_version: 3
+specification_version: 4
 summary: Fast and reliable link checker.
-test_files: []
+test_files:
+- spec/spec_helper.rb
+- spec/url_processor/base_spec.rb
+- spec/url_processor/config_spec.rb
+- spec/url_processor/runner_spec.rb
+- spec/url_processor_spec.rb

data/.document

@@ -1,5 +0,0 @@
-lib/**/*.rb
-bin/*
-- 
-features/**/*.feature
-LICENSE.txt

data/Gemfile.lock

@@ -1,72 +0,0 @@
-GEM
-  remote: http://rubygems.org/
-  specs:
-    addressable (2.3.5)
-    builder (3.2.2)
-    descendants_tracker (0.0.3)
-    diff-lcs (1.1.3)
-    docile (1.1.3)
-    faraday (0.9.0)
-      multipart-post (>= 1.2, < 3)
-    git (1.2.14)
-    github_api (0.11.2)
-      addressable (~> 2.3)
-      descendants_tracker (~> 0.0.1)
-      faraday (~> 0.8, < 0.10)
-      hashie (>= 1.2)
-      multi_json (>= 1.7.5, < 2.0)
-      nokogiri (~> 1.6.0)
-      oauth2
-    hashie (2.0.5)
-    highline (1.6.20)
-    jeweler (2.0.1)
-      builder
-      bundler (>= 1.0)
-      git (>= 1.2.5)
-      github_api
-      highline (>= 1.6.15)
-      nokogiri (>= 1.5.10)
-      rake
-      rdoc
-    json (1.8.1)
-    jwt (0.1.11)
-      multi_json (>= 1.5)
-    mini_portile (0.5.2)
-    multi_json (1.8.4)
-    multi_xml (0.5.5)
-    multipart-post (2.0.0)
-    nokogiri (1.6.1)
-      mini_portile (~> 0.5.0)
-    oauth2 (0.9.3)
-      faraday (>= 0.8, < 0.10)
-      jwt (~> 0.1.8)
-      multi_json (~> 1.3)
-      multi_xml (~> 0.5)
-      rack (~> 1.2)
-    rack (1.5.2)
-    rake (10.1.1)
-    rdoc (3.12.2)
-      json (~> 1.4)
-    rspec (2.8.0)
-      rspec-core (~> 2.8.0)
-      rspec-expectations (~> 2.8.0)
-      rspec-mocks (~> 2.8.0)
-    rspec-core (2.8.0)
-    rspec-expectations (2.8.0)
-      diff-lcs (~> 1.1.2)
-    rspec-mocks (2.8.0)
-    simplecov (0.8.2)
-      docile (~> 1.1.0)
-      multi_json
-      simplecov-html (~> 0.8.0)
-    simplecov-html (0.8.0)
-
-PLATFORMS
-  ruby
-
-DEPENDENCIES
-  bundler (~> 1.0)
-  jeweler (~> 2.0.1)
-  rdoc (~> 3.12)
-  rspec (~> 2.8.0)
-  simplecov

data/README.rdoc

@@ -1,19 +0,0 @@
-= url_processor
-
-Description goes here.
-
-== Contributing to url_processor
- 
-* Check out the latest master to make sure the feature hasn't been implemented or the bug hasn't been fixed yet.
-* Check out the issue tracker to make sure someone already hasn't requested it and/or contributed it.
-* Fork the project.
-* Start a feature/bugfix branch.
-* Commit and push until you are happy with your contribution.
-* Make sure to add tests for it. This is important so I don't break it in a future version unintentionally.
-* Please try not to mess with the Rakefile, version, or history. If you want to have your own version, or is otherwise necessary, that is fine, but please isolate to its own commit so I can cherry-pick around it.
-
-== Copyright
-
-Copyright (c) 2014 Eric Hayes. See LICENSE.txt for
-further details.
-

data/VERSION

@@ -1 +0,0 @@
-0.1.0