recluse 1.0.2 → 2.0.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 396ec13b808c57e16b103fcc9dd55c228fdef9b5
-  data.tar.gz: 89b5a9aa4d946fb4d3653a7422407b63fa8263fc
+  metadata.gz: 4bc973f210fae9eda00a82ccc421c957098182b3
+  data.tar.gz: 27b8c662b5e24f3b48e8d29277ef82d027d128a7
 SHA512:
-  metadata.gz: b43648a734c3330cf4f536e0572e02069922574a275ec83c3fc7f2b60019274741d361ce61acc469b26222211c27dc630ec975ae7f59bf8bf313c906edd1a079
-  data.tar.gz: 04f1f680c9dc0f6fad64f97ae410ca1233a28a52276059d652449c3eac4baa53da1068fe487c5d88f03ed791e4b8cb0011782c53051c6f182990a8869f2139d5
+  metadata.gz: 528286fe546e0862abd045b13abef5722a102aa3e9a660d93b08206683cf01a8a61593de6bfd40e93bbeff765a82890ebf593a8b70d97125d683ce7bead4c38f
+  data.tar.gz: cb4fcf8872a03d57635756616813603556a9099498f8376dc1dfc9e8e7363fbe71017af2c95b61801f6b15691746c1d4fc909c2de9537e8021be113c5e3873e4

data/README.md

@@ -228,6 +228,43 @@ List the YAML info of the profile.
 
 Bug reports and pull requests are welcome on GitHub.
 
+## Extending
+
+Recluse is modular so you can add tasks if you want. Below is an example of adding your own task to Recluse.
+
+```ruby
+require 'recluse'
+
+module MyModule
+  ##
+  # Create a task object
+  class MyTask < Recluse::Tasks::Task
+    ##
+    # First argument must be the profile. The rest are hash arguments specific for the task.
+    def initialize(profile, option1: false, option2: true, results: nil)
+      # Sets up everything based on the profile, queue-specific options, and can also prepopulate results.
+      super(profile, queue_options, results: results)
+      @queue.run_if do |link|
+      	# Run a link if this function returns true.
+      	# Link is a Recluse::Link object.
+      end
+      @queue.on_complete do |link, response|
+        # Run this function after the page has either successfully been retrieved, or failed to be retrieved.
+        # Link is a Recluse::Link object.
+        # Response is a Recluse::Response object.
+      end
+    end
+  end
+end
+
+# Add your task to the task list under the key 'my_task'.
+Recluse::Tasks.add_task(:my_task, MyModule::MyTask)
+
+# You can now access 'my_task' like you would the default Recluse tasks.
+my_profile = Recluse::Profile.load('my_profile')
+my_profile.test(:my_task, option1: true, option2: true)
+results = my_profile.results[:my_task]
+```
 
 ## License
 

data/lib/recluse/cli/main.rb

@@ -24,7 +24,7 @@ module Recluse
           parent_count = 0
           case group_by
           when :page
-            report = profile.results.parents
+            report = profile.results[:find].parents
             CSV.open(csv_path, 'w+') do |csv|
               csv << ['Page', 'Matching URLs']
               report.each do |parent, children|
@@ -35,7 +35,7 @@ module Recluse
               end
             end
           when :none
-            report = profile.results.parents
+            report = profile.results[:find].parents
             CSV.open(csv_path, 'w+') do |csv|
               csv << ['Matching URL', 'Page']
               report.each do |parent, children|
@@ -47,7 +47,7 @@ module Recluse
               end
             end
           when :url
-            report = profile.results.children
+            report = profile.results[:find].children
             CSV.open(csv_path, 'w+') do |csv|
               csv << ['Matching URL', 'Pages']
               parents = Set.new
@@ -61,7 +61,7 @@ module Recluse
               parent_count = parents.length
             end
           end
-          total = profile.results.parents.keys.length
+          total = profile.results[:find].parents.keys.length
           puts "Total pages:\t#{total}"
           puts "Matched URLs:\t#{child_count}"
           puts "Pages with matches:\t#{parent_count}\t#{perc parent_count, total}%"
@@ -87,7 +87,7 @@ module Recluse
           valid_status = proc do |code|
             (includes.any? { |include_code| include_code.equal?(code) }) && (excludes.none? { |exclude_code| exclude_code.equal?(code) })
           end
-          report = profile.results.children
+          report = profile.results[:status].children
           CSV.open(csv_path, 'w+') do |csv|
             csv << ['Status code', 'URL', page_label, 'With error']
             report.each do |child, info|
@@ -118,7 +118,7 @@ module Recluse
 
         def assert_save(profile, csv_path, report_vals)
           puts 'Saving report...'
-          report = profile.results.children
+          report = profile.results[:assert].children
           counts = {}
           CSV.open(csv_path, 'w+') do |csv|
             csv << ['Selector', 'Exists', 'On page']
@@ -182,8 +182,8 @@ module Recluse
           Signal.trap sig, &ending
         end
         (0...profile_queue.length).each do |i|
-          profile.results = profile_queue[i - 1].results unless i.zero?
-          profile.status
+          profile.results[:status] = profile_queue[i - 1].results[:status] unless i.zero?
+          profile.test :status
           profile = profile_queue[i + 1] if i + 1 < profile_queue.length
         end
         %w(INT TERM).each do |sig|
@@ -219,8 +219,8 @@ module Recluse
           Signal.trap sig, &ending
         end
         (0...profile_queue.length).each do |i|
-          profile.results = profile_queue[i - 1].results unless i.zero?
-          profile.find options['globs']
+          profile.results[:find] = profile_queue[i - 1].results[:find] unless i.zero?
+          profile.test(:find, globs: options['globs'])
           profile = profile_queue[i + 1] if i + 1 < profile_queue.length
         end
         %w(INT TERM).each do |sig|
@@ -265,9 +265,10 @@ module Recluse
         %w(INT TERM).each do |sig|
           Signal.trap sig, &ending
         end
+
         (0...profile_queue.length).each do |i|
-          profile.results = profile_queue[i - 1].results unless i.zero?
-          profile.assert options['exists']
+          profile.results[:assert] = profile_queue[i - 1].results[:assert] unless i.zero?
+          profile.test(:assert, selectors: options['exists'])
           profile = profile_queue[i + 1] if i + 1 < profile_queue.length
         end
         %w(INT TERM).each do |sig|

data/lib/recluse/info.rb

@@ -1,7 +1,7 @@
 module Recluse
   ##
   # Version of the gem.
-  VERSION = '1.0.2'.freeze
+  VERSION = '2.0.0'.freeze
 
   ##
   # Homepage of the gem (also used in user-agent).

data/lib/recluse/link.rb

@@ -42,6 +42,12 @@ module Recluse
       @absolute
     end
 
+    ##
+    # Inspection
+    def inspect
+      to_s
+    end
+
     ##
     # Is the link internal compared to +Addressable::URI+ roots?
     def internal?(addrroots, scheme_squash: false)

data/lib/recluse/profile.rb

@@ -2,6 +2,7 @@ require 'recluse/hashtree'
 require 'recluse/link'
 require 'recluse/result'
 require 'recluse/info'
+require 'recluse/tasks/list'
 require 'addressable/uri'
 require 'mechanize'
 require 'colorize'
@@ -45,14 +46,18 @@ module Recluse
     # HTTP and HTTPS schemed URLs are treated as equal. Optional. Defaults to +false+.
     attr_accessor :scheme_squash
 
-    ##
-    # +HashTree+ representation of results.
-    attr_accessor :results
-
     ##
     # When enabled, will follow redirects and report only the status code for the page that is landed upon. When disabled, will report the redirect status code. Defaults to +false+.
     attr_accessor :redirect
 
+    ##
+    # The list of run tests.
+    attr_accessor :tasks
+
+    ##
+    # Hash of resulting +HashTree+s.
+    attr_accessor :results
+
     ##
     # Create a profile.
     def initialize(
@@ -68,17 +73,20 @@ module Recluse
       raise ProfileError, 'Profile needs roots for starting point' if roots.empty?
       @name = name
       @email = email
-      @roots = roots
+      @roots = roots.map do |root|
+        if root.class == Link
+          root
+        else
+          Link.new(root, :root)
+        end
+      end
       @blacklist = blacklist
       @whitelist = whitelist
       @internal_only = internal_only
       @scheme_squash = scheme_squash
       @redirect = redirect
-      @results = HashTree.new do |url1, url2|
-        url1, url2 = url2, url1 if url2.length > url1.length
-        # Detect if URL exists already, but just has a slash at end
-        (url1 == url2 || (url1.length == (url2.length + 1) && url1[-1] == '/' && url2[-1] != '/' && url1[0...-1] == url2))
-      end
+      @tasks = {}
+      @results = {}
     end
 
     ##
@@ -96,150 +104,18 @@ module Recluse
     end
 
     ##
-    # Starting from the roots, goes through each runnable link and records the referrer, the status code, and any errors.
-    # Results are saved in <tt>@results</tt>.
-    def status(quiet: false)
-      queue = @roots.map { |url| Link.new(url, :root) }
-      addrroot = @roots.map { |url| Addressable::URI.parse url }
-      raise ProfileError, 'No roots to start from' if queue.empty?
-      agent = create_agent
-      while queue.length >= 1
-        element = queue.shift
-        next unless element.run?(@blacklist, @whitelist)
-        internal = element.internal?(addrroot)
-        next if @internal_only && !internal
-        if @results.child?(element.absolute)
-          @results.add element.absolute, element.parent
-          next
-        end
-        @results.add element.absolute, element.parent
-        if @scheme_squash
-          alt = element.address
-          alt.scheme = alt.scheme == 'http' ? 'https' : 'http'
-          if @results.child?(alt.to_s)
-            @results.set_child_value element.absolute, @results.get_child_value(alt.to_s)
-            next
-          end
-        end
-        result = Result.new 'idk', false
-        begin
-          page = agent.get element.absolute
-          result.code = page.code
-          if @redirect
-            result_link = Link.new(page.uri.to_s, element.parent)
-            internal = result_link.internal?(addrroot)
-          end
-          queue += page.links.map { |link| Link.new(link.uri.to_s, element.absolute) } if internal && (page.class != Mechanize::File) && (page.class != Mechanize::Image)
-        rescue Mechanize::ResponseCodeError => code
-          result.code = code.response_code
-        rescue => e
-          result.error = e
-        end
-        @results.set_child_value element.absolute, result
-        unless quiet
-          puts "[#{@name.colorize(mode: :bold)}][#{result.code.colorize(color: result.color, mode: :bold)}][#{(internal ? 'internal' : 'external').colorize(mode: :bold)}] #{element.absolute}"
-          puts "\a^ #{'Error'.colorize(mode: :bold, color: :red)}: #{result.error}" unless result.error == false
-        end
-      end
-    end
-
-    ##
-    # Find links matching glob patterns, starting from the roots. Overrides (but does not overwrite) +internal_only+ behavior to +true+.
-    def find(glob, quiet: false)
-      queue = @roots.map { |url| Link.new(url, :root) }
-      addrroot = @roots.map { |url| Addressable::URI.parse url }
-      raise ProfileError, 'No roots to start from' if queue.empty?
-      progress = ProgressBar.create(total: nil, format: '|%B|') unless quiet
-      agent = create_agent
-      while queue.length >= 1
-        element = queue.shift
-        match = element.match? glob
-        if match
-          @results.add element.absolute, element.parent
-          progress.log "[#{@name.colorize(mode: :bold)}][#{'found'.colorize(color: :green, mode: :bold)}] #{element.parent} => #{element.absolute}" unless quiet
-        end
-        next unless element.run?(@blacklist, @whitelist)
-        internal = element.internal?(addrroot)
-        next unless internal
-        next if @results.parent?(element.absolute)
-        if @scheme_squash
-          alt = element.address
-          alt.scheme = alt.scheme == 'http' ? 'https' : 'http'
-          next if @results.parent?(alt.to_s)
-        end
-        @results.add_parent element.absolute
-        result = Result.new 'idk', false
-        begin
-          page = agent.get element.absolute
-          result.code = page.code
-          if @redirect
-            result_link = Link.new(page.uri.to_s, element.parent)
-            next unless result_link.internal?(addrroot)
-          end
-          queue += page.links.map { |link| Link.new(link.uri.to_s, element.absolute) } unless (page.class == Mechanize::File) || (page.class == Mechanize::Image)
-        rescue Mechanize::ResponseCodeError => code
-          result.code = code.response_code
-        rescue => e
-          result.error = e
-        end
-        progress.increment unless quiet
-        unless quiet || (result.error == false)
-          progress.log "[#{@name.colorize(mode: :bold)}][#{result.code.colorize(color: result.color, mode: :bold)}] #{element.absolute}"
-          progress.log "\a^ #{'Error'.colorize(mode: :bold, color: :red)}: #{result.error}"
-        end
-      end
-    end
-
-    ##
-    # Asserts existence of CSS selectors.
-    def assert(selectors, quiet: false)
-      queue = @roots.map { |url| Link.new(url, :root) }
-      addrroot = @roots.map { |url| Addressable::URI.parse url }
-      raise ProfileError, 'No roots to start from' if queue.empty?
-      agent = create_agent
-      while queue.length >= 1
-        element = queue.shift
-        internal = element.internal?(addrroot)
-        next unless element.run?(@blacklist, @whitelist) && internal && !@results.child?(element.absolute)
-        if @scheme_squash
-          alt = element.address
-          alt.scheme = alt.scheme == 'http' ? 'https' : 'http'
-          next if @results.child?(alt.to_s)
-        end
-        @results.add_child element.absolute
-        existence = nil
-        result = Result.new 'idk', false
-        begin
-          page = agent.get element.absolute
-          result.code = page.code
-          if @redirect
-            result_link = Link.new(page.uri.to_s, element.parent)
-            next unless result_link.internal?(addrroot)
-          end
-          unless (page.class == Mechanize::File) || (page.class == Mechanize::Image)
-            existence = {}
-            selectors.each do |selector|
-              existence[selector] = !page.css(selector).empty?
-            end
-            @results.set_child_value element.absolute, existence
-            queue += page.links.map { |link| Link.new(link.uri.to_s, element.absolute) }
-          end
-        rescue Mechanize::ResponseCodeError => code
-          result.code = code.response_code
-        rescue => e
-          result.error = e
-        end
-        unless quiet
-          if result.error != false
-            puts "[#{@name.colorize(mode: :bold)}][#{result.code.colorize(color: result.color, mode: :bold)}] #{element.absolute}"
-            puts "\a^ #{'Error'.colorize(mode: :bold, color: :red)}: #{result.error}"
-          elsif !existence.nil?
-            existence.each do |selector, exists|
-              puts "[#{@name.colorize(mode: :bold)}][#{selector.colorize(mode: :bold)}][#{exists.to_s.colorize(color: (exists ? :green : :red), mode: :bold)}] #{element.absolute}"
-            end
-          end
+    # Runs test.
+    def test(key, options = {})
+      unless @results.key?(key) && @results[key].class == Recluse::HashTree
+        @results[key] = Recluse::HashTree.new do |url1, url2|
+          url1, url2 = url2, url1 if url2.length > url1.length
+          # Detect if URL exists already, but just has a slash at end
+          (url1 == url2 || (url1.length == (url2.length + 1) && url1[-1] == '/' && url2[-1] != '/' && url1[0...-1] == url2))
         end
       end
+      @tasks[key] = Recluse::Tasks.get(key).new(self, options.merge(results: @results[key]))
+      @tasks[key].run
+      @results[key]
     end
 
     ##
@@ -249,7 +125,7 @@ module Recluse
       fname = "#{@name}.yaml"
       options = uconf[fname]
       options['name'] = @name
-      options['roots'] = @roots
+      options['roots'] = @roots.map(&:to_s)
       options['email'] = @email
       options['blacklist'] = @blacklist
       options['whitelist'] = @whitelist
@@ -264,7 +140,9 @@ module Recluse
     def ==(other)
       return false if other.class != self.class
       instance_variables.all? do |ivar|
-        ivar == '@results'.to_sym || instance_variable_get(ivar) == other.instance_variable_get(ivar)
+        next true if ivar == '@results'.to_sym
+        next true if ivar == '@roots' && instance_variable_get(ivar).map(&:to_s) == other.instance_variable_get(ivar).map(&:to_s)
+        instance_variable_get(ivar) == other.instance_variable_get(ivar)
       end
     end
 

data/lib/recluse/queue.rb

@@ -0,0 +1,74 @@
+require 'mechanize'
+require 'recluse/response'
+
+module Recluse
+  ##
+  # Link checker
+  class Queue
+    ##
+    # Create an empty queue
+    def initialize(email, redirect: false)
+      @links = []
+      @run_if = proc { true }
+      @on_complete = proc { |link, response| }
+      @redirect = redirect
+      @email = email
+      @agent = Mechanize.new do |a|
+        a.ssl_version = 'TLSv1'
+        a.verify_mode = OpenSSL::SSL::VERIFY_NONE
+        a.max_history = nil
+        a.follow_meta_refresh = true
+        a.keep_alive = false
+        a.redirect_ok = @redirect
+        a.user_agent = "Mozilla/5.0 (compatible; recluse/#{Recluse::VERSION}; +#{Recluse::URL}) #{@email}"
+      end
+    end
+
+    ##
+    # Add to queue.
+    def add(link)
+      @links += [*link]
+    end
+
+    ##
+    # If the test is true, run the link. Procedure takes the link as input.
+    def run_if(&block)
+      @run_if = block
+    end
+
+    ##
+    # Run when a link has been checked. Procedure takes the link and response as inputs.
+    def on_complete(&block)
+      @on_complete = block
+    end
+
+    ##
+    # Run a link
+    def run_link(link)
+      response = Response.new
+      return nil unless @run_if.call(link)
+      begin
+        response.page = @agent.get link.absolute
+        response.code = response.page.code
+        response.success = true
+      rescue Mechanize::ResponseCodeError => code
+        response.code = code.response_code
+        response.success = false
+      rescue => error
+        response.errors = error
+        response.success = false
+      end
+      @on_complete.call link, response
+      response
+    end
+
+    ##
+    # Run queue
+    def run
+      until @links.empty?
+        link = @links.shift
+        run_link link
+      end
+    end
+  end
+end

data/lib/recluse/response.rb

@@ -0,0 +1,39 @@
+require 'mechanize'
+require 'recluse/statuscode'
+
+module Recluse
+  ##
+  # Response wrapper.
+  class Response
+    ##
+    # +Mechanize::Page+ of the response page. Might be +nil+.
+    attr_accessor :page
+
+    ##
+    # +StatusCode+ of the response.
+    attr_reader :code
+
+    ##
+    # Error string if any.
+    attr_accessor :errors
+
+    ##
+    # Whether the page was successfully accessed or not.
+    attr_accessor :success
+
+    ##
+    # Create new response.
+    def initialize(page: nil, errors: false, code: StatusCode.new('idk'), success: false)
+      @page = page
+      @code = code
+      @errors = errors
+      @success = success
+    end
+
+    ##
+    # Set a new status code.
+    def code=(new_code)
+      @code = StatusCode.new new_code
+    end
+  end
+end

data/lib/recluse/result.rb

@@ -20,7 +20,7 @@ module Recluse
     ##
     # Returns the HTTP status code.
     def inspect
-      @code
+      @code.to_s
     end
 
     ##

data/lib/recluse/tasks/assert.rb

@@ -0,0 +1,59 @@
+require 'recluse/tasks/task'
+require 'recluse/link'
+require 'recluse/result'
+require 'addressable/uri'
+require 'colorize'
+
+module Recluse
+  module Tasks
+    ##
+    # Asserts existence of CSS selectors.
+    class Assert < Task
+      ##
+      # Create new assertion task.
+      def initialize(profile, selectors: [], quiet: false, results: nil)
+        super(profile, queue_options: { redirect: profile.redirect }, results: results)
+        addr_roots = profile.roots.map { |root| Addressable::URI.parse(root.url) }
+        @queue.run_if do |link|
+          internal = link.internal?(addr_roots)
+          next false unless link.run?(profile.blacklist, profile.whitelist) && internal && !@results.child?(link.absolute)
+          if profile.scheme_squash
+            alt = link.address
+            alt.scheme = alt.scheme == 'http' ? 'https' : 'http'
+            next false if @results.child?(alt.to_s)
+          end
+          @results.add_child link.absolute
+          true
+        end
+        @queue.on_complete do |link, response|
+          existence = nil
+          result = Recluse::Result.new response.code.to_s, response.errors
+          if response.success
+            if profile.redirect
+              result_link = Link.new(response.page.uri.to_s, link.parent)
+              next unless result_link.internal?(addr_roots)
+            end
+            unless (response.page.class == Mechanize::File) || (response.page.class == Mechanize::Image)
+              existence = {}
+              selectors.each do |selector|
+                existence[selector] = !response.page.css(selector).empty?
+              end
+              @results.set_child_value link.absolute, existence
+              @queue.add(response.page.links.map { |new_link| Link.new(new_link.uri.to_s, link.absolute) })
+            end
+          end
+          unless quiet
+            if result.error != false
+              puts "[#{profile.name.colorize(mode: :bold)}][#{result.code.colorize(color: result.color, mode: :bold)}] #{link.absolute}"
+              puts "\a^ #{'Error'.colorize(mode: :bold, color: :red)}: #{result.error}"
+            elsif !existence.nil?
+              existence.each do |selector, exists|
+                puts "[#{profile.name.colorize(mode: :bold)}][#{selector.colorize(mode: :bold)}][#{exists.to_s.colorize(color: (exists ? :green : :red), mode: :bold)}] #{link.absolute}"
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/recluse/tasks/find.rb

@@ -0,0 +1,55 @@
+require 'recluse/tasks/task'
+require 'recluse/link'
+require 'recluse/result'
+require 'addressable/uri'
+require 'colorize'
+require 'ruby-progressbar'
+
+module Recluse
+  module Tasks
+    ##
+    # Find links matching glob patterns, starting from the roots. Overrides (but does not overwrite) +internal_only+ behavior to +true+.
+    class Find < Task
+      ##
+      # Create new find task.
+      def initialize(profile, globs: [], quiet: false, results: nil)
+        super(profile, queue_options: { redirect: profile.redirect }, results: results)
+        addr_roots = profile.roots.map { |root| Addressable::URI.parse(root.url) }
+        progress = ProgressBar.create(total: nil, format: '|%B|') unless quiet
+        @queue.run_if do |link|
+          match = link.match? globs
+          if match
+            @results.add link.absolute, link.parent
+            progress.log "[#{profile.name.colorize(mode: :bold)}][#{'found'.colorize(color: :green, mode: :bold)}] #{link.parent} => #{link.absolute}" unless quiet
+          end
+          next false unless link.run?(profile.blacklist, profile.whitelist)
+          internal = link.internal?(addr_roots)
+          next false unless internal
+          next false if @results.parent?(link.absolute)
+          if profile.scheme_squash
+            alt = link.address
+            alt.scheme = alt.scheme == 'http' ? 'https' : 'http'
+            next false if @results.parent?(alt.to_s)
+          end
+          @results.add_parent link.absolute
+          true
+        end
+        @queue.on_complete do |link, response|
+          result = Recluse::Result.new response.code.to_s, response.errors
+          if response.success
+            if profile.redirect
+              result_link = Recluse::Link.new(response.page.uri.to_s, link.parent)
+              next unless result_link.internal?(addr_roots)
+            end
+            @queue.add(response.page.links.map { |new_link| Link.new(new_link.uri.to_s, link.absolute) }) unless (response.page.class == Mechanize::File) || (response.page.class == Mechanize::Image)
+          end
+          progress.increment unless quiet
+          unless quiet || (result.error == false)
+            progress.log "[#{profile.name.colorize(mode: :bold)}][#{result.code.colorize(color: result.color, mode: :bold)}] #{link.absolute}"
+            progress.log "\a^ #{'Error'.colorize(mode: :bold, color: :red)}: #{result.error}"
+          end
+        end
+      end
+    end
+  end
+end

data/lib/recluse/tasks/list.rb

@@ -0,0 +1,42 @@
+require 'recluse/tasks/status'
+require 'recluse/tasks/assert'
+require 'recluse/tasks/find'
+
+module Recluse
+  ##
+  # Tasks are tests for Recluse.
+  module Tasks
+    ##
+    # Hash of available tasks.
+    @@list = {
+      status: Recluse::Tasks::Status,
+      assert: Recluse::Tasks::Assert,
+      find: Recluse::Tasks::Find
+    }
+    class << self
+      ##
+      # Add task to the list.
+      def add_task(key, task_class)
+        list[key] = task_class
+      end
+
+      ##
+      # Hash of available tasks.
+      def list
+        @@list
+      end
+
+      ##
+      # Get task by key name.
+      def get(key)
+        @@list[key]
+      end
+
+      ##
+      # Do something for each task.
+      def each(&block)
+        @@list.each(&block)
+      end
+    end
+  end
+end

data/lib/recluse/tasks/status.rb

@@ -0,0 +1,55 @@
+require 'recluse/tasks/task'
+require 'recluse/link'
+require 'recluse/result'
+require 'addressable/uri'
+require 'colorize'
+
+module Recluse
+  module Tasks
+    ##
+    # Starting from the roots, goes through each runnable link and records the referrer, the status code, and any errors.
+    class Status < Task
+      ##
+      # Create new status task.
+      def initialize(profile, quiet: false, results: nil)
+        super(profile, queue_options: { redirect: profile.redirect }, results: results)
+        addr_roots = profile.roots.map { |root| Addressable::URI.parse(root.url) }
+        @queue.run_if do |link|
+          next false unless link.run?(profile.blacklist, profile.whitelist)
+          internal = link.internal?(addr_roots)
+          next false if profile.internal_only && !internal
+          if @results.child?(link.absolute)
+            @results.add link.absolute, link.parent
+            next false
+          end
+          @results.add link.absolute, link.parent
+          if profile.scheme_squash
+            alt = link.address
+            alt.scheme = alt.scheme == 'http' ? 'https' : 'http'
+            if @results.child?(alt.to_s)
+              @results.set_child_value link.absolute, @results.get_child_value(alt.to_s)
+              next false
+            end
+          end
+          true
+        end
+        @queue.on_complete do |link, response|
+          result = Recluse::Result.new response.code.to_s, response.errors
+          if response.success
+            internal = link.internal? addr_roots
+            if profile.redirect
+              result_link = Recluse::Link.new response.page.uri.to_s, link.parent
+              internal = result_link.internal? addr_roots
+            end
+            queue.add(response.page.links.map { |new_link| Link.new(new_link.uri.to_s, link.absolute) }) if internal && (response.page.class != Mechanize::File) && (response.page.class != Mechanize::Image)
+          end
+          @results.set_child_value link.absolute, result
+          unless quiet
+            puts "[#{profile.name.colorize(mode: :bold)}][#{result.code.colorize(color: result.color, mode: :bold)}][#{(internal ? 'internal' : 'external').colorize(mode: :bold)}] #{link.absolute}"
+            puts "\a^ #{'Error'.colorize(mode: :bold, color: :red)}: #{result.error}" unless result.error == false
+          end
+        end
+      end
+    end
+  end
+end

data/lib/recluse/tasks/task.rb

@@ -0,0 +1,46 @@
+require 'recluse/queue'
+require 'recluse/hashtree'
+
+module Recluse
+  module Tasks
+    ##
+    # Task interface. Runs the queue with customized behavior.
+    class Task
+      ##
+      # +HashTree+ representation of results.
+      attr_reader :results
+
+      ##
+      # +Queue+ of links to check.
+      attr_accessor :queue
+
+      ##
+      # Create new task.
+      def initialize(profile, queue_options: {}, results: nil)
+        @queue = Recluse::Queue.new(profile.email, queue_options)
+        if results.nil?
+          @results = Recluse::HashTree.new do |url1, url2|
+            url1, url2 = url2, url1 if url2.length > url1.length
+            # Detect if URL exists already, but just has a slash at end
+            (url1 == url2 || (url1.length == (url2.length + 1) && url1[-1] == '/' && url2[-1] != '/' && url1[0...-1] == url2))
+          end
+        else
+          @results = results
+        end
+        @queue.add profile.roots
+      end
+
+      ##
+      # Add link (or links) to the queue.
+      def add(link)
+        @queue.add link
+      end
+
+      ##
+      # Run the queue.
+      def run
+        @queue.run
+      end
+    end
+  end
+end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: recluse
 version: !ruby/object:Gem::Version
-  version: 1.0.2
+  version: 2.0.0
 platform: ruby
 authors:
 - James Anthony Bruno
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2017-03-16 00:00:00.000000000 Z
+date: 2017-03-28 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: thor
@@ -203,8 +203,15 @@ files:
 - lib/recluse/info.rb
 - lib/recluse/link.rb
 - lib/recluse/profile.rb
+- lib/recluse/queue.rb
+- lib/recluse/response.rb
 - lib/recluse/result.rb
 - lib/recluse/statuscode.rb
+- lib/recluse/tasks/assert.rb
+- lib/recluse/tasks/find.rb
+- lib/recluse/tasks/list.rb
+- lib/recluse/tasks/status.rb
+- lib/recluse/tasks/task.rb
 - recluse.gemspec
 homepage: https://github.com/czycha/recluse
 licenses: