recluse 1.0.0

data/lib/recluse/cli/profile.rb

@@ -0,0 +1,117 @@
+require 'thor'
+require 'recluse/profile'
+require 'recluse/cli/whitelist'
+require 'recluse/cli/blacklist'
+require 'recluse/cli/roots'
+require 'user_config'
+
+module Recluse
+  module CLI
+    ##
+    # Commands to edit/create/delete profiles.
+    class Profile < Thor #:nodoc: all
+      method_option :blacklist, type: :array, desc: 'Glob patterns for URLs to ignore', default: []
+      method_option :whitelist, type: :array, desc: 'Glob pattern exceptions to blacklist', default: []
+      method_option :internal_only, type: :boolean, desc: 'Only check internal URLs', default: false
+      method_option :scheme_squash, type: :boolean, desc: 'HTTP and HTTPS URLs are treated as equals', default: false
+      method_option :redirect, type: :boolean, desc: 'Follow redirects and report final status code', default: false
+      desc 'create [options] name email root1 [root2] ...', 'create profile'
+      def create(name, email, *roots)
+        uconf = UserConfig.new '.recluse'
+        if uconf.exist?("#{name}.yaml")
+          puts "Profile #{name} already exists"
+          exit(-1)
+        end
+        begin
+          profile = Recluse::Profile.new(
+            name,
+            roots,
+            email,
+            blacklist: options['blacklist'],
+            whitelist: options['whitelist'],
+            internal_only: options['internal_only'],
+            scheme_squash: options['scheme_squash'],
+            redirect: options['redirect']
+          )
+          profile.save
+        rescue ProfileError => e
+          puts e
+          exit(-1)
+        end
+      end
+      desc 'profile remove name', 'remove profile'
+      def remove(name)
+        uconf = UserConfig.new '.recluse'
+        if uconf.exist?("#{name}.yaml")
+          uconf.delete "#{name}.yaml"
+        else
+          exit(-1)
+        end
+      end
+      method_option :blacklist, type: :array, desc: 'Glob patterns for URLs to ignore'
+      method_option :whitelist, type: :array, desc: 'Glob pattern exceptions to blacklist'
+      method_option :internal_only, type: :boolean, desc: 'Only check internal URLs'
+      method_option :scheme_squash, type: :boolean, desc: 'HTTP and HTTPS URLs are treated as equals'
+      method_option :roots, type: :array, desc: 'Roots to start the spidering at'
+      method_option :email, type: :string, desc: 'Email to identify spider for system admins'
+      method_option :redirect, type: :boolean, desc: 'Follow redirects and report final status code'
+      desc 'edit name [options]', 'edit profile'
+      def edit(name)
+        begin
+          profile = Recluse::Profile.load name
+        rescue ProfileError => e
+          puts e
+          exit(-1)
+        end
+        profile.roots = options['roots'] if options.key? 'roots'
+        profile.blacklist = options['blacklist'] if options.key? 'blacklist'
+        profile.whitelist = options['whitelist'] if options.key? 'whitelist'
+        profile.internal_only = options['internal_only'] if options.key? 'internal_only'
+        profile.scheme_squash = options['scheme_squash'] if options.key? 'scheme_squash'
+        profile.redirect = options['redirect'] if options.key? 'redirect'
+        profile.email = options['email'] if options.key? 'email'
+        profile.save
+      end
+      desc 'rename old_name new_name', 'rename profile'
+      def rename(old_name, new_name)
+        uconf = UserConfig.new '.recluse'
+        if uconf.exist?("#{new_name}.yaml")
+          puts "Profile #{new_name} already exists"
+          exit(-1)
+        end
+        return unless uconf.exist?("#{old_name}.yaml")
+        old_profile = uconf["#{old_name}.yaml"]
+        old_profile['name'] = new_name
+        new_profile = uconf["#{new_name}.yaml"]
+        old_profile.each do |key, value|
+          new_profile[key] = value
+        end
+        new_profile.save
+        uconf.delete "#{old_name}.yaml"
+      end
+      desc 'list', 'list profiles'
+      def list
+        uconf = UserConfig.new '.recluse'
+        files = uconf.list_in_directory '.'
+        files.each do |file|
+          puts file.gsub(/\.yaml$/, '')
+        end
+      end
+      desc 'info name', 'profile information'
+      def info(name)
+        uconf = UserConfig.new '.recluse'
+        unless uconf.exist?("#{name}.yaml")
+          puts "Profile #{name} doesn't exist"
+          exit(-1)
+        end
+        puts uconf["#{name}.yaml"].to_yaml
+      end
+      desc 'blacklist [subcommand] [options]', 'edit blacklist'
+      subcommand 'blacklist', Blacklist
+      desc 'roots [subcommand] [options]', 'edit roots'
+      subcommand 'roots', Roots
+      desc 'whitelist [subcommand] [options]', 'edit whitelist'
+      subcommand 'whitelist', Whitelist
+    end
+  end
+end

data/lib/recluse/cli/roots.rb

@@ -0,0 +1,59 @@
+require 'thor'
+require 'user_config'
+
+module Recluse
+  module CLI
+    ##
+    # Roots related commands.
+    class Roots < Thor #:nodoc: all
+      desc 'add profile pattern1 [pattern2] ...', 'add to roots'
+      def add(name, *roots)
+        uconf = UserConfig.new '.recluse'
+        unless uconf.exist?("#{name}.yaml")
+          puts "Profile #{name} doesn't exist"
+          exit(-1)
+        end
+        profile = uconf["#{name}.yaml"]
+        if profile.key?('roots')
+          profile['roots'] += roots
+        else
+          profile['roots'] = roots
+        end
+        profile.save
+      end
+      desc 'remove profile pattern1 [pattern2] ...', 'remove from roots'
+      def remove(name, *roots)
+        uconf = UserConfig.new '.recluse'
+        unless uconf.exist?("#{name}.yaml")
+          puts "Profile #{name} doesn't exist"
+          exit(-1)
+        end
+        profile = uconf["#{name}.yaml"]
+        return unless profile.key?('roots')
+        profile['roots'] -= roots
+        profile.save
+      end
+      desc 'clear profile', 'remove all roots'
+      def clear(name)
+        uconf = UserConfig.new '.recluse'
+        unless uconf.exist?("#{name}.yaml")
+          puts "Profile #{name} doesn't exist"
+          exit(-1)
+        end
+        profile = uconf["#{name}.yaml"]
+        profile['roots'] = []
+        profile.save
+      end
+      desc 'list profile', 'list roots'
+      def list(name)
+        uconf = UserConfig.new '.recluse'
+        unless uconf.exist?("#{name}.yaml")
+          puts "Profile #{name} doesn't exist"
+          exit(-1)
+        end
+        profile = uconf["#{name}.yaml"]
+        profile['roots'].each { |root| puts root } if profile.key?('roots')
+      end
+    end
+  end
+end

data/lib/recluse/cli/whitelist.rb

@@ -0,0 +1,59 @@
+require 'thor'
+require 'user_config'
+
+module Recluse
+  module CLI
+    ##
+    # Whitelist related commands.
+    class Whitelist < Thor #:nodoc: all
+      desc 'add profile pattern1 [pattern2] ...', 'add glob patterns to whitelist'
+      def add(name, *patterns)
+        uconf = UserConfig.new '.recluse'
+        unless uconf.exist?("#{name}.yaml")
+          puts "Profile #{name} doesn't exist"
+          exit(-1)
+        end
+        profile = uconf["#{name}.yaml"]
+        if profile.key?('whitelist')
+          profile['whitelist'] += patterns
+        else
+          profile['whitelist'] = patterns
+        end
+        profile.save
+      end
+      desc 'remove profile pattern1 [pattern2] ...', 'remove patterns from whitelist'
+      def remove(name, *patterns)
+        uconf = UserConfig.new '.recluse'
+        unless uconf.exist?("#{name}.yaml")
+          puts "Profile #{name} doesn't exist"
+          exit(-1)
+        end
+        profile = uconf["#{name}.yaml"]
+        return unless profile.key?('whitelist')
+        profile['whitelist'] -= patterns
+        profile.save
+      end
+      desc 'clear profile', 'remove all patterns in the whitelist'
+      def clear(name)
+        uconf = UserConfig.new '.recluse'
+        unless uconf.exist?("#{name}.yaml")
+          puts "Profile #{name} doesn't exist"
+          exit(-1)
+        end
+        profile = uconf["#{name}.yaml"]
+        profile['whitelist'] = []
+        profile.save
+      end
+      desc 'list profile', 'list patterns in whitelist'
+      def list(name)
+        uconf = UserConfig.new '.recluse'
+        unless uconf.exist?("#{name}.yaml")
+          puts "Profile #{name} doesn't exist"
+          exit(-1)
+        end
+        profile = uconf["#{name}.yaml"]
+        profile['whitelist'].each { |pattern| puts pattern } if profile.key?('whitelist')
+      end
+    end
+  end
+end

data/lib/recluse/hashtree.rb

@@ -0,0 +1,172 @@
+module Recluse
+  ##
+  # Sorta like a node tree but using two hashes for easy searching for parents and/or children.
+  # This way, it should have similar performance whether you're iterating over parents or children.
+  # Additionally, not every child will need a parent or they might not need a parent at initialization.
+  class HashTree
+    ##
+    # Create a hash tree.
+    def initialize(&block)
+      @parent_keys = {}
+      @child_keys = {}
+      @equivalence = block.nil? ? (proc { |a, b| a == b }) : block
+    end
+
+    ##
+    # Add child associated with parent(s).
+    def add(child, parents)
+      unless child?(child)
+        @child_keys[child] = {
+          value: nil,
+          parents: []
+        }
+      end
+      @child_keys[get_child_key(child)][:parents] += [*parents]
+      [*parents].each do |parent|
+        @parent_keys[parent] = [] unless parent?(parent)
+        @parent_keys[get_parent_key(parent)] << get_child_key(child)
+      end
+    end
+
+    ##
+    # Add parent with no children.
+    def add_parent(parents)
+      [*parents].each do |parent|
+        @parent_keys[parent] = [] unless parent?(parent)
+      end
+    end
+
+    ##
+    # Add child with no value and no parents.
+    def add_child(children)
+      [*children].each do |child|
+        next if child?(child)
+        @child_keys[child] = {
+          value: nil,
+          parents: []
+        }
+      end
+    end
+
+    ##
+    # Set value of child.
+    def set_child_value(child, value)
+      @child_keys[get_child_key(child)][:value] = value
+    end
+
+    ##
+    # Get value of child.
+    def get_child_value(child)
+      @child_keys[get_child_key(child)][:value]
+    end
+
+    ##
+    # Get child's parents
+    def get_parents(child)
+      @child_keys[get_child_key(child)][:parents]
+    end
+
+    ##
+    # Get parent's children
+    def get_children(parent)
+      @parent_keys[get_parent_key(parent)]
+    end
+
+    ##
+    # Collect values of children for parent.
+    def get_values(parent)
+      vals = {}
+      @parent_keys[get_parent_key(parent)].each do |child|
+        vals[child] = @child_keys[child][:value]
+      end
+      vals
+    end
+
+    ##
+    # Get parents hash.
+    def parents
+      @parent_keys.dup
+    end
+
+    ##
+    # Get children hash.
+    def children
+      @child_keys.dup
+    end
+
+    ##
+    # Does element exist as a child and/or parent key?
+    def has?(element)
+      child?(element) || parent?(element)
+    end
+
+    ##
+    # Is element a child?
+    def child?(element)
+      @child_keys.keys.any? { |key| @equivalence.call(key, element) }
+    end
+
+    ##
+    # Is element a parent?
+    def parent?(element)
+      @parent_keys.keys.any? { |key| @equivalence.call(key, element) }
+    end
+
+    ##
+    # Delete child. Removes references to child in associated parents.
+    def delete_child(element)
+      return false unless child?(element)
+      c_key = get_child_key(element)
+      @child_keys[c_key][:parents].each do |parent|
+        @parent_keys[parent] -= [c_key]
+      end
+      @child_keys.delete c_key
+      true
+    end
+
+    ##
+    # Delete parent. Removes references to parent in associated children.
+    def delete_parent(element)
+      return false unless parent?(element)
+      p_key = get_parent_key(element)
+      @parent_keys[p_key].each do |child|
+        @child_keys[child][:parents] -= [p_key]
+      end
+      @parent_keys.delete p_key
+      true
+    end
+
+    ##
+    # Delete from parents and children. Essentially removes all known references.
+    def delete(element)
+      delete_child(element)
+      delete_parent(element)
+    end
+
+    ##
+    # Finds children without parents. Returned as hash.
+    def orphans
+      @child_keys.select { |_key, info| info[:parents].empty? }
+    end
+
+    ##
+    # Finds parents without children. Returned as hash.
+    def childless
+      @parent_keys.select { |_key, children| children.empty? }
+    end
+
+    private
+
+    ##
+    # Get the child key (in case of alternative equivalence testing)
+    def get_child_key(child)
+      @child_keys.keys.find { |key| @equivalence.call(key, child) }
+    end
+
+    ##
+    # Get the parent key (in case of alternative equivalence testing)
+    def get_parent_key(parent)
+      @parent_keys.keys.find { |key| @equivalence.call(key, parent) }
+    end
+  end
+end

data/lib/recluse/info.rb

@@ -0,0 +1,9 @@
+module Recluse
+  ##
+  # Version of the gem.
+  VERSION = '1.0.0'.freeze
+
+  ##
+  # Homepage of the gem (also used in user-agent).
+  URL = 'https://github.com/czycha/recluse'.freeze
+end

data/lib/recluse/link.rb

@@ -0,0 +1,89 @@
+require 'addressable/uri'
+
+module Recluse
+  ##
+  # Errors related to links.
+  class LinkError < RuntimeError
+  end
+
+  ##
+  # A simple link container for a profile's queue.
+  class Link
+    ##
+    # URL of link. Can be relative.
+    attr_reader :url
+
+    ##
+    # Parent of link (i.e. the referrer). Can be +:root+ if no parent.
+    attr_reader :parent
+
+    ##
+    # The absolute URL of the link.
+    attr_reader :absolute
+
+    ##
+    # The +Addressable::URI+ representation of the link.
+    attr_reader :address
+
+    ##
+    # Create a link.
+    def initialize(url, parent)
+      raise LinkError, 'Incorrect parent URL. Expects :root or a string.' unless parent == :root || parent.class == String
+      @url = url
+      @parent = parent
+      @address = @parent == :root ? Addressable::URI.parse(@url) : Addressable::URI.join(@parent, @url)
+      @address.fragment = nil
+      @absolute = @address.to_s
+    end
+
+    ##
+    # Output as string.
+    def to_s
+      @absolute
+    end
+
+    ##
+    # Is the link internal compared to +Addressable::URI+ roots?
+    def internal?(addrroots, scheme_squash: false)
+      return true if @parent == :root
+      return addrroots.any? { |root| Link.internal_to?(root, @address) } unless scheme_squash
+      a2 = @address.dup
+      a2.scheme = a2.scheme == 'https' ? 'http' : 'https'
+      addrroots.any? { |root| (Link.internal_to?(root, @address) || Link.internal_to?(root, a2)) }
+    end
+
+    ##
+    # Is the link runnable compared to the black- and whitelists, and the link scheme?
+    def run?(blacklist, whitelist)
+      ((@address.scheme == 'http') || (@address.scheme == 'https')) && (!match?(blacklist) || match?(whitelist))
+    end
+
+    ##
+    # Does the link match any of the globs?
+    def match?(globs)
+      [*globs].any? { |glob| File.fnmatch(glob, @absolute) }
+    end
+
+    ##
+    # Check if +to+ is internal compared to +root+. Building block of +internal?+. Both +root+ and +to+ must be of type +Addressable::URI+.
+    #
+    # A link is internal compared to the root if it matches the following conditions:
+    #
+    # - Same scheme, subdomain, and domain. In other words, a relative URL can be built out of the link.
+    # - If +root+ is a directory and doesn't contain a filename (e.g. +http://example.com/test/+):
+    #   - Internal if link is below the root's path or is the same (e.g. +http://example.com/test/index.php+).
+    # - Otherwise if +root+ contains a filename (e.g. +http://example.com/test/index.php+):
+    #   - Internal if link is below parent directory of root (e.g. +http://example.com/test/about.php+).
+    def self.internal_to?(root, to)
+      route = root.route_to(to)
+      return false if route == to # can't be represented as relative url
+      route_internal = route.to_s[0...3] != '../'
+      has_slash = root.path[-1] == '/'
+      return route_internal if has_slash || !root.extname.empty?
+      slashed_root = root.dup
+      slashed_root.path = "#{root.path}/"
+      slashed_route = slashed_root.route_to(to)
+      (slashed_route.to_s[0...3] != '../')
+    end
+  end
+end