wordlist 0.1.0 → 0.1.1

data/lib/wordlist/builders/website.rb

@@ -6,33 +6,205 @@ module Wordlist
   module Builders
     class Website < Builder
 
+      # Proxy to use
+      attr_accessor :proxy
+
+      # User-Agent to use
+      attr_accessor :user_agent
+
+      # Referer URL to use
+      attr_accessor :referer
+
       # Host to spider
       attr_accessor :host
 
+      # HTTP Host Header to use in all requests.
+      attr_accessor :host_header
+
+      # Additional hosts that can be spidered
+      attr_reader :hosts
+
+      # Links to ignore while spidering
+      attr_reader :ignore_links
+
+      # Specifies whether the `content` attribute of `meta` tags will be
+      # parsed
+      attr_accessor :parse_meta
+
+      # Specifies whether `title` tags will be parsed
+      attr_accessor :parse_title
+
+      # Specifies whether `h1` tags will be parsed
+      attr_accessor :parse_h1
+
+      # Specifies whether `h2` tags will be parsed
+      attr_accessor :parse_h2
+
+      # Specifies whether `h3` tags will be parsed
+      attr_accessor :parse_h3
+
+      # Specifies whether `h4` tags will be parsed
+      attr_accessor :parse_h4
+
+      # Specifies whether `h5` tags will be parsed
+      attr_accessor :parse_h5
+
+      # Specifies whether `p` tags will be parsed
+      attr_accessor :parse_p
+
+      # Specifies whether `span` tags will be parsed
+      attr_accessor :parse_span
+
+      # Specifies whether the `alt` attributes of `img` tags will be parsed
+      attr_accessor :parse_alt
+
+      # Specifies whether HTML comment tags will be parsed
+      attr_accessor :parse_comments
+
+      # Additional XPath expressions to use to parse spidered pages
+      attr_reader :xpaths
+
+      #
+      # Creates a new Website builder object.
+      #
+      # @param [String] path
+      #   The path to the word-list to build.
+      #
+      # @param [Hash] options
+      #   Additional options.
+      #
+      # @option options [Hash] :proxy
+      #   The Hash of proxy information to use.
+      #
+      # @option options [String] :user_agent
+      #   The User-Agent string to send with each request.
+      #
+      # @option options [String] :referer
+      #   The Referer URL to send with each request.
       #
-      # Creates a new Website builder object with the specified _path_
-      # and _host_. If a _block_ is given, it will be passed the new created
-      # Website builder object.
+      # @option options [String] :host_header
+      #   The HTTP Host header to use in all requests.
       #
-      def initialize(path,host,&block)
-        @host = host
+      # @option options [Array<String, Regexp, Proc>] :ignore_links
+      #   Links to ignore while spidering.
+      #
+      # @option options [Boolean] :parse_meta (true)
+      #   Specifies whether the `content` attribute of `meta` tags will be
+      #   parsed.
+      #
+      # @option options [Boolean] :parse_title (true)
+      #   Specifies whether `title` tags will be parsed.
+      #
+      # @option options [Boolean] :parse_h1 (true)
+      #   Specifies whether `h1` tags will be parsed.
+      #
+      # @option options [Boolean] :parse_h2 (true)
+      #   Specifies whether `h2` tags will be parsed.
+      #
+      # @option options [Boolean] :parse_h3 (true)
+      #   Specifies whether `h3` tags will be parsed.
+      #
+      # @option options [Boolean] :parse_h4 (true)
+      #   Specifies whether `h4` tags will be parsed.
+      #
+      # @option options [Boolean] :parse_h5 (true)
+      #   Specifies whether `h5` tags will be parsed.
+      #
+      # @option options [Boolean] :parse_p (true)
+      #   Specifies whether `p` tags will be parsed.
+      #
+      # @option options [Boolean] :parse_span (true)
+      #   Specifies whether `span` tags will be parsed.
+      #
+      # @option options [Boolean] :parse_alt (true)
+      #   Specifies whether the `alt` attributes of `img` tags will be
+      #   parsed.
+      #
+      # @option options [Boolean] :parse_comments (false)
+      #   Specifies whether HTML comment tags will be parsed.
+      #
+      # @option options [Array<String>] :xpaths
+      #   Additional list of XPath expressions, to use when parsing
+      #   spidered pages.
+      #
+      def initialize(path,options={},&block)
+        @proxy      = options.fetch(:proxy,Spidr.proxy)
+        @user_agent = options[:user_agent]
+        @referer    = options[:referer]
+
+        @host        = options[:host]
+        @host_header = options[:host_header]
+        @hosts       = Array(options[:hosts])
+
+        @ignore_links = Array(options[:ignore_links])
 
-        super(path,&block)
+        @parse_meta     = options.fetch(:parse_meta,true)
+        @parse_title    = options.fetch(:parse_title,true)
+        @parse_h1       = options.fetch(:parse_h1,true)
+        @parse_h2       = options.fetch(:parse_h2,true)
+        @parse_h3       = options.fetch(:parse_h3,true)
+        @parse_h4       = options.fetch(:parse_h4,true)
+        @parse_h5       = options.fetch(:parse_h5,true)
+        @parse_p        = options.fetch(:parse_p,true)
+        @parse_span     = options.fetch(:parse_span,true)
+        @parse_alt      = options.fetch(:parse_alt,true)
+        @parse_comments = options.fetch(:parse_comments,false)
+
+        @xpaths = Array(options[:xpaths])
+
+        super(path,options,&block)
       end
 
       #
-      # Builds the word-list file by spidering the +host+ and parsing the
-      # inner-text from all HTML pages. If a _block_ is given, it will be
-      # called before all HTML pages on the +host+ have been parsed.
+      # Builds the word-list file by spidering the `host` and parsing the
+      # inner-text from all HTML pages.
+      #
+      # @yield [builder]
+      #   If a block is given, it will be called before all HTML pages on
+      #   the `host` have been parsed.
+      #
+      # @yieldparam [Website] builder
+      #   The website word-list builder.
       #
       def build!(&block)
         super(&block)
 
-        Spidr.host(@host) do |spidr|
+        options = {
+          :proxy        => @proxy,
+          :user_agent   => @user_agent,
+          :referer      => @referer,
+          :hosts        => @hosts,
+          :ignore_links => @ignore_links
+        }
+
+        xpaths = []
+        xpaths << '//meta/@content' if @parse_meta
+        xpaths << '//title'         if @parse_title
+        xpaths << '//h1'            if @parse_h1
+        xpaths << '//h2'            if @parse_h2
+        xpaths << '//h3'            if @parse_h3
+        xpaths << '//h4'            if @parse_h4
+        xpaths << '//h5'            if @parse_h5
+        xpaths << '//p'             if @parse_p
+        xpaths << '//span'          if @parse_span
+        xpaths << '//img/@alt'      if @parse_alt
+        xpaths += @xpaths
+
+        Spidr.host(@host,options) do |spidr|
           spidr.every_page do |page|
             if page.html?
-              page.doc.search('//h1|//h2|//h3|//h4|//h5|//p|//span').each do |element|
-                parse(element.inner_text)
+              if page.doc
+                xpaths.each do |xpath|
+                  page.doc.search(xpath).each do |element|
+                    parse(element.inner_text)
+                  end
+                end
+              end
+
+              if (@parse_comments && page.doc)
+                page.doc.traverse do |element|
+                  parse(element.inner_text) if element.comment?
+                end
               end
             end
           end

data/lib/wordlist/flat_file.rb

@@ -7,8 +7,13 @@ module Wordlist
     attr_accessor :path
 
     #
-    # Creates a new FlatFile list with the specified _path_ and given
-    # _options_.
+    # Opens a new FlatFile list.
+    #
+    # @param [String] path
+    #   The path to the flat file word-list read from.
+    #
+    # @param [Hash] options
+    #   Additional options.
     #
     def initialize(path,options={},&block)
       @path = path
@@ -17,9 +22,15 @@ module Wordlist
     end
 
     #
-    # Enumerates through every word in the flat-file, passing each
-    # word to the given _block_.
+    # Enumerates through every word in the flat-file.
+    #
+    # @yield [word]
+    #   The given block will be passed every word from the word-list.
+    #
+    # @yieldparam [String] word
+    #   A word from the word-list.
     #
+    # @example
     #   flat_file.each_word do |word|
     #     puts word
     #   end

data/lib/wordlist/list.rb

@@ -13,42 +13,55 @@ module Wordlist
     attr_accessor :min_length
 
     #
-    # Creates a new List object with the given _options_. If a _block_
-    # is given, it will be passed the newly created List object.
+    # Creates a new List object.
     #
-    # _options_ may include the following keys:
-    # <tt>:max_length</tt>:: The maximum length of words produced by the
-    #                        list.
-    # <tt>:min_length</tt>:: The minimum length of words produced by the
-    #                        list.
+    # @param [Hash] options
+    #   Additional options.
     #
-    def initialize(options={},&block)
+    # @option options [Integer] :max_length
+    #   The maximum length of words produced by the list.
+    #
+    # @option options [Integer] :min_length
+    #   The minimum length of words produced by the list.
+    #
+    # @yield [list]
+    #   If a block is given, it will be passed the new list object.
+    #
+    # @yieldparam [List] list
+    #   The new list object.
+    #
+    def initialize(options={})
       @mutators = []
 
-      @max_length = nil
-      @min_length = 0
-
-      if options[:max_length]
-        @max_length = options[:max_length]
-      end
-
-      if options[:min_length]
-        @min_length = options[:min_length]
-      end
+      @max_length = options[:max_length]
+      @min_length = options.fetch(:min_length,0)
 
-      block.call(self) if block
+      yield self if block_given?
     end
 
     #
-    # Adds a mutation rule for the specified _pattern_, to be replaced
-    # using the specified _substitute_. If a _block_ is given, and the
-    # _substitute_ data omitted, then the _block_ will be used to
-    # replace data matched by the _pattern_.
+    # Adds a mutation rule for the specified pattern, to be replaced
+    # using the specified substitute.
     #
-    #   list.mutate 'o', '0'
+    # @param [String, Regexp] pattern
+    #   The pattern to recognize text to mutate.
     #
-    #   list.mutate '0', 0x41
+    # @param [String, Integer, nil] substitute
+    #   The optional text to replace recognized text.
     #
+    # @yield [match]
+    #   If a block is given, it will be passed the recognized text to be
+    #   mutated. The return value of the block will be used to replace
+    #   the recognized text.
+    #
+    # @yieldparam [String] match
+    #   The recognized text to be mutated.
+    #
+    # @example
+    #   list.mutate 'o', '0'
+    #   
+    #   list.mutate '0', 0x41
+    #   
     #   list.mutate(/[oO]/) do |match|
     #     match.swapcase
     #   end
@@ -58,10 +71,15 @@ module Wordlist
     end
 
     #
-    # Enumerate through every word in the list, passing each word to
-    # the given block. By default this method passes nothing to the given
-    # _block_.
+    # Enumerate through every word in the list.
+    #
+    # @yield [word]
+    #   The given block will be passed each word in the list.
     #
+    # @yieldparam [String] word
+    #   A word from the list.
+    #
+    # @example
     #   list.each_word do |word|
     #     puts word
     #   end
@@ -70,9 +88,15 @@ module Wordlist
     end
 
     #
-    # Enumerates through every unique word in the list, passing each
-    # unique word to the given block.
+    # Enumerates through every unique word in the list.
+    #
+    # @yield [word]
+    #   The given block will be passed each unique word in the list.
+    #
+    # @yieldparam [String] word
+    #   A unique word from the list.
     #
+    # @example
     #   list.each_unique do |word|
     #     puts word
     #   end
@@ -91,9 +115,16 @@ module Wordlist
 
     #
     # Enumerates through every unique mutation, of every unique word, using
-    # the mutator rules define for the list. Every possible unique mutation
-    # will be passed to the given _block_.
+    # the mutator rules define for the list.
+    #
+    # @yield [word]
+    #   The given block will be passed every mutation of every unique
+    #   word in the list.
+    #
+    # @yieldparam [String] word
+    #   A mutation of a unique word from the list.
     #
+    # @example
     #   list.each_mutation do |word|
     #     puts word
     #   end

data/lib/wordlist/mutator.rb

@@ -10,18 +10,33 @@ module Wordlist
     attr_accessor :substitute
 
     #
-    # Creates a new Mutator with the specified _pattern_ and _substitute_
-    # data. If a _block_ is given, and the _substitute_ data is omitted, then
-    # the _block_ will be used to replace data matched by the _pattern_.
+    # Creates a new Mutator object.
+    #
+    # @param [String, Regexp] pattern
+    #   The pattern which recognizes text to mutate.
+    #
+    # @param [String, Integer] substitute
+    #   The optional text to replace recognized text.
+    #
+    # @yield [match]
+    #   If a block is given, it will be used to mutate recognized text.
+    #
+    # @yieldparam [String] match
+    #   The match text to mutate.
     #
     def initialize(pattern,substitute=nil,&block)
-      @pattern = pattern
+      @pattern    = pattern
       @substitute = (substitute || block)
     end
 
     #
-    # Replaces the specified _matched_ data using the +substitute+, which
-    # may be either a String, Integer or Proc.
+    # Mutates the given text.
+    #
+    # @param [String] matched
+    #   The recognized text to be mutated.
+    #
+    # @return [String]
+    #   The mutated text.
     #
     def replace(matched)
       result = if @substitute.kind_of?(Proc)
@@ -40,9 +55,20 @@ module Wordlist
     end
 
     #
-    # Performs every possible replacement of data, which matches the
-    # mutators +pattern+ using the replace method, on the specified _word_
-    # passing each variation to the given _block_.
+    # Enumerates over every possible mutation of the given word.
+    #
+    # @param [String] word
+    #   The word to mutate.
+    #
+    # @yield [mutation]
+    #   The given block will be passed every possible mutation of the
+    #   given word.
+    #
+    # @yieldparam [String] mutation
+    #   One possible mutation of the given word.
+    #
+    # @return [String]
+    #   The original word.
     #
     def each(word)
       choices = 0
@@ -76,6 +102,9 @@ module Wordlist
     #
     # Inspects the mutator.
     #
+    # @return [String]
+    #   The inspected mutator.
+    #
     def inspect
       "#{@pattern.inspect} -> #{@substitute.inspect}"
     end