wordlist 0.1.0

data/History.txt

@@ -0,0 +1,9 @@
+=== 0.1.0 / 2009-08-31
+
+* Initial release:
+  * Supports building word-lists from arbitrary text.
+  * Supports building word-lists from files.
+  * Supports building word-lists from websites.
+  * Supports enumerating through flat-file word-lists.
+  * Supports applying multiple mutation rules to each word in a word-list.
+

data/Manifest.txt

@@ -0,0 +1,30 @@
+History.txt
+Manifest.txt
+README.txt
+Rakefile
+lib/wordlist.rb
+lib/wordlist/unique_filter.rb
+lib/wordlist/parsers.rb
+lib/wordlist/builder.rb
+lib/wordlist/builders.rb
+lib/wordlist/builders/website.rb
+lib/wordlist/mutator.rb
+lib/wordlist/list.rb
+lib/wordlist/flat_file.rb
+lib/wordlist/version.rb
+tasks/spec.rb
+scripts/benchmark
+scripts/text/comedy_of_errors.txt
+spec/classes/parser_class.rb
+spec/classes/test_list.rb
+spec/text/previous_wordlist.txt
+spec/text/sample.txt
+spec/text/flat_file.txt
+spec/spec_helper.rb
+spec/unique_filter_spec.rb
+spec/parsers_spec.rb
+spec/mutator_spec.rb
+spec/builder_spec.rb
+spec/list_spec.rb
+spec/flat_file_spec.rb
+spec/wordlist_spec.rb

data/README.txt

@@ -0,0 +1,103 @@
+= Wordlist
+
+* http://wordlist.rubyforge.org/
+* http://github.com/sophsec/wordlist/
+* Postmodern (postmodern.mod3 at gmail.com)
+
+== DESCRIPTION:
+
+A Ruby library for generating and working with word-lists. Wordlist allows
+one to efficiently generate unique word-lists from arbitrary text or
+other sources, such as website content. Wordlist can also quickly enumerate
+through words within an existing word-list, applying multiple mutation
+rules to each word in the list.
+
+== FEATURES:
+
+* Uses a bucket system of CRC32 hashes for efficient filtering of duplicate
+  words.
+* Supports adding mutation rules to a word-list, which are applied to
+  words as the list is enumerated.
+* Supports building word-lists from arbitrary text.
+* Supports custom word-list builders:
+  * Wordlist::Builders::Website: Build word-lists from website content.
+* Supports custom word-list formats:
+  * Wordlist::FlatFile: Enumerates through the words in a flat-file
+    word-list.
+
+== EXAMPLES:
+
+* Build a word-list from arbitrary text:
+
+    Wordlist::Builder.build('list.txt') do |builder|
+      builder.parse(some_text)
+    end
+
+* Build a word-list from another file:
+
+    Wordlist::Builder.build('list.txt') do |builder|
+      builder.parse_file('some/file.txt')
+    end
+
+* Build a word-list from content off a website:
+
+    require 'wordlist/builders/website'
+
+    Wordlist::Builders::Website.build('list.txt','www.example.com')
+
+* Enumerate through each word in a flat-file word-list:
+
+    list = Wordlist::FlatFile.new('list.txt')
+    list.each_word do |word|
+      puts word
+    end
+
+* Enumerate through each unique word in a flat-file word-list:
+
+    list.each_unique do |word|
+      puts word
+    end
+
+* Define mutation rules, and enumerate through each unique mutation of each
+  unique word in the word-list:
+
+    list.mutate 'o', '0'
+    list.mutate 'a', 0x41
+    list.mutate(/[hax]/i) { |match| match.swapcase }
+
+    list.each_mutation do |word|
+      puts word
+    end
+
+== REQUIREMENTS:
+
+* {spidr}[http://spidr.rubyforge.org] >= 0.1.9
+
+== INSTALL:
+
+  $ sudo gem install wordlist
+
+== LICENSE:
+
+Wordlist - A Ruby library for generating and working with word-lists.
+
+Copyright (c) 2009 Hal Brodigan
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+'Software'), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED 'AS IS', WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
+IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
+CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
+TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
+SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/Rakefile

@@ -0,0 +1,22 @@
+# -*- ruby -*-
+
+require 'rubygems'
+require 'hoe'
+require 'hoe/signing'
+require './tasks/spec.rb'
+require './lib/wordlist/version.rb'
+
+Hoe.spec('wordlist') do
+  self.rubyforge_name = 'wordlist'
+  self.developer('Postmodern','postmodern.mod3@gmail.com')
+  self.remote_rdoc_dir = '/'
+  self.extra_deps = [
+    ['spidr', '>=0.1.9']
+  ]
+
+  self.extra_dev_deps = [
+    ['rspec', '>=1.1.12']
+  ]
+end
+
+# vim: syntax=Ruby

data/lib/wordlist.rb

@@ -0,0 +1,4 @@
+require 'wordlist/builder'
+require 'wordlist/list'
+require 'wordlist/flat_file'
+require 'wordlist/version'

data/lib/wordlist/builder.rb

@@ -0,0 +1,128 @@
+require 'wordlist/unique_filter'
+require 'wordlist/parsers'
+
+module Wordlist
+  class Builder
+
+    include Parsers
+
+    # Path of the word-list
+    attr_reader :path
+
+    # File for the word-list
+    attr_reader :file
+
+    #
+    # Creates a new word-list Builder object with the specified _path_.
+    # If a _block_ is given, it will be passed the newly created
+    # Builder object.
+    #
+    def initialize(path,&block)
+      super()
+
+      @path = File.expand_path(path)
+      @file = nil
+      @filter = nil
+
+      block.call(self) if block
+    end
+
+    #
+    # Creates a new Builder object with the given _arguments_, opens the
+    # word-list file, passes the builder object to the given _block_
+    # then finally closes the word-list file.
+    #
+    #   Builder.build('some/path') do |builder|
+    #     builder.parse(readline)
+    #   end
+    #
+    def self.build(*arguments,&block)
+      self.new(*arguments) do |builder|
+        builder.open!
+        builder.build!(&block)
+        builder.close!
+      end
+    end
+
+    #
+    # Opens the word-list file for writing. If the file already exists, the
+    # previous words will be used to filter future duplicate words.
+    #
+    def open!
+      @filter = UniqueFilter.new
+
+      if File.file?(@path)
+        File.open(@path) do |file|
+          file.each_line do |line|
+            @filter.saw!(line.chomp)
+          end
+        end
+      end
+
+      @file = File.new(@path,File::RDWR | File::CREAT | File::APPEND)
+    end
+
+    #
+    # Default to be called when the word-list is to be built, simply
+    # calls the given _block_.
+    #
+    def build!(&block)
+      block.call(self) if block
+    end
+
+    #
+    # Appends the specified _word_ to the word-list file, only if it has not
+    # been previously seen.
+    #
+    def <<(word)
+      if @file
+        @filter.pass(word) do |unique|
+          @file.puts unique
+        end
+      end
+
+      return self
+    end
+
+    #
+    # Add the specified _words_ to the word-list.
+    #
+    def +(words)
+      words.each { |word| self << word }
+      return self
+    end
+
+    #
+    # Parses the specified _text_ adding each unique word to the word-list
+    # file.
+    #
+    def parse(text)
+      super(text).each { |word| self << word }
+    end
+
+    #
+    # Parses the contents of the file at the specified _path_, adding
+    # each unique word to the word-list file.
+    #
+    def parse_file(path)
+      File.open(path) do |file|
+        file.each_line do |line|
+          parse(line)
+        end
+      end
+    end
+
+    #
+    # Closes the word-list file.
+    #
+    def close!
+      if @file
+        @file.close
+
+        @file = nil
+        @filter = nil
+      end
+    end
+
+  end
+end

data/lib/wordlist/builders.rb

@@ -0,0 +1 @@
+require 'wordlist/builders/website'

data/lib/wordlist/builders/website.rb

@@ -0,0 +1,44 @@
+require 'wordlist/builder'
+
+require 'spidr'
+
+module Wordlist
+  module Builders
+    class Website < Builder
+
+      # Host to spider
+      attr_accessor :host
+
+      #
+      # Creates a new Website builder object with the specified _path_
+      # and _host_. If a _block_ is given, it will be passed the new created
+      # Website builder object.
+      #
+      def initialize(path,host,&block)
+        @host = host
+
+        super(path,&block)
+      end
+
+      #
+      # Builds the word-list file by spidering the +host+ and parsing the
+      # inner-text from all HTML pages. If a _block_ is given, it will be
+      # called before all HTML pages on the +host+ have been parsed.
+      #
+      def build!(&block)
+        super(&block)
+
+        Spidr.host(@host) do |spidr|
+          spidr.every_page do |page|
+            if page.html?
+              page.doc.search('//h1|//h2|//h3|//h4|//h5|//p|//span').each do |element|
+                parse(element.inner_text)
+              end
+            end
+          end
+        end
+      end
+
+    end
+  end
+end

data/lib/wordlist/flat_file.rb

@@ -0,0 +1,36 @@
+require 'wordlist/list'
+
+module Wordlist
+  class FlatFile < List
+
+    # The path to the flat-file
+    attr_accessor :path
+
+    #
+    # Creates a new FlatFile list with the specified _path_ and given
+    # _options_.
+    #
+    def initialize(path,options={},&block)
+      @path = path
+
+      super(options,&block)
+    end
+
+    #
+    # Enumerates through every word in the flat-file, passing each
+    # word to the given _block_.
+    #
+    #   flat_file.each_word do |word|
+    #     puts word
+    #   end
+    #
+    def each_word(&block)
+      File.open(@path) do |file|
+        file.each_line do |line|
+          yield line.chomp
+        end
+      end
+    end
+
+  end
+end

data/lib/wordlist/list.rb

@@ -0,0 +1,131 @@
+require 'wordlist/unique_filter'
+require 'wordlist/mutator'
+
+module Wordlist
+  class List
+
+    include Enumerable
+
+    # Maximum length of words
+    attr_accessor :max_length
+
+    # Minimum length of words
+    attr_accessor :min_length
+
+    #
+    # Creates a new List object with the given _options_. If a _block_
+    # is given, it will be passed the newly created List object.
+    #
+    # _options_ may include the following keys:
+    # <tt>:max_length</tt>:: The maximum length of words produced by the
+    #                        list.
+    # <tt>:min_length</tt>:: The minimum length of words produced by the
+    #                        list.
+    #
+    def initialize(options={},&block)
+      @mutators = []
+
+      @max_length = nil
+      @min_length = 0
+
+      if options[:max_length]
+        @max_length = options[:max_length]
+      end
+
+      if options[:min_length]
+        @min_length = options[:min_length]
+      end
+
+      block.call(self) if block
+    end
+
+    #
+    # Adds a mutation rule for the specified _pattern_, to be replaced
+    # using the specified _substitute_. If a _block_ is given, and the
+    # _substitute_ data omitted, then the _block_ will be used to
+    # replace data matched by the _pattern_.
+    #
+    #   list.mutate 'o', '0'
+    #
+    #   list.mutate '0', 0x41
+    #
+    #   list.mutate(/[oO]/) do |match|
+    #     match.swapcase
+    #   end
+    #
+    def mutate(pattern,substitute=nil,&block)
+      @mutators << Mutator.new(pattern,substitute,&block)
+    end
+
+    #
+    # Enumerate through every word in the list, passing each word to
+    # the given block. By default this method passes nothing to the given
+    # _block_.
+    #
+    #   list.each_word do |word|
+    #     puts word
+    #   end
+    #
+    def each_word(&block)
+    end
+
+    #
+    # Enumerates through every unique word in the list, passing each
+    # unique word to the given block.
+    #
+    #   list.each_unique do |word|
+    #     puts word
+    #   end
+    #
+    def each_unique
+      unique_filter = UniqueFilter.new()
+
+      each_word do |word|
+        if unique_filter.saw!(word)
+          yield word
+        end
+      end
+
+      unique_filter = nil
+    end
+
+    #
+    # Enumerates through every unique mutation, of every unique word, using
+    # the mutator rules define for the list. Every possible unique mutation
+    # will be passed to the given _block_.
+    #
+    #   list.each_mutation do |word|
+    #     puts word
+    #   end
+    #
+    def each_mutation(&block)
+      mutation_filter = UniqueFilter.new()
+
+      mutator_stack = [lambda { |mutated_word|
+        # skip words shorter than the minimum length
+        next if mutated_word.length < @min_length
+
+        # truncate words longer than the maximum length
+        mutated_word = mutated_word[0,@max_length] if @max_length
+
+        if mutation_filter.saw!(mutated_word)
+          yield mutated_word
+        end
+      }]
+
+      (@mutators.length-1).downto(0) do |index|
+        mutator_stack.unshift(lambda { |word|
+          prev_mutator = @mutators[index]
+          next_mutator = mutator_stack[index+1]
+
+          prev_mutator.each(word,&next_mutator)
+        })
+      end
+
+      each_unique(&(mutator_stack.first))
+    end
+
+    alias each each_mutation
+
+  end
+end