wordlist 0.1.0 → 0.1.1

data/.document

@@ -0,0 +1,3 @@
+-
+ChangeLog.md
+LICENSE.txt

data/.gitignore

@@ -0,0 +1,11 @@
+doc
+pkg
+tmp/*
+scripts/*_wordlist.txt
+.DS_Store
+.bundle
+.yardoc
+*.db
+*.log
+*.swp
+*~

data/.rspec

@@ -0,0 +1 @@
+--colour --format documentation

data/.yardopts

@@ -0,0 +1 @@
+--markup markdown --title 'Wordlist Documentation' --protected

data/{History.txt → ChangeLog.md}

@@ -1,4 +1,8 @@
-=== 0.1.0 / 2009-08-31
+### 0.1.1 / 2012-06-11
+
+* Default {Wordlist::Builders::Website#proxy} to `Spidr.proxy`.
+
+### 0.1.0 / 2009-08-31
 
 * Initial release:
   * Supports building word-lists from arbitrary text.

data/LICENSE.txt

@@ -0,0 +1,22 @@
+Wordlist - A Ruby library for generating and working with word-lists.
+
+Copyright (c) 2009-2012 Hal Brodigan
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+'Software'), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED 'AS IS', WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
+IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
+CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
+TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
+SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.md

@@ -0,0 +1,96 @@
+# Wordlist
+
+* [Source](https://github.com/sophsec/wordlist#readme)
+* [Issues](https://github.com/sophsec/wordlist/issues)
+* [Email](mailto:postmodern.mod3 at gmail.com)
+
+## Description
+
+A Ruby library for generating and working with word-lists. Wordlist allows
+one to efficiently generate unique word-lists from arbitrary text or
+other sources, such as website content. Wordlist can also quickly enumerate
+through words within an existing word-list, applying multiple mutation
+rules to each word in the list.
+
+## Features
+
+* Uses a bucket system of CRC32 hashes for efficient filtering of duplicate
+  words.
+* Can build wordlists containing multi-word phrases.
+* Can build wordlists containing phrases containing a minimum and maximum
+  number of words.
+* Supports adding mutation rules to a word-list, which are applied to
+  words as the list is enumerated.
+* Supports building word-lists from arbitrary text.
+* Supports custom word-list builders:
+  * Wordlist::Builders::Website: Build word-lists from website content.
+* Supports custom word-list formats:
+  * Wordlist::FlatFile: Enumerates through the words in a flat-file
+    word-list.
+
+## Examples
+
+Build a word-list from arbitrary text:
+
+    Wordlist::Builder.build('list.txt') do |builder|
+      builder.parse(some_text)
+    end
+
+Build a word-list from another file:
+
+    Wordlist::Builder.build('list.txt') do |builder|
+      builder.parse_file('some/file.txt')
+    end
+
+Build a word-list of phrases containing at most three words, from the
+arbitrary text:
+
+    Wordlist::Builder.build('list.txt', :max_words => 3) do |builder|
+      builder.parse(some_text)
+    end
+
+Build a word-list from content off a website:
+
+    require 'wordlist/builders/website'
+
+    Wordlist::Builders::Website.build(
+      'list.txt',
+      :host => 'www.example.com'
+    )
+
+Enumerate through each word in a flat-file word-list:
+
+    list = Wordlist::FlatFile.new('list.txt')
+    list.each_word do |word|
+      puts word
+    end
+
+Enumerate through each unique word in a flat-file word-list:
+
+    list.each_unique do |word|
+      puts word
+    end
+
+Define mutation rules, and enumerate through each unique mutation of each
+unique word in the word-list:
+
+    list.mutate 'o', '0'
+    list.mutate '@', 0x41
+    list.mutate(/[hax]/i) { |match| match.swapcase }
+
+    list.each_mutation do |word|
+      puts word
+    end
+
+## Requirements
+
+* [spidr](http://spidr.rubyforge.org) >= 0.1.9
+
+## Install
+
+    $ gem install wordlist
+
+## License
+
+See {file:LICENSE.txt} for license information.
+

data/Rakefile

@@ -1,22 +1,35 @@
-# -*- ruby -*-
-
 require 'rubygems'
-require 'hoe'
-require 'hoe/signing'
-require './tasks/spec.rb'
-require './lib/wordlist/version.rb'
+require 'rake'
+
+begin
+  gem 'rubygems-tasks', '~> 0.1'
+  require 'rubygems/tasks'
+
+  Gem::Tasks.new
+rescue LoadError => e
+  warn e.message
+  warn "Run `gem install rubygems-tasks` to install 'rubygems/tasks'."
+end
 
-Hoe.spec('wordlist') do
-  self.rubyforge_name = 'wordlist'
-  self.developer('Postmodern','postmodern.mod3@gmail.com')
-  self.remote_rdoc_dir = '/'
-  self.extra_deps = [
-    ['spidr', '>=0.1.9']
-  ]
+begin
+  gem 'rspec', '~> 2.4'
+  require 'rspec/core/rake_task'
 
-  self.extra_dev_deps = [
-    ['rspec', '>=1.1.12']
-  ]
+  RSpec::Core::RakeTask.new
+rescue LoadError => e
+  task :spec do
+    abort "Please run `gem install rspec` to install RSpec."
+  end
 end
+task :default => :spec
+
+begin
+  gem 'yard', '~> 0.8'
+  require 'yard'
 
-# vim: syntax=Ruby
+  YARD::Rake::YardocTask.new  
+rescue LoadError => e
+  task :yard do
+    abort "Please run `gem install yard` to install YARD."
+  end
+end

data/bin/wordlist

@@ -0,0 +1,10 @@
+#!/usr/bin/env ruby
+
+lib_dir = File.expand_path(File.join(File.dirname(__FILE__),'..','lib'))
+unless $LOAD_PATH.include?(lib_dir)
+  $LOAD_PATH << lib_dir
+end
+
+require 'wordlist/runners/list'
+
+Wordlist::Runners::List.run(*ARGV)

data/gemspec.yml

@@ -0,0 +1,22 @@
+name: wordlist
+summary: A Ruby library for generating and working with word-lists.
+description:
+  A Ruby library for generating and working with word-lists. Wordlist
+  allows one to efficiently generate unique word-lists from arbitrary text
+  or other sources, such as website content. Wordlist can also quickly
+  enumerate through words within an existing word-list, applying multiple
+  mutation rules to each word in the list.
+
+license: MIT
+authors: Postmodern
+email: postmodern.mod3@gmail.com
+homepage: https://github.com/sophsec/wordlist
+has_yard: true
+
+dependencies:
+  spidr: ~> 0.2
+
+development_dependencies:
+  rubygems-tasks: ~> 0.1
+  rspec: ~> 2.4
+  yard: ~> 0.8

data/lib/wordlist/builder.rb

@@ -9,29 +9,70 @@ module Wordlist
     # Path of the word-list
     attr_reader :path
 
+    # Minimum number of words
+    attr_reader :min_words
+
+    # Maximum number of words
+    attr_reader :max_words
+
     # File for the word-list
     attr_reader :file
 
+    # The unique word filter
+    attr_reader :filter
+
+    # The queue of words awaiting processing
+    attr_reader :word_queue
+
+    #
+    # Creates a new word-list Builder object.
+    #
+    # @param [String] path
+    #   The path of the word-list file.
     #
-    # Creates a new word-list Builder object with the specified _path_.
-    # If a _block_ is given, it will be passed the newly created
-    # Builder object.
+    # @param [Hash] options
+    #   Additional options.
     #
-    def initialize(path,&block)
+    # @option options [Integer] :min_words (1)
+    #   The minimum number of words each line of the word-list must contain.
+    #
+    # @option options [Integer] :max_words
+    #   The maximum number of words each line of the word-list may contain.
+    #   Defaults to the value of `:min_words`, if not given.
+    #
+    def initialize(path,options={})
       super()
 
       @path = File.expand_path(path)
-      @file = nil
-      @filter = nil
 
-      block.call(self) if block
+      @min_words = options.fetch(:min_words,1)
+      @max_words = options.fetch(:max_words,@min_words)
+
+      @file       = nil
+      @filter     = UniqueFilter.new
+      @word_queue = []
+
+      yield self if block_given?
     end
 
     #
-    # Creates a new Builder object with the given _arguments_, opens the
-    # word-list file, passes the builder object to the given _block_
+    # Creates a new Builder object with the given arguments, opens the
+    # word-list file, passes the builder object to the given block
     # then finally closes the word-list file.
     #
+    # @param [Array] arguments
+    #   Additional arguments to pass to {#initialize}.
+    #
+    # @yield [builder]
+    #   If a block is given, it will be passed the new builder.
+    #
+    # @yieldparam [Builder] builder
+    #   The newly created builer object.
+    #
+    # @return [Builder]
+    #   The newly created builder object.
+    #
+    # @example
     #   Builder.build('some/path') do |builder|
     #     builder.parse(readline)
     #   end
@@ -48,9 +89,10 @@ module Wordlist
     # Opens the word-list file for writing. If the file already exists, the
     # previous words will be used to filter future duplicate words.
     #
+    # @return [File]
+    #   The open word-list file.
+    #
     def open!
-      @filter = UniqueFilter.new
-
       if File.file?(@path)
         File.open(@path) do |file|
           file.each_line do |line|
@@ -63,21 +105,86 @@ module Wordlist
     end
 
     #
-    # Default to be called when the word-list is to be built, simply
-    # calls the given _block_.
+    # Default to be called when the word-list is to be built.
+    #
+    # @yield [builder]
+    #   If a block is given, it will be passed the new builder object.
+    #
+    def build!
+      yield self if block_given?
+    end
+
+    #
+    # Enqueues a given word for processing.
+    #
+    # @param [String] word
+    #   The word to enqueue.
+    #
+    # @return [String]
+    #   The enqueued word.
+    #
+    def enqueue(word)
+      # enqueue the word
+      if @max_words == 1
+        @word_queue[0] = word.to_s
+      else
+        @word_queue << word.to_s
+
+        # make sure the queue does not overflow
+        if @word_queue.length > @max_words
+          @word_queue.shift
+        end
+      end
+
+      return word
+    end
+
+    #
+    # Enumerates over the combinations of previously seen words.
+    #
+    # @yield [combination]
+    #   The given block will be passed the combinations of previously
+    #   seen words.
+    #
+    # @yieldparam [String] combination
+    #   A combination of one or more space-separated words.
     #
-    def build!(&block)
-      block.call(self) if block
+    def word_combinations
+      if @max_words == 1
+        yield @word_queue[0]
+      else
+        current_words = @word_queue.length
+
+        # we must have atleast the minimum amount of words
+        if current_words >= @min_words
+          upper_bound = (current_words - @min_words)
+
+          # combine the words
+          upper_bound.downto(0) do |i|
+            yield @word_queue[i..-1].join(' ')
+          end
+        end
+      end
     end
 
     #
-    # Appends the specified _word_ to the word-list file, only if it has not
+    # Appends the given word to the word-list file, only if it has not
     # been previously seen.
     #
+    # @param [String] word
+    #   The word to append.
+    #
+    # @return [Builder]
+    #   The builder object.
+    #
     def <<(word)
+      enqueue(word)
+
       if @file
-        @filter.pass(word) do |unique|
-          @file.puts unique
+        word_combinations do |words|
+          @filter.pass(words) do |unique|
+            @file.puts unique
+          end
         end
       end
 
@@ -85,7 +192,13 @@ module Wordlist
     end
 
     #
-    # Add the specified _words_ to the word-list.
+    # Add the given words to the word-list.
+    #
+    # @param [Array<String>] words
+    #   The words to add to the list.
+    #
+    # @return [Builder]
+    #   The builder object.
     #
     def +(words)
       words.each { |word| self << word }
@@ -93,16 +206,21 @@ module Wordlist
     end
 
     #
-    # Parses the specified _text_ adding each unique word to the word-list
-    # file.
+    # Parses the given text, adding each unique word to the word-list file.
+    #
+    # @param [String] text
+    #   The text to parse.
     #
     def parse(text)
       super(text).each { |word| self << word }
     end
 
     #
-    # Parses the contents of the file at the specified _path_, adding
-    # each unique word to the word-list file.
+    # Parses the contents of the file at the given path, adding each unique
+    # word to the word-list file.
+    #
+    # @param [String] path
+    #   The path of the file to parse.
     #
     def parse_file(path)
       File.open(path) do |file|
@@ -118,9 +236,10 @@ module Wordlist
     def close!
       if @file
         @file.close
-
         @file = nil
-        @filter = nil
+
+        @filter.clear
+        @word_queue.clear
       end
     end